turbot-runner 0.1.23 → 0.1.24

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDA3ZmUyYTExMzFhNjIxOGE1MjkwMDgzNWEwNjM2MzQzY2UwYWJlNw==
+    MzkxMDdjODdjYjg3NDFkZTVjZjJmMzI4M2RhZmQ1NWI5OTBmNzFiOA==
   data.tar.gz: !binary |-
-    Y2EwNGZmNTE2YzY2NjAwOWZjMzYxMGNiYzg5ZDdmOTA2NzQ5MDZhYw==
+    ZmU3ODUzNmMxODg5NjZlMDhjZDMzYzk0MTY4YzA4ODVlZjEzZTVlOA==
 SHA512:
   metadata.gz: !binary |-
-    ODg4NGFlZTRmY2MwNjJhZDNiMjM3YWIwMGMyMTcyMWZmYjcyZjJhNDY5MTg1
-    MGUzNzE3M2RjZGMyZThjYTQ0ZTBkZmJkNzc4N2ZjYzY3ZWRhOWE0YzAwZDYy
-    MGUxNjU4MmY4ZGMxM2RjNzE1OTNmZmNiM2Y3YTM5MGQ2M2EwYTc=
+    Mzc3NTg0ZTY2ZWM5YmU4MWM4ODM0NmRkZjdhNDk2NWE4YjYxMDllMjJmOTlm
+    MmQwMzEwMGJkMThiMDFlYzFlYjg0ODQyMjkzMDBlMzgwMWM4YmM3MTI1NjA4
+    OGMxMjExNjIxNDI2N2FmZjYyODU2Y2IwMDEzMTkwZWJjNzVlODM=
   data.tar.gz: !binary |-
-    MWYxZjZkYjA4MGU4ZTk2MzU2MDg4ZWFhMDQ0ZjA2OGJkODlmNDA1OWMxYmQy
-    NTcxYjQyYTBmZWI2NGY3YTYyOWNkMDRlNjc1MzAyMTY5MWJhYmE0OGY2Y2E3
-    ODE2MDU1MDI5Nzc1ZDQ3OGIxM2JlYTYwNWZmZTk3ZDcyOTllNGE=
+    MDk3NzFhODNlZDA3NGUyMmZlMTBmYzEzM2UyNjFmMDc5NzJjYTJkMjQ0NGRh
+    M2U3MDlmOGYwNGI4Y2U3YzNkZDEwODk4NzA4ZDY1Y2NhNGJlYWE1MzkyNDYw
+    M2MyYTNkNjI0YzEwMTMyOTdiMzMwZmU1NTQ2Njk0YTQ5NjZkMGM=

data/lib/turbot_runner.rb

@@ -4,8 +4,6 @@ require 'turbot_runner/processor'
 require 'turbot_runner/runner'
 require 'turbot_runner/script_runner'
 require 'turbot_runner/utils'
-require 'turbot_runner/validator'
-require 'turbot_runner/validators'
 require 'turbot_runner/version'
 
 module TurbotRunner

data/lib/turbot_runner/processor.rb

@@ -1,5 +1,4 @@
-require 'json'
-require 'json-schema'
+require 'openc/json_schema'
 
 module TurbotRunner
   class Processor
@@ -16,18 +15,13 @@ module TurbotRunner
           @record_handler.handle_run_ended
           @runner.interrupt if @runner
         else
-          record = JSON.parse(line)
+          record = Openc::JsonSchema.convert_dates(schema_path, JSON.parse(line))
 
-          begin
-            converted_record = convert_record(record)
-            error_message = validate(converted_record)
-          rescue ConversionError => e
-            error_message = e.message
-          end
+          error_message = validate(record)
 
           if error_message.nil?
             begin
-              @record_handler.handle_valid_record(converted_record, @data_type)
+              @record_handler.handle_valid_record(record, @data_type)
             rescue InterruptRun
               @runner.interrupt if @runner
             end
@@ -46,60 +40,8 @@ module TurbotRunner
       @runner.interrupt
     end
 
-    def convert_record(record)
-      converted_record = Utils.deep_copy(record)
-
-      date_paths.each do |path|
-        begin
-          tmp = converted_record
-
-          path[0...-1].each do |path_item|
-            tmp = tmp[path_item]
-          end
-
-          value = tmp[path[-1]]
-        rescue NoMethodError
-          next
-        end
-
-        next unless value.is_a?(String)
-
-        if value == ''
-          tmp.delete(path[-1])
-        else
-          begin
-            tmp[path[-1]] = Date.strptime(value, '%Y-%m-%d').strftime('%Y-%m-%d')
-          rescue ArgumentError
-            raise ConversionError.new("Property not a valid date: #{path.join('.')}")
-          end
-        end
-      end
-
-      converted_record
-    end
-
-    def date_paths
-      @date_paths ||= get_date_paths(schema['properties'])
-    end
-
-    def get_date_paths(properties)
-      date_paths = []
-
-      properties.each do |name, attrs|
-        if attrs['format'] == 'date'
-          date_paths << [name]
-        elsif attrs['type'] == 'object'
-          get_date_paths(attrs['properties']).each do |path|
-            date_paths << [name] + path
-          end
-        end
-      end
-
-      date_paths
-    end
-
     def validate(record)
-      error = Validator.validate(schema, record)
+      error = Openc::JsonSchema.validate(schema_path, record)
 
       message = nil
 
@@ -137,14 +79,9 @@ module TurbotRunner
       message
     end
 
-    def schema
-      @schema ||= load_schema
-    end
-
-    def load_schema
+    def schema_path
       hyphenated_name = @data_type.to_s.gsub("_", "-").gsub(" ", "-")
-      path = File.join(SCHEMAS_PATH, "#{hyphenated_name}-schema.json")
-      JSON.load(File.read(path))
+      File.join(SCHEMAS_PATH, "#{hyphenated_name}-schema.json")
     end
 
     class ConversionError < StandardError; end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.23'
+  VERSION = '0.1.24'
 end

data/schema/schemas/company-schema.json

@@ -1,9 +1,7 @@
-
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Company Schema",
-  "type": "object",
   "description": "A company in OpenCorporates",
+  "type": "object",
   "properties": {
     "company_number": {
       "type": "string",
@@ -22,13 +20,16 @@
       "maxLength": 5
     },
     "incorporation_date": {
-      "type": "date"
+      "type": "string",
+      "format": "date"
     },
     "dissolution_date": {
-      "type": "date"
+      "type": "string",
+      "format": "date"
     },
     "retrieved_at": {
-      "type": "date-time"
+      "type": "string",
+      "format": "date-time"
     },
     "current_status": {
       "type": "string"
@@ -100,43 +101,71 @@
       }
     },
     "branch": {
-      "type": ["string","null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "description": "A flag to denote whether a company is a branch entity. This should only be set if the company is a type of branch (otherwise should be null). In general the only option here is 'F' for a 'Foreign' branch, i.e. an out-of-jurisdiction entity that has registered as having a presence in the jurisdiction. In the US this is sometimes called a Foreign Corporation",
-      "enum": ["F", "L",null]
+      "enum": [
+        "F",
+        "L",
+        null
+      ]
     },
     "all_attributes": {
       "type": "object",
       "description": "Other arbitrary attributes for a given company",
       "properties": {
         "jurisdiction_of_origin": {
-          "type": ["string","null"],
+          "type": [
+            "string",
+            "null"
+          ],
           "description": "The jurisdiction of the 'home' company if this is a branch",
           "minLength": 1
         },
         "home_company_number": {
-          "type": ["string","null"],
+          "type": [
+            "string",
+            "null"
+          ],
           "description": "If the entity is a 'branch', this is the company_number of the 'home' company in the home company's jurisdiction",
           "minLength": 1
         },
         "home_legal_name": {
-          "type": ["string","null"],
+          "type": [
+            "string",
+            "null"
+          ],
           "description": "The legal name of the 'home' company in its jurisdiction if this is a branch, and the name is different from the legal name of the branch",
           "minLength": 1
         },
         "registered_agent_address": {
-          "type": ["string","null"],
+          "type": [
+            "string",
+            "null"
+          ],
           "description": "The address of the 'Agent', a public address to which legal papers can be served",
           "minLength": 1
         },
         "registered_agent_name": {
-          "type": ["string","null"],
-          "description": "The 'Agent' of the company \u2013 a person or entity that is empowered to accept service for the company",
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "The 'Agent' of the company – a person or entity that is empowered to accept service for the company",
           "minLength": 1
         },
         "number_of_employees": {
           "anyOf": [
-            {"type":"string","minLength": 1},
-            {"type":"number","minimum": 0}
+            {
+              "type": "string",
+              "minLength": 1
+            },
+            {
+              "type": "number",
+              "minimum": 0
+            }
           ],
           "description": "The number of employees"
         },
@@ -145,7 +174,9 @@
           "description": "Information on the merger of this company into a 'survivor' company",
           "properties": {
             "surviving_company": {
-              "type": ["object"],
+              "type": [
+                "object"
+              ],
               "description": "the surviving_company that this has been merged into",
               "properties": {
                 "name": {
@@ -164,8 +195,18 @@
                   "minLength": 2
                 }
               },
-              "anyOf": [{"required": ["name"]},
-                        {"required": ["company_number"]}]
+              "anyOf": [
+                {
+                  "required": [
+                    "name"
+                  ]
+                },
+                {
+                  "required": [
+                    "company_number"
+                  ]
+                }
+              ]
             },
             "effective_date": {
               "type": "string",
@@ -174,11 +215,17 @@
             }
           },
           "additionalProperties": false,
-          "required": ["surviving_company"]
+          "required": [
+            "surviving_company"
+          ]
         }
       }
     }
   },
   "additionalProperties": false,
-  "required": ["company_number", "name", "jurisdiction_code"]
-}
+  "required": [
+    "company_number",
+    "name",
+    "jurisdiction_code"
+  ]
+}

data/schema/schemas/financial-payment-schema.json

@@ -1,10 +1,12 @@
 {
-  "title": "Financial-Payment",
-  "description": "A Financial-Payment is a payment from government to a recipient",
   "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "A Financial-Payment is a payment from government to a recipient",
   "type": "object",
   "properties": {
-    "sample_date": {"type": "string", "format": "date"},
+    "sample_date": {
+      "type": "string",
+      "format": "date"
+    },
     "source_jurisdiction": {
       "description": "Jurisdiction of the source of the data",
       "type": "string"
@@ -21,5 +23,10 @@
       "additionalItems": false
     }
   },
-  "required": ["company", "data", "sample_date"]
-}
+  "additionalProperties": false,
+  "required": [
+    "company",
+    "data",
+    "sample_date"
+  ]
+}

data/schema/schemas/includes/address.json

@@ -1,23 +1,46 @@
-{ "oneOf":
-  [
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "oneOf": [
     {
-      "type": ["string","null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "minLength": 2
     },
     {
       "name": "Address",
       "description": "An address object",
       "type": "object",
-      "properties": { "street_address": { "type": "string" },
-                      "locality": {"type":"string"},
-                      "region":   { "type": "string" },
-                      "postal_code":   { "type": "string" },
-                      "country":   { "type": "string" }
+      "properties": {
+        "street_address": {
+          "type": "string"
+        },
+        "locality": {
+          "type": "string"
+        },
+        "region": {
+          "type": "string"
+        },
+        "postal_code": {
+          "type": "string"
+        },
+        "country": {
+          "type": "string"
+        }
       },
-      "anyOf": [ { "required": ["street_address"] },
-                 { "required": ["postal_code"] }
-               ]
+      "anyOf": [
+        {
+          "required": [
+            "street_address"
+          ]
+        },
+        {
+          "required": [
+            "postal_code"
+          ]
+        }
+      ]
     }
   ]
-}
-
+}

data/schema/schemas/includes/alternative_name.json

@@ -1,13 +1,36 @@
 {
-  "name": "AlternativeName",
+  "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "An alternative name of a company, e.g. abbreviation, trading (including dba or doing business as), legal. As well as trading names etc, it can be used for storing alternative language representations of the legal name, in which case the language should be represented as two-letter ISO-639 code",
   "type": "object",
-  "properties": { "company_name": { "type": "string", "minLength": 1 },
-                  "start_date": {"type":"string","format": "date"},
-                  "end_date": {"type":"string","format": "date"},
-                  "language": { "type": "string", "minLength": 2, "maxLength": 2 },
-                  "type": { "type": "string", "enum": ["trading", "abbreviation", "legal"] }
+  "properties": {
+    "company_name": {
+      "type": "string",
+      "minLength": 1
+    },
+    "start_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "end_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "language": {
+      "type": "string",
+      "minLength": 2,
+      "maxLength": 2
+    },
+    "type": {
+      "type": "string",
+      "enum": [
+        "trading",
+        "abbreviation",
+        "legal"
+      ]
+    }
   },
-  "required": [ "company_name", "type" ]
-}
-
+  "required": [
+    "company_name",
+    "type"
+  ]
+}