turbot-runner 0.1.23 → 0.1.24

data/schema/schemas/share-parcel-schema.json

@@ -1,21 +1,22 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Share Parcel Schema",
   "type": "object",
-  "allOf" : [
-    // The following is a basic statement with sample_date, etc
-    { "$ref": "includes/base-statement.json" },
-    // And this overrides it to provide data-type-specific information
+  "allOf": [
+    {
+      "$ref": "includes/base-statement.json"
+    },
     {
       "properties": {
         "data": {
           "items": {
             "allOf": [
-              { "$ref": "includes/share-parcel-data.json" }
+              {
+                "$ref": "includes/share-parcel-data.json"
+              }
             ]
           }
         }
       }
     }
   ]
-}
+}

data/schema/schemas/simple-financial-payment-schema.json

@@ -1,6 +1,5 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Simple Financial Payment",
   "description": "A Financial Payment is a payment from government to a recipient",
   "type": "object",
   "properties": {
@@ -16,7 +15,11 @@
     "confidence": {
       "description": "Confidence in accuracy of data",
       "type": "string",
-      "enum": ["HIGH", "MEDIUM", "LOW"]
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
     },
     "company_name": {
       "description": "Name of the company that received the money",
@@ -52,19 +55,32 @@
     },
     "more_details_url": {
       "description": "A url from which more details can be seen (may be the same as the source_url)",
-      "type": ["string",null]
+      "type": [
+        "string",
+        null
+      ]
     },
     "description": {
       "description": "The description of the transaction as given in the raw data",
-      "type": ["string",null]
+      "type": [
+        "string",
+        null
+      ]
     },
     "expense_type": {
       "description": "The type of expense -- can be either capital, revenue (i.e. current expenditure) or null",
-      "enum": ["capital","revenue",null]
+      "enum": [
+        "capital",
+        "revenue",
+        null
+      ]
     },
     "expense_area": {
       "description": "category (in words) of the expenditure",
-      "type": ["string",null]
+      "type": [
+        "string",
+        null
+      ]
     },
     "entity_name": {
       "description": "The name of the government entity that made the payment, e.g. Environment Agency",
@@ -72,15 +88,35 @@
     },
     "entity_uri": {
       "description": "A unique URL (ideally a dereferencable URI) for the government entity",
-      "type": ["string",null]
+      "type": [
+        "string",
+        null
+      ]
     },
     "department_name": {
       "description": "The name of the government department which the entity belongs to (if relevant), e.g. Department of Health",
-      "type": ["string",null]
+      "type": [
+        "string",
+        null
+      ]
     },
     "csv_line_number": {
       "description": "If the source for the data is a CSV file, you can optionally include the line number of the CSV from which this data was retrieved",
-      "type": ["string",null]}
+      "type": [
+        "string",
+        null
+      ]
+    }
   },
-  "required": ["source_url", "sample_date", "company_name", "company_jurisdiction", "value", "payee_name", "date", "currency"]
-}
+  "additionalProperties": false,
+  "required": [
+    "source_url",
+    "sample_date",
+    "company_name",
+    "company_jurisdiction",
+    "value",
+    "payee_name",
+    "date",
+    "currency"
+  ]
+}

data/schema/schemas/simple-licence-schema.json

@@ -1,6 +1,5 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Simple Licence Schema",
   "type": "object",
   "properties": {
     "source_url": {
@@ -16,7 +15,11 @@
     "confidence": {
       "description": "Confidence in accuracy of data",
       "type": "string",
-      "enum": ["HIGH", "MEDIUM", "LOW"]
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
     },
     "company_name": {
       "description": "Name of the company holding the licence",
@@ -34,7 +37,10 @@
     },
     "jurisdiction_classification": {
       "description": "Description of how regulator classifies licence",
-      "type": ["string", "array"]
+      "type": [
+        "string",
+        "array"
+      ]
     },
     "regulator": {
       "description": "The regulating body that issued the licence",
@@ -52,8 +58,17 @@
     "category": {
       "description": "Category of licence",
       "type": "string",
-      "enum": ["Financial", "Business"]
+      "enum": [
+        "Financial",
+        "Business"
+      ]
     }
   },
-  "required": ["source_url", "sample_date", "company_name", "company_jurisdiction"]
-}
+  "additionalProperties": false,
+  "required": [
+    "source_url",
+    "sample_date",
+    "company_name",
+    "company_jurisdiction"
+  ]
+}

data/schema/schemas/simple-subsidiary-schema.json

@@ -1,6 +1,5 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Simple Subsidiary Schema",
   "type": "object",
   "properties": {
     "source_url": {
@@ -26,7 +25,11 @@
     "confidence": {
       "description": "Confidence in accuracy of data",
       "type": "string",
-      "enum": ["HIGH", "MEDIUM", "LOW"]
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
     },
     "parent_name": {
       "description": "Name of the controlling company",
@@ -66,5 +69,13 @@
       "type": "string"
     }
   },
-  "required": ["source_url", "sample_date", "parent_name", "parent_jurisdiction", "subsidiary_name", "subsidiary_jurisdiction"]
-}
+  "additionalProperties": false,
+  "required": [
+    "source_url",
+    "sample_date",
+    "parent_name",
+    "parent_jurisdiction",
+    "subsidiary_name",
+    "subsidiary_jurisdiction"
+  ]
+}

data/schema/schemas/subsidiary-relationship-schema.json

@@ -1,12 +1,26 @@
 {
-  "title": "Subsidiary Relationship",
-  "description": "A relationship of control between two companies",
   "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "A relationship of control between two companies",
   "type": "object",
   "properties": {
-    "sample_date": {"type": "string", "format": "date"},
-    "start_date": {"type": "string", "format": "date"},
-    "end_date": {"type": "string", "format": "date"},
+    "sample_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "start_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "start_date_type": {
+      "type": "string"
+    },
+    "end_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "end_date_type": {
+      "type": "string"
+    },
     "source_jurisdiction": {
       "description": "Jurisdiction of the source of the data",
       "type": "string"
@@ -23,5 +37,10 @@
       "additionalItems": false
     }
   },
-  "required": ["company", "data", "sample_date"]
-}
+  "additionalProperties": false,
+  "required": [
+    "company",
+    "data",
+    "sample_date"
+  ]
+}

data/spec/lib/processor_spec.rb

@@ -134,7 +134,7 @@ describe TurbotRunner::Processor do
             'number' => 123
           }
 
-          expected_error = 'Missing required property: sample_date'
+          expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
           expect(@handler).to receive(:handle_invalid_record).
             with(record, @data_type, expected_error)
           @processor.process(record.to_json)
@@ -149,7 +149,7 @@ describe TurbotRunner::Processor do
             'number' => 123
           }
 
-          expected_error = 'Property not a valid date: sample_date'
+          expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
           expect(@handler).to receive(:handle_invalid_record).
             with(record, @data_type, expected_error)
           @processor.process(record.to_json)
@@ -160,7 +160,7 @@ describe TurbotRunner::Processor do
     it 'can handle schemas with $refs' do
       handler = TurbotRunner::BaseHandler.new
       script_config = {
-        :data_type => 'rich-licence',
+        :data_type => 'licence',
         :identifying_fields => ['licence_number']
       }
 
@@ -188,76 +188,4 @@ describe TurbotRunner::Processor do
       processor.process(record.to_json)
     end
   end
-
-  describe '#convert_record' do
-    before do
-      schema = {
-        '$schema' => 'http://json-schema.org/draft-04/schema#',
-        'type' => 'object',
-        'properties' => {
-          'aaa' => {'format' => 'date'},
-          'bbb' => {'format' => 'not-date'},
-        }
-      }
-
-      @processor = TurbotRunner::Processor.new(nil, {}, nil)
-      allow(@processor).to receive(:schema).and_return(schema)
-    end
-
-    context 'when date field is YYYY-MM-DD' do
-      it 'leaves date field alone' do
-        record = {'aaa' => '2015-01-26', 'bbb' => 'cabbage'}
-        expect(@processor.convert_record(record)).to eq({'aaa' => '2015-01-26', 'bbb' => 'cabbage'})
-      end
-    end
-
-    context 'when date field with YYYY-MM-DD HH:MM:SS' do
-      it 'replaces value with YYYY-MM-DD' do
-        record = {'aaa' => '2015-01-26 12:34:56', 'bbb' => 'cabbage'}
-        expect(@processor.convert_record(record)).to eq({'aaa' => '2015-01-26', 'bbb' => 'cabbage'})
-      end
-    end
-
-    context 'when date field is empty string' do
-      it 'replaces removes field' do
-        record = {'aaa' => '', 'bbb' => 'cabbage'}
-        expect(@processor.convert_record(record)).to eq({'bbb' => 'cabbage'})
-      end
-    end
-
-    context 'when date field is invalid date' do
-      it 'rasies ConversionError' do
-        record = {'aaa' => 'cabbage', 'bbb' => 'cabbage'}
-        expect{@processor.convert_record(record)}.to raise_error(TurbotRunner::Processor::ConversionError)
-      end
-    end
-  end
-
-  specify '#get_date_paths' do
-    schema = {
-      '$schema' => 'http://json-schema.org/draft-04/schema#',
-      'type' => 'object',
-      'properties' => {
-        'aaa' => {'format' => 'date'},
-        'bbb' => {'format' => 'not-date'},
-        'ccc' => {
-          'type' => 'object',
-          'properties' => {
-            'ddd' => {'format' => 'date'},
-            'eee' => {'format' => 'not-date'},
-            'fff' => {
-              'type' => 'object',
-              'properties' => {
-                'ggg' => {'format' => 'date'},
-                'hhh' => {'format' => 'not-date'},
-              }
-            }
-          }
-        }
-      }
-    }
-
-    processor = TurbotRunner::Processor.new(nil, {}, nil)
-    expect(processor.get_date_paths(schema['properties'])).to eq([['aaa'], ['ccc', 'ddd'], ['ccc', 'fff', 'ggg']])
-  end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.23
+  version: 0.1.24
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-29 00:00:00.000000000 Z
+date: 2015-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: json-schema
+  name: openc-json_schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 0.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 0.0.5
 description: 
 email: bots@opencorporates.com
 executables: []
@@ -39,8 +39,6 @@ files:
 - lib/turbot_runner/runner.rb
 - lib/turbot_runner/script_runner.rb
 - lib/turbot_runner/utils.rb
-- lib/turbot_runner/validator.rb
-- lib/turbot_runner/validators.rb
 - lib/turbot_runner/version.rb
 - schema/schemas/company-schema.json
 - schema/schemas/financial-payment-schema.json
@@ -66,7 +64,6 @@ files:
 - schema/schemas/includes/unknown_entity_type.json
 - schema/schemas/licence-schema.json
 - schema/schemas/primary-data-schema.json
-- schema/schemas/rich-licence-schema.json
 - schema/schemas/share-parcel-schema.json
 - schema/schemas/simple-financial-payment-schema.json
 - schema/schemas/simple-licence-schema.json
@@ -112,8 +109,6 @@ files:
 - spec/bots/slow-bot/scraper.rb
 - spec/lib/processor_spec.rb
 - spec/lib/runner_spec.rb
-- spec/lib/validator_spec.rb
-- spec/lib/validators_spec.rb
 - spec/manual_spec.rb
 - spec/outputs/full-scraper.out
 - spec/outputs/full-transformer.out

data/lib/turbot_runner/validator.rb

@@ -1,77 +0,0 @@
-require 'json-schema'
-
-module TurbotRunner
-  module Validator
-    extend self
-
-    def validate(schema, record)
-      # We must change directory for the relative paths in schemas to make sense.
-      errors = Dir.chdir(SCHEMAS_PATH) do
-        JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
-      end
-
-      # For now, we just handle the first error.
-      error = errors[0]
-      return if error.nil?
-
-      case error[:failed_attribute]
-      when 'Required'
-        match = error[:message].match(/required property of '(.*)'/)
-        missing_property = match[1]
-        path = fragment_to_path("#{error[:fragment]}/#{missing_property}")
-
-        {:type => :missing, :path => path}
-      when 'OneOf'
-        if error[:message].match(/did not match any/)
-          path_elements = fragment_to_path(error[:fragment]).split('.')
-
-          raise "Deeply nested OneOf error at: #{error[:fragment]}" unless path_elements.size == 1
-
-          record_fragment = record[path_elements[0]]
-          schema_fragments = schema['properties'][path_elements[0]]['oneOf']
-
-          schema_fragments.each do |s|
-            s['properties'].each do |k, v|
-              next if v['enum'].nil?
-
-              if v['enum'].include?(record_fragment[k])
-                error1 = validate(s, record_fragment)
-                return error1.merge(:path => "#{path_elements[0]}.#{error1[:path]}")
-              end
-            end
-          end
-
-          {:type => :one_of_no_matches, :path => fragment_to_path(error[:fragment])}
-        else
-          {:type => :one_of_many_matches, :path => fragment_to_path(error[:fragment])}
-        end
-      when 'MinLength'
-        match = error[:message].match(/minimum string length of (\d+) in/)
-        min_length = match[1].to_i
-        {:type => :too_short, :path => fragment_to_path(error[:fragment]), :length => min_length}
-      when 'MaxLength'
-        match = error[:message].match(/maximum string length of (\d+) in/)
-        max_length = match[1].to_i
-        {:type => :too_long, :path => fragment_to_path(error[:fragment]), :length => max_length}
-      when 'TypeV4'
-        match = error[:message].match(/the following types?: ([\w\s,]+) in schema/)
-        allowed_types = match[1].split(',').map(&:strip)
-        {:type => :type_mismatch, :path => fragment_to_path(error[:fragment]), :allowed_types => allowed_types}
-      when 'Enum'
-        match = error[:message].match(/the following values: ([\w\s,]+) in schema/)
-        allowed_values = match[1].split(',').map(&:strip)
-        {:type => :enum_mismatch, :path => fragment_to_path(error[:fragment]), :allowed_values => allowed_values}
-      else
-        if error[:message].match(/must be of format yyyy-mm-dd/)
-          {:type => :format_mismatch, :path => fragment_to_path(error[:fragment]), :expected_format => 'yyyy-mm-dd'}
-        else
-          {:type => :unknown, :path => fragment_to_path(error[:fragment]), :failed_attribute => error[:failed_attribute], :message => error[:message]}
-        end
-      end
-    end
-
-    def fragment_to_path(fragment)
-      fragment.sub(/^#?\/*/, '').gsub('/', '.')
-    end
-  end
-end