turbot-runner 0.1.20 → 0.1.21

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OTBiNjljYWU5MzdhMzk4ZjJiMGI4MmNkMzJiZTU4MjIwYjIyNGZhZg==
+  data.tar.gz: !binary |-
+    ODEzZmE5ZjBlMDQ0MzY5MWQyODlkN2RjZDU0Y2UyODI3ODVmOWZkYw==
+SHA512:
+  metadata.gz: !binary |-
+    ZGQwMzJhZWY1NzUwOGYyZDQ1YjEyZjEyY2M1OGQ5YzY2YmU1ZjhlZjZlN2Y5
+    YmNjN2FiYTFkZWUxNzUyMDQ0MDIyYzA2OTI1MzEyNzA3ZGNjM2ViMDY4NGQ3
+    MWExM2JiMzkyOWE3MzExODMyNDMyNzYxMTEzNGU1YjNmYTcyODU=
+  data.tar.gz: !binary |-
+    Y2U0NGY0MDFmMzk4MTNhMWYxYjQwMzJhYmVlMjg5MGVkNDRhYWQ0NDBkZGQy
+    MzlkMmQyYjY0NTFkMjRjOGQ0MjFlN2Y0NDkxZWVkMjRiNjcyYjg5ODZiZGNi
+    MzZmNDQxOWM2YTg0YTI5NjQ5MGE0NzAwZWFiNmVmY2I5NWRkNTg=

data/lib/turbot_runner/base_handler.rb

@@ -6,7 +6,7 @@ module TurbotRunner
     def handle_run_ended
     end
 
-    def handle_invalid_record(record, data_type, line)
+    def handle_invalid_record(record, data_type, error_message)
     end
 
     def handle_invalid_json(line)

data/lib/turbot_runner/processor.rb

@@ -17,16 +17,22 @@ module TurbotRunner
           @runner.interrupt if @runner
         else
           record = JSON.parse(line)
-          errors = validate(record)
 
-          if errors.empty?
+          begin
+            converted_record = convert_record(record)
+            error_message = validate(converted_record)
+          rescue ConversionError => e
+            error_message = e.message
+          end
+
+          if error_message.nil?
             begin
-              @record_handler.handle_valid_record(record, @data_type)
+              @record_handler.handle_valid_record(converted_record, @data_type)
             rescue InterruptRun
               @runner.interrupt if @runner
             end
           else
-            @record_handler.handle_invalid_record(record, @data_type, errors)
+            @record_handler.handle_invalid_record(record, @data_type, error_message)
             @runner.interrupt_and_mark_as_failed if @runner
           end
         end
@@ -40,37 +46,107 @@ module TurbotRunner
       @runner.interrupt
     end
 
-    def validate(record)
-      errors = JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
-      messages = errors.map do |error|
-        case error[:message]
-        when /The property '#\/' did not contain a required property of '(\w+)'/
-          "Missing required attribute: #{Regexp.last_match(1)}"
+    def convert_record(record)
+      converted_record = Utils.deep_copy(record)
+
+      date_paths.each do |path|
+        begin
+          tmp = converted_record
+
+          path[0...-1].each do |path_item|
+            tmp = tmp[path_item]
+          end
+
+          value = tmp[path[-1]]
+        rescue NoMethodError
+          next
+        end
+
+        next unless value.is_a?(String)
+
+        if value == ''
+          tmp.delete(path[-1])
         else
-          error[:message]
+          begin
+            tmp[path[-1]] = Date.strptime(value, '%Y-%m-%d').strftime('%Y-%m-%d')
+          rescue ArgumentError
+            raise ConversionError.new("Property not a valid date: #{path.join('.')}")
+          end
         end
       end
 
-      if messages.empty?
+      converted_record
+    end
+
+    def date_paths
+      @date_paths ||= get_date_paths(schema['properties'])
+    end
+
+    def get_date_paths(properties)
+      date_paths = []
+
+      properties.each do |name, attrs|
+        if attrs['format'] == 'date'
+          date_paths << [name]
+        elsif attrs['type'] == 'object'
+          get_date_paths(attrs['properties']).each do |path|
+            date_paths << [name] + path
+          end
+        end
+      end
+
+      date_paths
+    end
+
+    def validate(record)
+      error = Validator.validate(schema, record)
+
+      message = nil
+
+      if error.nil?
         identifying_attributes = record.reject do |k, v|
           !@identifying_fields.include?(k) || v.nil? || v == ''
         end
 
         if identifying_attributes.empty?
-          messages << "There were no values provided for any of the identifying fields: #{@identifying_fields.join(', ')}"
+          message = "There were no values provided for any of the identifying fields: #{@identifying_fields.join(', ')}"
+        end
+      else
+        message = case error[:type]
+        when :missing
+          "Missing required property: #{error[:path]}"
+        when :one_of_no_matches
+          "No match for property: #{error[:path]}"
+        when :one_of_many_matches
+          "Multiple possible matches for property: #{error[:path]}"
+        when :too_short
+          "Property too short: #{error[:path]} (must be at least #{error[:length]} characters)"
+        when :too_long
+          "Property too long: #{error[:path]} (must be at most #{error[:length]} characters)"
+        when :type_mismatch
+          "Property of wrong type: #{error[:path]} (must be of type #{error[:allowed_types].join(', ')})"
+        when :enum_mismatch
+          "Property not an allowed value: #{error[:path]} (must be one of #{error[:allowed_values].join(', ')})"
+        when :format_mismatch
+          "Property not of expected format: #{error[:path]} (must be of format #{error[:expected_format]})"
+        when :unknown
+          error[:message]
         end
       end
 
-      messages
+      message
     end
 
     def schema
-      @schema ||= get_schema
+      @schema ||= load_schema
     end
 
-    def get_schema
+    def load_schema
       hyphenated_name = @data_type.to_s.gsub("_", "-").gsub(" ", "-")
-      File.expand_path("../../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
+      path = File.expand_path("../../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
+      JSON.load(File.read(path))
     end
+
+    class ConversionError < StandardError; end
   end
 end

data/lib/turbot_runner/utils.rb

@@ -0,0 +1,9 @@
+module TurbotRunner
+  module Utils
+    extend self
+
+    def deep_copy(thing)
+      Marshal.load(Marshal.dump(thing))
+    end
+  end
+end

data/lib/turbot_runner/validator.rb

@@ -0,0 +1,74 @@
+require 'json-schema'
+
+module TurbotRunner
+  module Validator
+    extend self
+
+    def validate(schema, record)
+      errors = JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
+      
+      # For now, we just handle the first error.
+      error = errors[0]
+      return if error.nil?
+
+      case error[:failed_attribute]
+      when 'Required'
+        match = error[:message].match(/required property of '(.*)'/)
+        missing_property = match[1]
+        path = fragment_to_path("#{error[:fragment]}/#{missing_property}")
+
+        {:type => :missing, :path => path}
+      when 'OneOf'
+        if error[:message].match(/did not match any/)
+          path_elements = fragment_to_path(error[:fragment]).split('.')
+
+          raise "Deeply nested OneOf error at: #{error[:fragment]}" unless path_elements.size == 1
+
+          record_fragment = record[path_elements[0]]
+          schema_fragments = schema['properties'][path_elements[0]]['oneOf']
+
+          schema_fragments.each do |s|
+            s['properties'].each do |k, v|
+              next if v['enum'].nil?
+
+              if v['enum'].include?(record_fragment[k])
+                error1 = validate(s, record_fragment)
+                return error1.merge(:path => "#{path_elements[0]}.#{error1[:path]}")
+              end
+            end
+          end
+
+          {:type => :one_of_no_matches, :path => fragment_to_path(error[:fragment])}
+        else
+          {:type => :one_of_many_matches, :path => fragment_to_path(error[:fragment])}
+        end
+      when 'MinLength'
+        match = error[:message].match(/minimum string length of (\d+) in/)
+        min_length = match[1].to_i
+        {:type => :too_short, :path => fragment_to_path(error[:fragment]), :length => min_length}
+      when 'MaxLength'
+        match = error[:message].match(/maximum string length of (\d+) in/)
+        max_length = match[1].to_i
+        {:type => :too_long, :path => fragment_to_path(error[:fragment]), :length => max_length}
+      when 'TypeV4'
+        match = error[:message].match(/the following types?: ([\w\s,]+) in schema/)
+        allowed_types = match[1].split(',').map(&:strip)
+        {:type => :type_mismatch, :path => fragment_to_path(error[:fragment]), :allowed_types => allowed_types}
+      when 'Enum'
+        match = error[:message].match(/the following values: ([\w\s,]+) in schema/)
+        allowed_values = match[1].split(',').map(&:strip)
+        {:type => :enum_mismatch, :path => fragment_to_path(error[:fragment]), :allowed_values => allowed_values}
+      else
+        if error[:message].match(/must be of format yyyy-mm-dd/)
+          {:type => :format_mismatch, :path => fragment_to_path(error[:fragment]), :expected_format => 'yyyy-mm-dd'}
+        else
+          {:type => :unknown, :path => fragment_to_path(error[:fragment]), :failed_attribute => error[:failed_attribute], :message => error[:message]}
+        end
+      end
+    end
+
+    def fragment_to_path(fragment)
+      fragment.sub(/^#?\/*/, '').gsub('/', '.')
+    end
+  end
+end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.20'
+  VERSION = '0.1.21'
 end

data/lib/turbot_runner.rb

@@ -3,5 +3,7 @@ require 'turbot_runner/exceptions'
 require 'turbot_runner/processor'
 require 'turbot_runner/runner'
 require 'turbot_runner/script_runner'
+require 'turbot_runner/utils'
+require 'turbot_runner/validator'
 require 'turbot_runner/validators'
 require 'turbot_runner/version'

data/schema/schemas/financial-payment-schema.json

@@ -4,6 +4,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
   "properties": {
+    "sample_date": {"type": "string", "format": "date"},
     "source_jurisdiction": {
       "description": "Jurisdiction of the source of the data",
       "type": "string"
@@ -20,8 +21,5 @@
       "additionalItems": false
     }
   },
-  "required": [
-    "company",
-    "data"
-  ]
+  "required": ["company", "data", "sample_date"]
 }

data/schema/schemas/licence-schema.json

@@ -4,6 +4,9 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
   "properties": {
+    "sample_date": {"type": "string", "format": "date"},
+    "start_date": {"type": "string", "format": "date"},
+    "end_date": {"type": "string", "format": "date"},
     "source_jurisdiction": {
       "description": "Jurisdiction of the source of the data",
       "type": "string"
@@ -20,8 +23,5 @@
       "additionalItems": false
     }
   },
-  "required": [
-    "company",
-    "data"
-  ]
+  "required": ["company", "data", "sample_date"]
 }

data/schema/schemas/primary-data-schema.json

@@ -9,7 +9,8 @@
         },
         "sample_date": {
             "description": "Date on which this fact was known to be true",
-            "type": "string"
+            "type": "string",
+            "format": "date"
         }
     },
     "required": ["source_url", "sample_date"],

data/schema/schemas/rich-licence-schema.json

@@ -3,6 +3,9 @@
   "description": "A Licence is a permission for an entity to do something that would otherwise not be permitted. Note that such permissions are recorded in multiple different ways and to different granularity. In addition categories cross over, for example a pornborker may be regulated as a retail establishment by a city, but also as a financial institution by a national or regional financial regulator. Sometimes the licence is a simple statement – the government of Rwanda has given a banking licence to a certain bank -- other times it is fine-grained and highly complex (e.g. an extractives licence or the UK FCA licencing scheme). This schema tries to make it easy to submit the former, while not losing granularity of the latter. The object has the name of Rich Licence to distinguish it from Simple Licence, which it is expected to ultimately replace.",
   "$schema": "http://json-schema.org/draft-04/schema#",
 	"properties": {
+          "sample_date": {"type": "string", "format": "date"},
+          "start_date": {"type": "string", "format": "date"},
+          "end_date": {"type": "string", "format": "date"},
 		"licence_holder": {
       "entity": {
         "oneOf": [

data/schema/schemas/simple-financial-payment-schema.json

@@ -10,7 +10,8 @@
     },
     "sample_date": {
       "description": "Date on which this fact was known to be true",
-      "type": "string"
+      "type": "string",
+      "format": "date"
     },
     "confidence": {
       "description": "Confidence in accuracy of data",

data/schema/schemas/simple-licence-schema.json

@@ -10,8 +10,8 @@
     },
     "sample_date": {
       "description": "Date on which this fact was known to be true",
-      "type": "date",
-      "minLength": 1
+      "type": "string",
+      "format": "date"
     },
     "confidence": {
       "description": "Confidence in accuracy of data",

data/schema/schemas/simple-subsidiary-schema.json

@@ -11,15 +11,17 @@
     "sample_date": {
       "description": "Date on which this fact was known to be true",
       "type": "string",
-      "minLength": 1
+      "format": "date"
     },
     "start_date": {
       "description": "Earliest known date this was known to be a subsidiary",
-      "type": "string"
+      "type": "string",
+      "format": "date"
     },
     "end_date": {
       "description": "Latest known date this was known to be a subsidiary",
-      "type": "string"
+      "type": "string",
+      "format": "date"
     },
     "confidence": {
       "description": "Confidence in accuracy of data",

data/schema/schemas/subsidiary-relationship-schema.json

@@ -4,6 +4,9 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
   "properties": {
+    "sample_date": {"type": "string", "format": "date"},
+    "start_date": {"type": "string", "format": "date"},
+    "end_date": {"type": "string", "format": "date"},
     "source_jurisdiction": {
       "description": "Jurisdiction of the source of the data",
       "type": "string"
@@ -20,8 +23,5 @@
       "additionalItems": false
     }
   },
-  "required": [
-    "company",
-    "data"
-  ]
+  "required": ["company", "data", "sample_date"]
 }

data/spec/lib/processor_spec.rb

@@ -26,9 +26,9 @@ describe TurbotRunner::Processor do
         end
 
         it 'calls Handler#handle_invalid_record' do
-          expected_errors = ['Missing required attribute: source_url']
+          expected_error = 'Missing required property: source_url'
           expect(@handler).to receive(:handle_invalid_record).
-            with(@record, @data_type, expected_errors)
+            with(@record, @data_type, expected_error)
           @processor.process(@record.to_json)
         end
       end
@@ -62,9 +62,9 @@ describe TurbotRunner::Processor do
         end
 
         it 'calls Handler#handle_invalid_record' do
-          expected_errors = ['Missing required attribute: source_url']
+          expected_error = 'Missing required property: source_url'
           expect(@handler).to receive(:handle_invalid_record).
-            with(@record, @data_type, expected_errors)
+            with(@record, @data_type, expected_error)
           @processor.process(@record.to_json)
         end
       end
@@ -78,9 +78,9 @@ describe TurbotRunner::Processor do
         end
 
         it 'calls Handler#handle_invalid_record' do
-          expected_errors = ['There were no values provided for any of the identifying fields: number']
+          expected_error = 'There were no values provided for any of the identifying fields: number'
           expect(@handler).to receive(:handle_invalid_record).
-            with(@record, @data_type, expected_errors)
+            with(@record, @data_type, expected_error)
           @processor.process(@record.to_json)
         end
       end
@@ -92,6 +92,141 @@ describe TurbotRunner::Processor do
           @processor.process(line)
         end
       end
+
+      context 'with record with sample_date from Time.now' do
+        it 'calls Handler#handle_valid_record with converted sample_date' do
+          record = {
+            'sample_date' => '2014-06-01 12:34:56 +0000',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expected_converted_record = {
+            'sample_date' => '2014-06-01',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+          expect(@handler).to receive(:handle_valid_record).
+            with(expected_converted_record, @data_type)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with record with missing sample_date' do
+        it 'calls Handler#handle_invalid_record' do
+          record = {
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expected_error = 'Missing required property: sample_date'
+          expect(@handler).to receive(:handle_invalid_record).
+            with(record, @data_type, expected_error)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with record with empty sample_date' do
+        it 'calls Handler#handle_invalid_record' do
+          record = {
+            'sample_date' => '',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expected_error = 'Missing required property: sample_date'
+          expect(@handler).to receive(:handle_invalid_record).
+            with(record, @data_type, expected_error)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with record with invalid sample_date' do
+        it 'calls Handler#handle_invalid_record' do
+          record = {
+            'sample_date' => '2014-06-00',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expected_error = 'Property not a valid date: sample_date'
+          expect(@handler).to receive(:handle_invalid_record).
+            with(record, @data_type, expected_error)
+          @processor.process(record.to_json)
+        end
+      end
+    end
+  end
+
+  describe '#convert_record' do
+    before do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'format' => 'date'},
+          'bbb' => {'format' => 'not-date'},
+        }
+      }
+
+      @processor = TurbotRunner::Processor.new(nil, {}, nil)
+      allow(@processor).to receive(:schema).and_return(schema)
     end
+
+    context 'when date field is YYYY-MM-DD' do
+      it 'leaves date field alone' do
+        record = {'aaa' => '2015-01-26', 'bbb' => 'cabbage'}
+        expect(@processor.convert_record(record)).to eq({'aaa' => '2015-01-26', 'bbb' => 'cabbage'})
+      end
+    end
+
+    context 'when date field with YYYY-MM-DD HH:MM:SS' do
+      it 'replaces value with YYYY-MM-DD' do
+        record = {'aaa' => '2015-01-26 12:34:56', 'bbb' => 'cabbage'}
+        expect(@processor.convert_record(record)).to eq({'aaa' => '2015-01-26', 'bbb' => 'cabbage'})
+      end
+    end
+
+    context 'when date field is empty string' do
+      it 'replaces removes field' do
+        record = {'aaa' => '', 'bbb' => 'cabbage'}
+        expect(@processor.convert_record(record)).to eq({'bbb' => 'cabbage'})
+      end
+    end
+
+    context 'when date field is invalid date' do
+      it 'rasies ConversionError' do
+        record = {'aaa' => 'cabbage', 'bbb' => 'cabbage'}
+        expect{@processor.convert_record(record)}.to raise_error(TurbotRunner::Processor::ConversionError)
+      end
+    end
+  end
+
+  specify '#get_date_paths' do
+    schema = {
+      '$schema' => 'http://json-schema.org/draft-04/schema#',
+      'type' => 'object',
+      'properties' => {
+        'aaa' => {'format' => 'date'},
+        'bbb' => {'format' => 'not-date'},
+        'ccc' => {
+          'type' => 'object',
+          'properties' => {
+            'ddd' => {'format' => 'date'},
+            'eee' => {'format' => 'not-date'},
+            'fff' => {
+              'type' => 'object',
+              'properties' => {
+                'ggg' => {'format' => 'date'},
+                'hhh' => {'format' => 'not-date'},
+              }
+            }
+          }
+        }
+      }
+    }
+
+    processor = TurbotRunner::Processor.new(nil, {}, nil)
+    expect(processor.get_date_paths(schema['properties'])).to eq([['aaa'], ['ccc', 'ddd'], ['ccc', 'fff', 'ggg']])
   end
 end

data/spec/lib/runner_spec.rb

@@ -223,7 +223,7 @@ describe TurbotRunner::Runner do
       end
 
       it 'returns false' do
-        expect(@runner.run).to be(true)
+        expect(@runner.run).to be(false)
       end
     end
   end

data/spec/lib/validator_spec.rb

@@ -0,0 +1,235 @@
+require 'spec_helper'
+
+describe TurbotRunner::Validator do
+  describe 'validation' do
+    specify 'when record is valid' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'required' => ['aaa'],
+      }
+      record = {'aaa' => 'zzz'}
+
+      expect([schema, record]).to be_valid
+    end
+
+    specify 'when required top-level property missing' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'required' => ['aaa'],
+      }
+      record = {}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :missing,
+        :path => 'aaa'
+      )
+    end
+
+    specify 'when required nested property missing' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'required' => ['aaa'],
+        'properties' => {
+          'aaa' => {
+            'type' => 'object',
+            'required' => ['bbb'],
+          }
+        }
+      }
+      record = {'aaa' => {}}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :missing,
+        :path => 'aaa.bbb'
+      )
+    end
+
+    context 'when none of oneOf options match' do
+      specify 'and we are switching on an enum field' do
+        schema = {
+          '$schema' => 'http://json-schema.org/draft-04/schema#',
+          'type' => 'object',
+          'required' => ['aaa'],
+          'properties' => {
+            'aaa' => {
+              'type' => 'object',
+              'oneOf' => [{
+                'properties' => {
+                  'a_type' => {
+                    'enum' => ['a1']
+                  },
+                  'a_properties' => {
+                    'type' => 'object',
+                    'required' => ['bbb'],
+                  }
+                }
+              }, {
+                'properties' => {
+                  'a_type' => {
+                    'enum' => ['a2']
+                  },
+                  'a_properties' => {
+                    'type' => 'object',
+                    'required' => ['ccc']
+                  }
+                }
+              }]
+            }
+          }
+        }
+      
+        record = {'aaa' => {'a_type' => 'a1', 'a_properties' => {}}}
+
+        expect([schema, record]).to fail_validation_with(
+          :type => :missing,
+          :path => 'aaa.a_properties.bbb'
+        )
+      end
+
+      specify 'and we are not switching on an enum field' do
+        schema = {
+          '$schema' => 'http://json-schema.org/draft-04/schema#',
+          'type' => 'object',
+          'required' => ['aaa'],
+          'properties' => {
+            'aaa' => {
+              'type' => 'object',
+              'oneOf' => [{
+                'properties' => {
+                  'bbb' => {
+                    'type' => 'object',
+                    'required' => ['ccc'],
+                  }
+                }
+              }, {
+                'properties' => {
+                  'bbb' => {
+                    'type' => 'object',
+                    'required' => ['ddd']
+                  }
+                }
+              }]
+            }
+          }
+        }
+      
+        record = {'aaa' => {'bbb' => {}}}
+
+        expect([schema, record]).to fail_validation_with(
+          :type => :one_of_no_matches,
+          :path => 'aaa'
+        )
+      end
+    end
+
+    specify 'when top-level property too short' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'minLength' => 2}
+        }
+      }
+      record = {'aaa' => 'x'}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :too_short,
+        :path => 'aaa',
+        :length => 2
+      )
+    end
+
+    specify 'when nested property too short' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {
+            'type' => 'object',
+            'properties' => {
+              'bbb' => {'minLength' => 2}
+            }
+          }
+        }
+      }
+      record = {'aaa' => {'bbb' => 'x'}}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :too_short,
+        :path => 'aaa.bbb',
+        :length => 2
+      )
+    end
+
+    specify 'when property too long' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'maxLength' => 2}
+        }
+      }
+      record = {'aaa' => 'xxx'}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :too_long,
+        :path => 'aaa',
+        :length => 2
+      )
+    end
+
+    specify 'when property of wrong type and many types allowed' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'type' => ['number', 'string']}
+        }
+      }
+      record = {'aaa' => ['xxx']}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :type_mismatch,
+        :path => 'aaa',
+        :allowed_types => ['number', 'string']
+      )
+    end
+
+    specify 'when property of wrong type and single type allowed' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'type' => 'number'}
+        }
+      }
+      record = {'aaa' => 'xxx'}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :type_mismatch,
+        :path => 'aaa',
+        :allowed_types => ['number']
+      )
+    end
+
+    specify 'when property not in enum' do
+      schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'enum' => ['a', 'b', 'c']}
+        }
+      }
+      record = {'aaa' => 'z'}
+
+      expect([schema, record]).to fail_validation_with(
+        :type => :enum_mismatch,
+        :path => 'aaa',
+        :allowed_values => ['a', 'b', 'c']
+      )
+    end
+  end
+end

data/spec/lib/validators_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe 'custom validators' do
+  describe 'for date' do
+    before do
+      @schema = {
+        '$schema' => 'http://json-schema.org/draft-04/schema#',
+        'type' => 'object',
+        'properties' => {
+          'aaa' => {'format' => 'date'}
+        }
+      }
+    end
+
+    specify 'validate valid dates' do
+      strings = [
+        '2015-01-10',
+        '2015-01-10T10:15:57',
+        '2015-01-10T10:15:57Z',
+        '2015-01-10T10:15:57+00:00',
+        '2015-01-10T11:15:57+01:00',
+        '2015-01-10T09:15:57-01:00',
+        '2015-01-10 10:15:57 +0000',
+        '2015-01-10 11:15:57 +0100',
+        '2015-01-10 09:15:57 -0100',
+      ]
+
+      strings.each do |string|
+        record = {'aaa' => string}
+        expect([@schema, record]).to be_valid
+      end
+    end
+
+    specify 'do not validator invalid dates' do
+      strings = [
+        'nonsense',
+        '2015-01-nonsense',
+        '2015:01:10',
+        '2015/01/10',
+      ]
+
+      strings.each do |string|
+        record = {'aaa' => string}
+        expect([@schema, record]).not_to be_valid
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'turbot_runner'
+
+RSpec::Matchers.define(:fail_validation_with) do |expected|
+  match do |actual|
+    schema, record = actual
+
+    error = TurbotRunner::Validator.validate(schema, record)
+    expect(error).to eq(expected)
+  end
+end
+
+RSpec::Matchers.define(:be_valid) do
+  match do |actual|
+    schema, record = actual
+
+    error = TurbotRunner::Validator.validate(schema, record)
+    expect(error).to eq(nil)
+  end
+end
+

metadata

@@ -1,46 +1,35 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: turbot-runner
-version: !ruby/object:Gem::Version 
-  hash: 51
-  prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 20
-  version: 0.1.20
+version: !ruby/object:Gem::Version
+  version: 0.1.21
 platform: ruby
-authors: 
+authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2015-01-15 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: json-schema
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 4
-        version: "2.4"
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.5.0
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.5.0
 description: 
 email: bots@opencorporates.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - bin/rspec
 - lib/turbot_runner.rb
 - lib/turbot_runner/base_handler.rb
@@ -49,8 +38,39 @@ files:
 - lib/turbot_runner/processor.rb
 - lib/turbot_runner/runner.rb
 - lib/turbot_runner/script_runner.rb
+- lib/turbot_runner/utils.rb
+- lib/turbot_runner/validator.rb
 - lib/turbot_runner/validators.rb
 - lib/turbot_runner/version.rb
+- schema/schemas/company-schema.json
+- schema/schemas/financial-payment-schema.json
+- schema/schemas/includes/address.json
+- schema/schemas/includes/alternative_name.json
+- schema/schemas/includes/company.json
+- schema/schemas/includes/filing.json
+- schema/schemas/includes/financial-payment-data-object.json
+- schema/schemas/includes/identifier.json
+- schema/schemas/includes/industry_code.json
+- schema/schemas/includes/licence-data-object.json
+- schema/schemas/includes/officer.json
+- schema/schemas/includes/organisation.json
+- schema/schemas/includes/permission.json
+- schema/schemas/includes/person.json
+- schema/schemas/includes/person_name.json
+- schema/schemas/includes/previous_name.json
+- schema/schemas/includes/share-parcel-data.json
+- schema/schemas/includes/share-parcel.json
+- schema/schemas/includes/subsidiary-relationship-data.json
+- schema/schemas/includes/total-shares.json
+- schema/schemas/includes/unknown_entity_type.json
+- schema/schemas/licence-schema.json
+- schema/schemas/primary-data-schema.json
+- schema/schemas/rich-licence-schema.json
+- schema/schemas/share-parcel-schema.json
+- schema/schemas/simple-financial-payment-schema.json
+- schema/schemas/simple-licence-schema.json
+- schema/schemas/simple-subsidiary-schema.json
+- schema/schemas/subsidiary-relationship-schema.json
 - spec/bots/bot-that-crashes-immediately/manifest.json
 - spec/bots/bot-that-crashes-immediately/scraper.rb
 - spec/bots/bot-that-crashes-immediately/transformer1.rb
@@ -91,73 +111,36 @@ files:
 - spec/bots/slow-bot/scraper.rb
 - spec/lib/processor_spec.rb
 - spec/lib/runner_spec.rb
+- spec/lib/validator_spec.rb
+- spec/lib/validators_spec.rb
 - spec/manual_spec.rb
 - spec/outputs/full-scraper.out
 - spec/outputs/full-transformer.out
 - spec/outputs/truncated-scraper.out
-- schema/schemas/company-schema.json
-- schema/schemas/financial-payment-schema.json
-- schema/schemas/includes/address.json
-- schema/schemas/includes/alternative_name.json
-- schema/schemas/includes/company.json
-- schema/schemas/includes/filing.json
-- schema/schemas/includes/financial-payment-data-object.json
-- schema/schemas/includes/identifier.json
-- schema/schemas/includes/industry_code.json
-- schema/schemas/includes/licence-data-object.json
-- schema/schemas/includes/officer.json
-- schema/schemas/includes/organisation.json
-- schema/schemas/includes/permission.json
-- schema/schemas/includes/person.json
-- schema/schemas/includes/person_name.json
-- schema/schemas/includes/previous_name.json
-- schema/schemas/includes/share-parcel-data.json
-- schema/schemas/includes/share-parcel.json
-- schema/schemas/includes/subsidiary-relationship-data.json
-- schema/schemas/includes/total-shares.json
-- schema/schemas/includes/unknown_entity_type.json
-- schema/schemas/licence-schema.json
-- schema/schemas/primary-data-schema.json
-- schema/schemas/rich-licence-schema.json
-- schema/schemas/share-parcel-schema.json
-- schema/schemas/simple-financial-payment-schema.json
-- schema/schemas/simple-licence-schema.json
-- schema/schemas/simple-subsidiary-schema.json
-- schema/schemas/subsidiary-relationship-schema.json
+- spec/spec_helper.rb
 homepage: http://turbot.opencorporates.com/
-licenses: 
+licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 55
-      segments: 
-      - 1
-      - 9
-      - 2
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.9.2
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Utilities for running bots with Turbot
 test_files: []
-
+has_rdoc: