turbot-runner 0.1.24 → 0.1.25

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzkxMDdjODdjYjg3NDFkZTVjZjJmMzI4M2RhZmQ1NWI5OTBmNzFiOA==
+    NWI1YjU1NDhkMjg2ZDkzNDJmNGYzNmU3NTg1OGRlMWExZGY5NDI0NA==
   data.tar.gz: !binary |-
-    ZmU3ODUzNmMxODg5NjZlMDhjZDMzYzk0MTY4YzA4ODVlZjEzZTVlOA==
+    ZDkyNTcxMDhmZjA5NDRjMjczNzdhMTA2MGJlNGJlZTQ3ZjM4NWZkOA==
 SHA512:
   metadata.gz: !binary |-
-    Mzc3NTg0ZTY2ZWM5YmU4MWM4ODM0NmRkZjdhNDk2NWE4YjYxMDllMjJmOTlm
-    MmQwMzEwMGJkMThiMDFlYzFlYjg0ODQyMjkzMDBlMzgwMWM4YmM3MTI1NjA4
-    OGMxMjExNjIxNDI2N2FmZjYyODU2Y2IwMDEzMTkwZWJjNzVlODM=
+    Y2Y5NjYyY2ExOWY1ZTg2OWFmMmUyMjlkNmMwZTI4N2NkYjEzZDY4YWY2Njlh
+    OTJlNWY2N2VjMDQzMDhmMTBjODM5NmI5ZDFjODc3YWQxMjc3ZjAxMjczN2U0
+    MjZmZmY2ODNiOTIxZmZiYTIyZGE2MDliMDMxNTk3NTQ1MTIxMTA=
   data.tar.gz: !binary |-
-    MDk3NzFhODNlZDA3NGUyMmZlMTBmYzEzM2UyNjFmMDc5NzJjYTJkMjQ0NGRh
-    M2U3MDlmOGYwNGI4Y2U3YzNkZDEwODk4NzA4ZDY1Y2NhNGJlYWE1MzkyNDYw
-    M2MyYTNkNjI0YzEwMTMyOTdiMzMwZmU1NTQ2Njk0YTQ5NjZkMGM=
+    ODE2N2ZhN2M2MWFjMzJhNDgyNGQzNDBlNGM2OGU2NmEyYjBkNDUyNTkyNzZh
+    MTU1OGM1YjM3YjJiZmE2OWEzNTNmODJlNTEzNGIxZjc5NmRiN2E3YmVmNWQy
+    OTFlODIwNGZhOWRlYjliYjJlZjBlMmIwYzhkNGNlOTlhOTFkZDE=

data/lib/turbot_runner/processor.rb

@@ -17,7 +17,7 @@ module TurbotRunner
         else
           record = Openc::JsonSchema.convert_dates(schema_path, JSON.parse(line))
 
-          error_message = validate(record)
+          error_message = Validator.validate(schema_path, record, @identifying_fields)
 
           if error_message.nil?
             begin
@@ -40,50 +40,9 @@ module TurbotRunner
       @runner.interrupt
     end
 
-    def validate(record)
-      error = Openc::JsonSchema.validate(schema_path, record)
-
-      message = nil
-
-      if error.nil?
-        identifying_attributes = record.reject do |k, v|
-          !@identifying_fields.include?(k) || v.nil? || v == ''
-        end
-
-        if identifying_attributes.empty?
-          message = "There were no values provided for any of the identifying fields: #{@identifying_fields.join(', ')}"
-        end
-      else
-        message = case error[:type]
-        when :missing
-          "Missing required property: #{error[:path]}"
-        when :one_of_no_matches
-          "No match for property: #{error[:path]}"
-        when :one_of_many_matches
-          "Multiple possible matches for property: #{error[:path]}"
-        when :too_short
-          "Property too short: #{error[:path]} (must be at least #{error[:length]} characters)"
-        when :too_long
-          "Property too long: #{error[:path]} (must be at most #{error[:length]} characters)"
-        when :type_mismatch
-          "Property of wrong type: #{error[:path]} (must be of type #{error[:allowed_types].join(', ')})"
-        when :enum_mismatch
-          "Property not an allowed value: #{error[:path]} (must be one of #{error[:allowed_values].join(', ')})"
-        when :format_mismatch
-          "Property not of expected format: #{error[:path]} (must be of format #{error[:expected_format]})"
-        when :unknown
-          error[:message]
-        end
-      end
-
-      message
-    end
-
     def schema_path
       hyphenated_name = @data_type.to_s.gsub("_", "-").gsub(" ", "-")
       File.join(SCHEMAS_PATH, "#{hyphenated_name}-schema.json")
     end
-
-    class ConversionError < StandardError; end
   end
 end

data/lib/turbot_runner/validator.rb

@@ -0,0 +1,44 @@
+module TurbotRunner
+  module Validator
+    extend self
+
+    def validate(schema_path, record, identifying_fields)
+      error = Openc::JsonSchema.validate(schema_path, record)
+
+      message = nil
+
+      if error.nil?
+        identifying_attributes = record.reject do |k, v|
+          !identifying_fields.include?(k) || v.nil? || v == ''
+        end
+
+        if identifying_attributes.empty?
+          message = "There were no values provided for any of the identifying fields: #{identifying_fields.join(', ')}"
+        end
+      else
+        message = case error[:type]
+        when :missing
+          "Missing required property: #{error[:path]}"
+        when :one_of_no_matches
+          "No match for property: #{error[:path]}"
+        when :one_of_many_matches
+          "Multiple possible matches for property: #{error[:path]}"
+        when :too_short
+          "Property too short: #{error[:path]} (must be at least #{error[:length]} characters)"
+        when :too_long
+          "Property too long: #{error[:path]} (must be at most #{error[:length]} characters)"
+        when :type_mismatch
+          "Property of wrong type: #{error[:path]} (must be of type #{error[:allowed_types].join(', ')})"
+        when :enum_mismatch
+          "Property not an allowed value: #{error[:path]} (must be one of #{error[:allowed_values].join(', ')})"
+        when :format_mismatch
+          "Property not of expected format: #{error[:path]} (must be of format #{error[:expected_format]})"
+        when :unknown
+          error[:message]
+        end
+      end
+
+      message
+    end
+  end
+end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.24'
+  VERSION = '0.1.25'
 end

data/lib/turbot_runner.rb

@@ -4,6 +4,7 @@ require 'turbot_runner/processor'
 require 'turbot_runner/runner'
 require 'turbot_runner/script_runner'
 require 'turbot_runner/utils'
+require 'turbot_runner/validator'
 require 'turbot_runner/version'
 
 module TurbotRunner

data/spec/lib/processor_spec.rb

@@ -17,28 +17,47 @@ describe TurbotRunner::Processor do
         @processor = TurbotRunner::Processor.new(nil, @script_config, @handler)
       end
 
-      context 'with record missing required field' do
-        before do
-          @record = {
+      context 'with valid record' do
+        it 'calls Handler#handle_valid_record' do
+          record = {
             'sample_date' => '2014-06-01',
+            'source_url' => 'http://example.com/123',
             'number' => 123
           }
+
+          expect(@handler).to receive(:handle_valid_record).with(record, @data_type)
+          @processor.process(record.to_json)
         end
+      end
 
+      context 'with invalid record' do
         it 'calls Handler#handle_invalid_record' do
+          record = {
+            'sample_date' => '2014-06-01',
+            'number' => 123
+          }
+
           expected_error = 'Missing required property: source_url'
           expect(@handler).to receive(:handle_invalid_record).
-            with(@record, @data_type, expected_error)
-          @processor.process(@record.to_json)
+            with(record, @data_type, expected_error)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with invalid JSON' do
+        it 'calls Handler#handle_invalid_json' do
+          line = 'this is not JSON'
+          expect(@handler).to receive(:handle_invalid_json).with(line)
+          @processor.process(line)
         end
       end
     end
 
     context 'with a runner passed in' do
       before do
-        script_runner = instance_double('ScriptRunner')
-        allow(script_runner).to receive(:interrupt_and_mark_as_failed)
-        @processor = TurbotRunner::Processor.new(script_runner, @script_config, @handler)
+        @script_runner = instance_double('ScriptRunner')
+        allow(@script_runner).to receive(:interrupt_and_mark_as_failed)
+        @processor = TurbotRunner::Processor.new(@script_runner, @script_config, @handler)
       end
 
       context 'with valid record' do
@@ -48,12 +67,13 @@ describe TurbotRunner::Processor do
             'source_url' => 'http://example.com/123',
             'number' => 123
           }
+
           expect(@handler).to receive(:handle_valid_record).with(record, @data_type)
           @processor.process(record.to_json)
         end
       end
 
-      context 'with record missing required field' do
+      context 'with invalid record' do
         before do
           @record = {
             'sample_date' => '2014-06-01',
@@ -67,93 +87,44 @@ describe TurbotRunner::Processor do
             with(@record, @data_type, expected_error)
           @processor.process(@record.to_json)
         end
-      end
-
-      context 'with record missing all identifying fields' do
-        before do
-          @record = {
-            'sample_date' => '2014-06-01',
-            'source_url' => 'http://example.com/123'
-          }
-        end
 
-        it 'calls Handler#handle_invalid_record' do
-          expected_error = 'There were no values provided for any of the identifying fields: number'
-          expect(@handler).to receive(:handle_invalid_record).
-            with(@record, @data_type, expected_error)
+        it 'interrupts runner' do
+          expect(@script_runner).to receive(:interrupt_and_mark_as_failed)
           @processor.process(@record.to_json)
         end
       end
 
       context 'with invalid JSON' do
-        it 'calls Handler#handle_invalid_json' do
-          line = 'this is not JSON'
-          expect(@handler).to receive(:handle_invalid_json).with(line)
-          @processor.process(line)
+        before do
+          @line = 'this is not JSON'
         end
-      end
-
-      context 'with record with sample_date from Time.now' do
-        it 'calls Handler#handle_valid_record with converted sample_date' do
-          record = {
-            'sample_date' => '2014-06-01 12:34:56 +0000',
-            'source_url' => 'http://example.com/123',
-            'number' => 123
-          }
 
-          expected_converted_record = {
-            'sample_date' => '2014-06-01',
-            'source_url' => 'http://example.com/123',
-            'number' => 123
-          }
-          expect(@handler).to receive(:handle_valid_record).
-            with(expected_converted_record, @data_type)
-          @processor.process(record.to_json)
+        it 'calls Handler#handle_invalid_json' do
+          expect(@handler).to receive(:handle_invalid_json).with(@line)
+          @processor.process(@line)
         end
-      end
-
-      context 'with record with missing sample_date' do
-        it 'calls Handler#handle_invalid_record' do
-          record = {
-            'source_url' => 'http://example.com/123',
-            'number' => 123
-          }
 
-          expected_error = 'Missing required property: sample_date'
-          expect(@handler).to receive(:handle_invalid_record).
-            with(record, @data_type, expected_error)
-          @processor.process(record.to_json)
+        it 'interrupts runner' do
+          expect(@script_runner).to receive(:interrupt_and_mark_as_failed)
+          @processor.process(@line)
         end
       end
 
-      context 'with record with empty sample_date' do
-        it 'calls Handler#handle_invalid_record' do
+      it 'converts date format' do
           record = {
-            'sample_date' => '',
+            'sample_date' => '2014-06-01 12:34:56 +0000',
             'source_url' => 'http://example.com/123',
             'number' => 123
           }
 
-          expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
-          expect(@handler).to receive(:handle_invalid_record).
-            with(record, @data_type, expected_error)
-          @processor.process(record.to_json)
-        end
-      end
-
-      context 'with record with invalid sample_date' do
-        it 'calls Handler#handle_invalid_record' do
-          record = {
-            'sample_date' => '2014-06-00',
+          converted_record = {
+            'sample_date' => '2014-06-01',
             'source_url' => 'http://example.com/123',
             'number' => 123
           }
 
-          expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
-          expect(@handler).to receive(:handle_invalid_record).
-            with(record, @data_type, expected_error)
+          expect(@handler).to receive(:handle_valid_record).with(converted_record, @data_type)
           @processor.process(record.to_json)
-        end
       end
     end
 

data/spec/lib/validator_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe TurbotRunner::Validator do
+  describe '.validate' do
+    specify 'with valid record' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expect(record).to be_valid
+    end
+
+    specify 'with record missing required field' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'number' => 123
+      }
+      expected_error = 'Missing required property: source_url'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record missing all identifying fields' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'source_url' => 'http://example.com/123'
+      }
+      expected_error = 'There were no values provided for any of the identifying fields: number'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record with empty sample_date' do
+      record = {
+        'sample_date' => '',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record with invalid sample_date' do
+      record = {
+        'sample_date' => '2014-06-00',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
+      expect(record).to fail_validation_with(expected_error)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,20 +1,17 @@
 require 'turbot_runner'
 
-RSpec::Matchers.define(:fail_validation_with) do |expected|
-  match do |actual|
-    schema, record = actual
-
-    error = TurbotRunner::Validator.validate(schema, record)
-    expect(error).to eq(expected)
+RSpec::Matchers.define(:fail_validation_with) do |expected_error|
+  match do |record|
+    schema_path = File.join(TurbotRunner::SCHEMAS_PATH, 'primary-data-schema.json')
+    identifying_fields = ['number']
+    expect(TurbotRunner::Validator.validate(schema_path, record, identifying_fields)).to eq(expected_error)
   end
 end
 
 RSpec::Matchers.define(:be_valid) do
-  match do |actual|
-    schema, record = actual
-
-    error = TurbotRunner::Validator.validate(schema, record)
-    expect(error).to eq(nil)
+  match do |record|
+    schema_path = File.join(TurbotRunner::SCHEMAS_PATH, 'primary-data-schema.json')
+    identifying_fields = ['number']
+    expect(TurbotRunner::Validator.validate(schema_path, record, identifying_fields)).to eq(nil)
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.24
+  version: 0.1.25
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-16 00:00:00.000000000 Z
+date: 2015-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openc-json_schema
@@ -39,6 +39,7 @@ files:
 - lib/turbot_runner/runner.rb
 - lib/turbot_runner/script_runner.rb
 - lib/turbot_runner/utils.rb
+- lib/turbot_runner/validator.rb
 - lib/turbot_runner/version.rb
 - schema/schemas/company-schema.json
 - schema/schemas/financial-payment-schema.json
@@ -109,6 +110,7 @@ files:
 - spec/bots/slow-bot/scraper.rb
 - spec/lib/processor_spec.rb
 - spec/lib/runner_spec.rb
+- spec/lib/validator_spec.rb
 - spec/manual_spec.rb
 - spec/outputs/full-scraper.out
 - spec/outputs/full-transformer.out