turbot-runner 0.1.38 → 0.1.39

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YmMwMGQ1NTRjODA5MDQ5NzEyMjc2NWEzNzQzNzMxNmI5ZmMzNTJmYQ==
+    MTk1NGQ2ZjNlYTZhODFkMDk4OWU4YzNhOTc3YzMzNjVmMDQwOWY1Ng==
   data.tar.gz: !binary |-
-    MjA3NGU1ZTk2ZDM3NDdjZjVjNTY5Zjc0MGE2ZDAxNjRiYTM1ZmY0NQ==
+    YzcyOTQyMTdkYTc0ZmNkNDYwNzY5YTM1YWIxYjc2M2UyOGNhY2FhYQ==
 SHA512:
   metadata.gz: !binary |-
-    ZjQwYTQwNzQ5YmNmZWJjYTY1NjI2ZTM3MjVlYjM1ZWZiMGY0ZGQ1M2U1MDY5
-    NTZiZDFiZDI3YzZhYzMwYmU0ZTMxZWZkOWQ4ZjYwZWI5YzgxMzZjZGJjMDc4
-    MmQ0OWRiZmEzY2Q2YmY3Yzg0OTMyNTY3NDUxMzA1ZTc0ODZiM2U=
+    NmMzYzJhYmIwMTkxMTUwMWM0MmI5ZjBiZDQwOWE1NjVkYzQ3OGVmNDZmZTZj
+    ZGI2YmNlODQxYTE2NzdiYTQ4YmQ5ZWRhMWY2NGZkOTZiYzM3NzNlYjNjZmJk
+    NWYzOWU2NTZjYzM3ZWVjNDIwMjliZDA5OTJmMGFmOWQ4ZTZhMDc=
   data.tar.gz: !binary |-
-    Y2RjYmVkZjk5YmI4MTY4MGEyYjNkZDUwZTAzNjhlZThkZWZjZjQ1OGZmOTY3
-    YTU4ZmFmMzZmNjhmODc4ZDI4YTQwNTEyNDE0NmRhZDdhODQzNThhZGYwNWQ2
-    YjlkZTMwNzQwN2U3YmI0NTAzNmI2ZDJiMjI3MzZmYzgwZWMyNzc=
+    OWZiNWM0YTNmZDBhMmE0NGIyOWY4N2VlMTM0NmE2YTFhMWYxODNmZGU5Y2Zl
+    MmYxYmVlMTZkZmE1MTM0MTIyZjFjZWQ2Yzc5YzhkMDE5NzE5M2E0MWQ2ODMy
+    ZTEwNzg2ZGE4Mzk4YTcyOGFiZTY4YjZhNTcyMmNlMWFiNWYxZTA=

data/lib/turbot_runner/processor.rb

@@ -7,6 +7,7 @@ module TurbotRunner
       @data_type = script_config[:data_type]
       @identifying_fields = script_config[:identifying_fields]
       @record_handler = record_handler
+      @seen_uids = script_config[:duplicates_allowed] ? nil : Set.new
     end
 
     def process(line)
@@ -22,7 +23,8 @@ module TurbotRunner
           error_message = Validator.validate(
             @data_type,
             record_to_validate,
-            @identifying_fields
+            @identifying_fields,
+            @seen_uids
           )
 
           if error_message.nil?

data/lib/turbot_runner/runner.rb

@@ -27,9 +27,11 @@ module TurbotRunner
       succeeded = run_script(scraper_config)
       # Run the transformers even if the scraper fails
       transformers.each do |transformer_config|
-        succeeded = run_script(
-          transformer_config.merge(:base_directory => @base_directory),
-          input_file=scraper_output_file) && succeeded
+        config = transformer_config.merge(
+          :base_directory => @base_directory,
+          :duplicates_allowed => duplicates_allowed
+        )
+        succeeded = run_script(config, input_file=scraper_output_file) && succeeded
       end
       succeeded
     end
@@ -142,7 +144,8 @@ module TurbotRunner
         :base_directory => @base_directory,
         :file => scraper_script,
         :data_type => scraper_data_type,
-        :identifying_fields => scraper_identifying_fields
+        :identifying_fields => scraper_identifying_fields,
+        :duplicates_allowed => duplicates_allowed
       }
     end
 
@@ -170,6 +173,10 @@ module TurbotRunner
       @config[:identifying_fields]
     end
 
+    def duplicates_allowed
+      @config[:duplicates_allowed]
+    end
+
     def assert_absolute_path(path)
       unless Pathname.new(path).absolute?
         raise "#{path} must be an absolute path"

data/lib/turbot_runner/validator.rb

@@ -1,28 +1,42 @@
+require 'active_support/core_ext/hash/slice'
+require 'active_support/core_ext/object/to_query'
+
 module TurbotRunner
   module Validator
     extend self
 
-    def validate(data_type, record, identifying_fields)
+    def validate(data_type, record, identifying_fields, seen_uids)
       schema_path = TurbotRunner.schema_path(data_type)
       error = Openc::JsonSchema.validate(schema_path, record)
 
-      message = nil
-
-      if error.nil?
-        flattened_record = TurbotRunner::Utils.flatten(record)
-
-        identifying_attributes = flattened_record.reject do |k, v|
-          !identifying_fields.include?(k) || v.nil? || v == ''
-        end
+      message = error.nil? ? nil : error[:message]
 
+      if message.nil?
+        identifying_hash = identifying_hash(record, identifying_fields)
+        identifying_attributes = identifying_hash.reject {|k, v| v.nil? || v == ''}
         if identifying_attributes.empty?
           message = "There were no values provided for any of the identifying fields: #{identifying_fields.join(', ')}"
         end
-      else
-        message = error[:message]
+      end
+
+      if message.nil? && !seen_uids.nil?
+        record_uid = record_uid(identifying_hash)
+        if seen_uids.include?(record_uid)
+          message = "Already seen record with these identifying fields: #{identifying_hash}"
+        else
+          seen_uids.add(record_uid)
+        end
       end
 
       message
     end
+
+    def identifying_hash(record, identifying_fields)
+      TurbotRunner::Utils.flatten(record).slice(*identifying_fields)
+    end
+
+    def record_uid(identifying_hash)
+      Digest::SHA1.hexdigest(identifying_hash.to_query)
+    end
   end
 end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.38'
+  VERSION = '0.1.39'
 end

data/spec/bots/bot-that-is-allowed-to-produce-duplicates/manifest.json

@@ -0,0 +1,9 @@
+{
+    "bot_id": "bot-that-is-allowed-to-produce-duplicates",
+    "description": "This bot produces duplicates, but that's ok",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"],
+    "duplicates_allowed": true
+}

data/spec/bots/bot-that-is-allowed-to-produce-duplicates/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n % 5}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-that-produces-duplicates/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-that-produces-duplicates",
+    "description": "This bot produces duplicates",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-that-produces-duplicates/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n % 5}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/lib/processor_spec.rb

@@ -142,7 +142,7 @@ describe TurbotRunner::Processor do
         }
 
         expect(TurbotRunner::Validator).to receive(:validate).
-          with('primary data', expected_record_to_validate, ['number'])
+          with('primary data', expected_record_to_validate, ['number'], Set.new)
         @processor.process(record.to_json)
       end
     end

data/spec/lib/runner_spec.rb

@@ -236,6 +236,26 @@ describe TurbotRunner::Runner do
         expect{@runner.run}.to raise_error(TurbotRunner::InvalidDataType)
       end
     end
+
+    context 'with a bot that produces duplicate data' do
+      before do
+        @runner = test_runner('bot-that-produces-duplicates')
+      end
+
+      it 'raises returns false' do
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a bot that is expected to produce duplicate data' do
+      before do
+        @runner = test_runner('bot-that-is-allowed-to-produce-duplicates')
+      end
+
+      it 'raises returns false' do
+        expect(@runner.run).to be(true)
+      end
+    end
   end
 
   describe '#process_output' do

data/spec/lib/validator_spec.rb

@@ -58,7 +58,7 @@ describe TurbotRunner::Validator do
           'four' => {}
         }
         identifying_fields = ['one.two.three', 'four.five.six']
-        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields, Set.new)
         expect(error).to eq('There were no values provided for any of the identifying fields: one.two.three, four.five.six')
       end
 
@@ -69,7 +69,7 @@ describe TurbotRunner::Validator do
           'one' => {'two' => {'three' => 123}}
         }
         identifying_fields = ['one.two.three', 'four.five.six']
-        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields, Set.new)
         expect(error).to eq(nil)
       end
 
@@ -81,9 +81,24 @@ describe TurbotRunner::Validator do
           'four' => {'five' => {'six' => 456}}
         }
         identifying_fields = ['one.two.three', 'four.five.six']
-        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields, Set.new)
         expect(error).to eq(nil)
       end
     end
+
+    specify 'with duplicate record' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+
+      seen_uids = Set.new
+      error = TurbotRunner::Validator.validate('primary-data', record, 'number', seen_uids)
+      expect(error).to eq(nil)
+
+      error = TurbotRunner::Validator.validate('primary-data', record, 'number', seen_uids)
+      expect(error).to eq('Already seen record with these identifying fields: {"number"=>123}')
+    end
   end
 end

data/spec/spec_helper.rb

@@ -3,7 +3,7 @@ require 'turbot_runner'
 RSpec::Matchers.define(:fail_validation_with) do |expected_error|
   match do |record|
     identifying_fields = ['number']
-    @error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+    @error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields, Set.new)
     expect(@error).to eq(expected_error)
   end
 
@@ -15,6 +15,6 @@ end
 RSpec::Matchers.define(:be_valid) do
   match do |record|
     identifying_fields = ['number']
-    expect(TurbotRunner::Validator.validate('primary-data', record, identifying_fields)).to eq(nil)
+    expect(TurbotRunner::Validator.validate('primary-data', record, identifying_fields, Set.new)).to eq(nil)
   end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.38
+  version: 0.1.39
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-12 00:00:00.000000000 Z
+date: 2015-03-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.1.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.1.4
 - !ruby/object:Gem::Dependency
   name: openc-json_schema
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +100,10 @@ files:
 - spec/bots/bot-that-expects-file/manifest.json
 - spec/bots/bot-that-expects-file/scraper.rb
 - spec/bots/bot-that-expects-file/something.txt
+- spec/bots/bot-that-is-allowed-to-produce-duplicates/manifest.json
+- spec/bots/bot-that-is-allowed-to-produce-duplicates/scraper.rb
+- spec/bots/bot-that-produces-duplicates/manifest.json
+- spec/bots/bot-that-produces-duplicates/scraper.rb
 - spec/bots/bot-with-invalid-data-type/manifest.json
 - spec/bots/bot-with-invalid-data-type/scraper.rb
 - spec/bots/bot-with-invalid-sample-date/manifest.json