turbot-runner 0.2.21 → 0.2.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2a9fdfb0bc2c0e85ec636a581d2fb4657f865daa
-  data.tar.gz: e89c05d192f3d2a503c05c41faa48dc99cd4e966
+  metadata.gz: cf3018be33a490d1220c5f1afaeda1d7c5fdd2f1
+  data.tar.gz: a53d4a8b5a69d1da8a596d4028b3210fa4a87294
 SHA512:
-  metadata.gz: f0ca31bdbb3df8c219c98e0598243aa6dbd71557b1f69c18405bbb8410d636fa0a213fb03726f9f07b02b2277cb914814e58fbe542331f0f6621f76aa8e077bb
-  data.tar.gz: f8e09a2e31d4e1dce2a1fc473e72437a58d25f7e2eec2054c3313de2cbaf474c2d6db2610e69e48da2a749ec1d85ab33346b5e494df532b982815a0f0a44aa96
+  metadata.gz: 1701ffbe66db6b127c620dc902db0c748acd872d315968bf3da04492ed3817534e1653d43c5deffb4ed083bf5844d0de25b7f8fb69b57fab7b0f18e11c9a2cc0
+  data.tar.gz: 7e50e47e1f2097daf2f83a42ca4d3bab58053ab8acc9e366deec6d81a23ed123411cb17b3a1256dd6357d3c4189fab60e5e4737abb0618728e3173048bc02daa

data/lib/turbot_runner/processor.rb

@@ -10,20 +10,23 @@ module TurbotRunner
       @seen_uids = script_config[:duplicates_allowed] ? nil : Set.new
     end
 
-    def process(line)
+    def process(line, opts={})
+      validate = opts[:validate].nil? ? true : opts[:validate]
       begin
         if line.strip == "SNAPSHOT ENDED" || line.strip == "RUN ENDED" # latter is legacy
           @record_handler.handle_snapshot_ended(@data_type)
           @runner.interrupt if @runner
         else
           record = Openc::JsonSchema.convert_dates(schema_path, JSON.parse(line))
-
-          error_message = Validator.validate(
-            @data_type,
-            record,
-            @identifying_fields,
-            @seen_uids
-          )
+          error_message = nil
+          if validate
+            error_message = Validator.validate(
+              @data_type,
+              record,
+              @identifying_fields,
+              @seen_uids
+            )
+          end
 
           if error_message.nil?
             begin

data/lib/turbot_runner/runner.rb

@@ -68,11 +68,11 @@ module TurbotRunner
       end
     end
 
-    def process_output
-      process_script_output(scraper_config)
+    def process_output(opts={})
+      process_script_output(scraper_config, opts)
 
       transformers.each do |transformer_config|
-        process_script_output(transformer_config.merge(:base_directory => @base_directory))
+        process_script_output(transformer_config.merge(:base_directory => @base_directory), opts)
       end
     end
 
@@ -115,7 +115,7 @@ module TurbotRunner
       script_runner.run # returns boolean indicating success
     end
 
-    def process_script_output(script_config)
+    def process_script_output(script_config, opts)
       # The first argument to the Processor constructor is a nil
       # Runner. This is because no running behaviour
       # (e.g. interruptions etc) is required; we just want to do
@@ -124,7 +124,7 @@ module TurbotRunner
       file = output_file(script_config[:file])
       File.open(file) do |f|
         f.each_line do |line|
-          processor.process(line)
+          processor.process(line, opts)
         end
       end
     rescue Errno::ENOENT => e

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.2.21'
+  VERSION = '0.2.22'
 end

data/spec/lib/processor_spec.rb

@@ -31,16 +31,26 @@ describe TurbotRunner::Processor do
       end
 
       context 'with invalid record' do
-        it 'calls Handler#handle_invalid_record' do
-          record = {
+        before do
+          @record = {
             'sample_date' => '2014-06-01',
             'number' => 123
           }
+        end
 
+        it 'calls Handler#handle_invalid_record' do
           expected_error = 'Missing required property: source_url'
           expect(@handler).to receive(:handle_invalid_record).
-            with(record, @data_type, expected_error)
-          @processor.process(record.to_json)
+            with(@record, @data_type, expected_error)
+          @processor.process(@record.to_json)
+        end
+
+        context 'requesting non-validation' do
+          it 'calls Handler#handle_valid_record' do
+            expect(@handler).to receive(:handle_valid_record).
+                                 with(@record, @data_type)
+            @processor.process(@record.to_json, validate: false)
+          end
         end
       end
 

data/spec/lib/runner_spec.rb

@@ -327,6 +327,18 @@ describe TurbotRunner::Runner do
       expect(@handler.records_seen['simple-licence']).to eq(10)
     end
 
+    it 'passes opts to processor.process' do
+      test_runner('bot-with-transformer').run
+      runner = test_runner('bot-with-transformer',
+        :record_handler => @handler
+      )
+      opts = {frob: 5}
+      processor = double('processor')
+      allow(TurbotRunner::Processor).to receive(:new).and_return(processor)
+      expect(processor).to receive(:process).with(anything, opts).at_least(:once)
+      runner.process_output(opts)
+    end
+
     it 'can cope when scraper has failed immediately' do
       test_runner('bot-that-crashes-immediately').run
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.2.21
+  version: 0.2.22
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-07 00:00:00.000000000 Z
+date: 2016-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -228,61 +228,4 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Utilities for running bots with Turbot
-test_files:
-- spec/bots/bot-that-crashes-immediately/manifest.json
-- spec/bots/bot-that-crashes-immediately/scraper.rb
-- spec/bots/bot-that-crashes-immediately/transformer1.rb
-- spec/bots/bot-that-crashes-in-scraper/manifest.json
-- spec/bots/bot-that-crashes-in-scraper/scraper.rb
-- spec/bots/bot-that-crashes-in-scraper/transformer1.rb
-- spec/bots/bot-that-crashes-in-transformer/manifest.json
-- spec/bots/bot-that-crashes-in-transformer/scraper.rb
-- spec/bots/bot-that-crashes-in-transformer/transformer1.rb
-- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
-- spec/bots/bot-that-emits-run-ended/manifest.json
-- spec/bots/bot-that-emits-run-ended/scraper.rb
-- spec/bots/bot-that-emits-snapshot-ended/manifest.json
-- spec/bots/bot-that-emits-snapshot-ended/scraper.rb
-- spec/bots/bot-that-expects-file/manifest.json
-- spec/bots/bot-that-expects-file/scraper.rb
-- spec/bots/bot-that-expects-file/something.txt
-- spec/bots/bot-that-is-allowed-to-produce-duplicates/manifest.json
-- spec/bots/bot-that-is-allowed-to-produce-duplicates/scraper.rb
-- spec/bots/bot-that-produces-duplicates/manifest.json
-- spec/bots/bot-that-produces-duplicates/scraper.rb
-- spec/bots/bot-with-invalid-data-type/manifest.json
-- spec/bots/bot-with-invalid-data-type/scraper.rb
-- spec/bots/bot-with-invalid-sample-date/manifest.json
-- spec/bots/bot-with-invalid-sample-date/scraper.rb
-- spec/bots/bot-with-pause/manifest.json
-- spec/bots/bot-with-pause/scraper.rb
-- spec/bots/bot-with-transformer/manifest.json
-- spec/bots/bot-with-transformer/scraper.rb
-- spec/bots/bot-with-transformer/transformer.rb
-- spec/bots/bot-with-transformers/manifest.json
-- spec/bots/bot-with-transformers/scraper.rb
-- spec/bots/bot-with-transformers/transformer1.rb
-- spec/bots/bot-with-transformers/transformer2.rb
-- spec/bots/invalid-json-bot/manifest.json
-- spec/bots/invalid-json-bot/scraper.rb
-- spec/bots/invalid-record-bot/manifest.json
-- spec/bots/invalid-record-bot/scraper.rb
-- spec/bots/logging-bot/manifest.json
-- spec/bots/logging-bot/scraper.rb
-- spec/bots/python-bot/manifest.json
-- spec/bots/python-bot/scraper.py
-- spec/bots/ruby-bot/manifest.json
-- spec/bots/ruby-bot/scraper.rb
-- spec/bots/slow-bot/manifest.json
-- spec/bots/slow-bot/scraper.rb
-- spec/lib/processor_spec.rb
-- spec/lib/runner_spec.rb
-- spec/lib/utils_spec.rb
-- spec/lib/validator_spec.rb
-- spec/manual.rb
-- spec/outputs/full-scraper.out
-- spec/outputs/full-transformer.out
-- spec/outputs/truncated-scraper.out
-- spec/spec_helper.rb
-- spec/support/custom_matchers.rb
-- spec/support/helpers.rb
+test_files: []