turbot-runner 0.1.3 → 0.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MWZiMTU5NGIzYjc4ZTdlNTJiOGRiMzkwNzI0NjRiY2MwYWU2Yzk0Ng==
+    M2Q4MGEyNmViMzEzODU2YWJkNTRkNjY2NzQ4N2JkODUzYmM3MTVhOQ==
   data.tar.gz: !binary |-
-    YWZkY2NhNTNhNzdjMWQ1MzBkNzc2NWNhNTY0ZDg2YzU0MWI0YTgxMA==
+    MjJhZTJjZjMwYmZjNTY5Mjk3YWU4OTUwMTA1M2I4MjNiZWFiNzVlYw==
 SHA512:
   metadata.gz: !binary |-
-    NzlhZjdmMjZhZTNjNDQyOTY1MWVmZGEzZDFmM2RjZWY1MjAwMDM5OTlmZDAx
-    N2E2ZGExNDljYTYxOTE3NDc3NTEyODYyMDkzOTUxYzVmODdmMzVjMTc5MmE3
-    NzUyNDhjNTRmNTk2NjRhZDU4M2IzY2VjZTAwOWJmMmFmNTRkYjc=
+    ZjYzMTZhODVkNmU3MWE3ZjkzNTNkMzRhNzZjMWUzODlhODc0YmFlODhlMWE3
+    YzUxMDU3Y2E5MTBhZTg2MTY3MjIyMDNkZmE2OTIwOWI1M2RmMTQ4MDQzODlm
+    MjE0MGMzNTc2NjZlZWI5MGI5NTgxZTQzZDRkYmE4YzUxZjAzOWE=
   data.tar.gz: !binary |-
-    MDA5NGJhOTBiZTIyZGYyNTEyN2NhNWEwOTVjYTVlNTQ2YTE2YTg1ODNlYTlj
-    NTJjYzAxZTMzYzkwMDgzN2MzNGExZGQwMWU1Mzk3YzAwMTVjNDQxMGFkMGU3
-    MWFlMzQ3Yjg1Njk4ODM3NWUwNGIwNTI0YTk3MWM4YThhM2YxNWY=
+    ZGE3NDZmZGRhYWI2NjNlZTM4MWExYzg3MmZiYzNiMzExMTFhZDlkYWVhOTIy
+    MTk3Mzk1NTU3Y2RhZDY2ZGQ1Yzc5ZTg0YTMxMmQyN2Y0YzVmMjA5ZjVlODk4
+    MDg1MDQ0ZWI4MDk1ZWZjZjk2ZmMyOTEwMDY5Mzc5NjFjZWRjZjg=

data/lib/turbot_runner/processor.rb

@@ -3,9 +3,10 @@ require 'json-schema'
 
 module TurbotRunner
   class Processor
-    def initialize(runner, data_type, record_handler)
+    def initialize(runner, script_config, record_handler)
       @runner = runner
-      @data_type = data_type
+      @data_type = script_config[:data_type]
+      @identifying_fields = script_config[:identifying_fields]
       @record_handler = record_handler
     end
 
@@ -44,6 +45,18 @@ module TurbotRunner
           error[:message]
         end
       end
+
+      if messages.empty?
+        identifying_attributes = record.reject do |k, v|
+          !@identifying_fields.include?(k) || v.nil? || v == ''
+        end
+
+        if identifying_attributes.empty?
+          messages << "There were no values provided for any of the identifying fields: #{@identifying_fields.join(', ')}"
+        end
+      end
+
+      messages
     end
 
     def schema

data/lib/turbot_runner/runner.rb

@@ -17,20 +17,20 @@ module TurbotRunner
     def run
       FileUtils.mkdir_p(@output_directory)
 
-      return false if not run_scraper
+      return false if not run_script(scraper_config)
 
-      transformers.each do |transformer|
-        return false if not run_transformer(transformer)
+      transformers.each do |transformer_config|
+        return false if not run_script(transformer_config, input_file=scraper_output_file)
       end
 
       true
     end
 
     def process_output
-      return false if not process_scraper_output
+      return false if not process_script_output(scraper_config)
 
-      transformers.each do |transformer|
-        return false if not process_transformer_output(transformer)
+      transformers.each do |transformer_config|
+        return false if not process_script_output(transformer_config)
       end
 
       true
@@ -50,25 +50,13 @@ module TurbotRunner
       end
     end
 
-    def run_scraper
-      run_script(scraper_script, scraper_data_type)
-    end
-
-    def run_transformer(transformer)
-      run_script(
-        transformer[:file],
-        transformer[:data_type],
-        input_file=scraper_output_file
-      )
-    end
-
-    def run_script(script, data_type, input_file=nil)
-      command = build_command(script, input_file)
+    def run_script(script_config, input_file=nil)
+      command = build_command(script_config[:file], input_file)
 
       runner = ScriptRunner.new(
         command,
-        output_file(script),
-        data_type,
+        output_file(script_config[:file]),
+        script_config,
         :record_handler => @record_handler,
         :timeout => @timeout
       )
@@ -76,18 +64,10 @@ module TurbotRunner
       runner.run
     end
 
-    def process_scraper_output
-      process_script_output(scraper_script, scraper_data_type)
-    end
-
-    def process_transformer_output(transformer)
-      process_script_output(transformer[:file], transformer[:data_type])
-    end
-
-    def process_script_output(script, data_type)
-      processor = Processor.new(nil, data_type, @record_handler)
+    def process_script_output(script_config)
+      processor = Processor.new(nil, script_config, @record_handler)
 
-      File.open(output_file(script)) do |f|
+      File.open(output_file(script_config[:file])) do |f|
         f.each_line do |line|
           processor.process(line)
         end
@@ -123,6 +103,14 @@ module TurbotRunner
       }[language]
     end
 
+    def scraper_config
+      {
+        :file => scraper_script,
+        :data_type => scraper_data_type,
+        :identifying_fields => scraper_identifying_fields
+      }
+    end
+
     def scraper_script
       "scraper#{script_extension}"
     end
@@ -142,5 +130,9 @@ module TurbotRunner
     def scraper_data_type
       @config[:data_type]
     end
+
+    def scraper_identifying_fields
+      @config[:identifying_fields]
+    end
   end
 end

data/lib/turbot_runner/script_runner.rb

@@ -8,12 +8,12 @@ trap('INT') {}
 
 module TurbotRunner
   class ScriptRunner
-    def initialize(command, output_file, data_type, options={})
+    def initialize(command, output_file, script_config, options={})
       @command = command
       @output_file = output_file
 
       record_handler = options[:record_handler] || BaseHandler.new  # A BaseHandler does nothing
-      @processor = Processor.new(self, data_type, record_handler)
+      @processor = Processor.new(self, script_config, record_handler)
 
       @timeout = options[:timeout] || 3600
     end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/spec/lib/{processor.rb → processor_spec.rb}

@@ -6,7 +6,13 @@ describe TurbotRunner::Processor do
     before do
       @handler = TurbotRunner::BaseHandler.new
       @data_type = 'primary data'
-      @processor = TurbotRunner::Processor.new(@handler, @data_type)
+      script_config = {
+        :data_type => @data_type,
+        :identifying_fields => ['number']
+      }
+      script_runner = instance_double('ScriptRunner')
+      allow(script_runner).to receive(:interrupt_and_mark_as_failed)
+      @processor = TurbotRunner::Processor.new(script_runner, script_config, @handler)
     end
 
     context 'with valid record' do
@@ -21,15 +27,15 @@ describe TurbotRunner::Processor do
       end
     end
 
-    context 'with invalid record' do
-      it 'calls Handler#handle_invalid_record' do
-        before do
-          @record = {
-            'sample_date' => '2014-06-01',
-            'number' => 123
-          }
-        end
+    context 'with record missing required field' do
+      before do
+        @record = {
+          'sample_date' => '2014-06-01',
+          'number' => 123
+        }
+      end
 
+      it 'calls Handler#handle_invalid_record' do
         expected_errors = ['Missing required attribute: source_url']
         expect(@handler).to receive(:handle_invalid_record).
           with(@record, @data_type, expected_errors)
@@ -37,6 +43,22 @@ describe TurbotRunner::Processor do
       end
     end
 
+    context 'with record missing all identifying fields' do
+      before do
+        @record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123'
+        }
+      end
+
+      it 'calls Handler#handle_invalid_record' do
+        expected_errors = ['There were no values provided for any of the identifying fields: number']
+        expect(@handler).to receive(:handle_invalid_record).
+          with(@record, @data_type, expected_errors)
+        @processor.process(@record.to_json)
+      end
+    end
+
     context 'with invalid JSON' do
       it 'calls Handler#handle_invalid_json' do
         line = 'this is not JSON'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-07 00:00:00.000000000 Z
+date: 2014-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
@@ -88,7 +88,7 @@ files:
 - spec/bots/ruby-bot/scraper.rb
 - spec/bots/slow-bot/manifest.json
 - spec/bots/slow-bot/scraper.rb
-- spec/lib/processor.rb
+- spec/lib/processor_spec.rb
 - spec/lib/runner_spec.rb
 - spec/manual_spec.rb
 - spec/outputs/full-scraper.out