turbot-runner 0.1.9 → 0.1.10

data/lib/turbot_runner/runner.rb

@@ -16,24 +16,20 @@ module TurbotRunner
 
     def run
       FileUtils.mkdir_p(@output_directory)
-
-      return false if not run_script(scraper_config)
-
+      succeeded = run_script(scraper_config)
+      # Run the transformers even if the scraper fails
       transformers.each do |transformer_config|
-        return false if not run_script(transformer_config, input_file=scraper_output_file)
+        succeeded = run_script(transformer_config, input_file=scraper_output_file) && succeeded
       end
-
-      true
+      succeeded
     end
 
     def process_output
-      return false if not process_script_output(scraper_config)
+      process_script_output(scraper_config)
 
       transformers.each do |transformer_config|
-        return false if not process_script_output(transformer_config)
+        process_script_output(transformer_config)
       end
-
-      true
     end
 
     private
@@ -53,7 +49,7 @@ module TurbotRunner
     def run_script(script_config, input_file=nil)
       command = build_command(script_config[:file], input_file)
 
-      runner = ScriptRunner.new(
+      script_runner = ScriptRunner.new(
         command,
         output_file(script_config[:file]),
         script_config,
@@ -61,7 +57,7 @@ module TurbotRunner
         :timeout => @timeout
       )
 
-      runner.run
+      script_runner.run # returns boolean indicating success
     end
 
     def process_script_output(script_config)

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.9'
+  VERSION = '0.1.10'
 end

data/spec/bots/bot-that-crashes-in-scraper/manifest.json

@@ -4,5 +4,12 @@
     "language": "ruby",
     "data_type": "primary data",
     "identifying_fields": ["licence_number"],
-    "files": ["scraper.rb"]
+    "files": ["scraper.rb", "transformer1.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
 }

data/spec/bots/bot-that-crashes-in-scraper/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/lib/runner_spec.rb

@@ -126,6 +126,14 @@ describe TurbotRunner::Runner do
         @runner.run
         expect([@runner, 'scraper']).to have_error_output_matching(/Oh no/)
       end
+
+      it 'still runs the transformers' do
+        expect(@runner).to receive(:run_script).once.with(
+          hash_including(:file=>"scraper.rb"))
+        expect(@runner).to receive(:run_script).once.with(
+          hash_including(:file=>"transformer1.rb"), anything)
+        @runner.run
+      end
     end
 
     context 'with a bot that crashes in transformer' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-18 00:00:00.000000000 Z
+date: 2014-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
@@ -44,6 +44,7 @@ files:
 - lib/turbot_runner/version.rb
 - spec/bots/bot-that-crashes-in-scraper/manifest.json
 - spec/bots/bot-that-crashes-in-scraper/scraper.rb
+- spec/bots/bot-that-crashes-in-scraper/transformer1.rb
 - spec/bots/bot-that-crashes-in-transformer/manifest.json
 - spec/bots/bot-that-crashes-in-transformer/scraper.rb
 - spec/bots/bot-that-crashes-in-transformer/transformer1.rb