turbot-runner 0.0.20 → 0.0.21

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.0.20'
+  VERSION = '0.0.21'
 end

data/lib/turbot_runner.rb

@@ -11,6 +11,7 @@ module TurbotRunner
   class BaseRunner
 
     attr_reader :wait_thread
+    attr_reader :error
 
     def initialize(bot_directory)
       @bot_directory = bot_directory
@@ -28,7 +29,6 @@ module TurbotRunner
       @status = :initialized
       @interrupted = false
       @schemas = {}
-#      @seen_uids = Set.new
     end
 
     def run(opts={})
@@ -49,7 +49,18 @@ module TurbotRunner
       begin
         until @interrupted do
           line = scraper_runner.get_next_line
-          break if line.nil?
+
+          if line.nil?
+            if scraper_runner.finished?
+              if scraper_runner.success?
+                break
+              else
+                scraper_runner.raise_if_failed!
+              end
+            else
+              next
+            end
+          end
 
           begin
             record = JSON.parse(line)
@@ -67,12 +78,29 @@ module TurbotRunner
               data_type1 = transformer['data_type']
 
               runner = transformer['runner']
+              runner.raise_if_failed!
+
               runner.send_line(line)
               line1 = runner.get_next_line
 
-              # A transformer should output an empty line if it doesn't make
+              if line1.nil?
+                if runner.finished?
+                  if runner.success?
+                    break
+                  else
+                    runner.raise_if_failed!
+                  end
+                else
+                  next
+                end
+              end
+
+              # A transformer can output an empty line if it doesn't make
               # sense to transform a record.
-              next if line1.strip.empty?
+              if line1.strip.empty?
+                puts
+                next
+              end
 
               begin
                 record1 = JSON.parse(line1)
@@ -106,6 +134,7 @@ module TurbotRunner
           handle_interrupted_run
         else
           @status = :failed
+          @error = e
           handle_failed_run
         end
       end
@@ -227,24 +256,48 @@ module TurbotRunner
   end
 
   class CommandRunner
+
+    def to_s
+      "CommandRunner #{@command}, pid #{@wait_thread.pid} (#{@wait_thread.status})"
+    end
+
     def initialize(command, opts={})
       @command = command
       @timeout = opts[:timeout] ||= 3600
-      @stdin, @stdout, @wait_thread = Open3.popen2(command)
+      @stdin, @stdout, @stderr, @wait_thread = Open3.popen3(command)
     end
 
     def get_next_line
       begin
         Timeout::timeout(@timeout) { @stdout.gets }
       rescue Timeout::Error
-        STDOUT.puts("#{@command} produced no output for #{@timeout} seconds")
-        raise ScriptError
+        raise TurbotRunner::ScriptError.new("#{@command} produced no output for #{@timeout} seconds")
       rescue EOFError
-        raise ScriptError unless @wait_thread.value.success?
+        raise_if_failed!
         return nil
       end
     end
 
+    def success?
+      if finished?
+        @wait_thread.value.success?
+      end
+    end
+
+    def failed?
+      if finished?
+        !@wait_thread.value.success?
+      end
+    end
+
+    def raise_if_failed!
+      raise TurbotRunner::ScriptError.new(@stderr.read()) if failed?
+    end
+
+    def finished?
+      !@wait_thread.status
+    end
+
     def send_line(line)
       @stdin.puts(line)
     end

data/spec/dummy-bot-ruby/transformer.rb

@@ -1,6 +1,6 @@
 require 'json'
-
 STDIN.each_line do |line|
+
   raw_record = JSON.parse(line)
 
   transformed_record = {

data/spec/dummy-broken-bot-ruby/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "dummy-bot",
+    "data_type": "hello",
+    "description": "This is a dummy bot",
+    "identifying_fields": ["number"],
+    "files": ["scraper.rb"],
+    "frequency": "monthly"
+}

data/spec/dummy-broken-bot-ruby/scraper.rb

@@ -0,0 +1,4 @@
+require 'json'
+
+puts({h: 1}.to_json)
+raise "oops"

data/spec/dummy-broken-bot-ruby/transformer.rb

@@ -0,0 +1,12 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :n => raw_record['n'],
+    :goodbye => raw_record['hello'].sub('hello', 'goodbye')
+  }
+
+  puts transformed_record.to_json
+end

data/spec/dummy-broken-bot-ruby-2/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "dummy-bot",
+    "data_type": "hello",
+    "description": "This is a dummy bot",
+    "identifying_fields": ["number"],
+    "files": ["scraper.rb"],
+    "transformers": [
+        {
+            "file": "transformer.rb",
+            "data_type": "goodbye",
+            "identifying_fields": [""]
+        }
+    ],
+    "frequency": "monthly"
+}

data/spec/dummy-broken-bot-ruby-2/scraper.rb

@@ -0,0 +1,4 @@
+require 'json'
+
+puts({h: 1}.to_json)
+raise "oops"

data/spec/dummy-broken-bot-ruby-2/transformer.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :p => raw_record['n'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/dummy-broken-bot-ruby-3/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "dummy-bot",
+    "data_type": "hello",
+    "description": "This is a dummy bot",
+    "identifying_fields": ["number"],
+    "files": ["scraper.rb"],
+    "transformers": [
+        {
+            "file": "transformer.rb",
+            "data_type": "goodbye",
+            "identifying_fields": [""]
+        }
+    ],
+    "frequency": "monthly"
+}

data/spec/dummy-broken-bot-ruby-3/scraper.rb

@@ -0,0 +1,5 @@
+require 'json'
+
+3.times do |n|
+  puts({h: n}.to_json)
+end

data/spec/dummy-broken-bot-ruby-3/transformer.rb

@@ -0,0 +1,5 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raise "oops"
+end

data/spec/turbot_runner_spec.rb

@@ -11,6 +11,19 @@ class SpecRunner < TurbotRunner::BaseRunner
   end
 end
 
+class BrokenRunner < TurbotRunner::BaseRunner
+  def validate(record, data_type)
+    []
+  end
+
+  def handle_valid_record(*args)
+  end
+
+  def handle_failed_run
+  end
+end
+
+
 describe TurbotRunner::BaseRunner do
   it 'can run a ruby bot' do
     runner = SpecRunner.new('spec/dummy-bot-ruby')
@@ -39,4 +52,40 @@ describe TurbotRunner::BaseRunner do
     expect(runner).to receive(:handle_successful_run)
     runner.run
   end
+
+  describe "broken bots" do
+    describe "failing bot without transformer" do
+      it 'should call handle_failed_run' do
+        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
+        expect(runner).to receive(:handle_valid_record)
+        expect(runner).to receive(:handle_failed_run)
+        runner.run
+      end
+
+      it 'should set `error` attribute of runner with exception message' do
+        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
+        runner.run
+        expect(runner.error.to_s).to match(/oops/)
+      end
+    end
+
+    describe "failing bot with successful transformer" do
+      it 'should call handle_failed_run' do
+        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby-2')
+        expect(runner).to receive(:handle_valid_record) # first record
+        expect(runner).to receive(:handle_valid_record) # first transform
+        expect(runner).to receive(:handle_failed_run)
+        runner.run
+      end
+    end
+
+    describe "sucessful bot with failing transformer" do
+      it 'should call handle_failed_run' do
+        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby-3')
+        expect(runner).to receive(:handle_valid_record) # the untransformed one
+        expect(runner).to receive(:handle_failed_run) # the transformer breaks immediately
+        runner.run
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.21
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-22 00:00:00.000000000 Z
+date: 2014-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
@@ -43,6 +43,15 @@ files:
 - spec/dummy-bot-ruby/manifest.json
 - spec/dummy-bot-ruby/scraper.rb
 - spec/dummy-bot-ruby/transformer.rb
+- spec/dummy-broken-bot-ruby-2/manifest.json
+- spec/dummy-broken-bot-ruby-2/scraper.rb
+- spec/dummy-broken-bot-ruby-2/transformer.rb
+- spec/dummy-broken-bot-ruby-3/manifest.json
+- spec/dummy-broken-bot-ruby-3/scraper.rb
+- spec/dummy-broken-bot-ruby-3/transformer.rb
+- spec/dummy-broken-bot-ruby/manifest.json
+- spec/dummy-broken-bot-ruby/scraper.rb
+- spec/dummy-broken-bot-ruby/transformer.rb
 - spec/turbot_runner_spec.rb
 - schema/schemas/company-schema.json
 - schema/schemas/financial-payment-schema.json