turbot-runner 0.0.8 → 0.0.9

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YzFlNTliNjE3MjU2NTExNzU2ZDA0OWJjZDcyZTE5ZmNlZWMyNWQ2ZQ==
+  data.tar.gz: !binary |-
+    MDhlOWNhZjNkMWJlZjVkOTdkMWI4MzYxZDc0NDFiOTNlNDJlNmQwYg==
+SHA512:
+  metadata.gz: !binary |-
+    YzIzNjRmZWVjZjY1ZThkMTY3NmRkNzE3NDg2MjExYjFhMjRhMDZkMjBhMGE1
+    ZDk5OGE3ZWFkNDJhNDhjOTY4ODBjYWRkMjEzNjA1OTkzMjliYTc3YWI1ZDZi
+    OTBmMzEzM2I5OWYxZWQyYzQ2YTQyYTAwNjk2NWQxOGJmOWU1ZTM=
+  data.tar.gz: !binary |-
+    YjA3ZmYzOTA3ZTZhYzAwMGFhNjEzZmZjMzg2NGFhNjRhMTgwMTk3Yjc4ZmQw
+    MDkzNTZlNTEyZmZkZWYzMGUyMDJmODA2NDE0NjU3MWIyMWJjODI5Y2I1N2Yx
+    MjBkMDgzMDVjNzVhNzNiYjNkNDFjZGRhMzAzZjI5M2I1YzdmZDM=

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
 end

data/lib/turbot_runner.rb

@@ -31,8 +31,21 @@ module TurbotRunner
 
       command = "#{interpreter_for(scraper_file)} #{scraper_file}"
       data_type = @config['data_type']
+
+      scraper_runner = CommandRunner.new(command)
+
+      transformers.each do |config|
+        file = File.join(@bot_directory, config['file'])
+        command = "#{interpreter_for(file)} #{file}"
+        transformer_runner = CommandRunner.new(command)
+        config['runner'] = transformer_runner
+      end
+
       begin
-        run_script_each_line(command) do |line|
+        until @interrupted do
+          line = scraper_runner.get_next_line
+          break if line.nil?
+
           begin
             record = JSON.parse(line)
           rescue JSON::ParserError
@@ -41,29 +54,30 @@ module TurbotRunner
           end
 
           errors = validate(record, data_type)
+
           if errors.empty?
             handle_valid_record(record, data_type)
 
             transformers.each do |transformer|
-              file = File.join(@bot_directory, transformer['file'])
-              command1 = "#{interpreter_for(file)} #{file}"
               data_type1 = transformer['data_type']
 
-              run_script_each_line(command1, :input => line) do |line1|
-                begin
-                  record1 = JSON.parse(line1)
-                rescue JSON::ParserError
-                  handle_non_json_output(line1)
-                  next
-                end
-
-                errors = validate(record1, data_type1)
-
-                if errors.empty?
-                  handle_valid_record(record1, data_type1)
-                else
-                  handle_invalid_record(record1, data_type1, errors)
-                end
+              runner = transformer['runner']
+              runner.send_line(line)
+              line1 = runner.get_next_line
+
+              begin
+                record1 = JSON.parse(line1)
+              rescue JSON::ParserError
+                handle_non_json_output(line1)
+                next
+              end
+
+              errors = validate(record1, data_type1)
+
+              if errors.empty?
+                handle_valid_record(record1, data_type1)
+              else
+                handle_invalid_record(record1, data_type1, errors)
               end
             end
           else
@@ -87,6 +101,11 @@ module TurbotRunner
           handle_failed_run
         end
       end
+    ensure
+      scraper_runner.close unless scraper_runner.nil?
+      transformers.each do |transformer|
+        transformer['runner'].close unless transformer['runner'].nil?
+      end
     end
 
     def successful?
@@ -146,32 +165,6 @@ module TurbotRunner
       raise NotImplementedError
     end
 
-    def run_script_each_line(command, options={})
-      # TODO: handle timeouts, errors
-      Open3::popen2(command) do |stdin, stdout, wait_thread|
-        @wait_thread = wait_thread
-        if options[:input]
-          stdin.puts(options[:input])
-          stdin.close
-        end
-
-        timeout = options[:timeout] || 3600
-
-        while !@interrupted do
-          begin
-            result = stdout.readline.strip
-            yield result unless result.empty?
-          rescue EOFError
-            break
-          end
-        end
-
-        if !wait_thread.value.success?
-          raise ScriptError
-        end
-      end
-    end
-
     def scraper_file
       candidates = Dir.glob(File.join(@bot_directory, 'scraper.{rb,py}'))
       case candidates.size
@@ -196,4 +189,33 @@ module TurbotRunner
       end
     end
   end
+
+  class CommandRunner
+    def initialize(command, opts={})
+      @command = command
+      @timeout = opts[:timeout] ||= 3600
+      @stdin, @stdout, @wait_thread = Open3.popen2(command)
+    end
+
+    def get_next_line
+      begin
+        Timeout::timeout(@timeout) { @stdout.gets }
+      rescue Timeout::Error
+        STDOUT.puts("#{@command} produced no output for #{@timeout} seconds")
+        raise ScriptError
+      rescue EOFError
+        raise ScriptError unless @wait_thread.value.success?
+        return nil
+      end
+    end
+
+    def send_line(line)
+      @stdin.puts(line)
+    end
+
+    def close
+      @stdin.close
+      @stdout.close
+    end
+  end
 end

data/spec/turbot_runner_spec.rb

@@ -3,9 +3,6 @@ require 'turbot_runner'
 
 describe TurbotRunner::BaseRunner do
   it 'can run a bot' do
-    # This test runs slowly - there seems to be some delay in subprocesses
-    # reading from their stdins, but this is not observed when the code is run
-    # outside of rspec.
 
     class SpecRunner < TurbotRunner::BaseRunner
       def validate(record, data_type)

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.0.8
-  prerelease: 
+  version: 0.0.9
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-08 00:00:00.000000000 Z
+date: 2014-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -37,10 +34,6 @@ files:
 - lib/prerun.rb
 - lib/turbot_runner.rb
 - lib/turbot_runner/version.rb
-- spec/dummy-bot/manifest.json
-- spec/dummy-bot/scraper.rb
-- spec/dummy-bot/transformer.rb
-- spec/turbot_runner_spec.rb
 - schema/schemas/company-schema.json
 - schema/schemas/includes/address.json
 - schema/schemas/includes/base-statement.json
@@ -58,29 +51,33 @@ files:
 - schema/schemas/share-parcel-schema.json
 - schema/schemas/simple-licence-schema.json
 - schema/schemas/subsidiary-relationship-schema.json
+- spec/dummy-bot/manifest.json
+- spec/dummy-bot/scraper.rb
+- spec/dummy-bot/transformer.rb
+- spec/turbot_runner_spec.rb
 homepage: http://turbot.opencorporates.com/
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Utilities for running bots with Turbot
 test_files: []
+has_rdoc: