turbot-runner 0.0.24 → 0.1.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YTNmMmJmZDNkNDllMTk0MjFmYTkxOTMxZWI3NzNiYjdlYzQ2NzgzMg==
+    YWYzYWQ4OTMwNGNiMDMxMjJlYzRlMDMyMGNjMmM0ZGNlZjA5MzdiNQ==
   data.tar.gz: !binary |-
-    MThhNmM4ZGM3OTU2NDAyZGRkM2EzZTE2ZTA4ZDQzMjFmNTE3ODFmNw==
+    ZTJjZGMxNDlkN2EyMmVjZDJjZWNmZWQ0ZWE5NTJjN2EyYjgxYjRkNQ==
 SHA512:
   metadata.gz: !binary |-
-    Yjc0NjdiZTc2NWE1NzAwMGIzMTBhNTIxNGE5YmQ4ZjFmMWMwZjcyMWJlYWFk
-    ZDM0YjcyNDRiYTE2MjMzOTQ4MDA4YzkwNjAzOGUyOGNjNmI3Y2U3YzEwODdk
-    YmQ1ZjYzYjdmNGY5ODIwZjZhMjQ5ODQ5OThlNWU4MDg1M2I5ODM=
+    Yjg3YThjYzk5NTBhMjFjN2Q1MzMxNmY2N2ZhNDhmNDM3MmUxN2ZiYzgwZTM0
+    ZTM0ZWJmNDA2NDljZDk4YTM0NDlmMjc3ODk3ZjY3NDk3MjZhZTM1ZjJlNzFi
+    Mzg5ODdkODIwNjU3YjVmNmI3OWQ2YjBjZjZlMmIzNTk1MTZmZDQ=
   data.tar.gz: !binary |-
-    YjNkMDAyMTBkYjY4YTg5ZWM5NDdlOGE2OWE5OGNkMzRkMDI1MjA4OWJjMDQ3
-    NTJiODM4MzIzNDQ2ZDg2ZTNlZDJkYTNmMzk4ODExODlhOTdjNTU0NDc5MmI1
-    NWIyMTY2MTVjOGI1OWYxNmNjMzJlOTdlZDU1Yzk0NzQxODc1ZmQ=
+    NDQ2ZjY4YmE1NTI5NjRiM2U3ZTVlYjhlOTY1NDc2NGJjMGM1ODdhZDAzZjE5
+    ZWFhZWFmOTIwYTUyNjhjZDNiOWI3ZDliNWU1ODdlZDViZGYzNjZiZGYzMTg3
+    Y2NkZjE3ODVhYWI4YWQ2NGQzZTI4YTc0NTI3NzA2OGJhMjhiZDM=

data/lib/turbot_runner/base_handler.rb

@@ -0,0 +1,13 @@
+module TurbotRunner
+  class BaseHandler
+    def handle_valid_record(record, data_type)
+      true
+    end
+
+    def handle_invalid_record(record, data_type, line)
+    end
+
+    def handle_invalid_json(line)
+    end
+  end
+end

data/lib/turbot_runner/processor.rb

@@ -0,0 +1,55 @@
+require 'json'
+require 'json-schema'
+
+module TurbotRunner
+  class Processor
+    def initialize(runner, data_type, record_handler)
+      @runner = runner
+      @data_type = data_type
+      @record_handler = record_handler
+    end
+
+    def process(line)
+      begin
+        record = JSON.parse(line)
+        errors = validate(record)
+
+        if errors.empty?
+          rc = @record_handler.handle_valid_record(record, @data_type)
+          @runner.interrupt unless rc
+        else
+          @record_handler.handle_invalid_record(record, @data_type, errors)
+          @runner.interrupt_and_mark_as_failed
+        end
+      rescue JSON::ParserError
+        @record_handler.handle_invalid_json(line)
+        @runner.interrupt_and_mark_as_failed
+      end
+    end
+
+    def interrupt
+      @runner.interrupt
+    end
+
+    def validate(record)
+      errors = JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
+      messages = errors.map do |error|
+        case error[:message]
+        when /The property '#\/' did not contain a required property of '(\w+)'/
+          "Missing required attribute: #{Regexp.last_match(1)}"
+        else
+          error[:message]
+        end
+      end
+    end
+
+    def schema
+      @schema ||= get_schema
+    end
+
+    def get_schema
+      hyphenated_name = @data_type.to_s.gsub("_", "-").gsub(" ", "-")
+      File.expand_path("../../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
+    end
+  end
+end

data/lib/turbot_runner/runner.rb

@@ -0,0 +1,150 @@
+require 'json'
+require 'fileutils'
+
+module TurbotRunner
+  class Runner
+    attr_reader :directory
+
+    def initialize(directory, options={})
+      @directory = directory
+      @config = load_config(directory)
+      @record_handler = options[:record_handler]
+      @log_to_file = options[:log_to_file]
+      @timeout = options[:timeout]
+    end
+
+    def run
+      FileUtils.rm_rf(output_directory)
+      FileUtils.mkdir_p(output_directory)
+
+      return false if not run_scraper
+
+      transformers.each do |transformer|
+        return false if not run_transformer(transformer)
+      end
+
+      true
+    end
+
+    def process_output
+      return false if not process_scraper_output
+
+      transformers.each do |transformer|
+        return false if not process_transformer_output(transformer)
+      end
+
+      true
+    end
+
+    private
+    def load_config(directory)
+      manifest_path = File.join(directory, 'manifest.json')
+      raise "Could not find #{manifest_path}" unless File.exist?(manifest_path)
+
+      begin
+        json = open(manifest_path) {|f| f.read}
+        JSON.parse(json, :symbolize_names => true)
+      rescue JSON::ParserError
+        # TODO provide better error message
+        raise "Could not parse #{manifest_path} as JSON"
+      end
+    end
+
+    def run_scraper
+      run_script(scraper_script, scraper_data_type)
+    end
+
+    def run_transformer(transformer)
+      run_script(
+        transformer[:file],
+        transformer[:data_type],
+        input_file=scraper_output_file
+      )
+    end
+
+    def run_script(script, data_type, input_file=nil)
+      command = build_command(script, input_file)
+
+      runner = ScriptRunner.new(
+        command,
+        output_file(script),
+        data_type,
+        :record_handler => @record_handler,
+        :timeout => @timeout
+      )
+
+      runner.run
+    end
+
+    def process_scraper_output
+      process_script_output(scraper_script, scraper_data_type)
+    end
+
+    def process_transformer_output(transformer)
+      process_script_output(transformer[:file], transformer[:data_type])
+    end
+
+    def process_script_output(script, data_type)
+      processor = Processor.new(nil, data_type, @record_handler)
+
+      File.open(output_file(script)) do |f|
+        f.each_line do |line|
+          processor.process(line)
+        end
+      end
+    end
+
+    def build_command(script, input_file=nil)
+      raise "Could not run #{script} with #{language}" unless script_extension == File.extname(script)
+      path_to_script = File.join(@directory, script)
+      command = "#{language} #{additional_args} #{path_to_script} >#{output_file(script)}"
+      command << " 2>#{output_file(script, '.err')}" if @log_to_file
+      command << " <#{input_file}" unless input_file.nil?
+
+      command
+    end
+
+    def output_file(script, extension='.out')
+      basename = File.basename(script, script_extension)
+      File.join(output_directory, basename) + extension
+    end
+
+    def script_extension
+      {
+        'ruby' => '.rb',
+        'python' => '.py',
+      }[language]
+    end
+
+    def additional_args
+      {
+        'ruby' => "-r#{File.expand_path('../prerun.rb', __FILE__)}",
+        'python' => '-u',
+      }[language]
+    end
+
+    def scraper_script
+      "scraper#{script_extension}"
+    end
+
+    def transformers
+      @config[:transformers] || []
+    end
+
+    def scraper_output_file
+      File.join(output_directory, 'scraper.out')
+    end
+
+    def language
+      @config[:language].downcase
+    end
+
+    def scraper_data_type
+      @config[:data_type]
+    end
+
+    def output_directory
+      File.join(@directory, 'output')
+    end
+  end
+end

data/lib/turbot_runner/script_runner.rb

@@ -0,0 +1,90 @@
+require 'timeout'
+
+# This is a useful blog post:
+# http://blog.robseaman.com/2008/12/12/sending-ctrl-c-to-a-subprocess-with-ruby
+
+# Ensure that SIGINT is ignored by the process running this.
+trap('INT') {}
+
+module TurbotRunner
+  class ScriptRunner
+    def initialize(command, output_file, data_type, options={})
+      @command = command
+      @output_file = output_file
+
+      record_handler = options[:record_handler] || BaseHandler.new  # A BaseHandler does nothing
+      @processor = Processor.new(self, data_type, record_handler)
+
+      @timeout = options[:timeout] || 3600
+    end
+
+    def run
+      @interrupted = false
+      @failed = false
+
+      # Start a thread that spawns a subprocess that runs the script and
+      # redirects the script's output to a file at a known location.
+      script_thread = Thread.new { run_command(@command) }
+
+      # Wait for the output file to be created, so that we can start to read
+      # from it.
+      begin
+        f = File.open(@output_file)
+      rescue Errno::ENOENT
+        sleep 0.1
+        retry
+      end
+        
+      # Read from output file line by line until either we reach the end of the
+      # file and the script has exited, or @interrupted becomes true.
+      until @interrupted do
+        begin
+          line = f.readline
+          @processor.process(line)
+        rescue EOFError
+          break unless script_thread.alive?
+          sleep 0.1
+        end
+      end
+
+      # script_thread may still be alive if we exited the loop above becuase
+      # @interrupted became true, and so we must kill it.
+      kill_running_processes if script_thread.alive?
+
+      @failed ? false : script_thread.join.value
+    ensure
+      f.close if f
+    end
+
+    def interrupt
+      @interrupted = true
+    end
+
+    def interrupt_and_mark_as_failed
+      @interrupted = true
+      @failed = true
+    end
+
+    private
+    def run_command(command)
+      begin
+        Timeout::timeout(@timeout) do
+          system(command)
+
+          # A nil exitstatus indicates that the script was interrupted.  A
+          # termsig of 2 indicates that the script was interrupted by a SIGINT.
+          $?.exitstatus == 0 || ($?.exitstatus.nil? && $?.termsig == 2)
+        end
+      rescue Timeout::Error
+        kill_running_processes
+        false
+      end
+    end
+
+    def kill_running_processes
+      # Send SIGINT to each process in the current proceess group, having
+      # already ensured that the current process itself ignores the signal.
+      Process.kill('INT', 0)
+    end
+  end
+end

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.0.24'
+  VERSION = '0.1.0'
 end

data/lib/turbot_runner.rb

@@ -1,335 +1,5 @@
-#require 'active_support/core_ext/hash/slice'
-#require 'active_support/core_ext/object/to_query'
-require 'json'
-require 'json-schema'
-require 'open3'
-require 'set'
-require 'timeout'
-require 'io/wait'
-
-module TurbotRunner
-  class ScriptError < StandardError; end
-
-  class BaseRunner
-
-    attr_reader :wait_thread
-    attr_reader :error
-
-    def initialize(bot_directory)
-      @bot_directory = bot_directory
-
-      manifest_path = File.join(bot_directory, 'manifest.json')
-      raise "Could not find #{manifest_path}" unless File.exist?(manifest_path)
-
-      begin
-        @config = JSON.parse(open(manifest_path) {|f| f.read})
-      rescue JSON::ParserError
-        # TODO provide better error message
-        raise "Could not parse #{manifest_path} as JSON"
-      end
-
-      @status = :initialized
-      @interrupted = false
-      @schemas = {}
-    end
-
-    def run(opts={})
-      @status = :running
-
-      command = "#{interpreter_for(scraper_file)} #{scraper_file}"
-      data_type = @config['data_type']
-
-      scraper_runner = CommandRunner.new(command)
-
-      transformers.each do |config|
-        file = File.join(@bot_directory, config['file'])
-        command = "#{interpreter_for(file)} #{file}"
-        transformer_runner = CommandRunner.new(command)
-        config['runner'] = transformer_runner
-      end
-
-      begin
-        until @interrupted do
-          line = scraper_runner.get_next_line
-
-          if line.nil?
-            if scraper_runner.finished?
-              if scraper_runner.success?
-                break
-              else
-                scraper_runner.raise_if_failed!
-              end
-            else
-              next
-            end
-          end
-
-          begin
-            record = JSON.parse(line)
-          rescue JSON::ParserError
-            handle_non_json_output(line)
-            next
-          end
-
-          errors = validate(record, data_type)
-
-          if errors.empty?
-            handle_valid_record(record, data_type)
-
-            transformers.each do |transformer|
-              data_type1 = transformer['data_type']
-
-              runner = transformer['runner']
-              runner.raise_if_failed!
-
-              runner.send_line(line)
-              line1 = runner.get_next_line
-
-              if line1.nil?
-                if runner.finished?
-                  if runner.success?
-                    break
-                  else
-                    runner.raise_if_failed!
-                  end
-                else
-                  next
-                end
-              end
-
-              # A transformer can output an empty line if it doesn't make
-              # sense to transform a record.
-              if line1.strip.empty?
-                puts
-                next
-              end
-
-              begin
-                record1 = JSON.parse(line1)
-              rescue JSON::ParserError
-                handle_non_json_output(line1)
-                next
-              end
-
-              errors = validate(record1, data_type1)
-
-              if errors.empty?
-                handle_valid_record(record1, data_type1)
-              else
-                handle_invalid_record(record1, data_type1, errors)
-              end
-            end
-          else
-            handle_invalid_record(record, data_type, errors)
-          end
-        end
-        if @interrupted
-          @status = :interrupted
-          handle_interrupted_run
-        else
-          @status = :successful
-          handle_successful_run
-        end
-
-        handle_stderr(scraper_runner.drain_stderr)
-
-        transformers.each do |transformer|
-          runner = transformer['runner']
-          handle_stderr(runner.drain_stderr)
-        end
-
-      rescue ScriptError => e
-        if @interrupted
-          @status = :interrupted
-          handle_interrupted_run
-        else
-          @status = :failed
-          handle_failed_run
-        end
-      end
-    ensure
-      handle_stderr(scraper_runner.drain_stderr)
-      scraper_runner.close unless scraper_runner.nil?
-
-      transformers.each do |transformer|
-        runner = transformer['runner']
-        if !runner.nil?
-          handle_stderr(runner.drain_stderr)
-          runner.close
-        end
-      end
-    end
-
-    def successful?
-      @status == :successful
-    end
-
-    def interrupt
-      @interrupted = true
-    end
-
-    private
-    def transformers
-      @config['transformers'] || []
-    end
-
-    def validate(record, data_type)
-      schema = get_schema(data_type)
-      errors = JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
-      messages = errors.map do |error|
-        case error[:message]
-        when /The property '#\/' did not contain a required property of '(\w+)'/
-          "Missing required attribute: #{Regexp.last_match(1)}"
-        else
-          error[:message]
-        end
-      end
-
-#      if messages.empty?
-#        identifying_fields = identifying_fields_for_data_type(data_type)
-#        identifying_hash = record.slice(*identifying_fields)
-#
-#        if identifying_hash.empty?
-#          messages << "Missing attributes for identifying fields: #{identifying_fields.join(', ')}"
-#        else
-#          record_uid = Digest::SHA1.hexdigest(identifying_hash.to_query)
-#          if @seen_uids.include?(record_uid)
-#            messages << "Values for identifying fields must be unique.  There has already been a record with: #{identifying_hash.to_json}"
-#          else
-#            @seen_uids << record_uid
-#          end
-#        end
-#      end
-
-      messages
-    end
-
-    def identifying_fields_for_data_type(data_type)
-      if data_type == @config['data_type']
-        @config['identifying_fields']
-      else
-        transformers = @config['transformers'].select {|transformer| transformer['data_type'] == data_type}
-        raise "Expected to find precisely 1 transformer matching #{data_type} in manifest.json" unless transformers.size == 1
-        transformers[0]['identifying_fields']
-      end
-    end
-
-    def get_schema(data_type)
-      if !@schemas.has_key?(data_type)
-        hyphenated_name = data_type.to_s.gsub("_", "-").gsub(" ", "-")
-        @schemas[data_type] = File.expand_path("../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
-      end
-
-      @schemas[data_type]
-    end
-
-    def handle_valid_record(record, data_type)
-      raise NotImplementedError
-    end
-
-    def handle_invalid_record(record, data_type, errors)
-      raise NotImplementedError
-    end
-
-    def handle_non_json_output(line)
-      raise NotImplementedError
-    end
-
-    def handle_successful_run
-    end
-
-    def handle_interrupted_run
-    end
-
-    def handle_stderr(data)
-      $stderr.write(data)
-    end
-
-    def scraper_file
-      candidates = Dir.glob(File.join(@bot_directory, 'scraper.{rb,py}'))
-      case candidates.size
-      when 0
-        raise 'Could not find scraper to run'
-      when 1
-        candidates.first
-      else
-        raise "Found multiple scrapers: #{candidates.join(', ')}"
-      end
-    end
-
-    def interpreter_for(file)
-      case file
-      when /\.rb$/
-        prerun = File.expand_path("../prerun.rb", __FILE__)
-        "ruby -r#{prerun}"
-      when /\.py$/
-        'python -u'
-      else
-        raise "Could not run #{file}"
-      end
-    end
-  end
-
-  class CommandRunner
-
-    def to_s
-      "CommandRunner #{@command}, pid #{@wait_thread.pid} (#{@wait_thread.status})"
-    end
-
-    def initialize(command, opts={})
-      @command = command
-      @timeout = opts[:timeout] ||= 3600
-      @stdin, @stdout, @stderr, @wait_thread = Open3.popen3(command)
-    end
-
-    def get_next_line
-      begin
-        Timeout::timeout(@timeout) { @stdout.gets }
-      rescue Timeout::Error
-        raise TurbotRunner::ScriptError.new("#{@command} produced no output for #{@timeout} seconds")
-      rescue EOFError
-        raise_if_failed!
-        return nil
-      end
-    end
-
-    def drain_stderr
-      output = ''
-      while @stderr.ready?
-        output += @stderr.read(256)
-      end
-      output
-    end
-
-    def success?
-      if finished?
-        @wait_thread.value.success?
-      end
-    end
-
-    def failed?
-      if finished?
-        !@wait_thread.value.success?
-      end
-    end
-
-    def raise_if_failed!
-      raise TurbotRunner::ScriptError if failed?
-    end
-
-    def finished?
-      !@wait_thread.status
-    end
-
-    def send_line(line)
-      @stdin.puts(line)
-    end
-
-    def close
-      @stdin.close
-      @stdout.read # drain pipe
-      @stdout.close
-      @wait_thread.kill
-    end
-  end
-end
+require 'turbot_runner/base_handler'
+require 'turbot_runner/processor'
+require 'turbot_runner/runner'
+require 'turbot_runner/script_runner'
+require 'turbot_runner/version'