turbot-runner 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MGQwYjkyYzJlYWE2MmM2YzRiMjljM2NlM2MxNGQyYmZkMDMzOWZjZQ==
+  data.tar.gz: !binary |-
+    ODRjOWJkZmIzYjNiNTliZmRjYjA2ZDNhZGQ0MmFlZGY4ZmEwZTc3ZA==
+SHA512:
+  metadata.gz: !binary |-
+    ZTYwYTdmOWY0OTdlYzEyZTNjZDMxOTc1ZGI2YmM2Nzg0ZDIwZTYwNzc1ZThi
+    NjJlNTVlNWUyMjFlM2RhOWQyMTU4MWM1MDJlYTRhZTAyNjY0MzJhMTBjOGZj
+    N2FkNzMwOTc3YmIxM2NmMjJhNWY5MWE1YTE1ZjIwMjcxZWMwODA=
+  data.tar.gz: !binary |-
+    MGNiZDlhYzNiYTM4ZjhmYjE3OWJmMzJhNzI5MThkMmM4ODBkMDQ4YmNmMmJi
+    MzE1ZDIzNmFkYmQ1OGRiNDA3MjY1OWZmZjdjN2FjNDMwNjYzNzg2NDhlYTc0
+    NmNkMmQwMTcwYWQ5YzE1YTY2NDJmZjFjYWY5Mjc4Zjk2NTZmZTQ=

data/bin/rspec

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rspec-core', 'rspec')

data/lib/turbot_runner.rb

@@ -0,0 +1,168 @@
+require 'open3'
+
+module TurbotRunner
+  class ScriptError < StandardError; end
+
+  class BaseRunner
+    def initialize(bot_directory)
+      @bot_directory = bot_directory
+
+      manifest_path = File.join(bot_directory, 'manifest.json')
+      raise "Could not find #{manifest_path}" unless File.exist?(manifest_path)
+
+      begin
+        @config = JSON.parse(open(manifest_path) {|f| f.read})
+      rescue JSON::ParserError
+        # TODO provide better error message
+        raise "Could not parse #{manifest_path} as JSON"
+      end
+
+      @interrupted = false
+      @schemas = {}
+    end
+
+    def run(opts={})
+      validation_required = opts[:validate] || true
+
+      command = "#{interpreter_for(scraper_file)} #{scraper_file}"
+      data_type = @config['data_type']
+
+      begin
+        run_script_each_line(command) do |line|
+          record = JSON.parse(line)
+          errors = validate(record, data_type)
+
+          if errors.empty?
+            handle_valid_record(record, data_type)
+
+            transformers.each do |transformer|
+              file = File.join(@bot_directory, transformer['file'])
+              command1 = "#{interpreter_for(file)} #{file}"
+              data_type1 = transformer['data_type']
+
+              run_script_each_line(command1, :input => line) do |line1|
+                record1 = JSON.parse(line1)
+
+                errors = validate(record1, data_type1)
+
+                if errors.empty?
+                  handle_valid_record(record1, data_type1)
+                else
+                  handle_invalid_record(record1, data_type1, errors)
+                end
+              end
+            end
+          else
+            handle_invalid_record(record, data_type, errors)
+          end
+        end
+
+        if @interrupted
+          handle_interrupted_run
+        else
+          handle_successful_run
+        end
+      rescue ScriptError => e
+        handle_failed_run(e.message)
+      end
+    end
+
+    def interrupt
+      @interrupted = true
+    end
+
+    private
+    def transformers
+      @config['transformers'] || []
+    end
+
+    def validate(record, data_type)
+      schema = get_schema(data_type)
+      errors = JSON::Validator.fully_validate(schema, record, :errors_as_objects => true)
+      errors.map do |error|
+        case error[:message]
+        when /The property '#\/' did not contain a required property of '(\w+)'/
+          "Missing required attribute: #{Regexp.last_match(1)}"
+        else
+          error[:message]
+        end
+      end
+    end
+
+    def get_schema(data_type)
+      if !@schemas.has_key?(data_type)
+        hyphenated_name = data_type.to_s.gsub("_", "-").gsub(" ", "-")
+        @schemas[data_type] = File.expand_path("../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
+      end
+
+      @schemas[data_type]
+    end
+
+    def handle_valid_record(record, data_type)
+      raise NotImplementedError
+    end
+
+    def handle_invalid_record(record, data_type, errors)
+      raise NotImplementedError
+    end
+
+    def handle_successful_run
+    end
+
+    def handle_interrupted_run
+    end
+
+    def handle_failed_run(output)
+      raise NotImplementedError
+    end
+
+    def run_script_each_line(command, options={})
+      # TODO: handle timeouts, errors
+      Open3::popen3(command) do |stdin, stdout, stderr, wait_thread|
+        if options[:input]
+          stdin.puts(options[:input])
+          stdin.close
+        end
+
+        timeout = options[:timeout] || 3600
+
+        while !@interrupted do
+          begin
+            result = stdout.readline.strip
+            yield result unless result.empty?
+          rescue EOFError
+            break
+          end
+        end
+
+        if !wait_thread.value.success?
+          output = stderr.read
+          raise ScriptError.new(output)
+        end
+      end
+    end
+
+    def scraper_file
+      candidates = Dir.glob(File.join(@bot_directory, 'scraper.{rb,py}'))
+      case candidates.size
+      when 0
+        raise 'Could not find scraper to run'
+      when 1
+        candidates.first
+      else
+        raise "Found multiple scrapers: #{candidates.join(', ')}"
+      end
+    end
+
+    def interpreter_for(file)
+      case file
+      when /\.rb$/
+        'ruby'
+      when /\.py$/
+        'python'
+      else
+        raise "Could not run #{file}"
+      end
+    end
+  end
+end

data/lib/turbot_runner/version.rb

@@ -0,0 +1,3 @@
+module TurbotRunner
+  VERSION = '0.0.1'
+end

data/spec/dummy-bot/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "dummy-bot",
+    "data_type": "hello",
+    "description": "This is a dummy bot",
+    "identifying_fields": ["number"],
+    "files": ["scraper.rb"],
+    "transformers": [
+        {
+            "file": "transformer.rb",
+            "data_type": "goodbye",
+            "identifying_fields": [""]
+        }
+    ],
+    "frequency": "monthly"
+}

data/spec/dummy-bot/scraper.rb

@@ -0,0 +1,6 @@
+require 'json'
+
+puts({:n => 1, :hello => 'hello, 1'}.to_json)
+puts({:n => 2, :hello => 'hello, 2'}.to_json)
+puts({:n => 3}.to_json)
+puts({:n => 4, :hello => 'hello, 4'}.to_json)

data/spec/dummy-bot/transformer.rb

@@ -0,0 +1,12 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :n => raw_record['n'],
+    :goodbye => raw_record['hello'].sub('hello', 'goodbye')
+  }
+
+  puts transformed_record.to_json
+end

data/spec/turbot_runner_spec.rb

@@ -0,0 +1,32 @@
+require 'json'
+require 'turbot_runner'
+
+describe TurbotRunner::BaseRunner do
+  it 'can run a bot' do
+    # This test runs slowly - there seems to be some delay in subprocesses
+    # reading from their stdins, but this is not observed when the code is run
+    # outside of rspec.
+
+    class SpecRunner < TurbotRunner::BaseRunner
+      def validate(record, data_type)
+        if record['n'] == 3
+          [:error]
+        else
+          []
+        end
+      end
+    end
+
+    runner = SpecRunner.new('spec/dummy-bot')
+
+    expect(runner).to receive(:handle_valid_record).with({'n' => 1, 'hello' => 'hello, 1'}, 'hello')
+    expect(runner).to receive(:handle_valid_record).with({'n' => 1, 'goodbye' => 'goodbye, 1'}, 'goodbye')
+    expect(runner).to receive(:handle_valid_record).with({'n' => 2, 'hello' => 'hello, 2'}, 'hello')
+    expect(runner).to receive(:handle_valid_record).with({'n' => 2, 'goodbye' => 'goodbye, 2'}, 'goodbye')
+    expect(runner).to receive(:handle_invalid_record).with({'n' => 3}, 'hello', [:error])
+    expect(runner).to receive(:handle_valid_record).with({'n' => 4, 'hello' => 'hello, 4'}, 'hello')
+    expect(runner).to receive(:handle_valid_record).with({'n' => 4, 'goodbye' => 'goodbye, 4'}, 'goodbye')
+    expect(runner).to receive(:handle_successful_run)
+    runner.run
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: turbot-runner
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- OpenCorporates
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+description: 
+email: bots@opencorporates.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/rspec
+- lib/turbot_runner.rb
+- lib/turbot_runner/version.rb
+- spec/dummy-bot/manifest.json
+- spec/dummy-bot/scraper.rb
+- spec/dummy-bot/transformer.rb
+- spec/turbot_runner_spec.rb
+homepage: http://turbot.opencorporates.com/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Utilities for running bots with Turbot
+test_files: []
+has_rdoc: