turbot-runner 0.0.24 → 0.1.0

data/spec/manual_spec.rb

@@ -0,0 +1,55 @@
+# These specs test behaviour that is most easily exercised or verified by hand.
+
+$:.unshift('lib')
+require 'turbot_runner'
+
+puts
+puts '-' * 80
+
+puts <<eos
+This tests whether stderr is directed to the console.
+When the scraper is run, you should see the following two lines in the console:
+
+doing...
+done
+
+Press <enter> to run the test.
+eos
+
+gets
+
+runner = TurbotRunner::Runner.new('spec/bots/logging-bot').run
+
+puts
+puts 'Did you see the expected lines? [y]/n'
+
+exit(1) unless ['Y', 'y', ''].include?(gets.chomp)
+
+puts
+puts '-' * 80
+
+puts <<eos
+This tests whether hitting Ctrl-C interrupts a running scraper correctly.  When
+the scraper is run, it will pause after producing five lines of output, and
+instruct you to interrupt it.  You will have ten seconds to do so.
+
+Press <enter> to run the test.
+eos
+
+gets
+
+runner = TurbotRunner::Runner.new('spec/bots/bot-with-pause').run
+
+expected_output = File.readlines('spec/outputs/truncated-scraper.out').map {|line| JSON.parse(line)}
+actual_output = File.readlines('spec/bots/bot-with-pause/output/scraper.out').map {|line| JSON.parse(line)}
+
+if expected_output == actual_output
+  puts 'Bot produced expected output'
+else
+  puts 'Bot did not produce expected output'
+  exit(1)
+end
+
+puts
+puts '-' * 80
+puts 'All tests passed!'

data/spec/outputs/full-scraper.out

@@ -0,0 +1,10 @@
+{"licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ5","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ6","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ7","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ8","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ9","source_url":"http://example.com","sample_date":"2014-06-01"}

data/spec/outputs/full-transformer.out

@@ -0,0 +1,10 @@
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ5","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ6","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ7","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ8","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ9","source_url":"http://example.com","sample_date":"2014-06-01"}

data/spec/outputs/truncated-scraper.out

@@ -0,0 +1,5 @@
+{"licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.1.0
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-31 00:00:00.000000000 Z
+date: 2014-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
@@ -31,8 +31,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/rspec
-- lib/prerun.rb
 - lib/turbot_runner.rb
+- lib/turbot_runner/base_handler.rb
+- lib/turbot_runner/prerun.rb
+- lib/turbot_runner/processor.rb
+- lib/turbot_runner/runner.rb
+- lib/turbot_runner/script_runner.rb
 - lib/turbot_runner/version.rb
 - schema/schemas/company-schema.json
 - schema/schemas/financial-payment-schema.json
@@ -56,22 +60,39 @@ files:
 - schema/schemas/simple-licence-schema.json
 - schema/schemas/simple-subsidiary-schema.json
 - schema/schemas/subsidiary-relationship-schema.json
-- spec/dummy-bot-python/manifest.json
-- spec/dummy-bot-python/scraper.py
-- spec/dummy-bot-python/transformer.py
-- spec/dummy-bot-ruby/manifest.json
-- spec/dummy-bot-ruby/scraper.rb
-- spec/dummy-bot-ruby/transformer.rb
-- spec/dummy-broken-bot-ruby-2/manifest.json
-- spec/dummy-broken-bot-ruby-2/scraper.rb
-- spec/dummy-broken-bot-ruby-2/transformer.rb
-- spec/dummy-broken-bot-ruby-3/manifest.json
-- spec/dummy-broken-bot-ruby-3/scraper.rb
-- spec/dummy-broken-bot-ruby-3/transformer.rb
-- spec/dummy-broken-bot-ruby/manifest.json
-- spec/dummy-broken-bot-ruby/scraper.rb
-- spec/dummy-broken-bot-ruby/transformer.rb
-- spec/turbot_runner_spec.rb
+- spec/bots/bot-that-crashes-in-scraper/manifest.json
+- spec/bots/bot-that-crashes-in-scraper/scraper.rb
+- spec/bots/bot-that-crashes-in-transformer/manifest.json
+- spec/bots/bot-that-crashes-in-transformer/scraper.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer1.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
+- spec/bots/bot-with-pause/manifest.json
+- spec/bots/bot-with-pause/scraper.rb
+- spec/bots/bot-with-transformer/manifest.json
+- spec/bots/bot-with-transformer/scraper.rb
+- spec/bots/bot-with-transformer/transformer.rb
+- spec/bots/bot-with-transformers/manifest.json
+- spec/bots/bot-with-transformers/scraper.rb
+- spec/bots/bot-with-transformers/transformer1.rb
+- spec/bots/bot-with-transformers/transformer2.rb
+- spec/bots/invalid-json-bot/manifest.json
+- spec/bots/invalid-json-bot/scraper.rb
+- spec/bots/invalid-record-bot/manifest.json
+- spec/bots/invalid-record-bot/scraper.rb
+- spec/bots/logging-bot/manifest.json
+- spec/bots/logging-bot/scraper.rb
+- spec/bots/python-bot/manifest.json
+- spec/bots/python-bot/scraper.py
+- spec/bots/ruby-bot/manifest.json
+- spec/bots/ruby-bot/scraper.rb
+- spec/bots/slow-bot/manifest.json
+- spec/bots/slow-bot/scraper.rb
+- spec/lib/processor.rb
+- spec/lib/runner_spec.rb
+- spec/manual_spec.rb
+- spec/outputs/full-scraper.out
+- spec/outputs/full-transformer.out
+- spec/outputs/truncated-scraper.out
 homepage: http://turbot.opencorporates.com/
 licenses:
 - MIT

data/spec/dummy-bot-python/manifest.json

@@ -1,15 +0,0 @@
-{
-    "bot_id": "dummy-bot",
-    "data_type": "hello",
-    "description": "This is a dummy bot",
-    "identifying_fields": ["number"],
-    "files": ["scraper.py"],
-    "transformers": [
-        {
-            "file": "transformer.py",
-            "data_type": "goodbye",
-            "identifying_fields": [""]
-        }
-    ],
-    "frequency": "monthly"
-}

data/spec/dummy-bot-python/scraper.py

@@ -1,11 +0,0 @@
-from __future__ import print_function
-
-import json
-import sys
-
-print('hello from python', file=sys.stderr)
-
-print(json.dumps({'n': 5, 'hello': 'hello, 5'}))
-print(json.dumps({'n': 6, 'hello': 'hello, 6'}))
-print(json.dumps({'n': 7}))
-print(json.dumps({'n': 8, 'hello': 'hello, 8'}))

data/spec/dummy-bot-python/transformer.py

@@ -1,15 +0,0 @@
-import json
-import sys
-
-while True:
-    line = sys.stdin.readline()
-    if not line:
-        break
-
-    raw_record = json.loads(line)
-    transformed_record = {
-        'n': raw_record['n'],
-        'goodbye': raw_record['hello'].replace('hello', 'goodbye')
-    }
-
-    print json.dumps(transformed_record)

data/spec/dummy-bot-ruby/manifest.json

@@ -1,15 +0,0 @@
-{
-    "bot_id": "dummy-bot",
-    "data_type": "hello",
-    "description": "This is a dummy bot",
-    "identifying_fields": ["number"],
-    "files": ["scraper.rb"],
-    "transformers": [
-        {
-            "file": "transformer.rb",
-            "data_type": "goodbye",
-            "identifying_fields": [""]
-        }
-    ],
-    "frequency": "monthly"
-}

data/spec/dummy-bot-ruby/scraper.rb

@@ -1,8 +0,0 @@
-require 'json'
-
-$stderr.puts('hello from ruby')
-
-puts({:n => 1, :hello => 'hello, 1'}.to_json)
-puts({:n => 2, :hello => 'hello, 2'}.to_json)
-puts({:n => 3}.to_json)
-puts({:n => 4, :hello => 'hello, 4'}.to_json)

data/spec/dummy-bot-ruby/transformer.rb

@@ -1,12 +0,0 @@
-require 'json'
-STDIN.each_line do |line|
-
-  raw_record = JSON.parse(line)
-
-  transformed_record = {
-    :n => raw_record['n'],
-    :goodbye => raw_record['hello'].sub('hello', 'goodbye')
-  }
-
-  puts transformed_record.to_json
-end

data/spec/dummy-broken-bot-ruby/manifest.json

@@ -1,8 +0,0 @@
-{
-    "bot_id": "dummy-bot",
-    "data_type": "hello",
-    "description": "This is a dummy bot",
-    "identifying_fields": ["number"],
-    "files": ["scraper.rb"],
-    "frequency": "monthly"
-}

data/spec/dummy-broken-bot-ruby/scraper.rb

@@ -1,6 +0,0 @@
-require 'json'
-
-$stderr.puts('hello')
-
-puts({h: 1}.to_json)
-raise "oops"

data/spec/dummy-broken-bot-ruby/transformer.rb

@@ -1,12 +0,0 @@
-require 'json'
-
-STDIN.each_line do |line|
-  raw_record = JSON.parse(line)
-
-  transformed_record = {
-    :n => raw_record['n'],
-    :goodbye => raw_record['hello'].sub('hello', 'goodbye')
-  }
-
-  puts transformed_record.to_json
-end

data/spec/dummy-broken-bot-ruby-2/manifest.json

@@ -1,15 +0,0 @@
-{
-    "bot_id": "dummy-bot",
-    "data_type": "hello",
-    "description": "This is a dummy bot",
-    "identifying_fields": ["number"],
-    "files": ["scraper.rb"],
-    "transformers": [
-        {
-            "file": "transformer.rb",
-            "data_type": "goodbye",
-            "identifying_fields": [""]
-        }
-    ],
-    "frequency": "monthly"
-}

data/spec/dummy-broken-bot-ruby-2/scraper.rb

@@ -1,4 +0,0 @@
-require 'json'
-
-puts({h: 1}.to_json)
-raise "oops"

data/spec/dummy-broken-bot-ruby-2/transformer.rb

@@ -1,11 +0,0 @@
-require 'json'
-
-STDIN.each_line do |line|
-  raw_record = JSON.parse(line)
-
-  transformed_record = {
-    :p => raw_record['n'],
-  }
-
-  puts transformed_record.to_json
-end

data/spec/dummy-broken-bot-ruby-3/manifest.json

@@ -1,15 +0,0 @@
-{
-    "bot_id": "dummy-bot",
-    "data_type": "hello",
-    "description": "This is a dummy bot",
-    "identifying_fields": ["number"],
-    "files": ["scraper.rb"],
-    "transformers": [
-        {
-            "file": "transformer.rb",
-            "data_type": "goodbye",
-            "identifying_fields": [""]
-        }
-    ],
-    "frequency": "monthly"
-}

data/spec/dummy-broken-bot-ruby-3/scraper.rb

@@ -1,5 +0,0 @@
-require 'json'
-
-3.times do |n|
-  puts({h: n}.to_json)
-end

data/spec/dummy-broken-bot-ruby-3/transformer.rb

@@ -1,5 +0,0 @@
-require 'json'
-
-STDIN.each_line do |line|
-  raise "oops"
-end

data/spec/turbot_runner_spec.rb

@@ -1,117 +0,0 @@
-require 'json'
-require 'turbot_runner'
-
-class SpecRunner < TurbotRunner::BaseRunner
-  def validate(record, data_type)
-    if record['n'] % 4 == 3
-      [:error]
-    else
-      []
-    end
-  end
-
-  def handle_failed_run
-  end
-end
-
-class BrokenRunner < TurbotRunner::BaseRunner
-  def validate(record, data_type)
-    []
-  end
-
-  def handle_valid_record(*args)
-  end
-
-  def handle_failed_run
-  end
-end
-
-
-describe TurbotRunner::BaseRunner do
-  before do
-    $stderr = StringIO.new
-  end
-
-  after do
-    $stderr = STDERR
-  end
-
-  it 'can run a ruby bot' do
-    runner = SpecRunner.new('spec/dummy-bot-ruby')
-
-    expect(runner).to receive(:handle_valid_record).with({'n' => 1, 'hello' => 'hello, 1'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 1, 'goodbye' => 'goodbye, 1'}, 'goodbye')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 2, 'hello' => 'hello, 2'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 2, 'goodbye' => 'goodbye, 2'}, 'goodbye')
-    expect(runner).to receive(:handle_invalid_record).with({'n' => 3}, 'hello', [:error])
-    expect(runner).to receive(:handle_valid_record).with({'n' => 4, 'hello' => 'hello, 4'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 4, 'goodbye' => 'goodbye, 4'}, 'goodbye')
-    expect(runner).to receive(:handle_successful_run)
-    runner.run
-    expect($stderr.string).to eq("hello from ruby\n")
-  end
-
-  it 'can run a python bot' do
-    runner = SpecRunner.new('spec/dummy-bot-python')
-
-    expect(runner).to receive(:handle_valid_record).with({'n' => 5, 'hello' => 'hello, 5'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 5, 'goodbye' => 'goodbye, 5'}, 'goodbye')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 6, 'hello' => 'hello, 6'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 6, 'goodbye' => 'goodbye, 6'}, 'goodbye')
-    expect(runner).to receive(:handle_invalid_record).with({'n' => 7}, 'hello', [:error])
-    expect(runner).to receive(:handle_valid_record).with({'n' => 8, 'hello' => 'hello, 8'}, 'hello')
-    expect(runner).to receive(:handle_valid_record).with({'n' => 8, 'goodbye' => 'goodbye, 8'}, 'goodbye')
-    expect(runner).to receive(:handle_successful_run)
-    runner.run
-    expect($stderr.string).to eq("hello from python\n")
-  end
-
-  describe "broken bots" do
-    describe "failing bot without transformer" do
-      it 'should call handle_failed_run' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
-        expect(runner).to receive(:handle_valid_record)
-        expect(runner).to receive(:handle_failed_run)
-        runner.run
-      end
-
-      it 'should write exception to stderr' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
-        runner.run
-        expect($stderr.string).to match(/^hello/)
-        expect($stderr.string).to match(/oops/)
-      end
-    end
-
-    describe "failing bot with successful transformer" do
-      it 'should call handle_failed_run' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby-2')
-        expect(runner).to receive(:handle_valid_record) # first record
-        expect(runner).to receive(:handle_valid_record) # first transform
-        expect(runner).to receive(:handle_failed_run)
-        runner.run
-      end
-
-      it 'should write exception to stderr' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
-        runner.run
-        expect($stderr.string).to match(/oops/)
-      end
-    end
-
-    describe "sucessful bot with failing transformer" do
-      it 'should call handle_failed_run' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby-3')
-        expect(runner).to receive(:handle_valid_record) # the untransformed one
-        expect(runner).to receive(:handle_failed_run) # the transformer breaks immediately
-        runner.run
-      end
-
-      it 'should write exception to stderr' do
-        runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
-        runner.run
-        expect($stderr.string).to match(/oops/)
-      end
-    end
-  end
-end