turbot-runner 0.0.24 → 0.1.0

data/spec/bots/bot-that-crashes-in-scraper/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-that-crashes-in-scraper",
+    "description": "This is a bot that crashes in the scraper",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-that-crashes-in-scraper/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+  raise 'Oh no' if n == 4
+end

data/spec/bots/bot-that-crashes-in-transformer/manifest.json

@@ -0,0 +1,20 @@
+{
+    "bot_id": "bot-that-crashes-in-transformer",
+    "description": "This is a bot that crashes in the transformer",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb", "transformer2.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        },
+        {
+            "file": "transformer2.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-that-crashes-in-transformer/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-that-crashes-in-transformer/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-that-crashes-in-transformer/transformer2.rb

@@ -0,0 +1,17 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+
+  raise 'Oh no' if raw_record['licence_number'] == 'XYZ4'
+end

data/spec/bots/bot-with-pause/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-with-pause",
+    "description": "This bot pauses for ten second halfway after outputting five lines",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-with-pause/scraper.rb

@@ -0,0 +1,16 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+
+  if n == 4
+    $stderr.puts 'The scraper will sleep for ten seconds...'
+    sleep 10
+    $stderr.puts 'The scraper is resuming...'
+  end
+end

data/spec/bots/bot-with-transformer/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "bot-with-transformer",
+    "description": "This is a bot with a transformer",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb", "transformer2.rb"],
+    "transformers": [
+        {
+            "file": "transformer.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-with-transformer/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-transformer/transformer.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-with-transformers/manifest.json

@@ -0,0 +1,20 @@
+{
+    "bot_id": "bot-with-transformers",
+    "description": "This is a bot with multiple transformers",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        },
+        {
+            "file": "transformer2.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-with-transformers/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-transformers/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-with-transformers/transformer2.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/invalid-json-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "invalid-json-bot",
+    "description": "This is a bot that produces invalid JSON",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/invalid-json-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  record = 'This is not JSON' if n == 5
+  puts(record.to_json)
+end

data/spec/bots/invalid-record-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "invalid-record-bot",
+    "description": "This is a bot that produces an invalid record",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/invalid-record-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  record.delete(:source_url) if n == 5
+  puts(record.to_json)
+end

data/spec/bots/logging-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "logging-bot",
+    "description": "This is a bot that logs",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/logging-bot/scraper.rb

@@ -0,0 +1,14 @@
+require 'json'
+
+$stderr.puts('doing...')
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end
+
+$stderr.puts('done')

data/spec/bots/python-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "python-bot",
+    "description": "This is a python bot",
+    "language": "python",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.py"]
+}

data/spec/bots/python-bot/scraper.py

@@ -0,0 +1,11 @@
+from __future__ import print_function
+
+import json
+
+for n in range(10):
+    record = {
+      'licence_number': 'XYZ{}'.format(n),
+      'source_url': 'http://example.com',
+      'sample_date': '2014-06-01'
+    }
+    print(json.dumps(record))

data/spec/bots/ruby-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "ruby-bot",
+    "description": "This is a ruby bot",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/ruby-bot/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/slow-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "slow-bot",
+    "description": "This bot sleeps between each record it produces",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/slow-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(99) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+  sleep 0.1
+end

data/spec/lib/processor.rb

@@ -0,0 +1,48 @@
+require 'json'
+require 'turbot_runner'
+
+describe TurbotRunner::Processor do
+  describe '#process' do
+    before do
+      @handler = TurbotRunner::BaseHandler.new
+      @data_type = 'primary data'
+      @processor = TurbotRunner::Processor.new(@handler, @data_type)
+    end
+
+    context 'with valid record' do
+      it 'calls Handler#handle_valid_record' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'number' => 123
+        }
+        expect(@handler).to receive(:handle_valid_record).with(record, @data_type)
+        @processor.process(record.to_json)
+      end
+    end
+
+    context 'with invalid record' do
+      it 'calls Handler#handle_invalid_record' do
+        before do
+          @record = {
+            'sample_date' => '2014-06-01',
+            'number' => 123
+          }
+        end
+
+        expected_errors = ['Missing required attribute: source_url']
+        expect(@handler).to receive(:handle_invalid_record).
+          with(@record, @data_type, expected_errors)
+        @processor.process(@record.to_json)
+      end
+    end
+
+    context 'with invalid JSON' do
+      it 'calls Handler#handle_invalid_json' do
+        line = 'this is not JSON'
+        expect(@handler).to receive(:handle_invalid_json).with(line)
+        @processor.process(line)
+      end
+    end
+  end
+end

data/spec/lib/runner_spec.rb

@@ -0,0 +1,244 @@
+require 'json'
+require 'turbot_runner'
+
+describe TurbotRunner::Runner do
+  after(:all) do
+    puts
+    puts 'If all specs passed, you should now run `ruby spec/manual_spec.rb`'
+  end
+
+  describe '#run' do
+    context 'with a bot written in ruby' do
+      before do
+        @runner = TurbotRunner::Runner.new('spec/bots/ruby-bot')
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot written in python' do
+      before do
+        @runner = TurbotRunner::Runner.new('spec/bots/python-bot')
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+      end
+    end
+
+    context 'with a bot with a transformer' do
+      before do
+        @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformer')
+      end
+
+      it 'produces expected outputs' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+        expect([@runner, 'transformer']).to have_output('full-transformer.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot with multiple transformers' do
+      before do
+        @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformers')
+      end
+
+      it 'produces expected outputs' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+        expect([@runner, 'transformer1']).to have_output('full-transformer.out')
+        expect([@runner, 'transformer2']).to have_output('full-transformer.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot that logs' do
+      context 'when logging to file enabled' do
+        it 'logs to file' do
+          expected_log = "doing...\ndone\n"
+          runner = TurbotRunner::Runner.new(
+            'spec/bots/logging-bot',
+            :log_to_file => true
+          )
+          runner.run
+          expect([runner, 'scraper']).to have_error_output_matching(expected_log)
+        end
+      end
+
+      context 'when logging to file not enabled' do
+        xit 'logs to stderr' do
+          # This is tested in manual_spec.rb
+        end
+      end
+    end
+
+    context 'with a bot that crashes in scraper' do
+      before do
+        @runner = TurbotRunner::Runner.new(
+          'spec/bots/bot-that-crashes-in-scraper',
+          :log_to_file => true
+        )
+      end
+
+      it 'returns false' do
+        expect(@runner.run).to be(false)
+      end
+
+      it 'writes error to stderr' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_error_output_matching(/Oh no/)
+      end
+    end
+
+    context 'with a bot that crashes in transformer' do
+      before do
+        @runner = TurbotRunner::Runner.new(
+          'spec/bots/bot-that-crashes-in-transformer',
+          :log_to_file => true
+        )
+      end
+
+      it 'returns false' do
+        expect(@runner.run).to be(false)
+      end
+
+      it 'writes error to stderr' do
+        @runner.run
+        expect([@runner, 'transformer2']).to have_error_output_matching(/Oh no/)
+      end
+    end
+
+    context 'with a bot that is interrupted in scraper' do
+      xit 'produces truncated output' do
+        # This is tested in manual_spec.rb
+      end
+    end
+
+    context 'with a handler that interrupts the runner' do
+      before do
+        class Handler < TurbotRunner::BaseHandler
+          def initialize(*)
+            @count = 0
+            super
+          end
+
+          def handle_valid_record(record, data_type)
+            @count += 1
+            @count < 5 ? true : false
+          end
+        end
+
+        @runner = TurbotRunner::Runner.new(
+          'spec/bots/slow-bot',
+          :record_handler => Handler.new,
+          :log_to_file => true
+        )
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('truncated-scraper.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a scraper that produces an invalid record' do
+      it 'returns false' do
+        @runner = TurbotRunner::Runner.new('spec/bots/invalid-record-bot')
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a scraper that produces invalid JSON' do
+      it 'returns false' do
+        @runner = TurbotRunner::Runner.new('spec/bots/invalid-json-bot')
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a scraper that hangs' do
+      it 'returns false' do
+        @runner = TurbotRunner::Runner.new(
+          'spec/bots/bot-with-pause',
+          :timeout => 1,
+          :log_to_file => true
+        )
+        expect(@runner.run).to be(false)
+      end
+    end
+  end
+
+  describe '#process_output' do
+    before do
+      # This creates the output to work with
+      TurbotRunner::Runner.new('spec/bots/bot-with-transformer').run
+    end
+
+    it 'calls handler once for each line of output' do
+      class Handler < TurbotRunner::BaseHandler
+        attr_reader :records_seen
+
+        def initialize(*)
+          @records_seen = Hash.new {|h, k| h[k] = 0}
+          super
+        end
+
+        def handle_valid_record(record, data_type)
+          @records_seen[data_type] += 1
+        end
+      end
+      
+      handler = Handler.new
+      runner = TurbotRunner::Runner.new(
+        'spec/bots/bot-with-transformer',
+        :record_handler => handler
+      )
+
+      runner.process_output
+      expect(handler.records_seen['primary data']).to eq(10)
+      expect(handler.records_seen['simple-licence']).to eq(10)
+    end
+  end
+end
+
+
+RSpec::Matchers.define :have_output do |expected|
+  match do |actual|
+    runner, script = actual
+
+    expected_path = File.join('spec', 'outputs', expected)
+    expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
+    actual_path = File.join(runner.directory, 'output', "#{script}.out")
+    actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
+    expect(expected_output).to eq(actual_output)
+  end
+end
+
+
+RSpec::Matchers.define :have_error_output_matching do |expected|
+  match do |actual|
+    runner, script = actual
+
+    actual_path = File.join(runner.directory, 'output', "#{script}.err")
+    actual_output = File.read(actual_path)
+    expect(actual_output).to match(expected)
+  end
+end