turbot-runner-morph 0.0.1

data/spec/bots/invalid-record-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "invalid-record-bot",
+    "description": "This is a bot that produces an invalid record",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/invalid-record-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  record.delete(:source_url) if n == 5
+  puts(record.to_json)
+end

data/spec/bots/logging-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "logging-bot",
+    "description": "This is a bot that logs",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/logging-bot/scraper.rb

@@ -0,0 +1,14 @@
+require 'json'
+
+$stderr.puts('doing...')
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end
+
+$stderr.puts('done')

data/spec/bots/python-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "python-bot",
+    "description": "This is a python bot",
+    "language": "python",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.py"]
+}

data/spec/bots/python-bot/scraper.py

@@ -0,0 +1,11 @@
+from __future__ import print_function
+
+import json
+
+for n in range(10):
+    record = {
+      'licence_number': 'XYZ{}'.format(n),
+      'source_url': 'http://example.com',
+      'sample_date': '2014-06-01'
+    }
+    print(json.dumps(record))

data/spec/bots/ruby-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "ruby-bot",
+    "description": "This is a ruby bot",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/ruby-bot/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/slow-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "slow-bot",
+    "description": "This bot sleeps between each record it produces",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/slow-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(99) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+  sleep 0.1
+end

data/spec/lib/processor_spec.rb

@@ -0,0 +1,181 @@
+require 'json'
+require 'turbot_runner'
+
+describe TurbotRunner::Processor do
+  describe '#process' do
+    before do
+      @handler = TurbotRunner::BaseHandler.new
+      @data_type = 'primary data'
+      @script_config = {
+        :data_type => @data_type,
+        :identifying_fields => ['number']
+      }
+    end
+
+    context 'with a nil runner passed in' do
+      before do
+        @processor = TurbotRunner::Processor.new(nil, @script_config, @handler)
+      end
+
+      context 'with valid record' do
+        it 'calls Handler#handle_valid_record' do
+          record = {
+            'sample_date' => '2014-06-01',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expect(@handler).to receive(:handle_valid_record).with(record, @data_type)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with invalid record' do
+        it 'calls Handler#handle_invalid_record' do
+          record = {
+            'sample_date' => '2014-06-01',
+            'number' => 123
+          }
+
+          expected_error = 'Missing required property: source_url'
+          expect(@handler).to receive(:handle_invalid_record).
+            with(record, @data_type, expected_error)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with invalid JSON' do
+        it 'calls Handler#handle_invalid_json' do
+          line = 'this is not JSON'
+          expect(@handler).to receive(:handle_invalid_json).with(line)
+          @processor.process(line)
+        end
+      end
+    end
+
+    context 'with a runner passed in' do
+      before do
+        @script_runner = instance_double('ScriptRunner')
+        allow(@script_runner).to receive(:interrupt_and_mark_as_failed)
+        @processor = TurbotRunner::Processor.new(@script_runner, @script_config, @handler)
+      end
+
+      context 'with valid record' do
+        it 'calls Handler#handle_valid_record' do
+          record = {
+            'sample_date' => '2014-06-01',
+            'source_url' => 'http://example.com/123',
+            'number' => 123
+          }
+
+          expect(@handler).to receive(:handle_valid_record).with(record, @data_type)
+          @processor.process(record.to_json)
+        end
+      end
+
+      context 'with invalid record' do
+        before do
+          @record = {
+            'sample_date' => '2014-06-01',
+            'number' => 123
+          }
+        end
+
+        it 'calls Handler#handle_invalid_record' do
+          expected_error = 'Missing required property: source_url'
+          expect(@handler).to receive(:handle_invalid_record).
+            with(@record, @data_type, expected_error)
+          @processor.process(@record.to_json)
+        end
+
+        it 'interrupts runner' do
+          expect(@script_runner).to receive(:interrupt_and_mark_as_failed)
+          @processor.process(@record.to_json)
+        end
+      end
+
+      context 'with invalid JSON' do
+        before do
+          @line = 'this is not JSON'
+        end
+
+        it 'calls Handler#handle_invalid_json' do
+          expect(@handler).to receive(:handle_invalid_json).with(@line)
+          @processor.process(@line)
+        end
+
+        it 'interrupts runner' do
+          expect(@script_runner).to receive(:interrupt_and_mark_as_failed)
+          @processor.process(@line)
+        end
+      end
+
+      it 'converts date format' do
+        record = {
+          'sample_date' => '2014-06-01 12:34:56 +0000',
+          'source_url' => 'http://example.com/123',
+          'number' => 123
+        }
+
+        converted_record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'number' => 123
+        }
+
+        expect(@handler).to receive(:handle_valid_record).with(converted_record, @data_type)
+        @processor.process(record.to_json)
+      end
+
+      it 'does not pass retrieved_at to validator' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'retrieved_at' => '2014-06-01 12:34:56 +0000',
+          'source_url' => 'http://example.com/123',
+          'number' => 123
+        }
+
+        expected_record_to_validate = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'number' => 123
+        }
+
+        expect(TurbotRunner::Validator).to receive(:validate).
+          with('primary data', expected_record_to_validate, ['number'])
+        @processor.process(record.to_json)
+      end
+    end
+
+    it 'can handle schemas with $refs' do
+      handler = TurbotRunner::BaseHandler.new
+      script_config = {
+        :data_type => 'licence',
+        :identifying_fields => ['licence_number']
+      }
+
+      script_runner = instance_double('ScriptRunner')
+      allow(script_runner).to receive(:interrupt_and_mark_as_failed)
+      processor = TurbotRunner::Processor.new(script_runner, script_config, handler)
+
+      record = {
+        :licence_holder => {
+          :entity_type => 'company',
+          :entity_properties => {
+            :name => 'Hairy Goat Breeding Ltd',
+            :jurisdiction_code => 'gb',
+          }
+        },
+        :licence_number => '1234',
+        :permissions => ['Goat breeding'],
+        :licence_issuer => 'Sheep and Goat Board of Bermuda',
+        :jurisdiction_of_licence => 'bm',
+        :source_url => 'http://example.com',
+        :sample_date => '2015-01-01'
+      }
+
+      expect(handler).to receive(:handle_valid_record)
+      processor.process(record.to_json)
+    end
+  end
+end

data/spec/lib/runner_spec.rb

@@ -0,0 +1,330 @@
+require 'json'
+require 'turbot_runner'
+
+describe TurbotRunner::Runner do
+  before(:each) do
+    Dir.glob('spec/bots/**/output/*').each {|f| File.delete(f)}
+  end
+
+  after(:all) do
+    puts
+    puts 'If all specs passed, you should now run `ruby spec/manual_spec.rb`'
+  end
+
+  describe '#run' do
+    context 'with a bot written in ruby' do
+      before do
+        @runner = test_runner('ruby-bot')
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot written in python' do
+      before do
+        @runner = test_runner('python-bot')
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+      end
+    end
+
+    context 'with a bot with a transformer' do
+      before do
+        @runner = test_runner('bot-with-transformer')
+      end
+
+      it 'produces expected outputs' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+        expect([@runner, 'transformer']).to have_output('full-transformer.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot with multiple transformers' do
+      before do
+        @runner = test_runner('bot-with-transformers')
+      end
+
+      it 'produces expected outputs' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('full-scraper.out')
+        expect([@runner, 'transformer1']).to have_output('full-transformer.out')
+        expect([@runner, 'transformer2']).to have_output('full-transformer.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot that logs' do
+      context 'when logging to file enabled' do
+        it 'logs to file' do
+          expected_log = "doing...\ndone\n"
+          runner = test_runner('logging-bot', :log_to_file => true)
+          runner.run
+          expect([runner, 'scraper']).to have_error_output_matching(expected_log)
+        end
+      end
+
+      context 'when logging to file not enabled' do
+        xit 'logs to stderr' do
+          # This is tested in manual_spec.rb
+        end
+      end
+    end
+
+    context 'with a bot that outputs RUN ENDED' do
+      before do
+        @runner = test_runner('bot-that-emits-run-ended', :log_to_file => true)
+      end
+      it 'calls handle_run_ended on the handler' do
+        expect_any_instance_of(TurbotRunner::BaseHandler).to receive(:handle_run_ended)
+        @runner.run
+      end
+
+      it 'interrupts the run' do
+        expect_any_instance_of(TurbotRunner::ScriptRunner).to receive(:interrupt)
+        @runner.run
+      end
+    end
+
+
+    context 'with a bot that crashes in scraper' do
+      before do
+        @runner = test_runner('bot-that-crashes-in-scraper', :log_to_file => true)
+      end
+
+      it 'returns false' do
+        expect(@runner.run).to be(false)
+      end
+
+      it 'writes error to stderr' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_error_output_matching(/Oh no/)
+      end
+
+      it 'still runs the transformers' do
+        expect(@runner).to receive(:run_script).once.with(
+          hash_including(:file=>"scraper.rb"))
+        expect(@runner).to receive(:run_script).once.with(
+          hash_including(:file=>"transformer1.rb"), anything)
+        @runner.run
+      end
+    end
+
+    context 'with a bot that expects a file to be present in the working directory' do
+      before do
+        @runner = test_runner('bot-that-expects-file')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a bot that crashes in transformer' do
+      before do
+        @runner = test_runner('bot-that-crashes-in-transformer', :log_to_file => true)
+      end
+
+      it 'returns false' do
+        expect(@runner.run).to be(false)
+      end
+
+      it 'writes error to stderr' do
+        @runner.run
+        expect([@runner, 'transformer2']).to have_error_output_matching(/Oh no/)
+      end
+    end
+
+    context 'with a bot that is interrupted in scraper' do
+      xit 'produces truncated output' do
+        # This is tested in manual_spec.rb
+      end
+    end
+
+    context 'with a handler that interrupts the runner' do
+      before do
+        class Handler < TurbotRunner::BaseHandler
+          def initialize(*)
+            @count = 0
+            super
+          end
+
+          def handle_valid_record(record, data_type)
+            @count += 1
+            raise TurbotRunner::InterruptRun if @count >= 5
+          end
+        end
+
+        @runner = test_runner('slow-bot',
+          :record_handler => Handler.new,
+          :log_to_file => true
+        )
+      end
+
+      it 'produces expected output' do
+        @runner.run
+        expect([@runner, 'scraper']).to have_output('truncated-scraper.out')
+      end
+
+      it 'returns true' do
+        expect(@runner.run).to be(true)
+      end
+    end
+
+    context 'with a scraper that produces an invalid record' do
+      it 'returns false' do
+        @runner = test_runner('invalid-record-bot')
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a scraper that produces invalid JSON' do
+      it 'returns false' do
+        @runner = test_runner('invalid-json-bot')
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a scraper that hangs' do
+      # XXX This spec fails because the loop in ScriptRunner#run that
+      # reads lines from the output file doesn't start until the
+      # output file is created; however, the way we're redirecting
+      # stdout using the shell means the file doesn't get created
+      # until
+      it 'returns false' do
+        @runner = test_runner('bot-with-pause',
+          :timeout => 1,
+          :log_to_file => true
+        )
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a bot that emits an invalid sample date' do
+      before do
+        @runner = test_runner('bot-with-invalid-sample-date')
+      end
+
+      it 'returns false' do
+        expect(@runner.run).to be(false)
+      end
+    end
+
+    context 'with a bot with an invalid data type' do
+      before do
+        @runner = test_runner('bot-with-invalid-data-type')
+      end
+
+      it 'raises InvalidDataType' do
+        expect{@runner.run}.to raise_error(TurbotRunner::InvalidDataType)
+      end
+    end
+  end
+
+  describe '#process_output' do
+    before do
+      class Handler < TurbotRunner::BaseHandler
+        attr_reader :records_seen
+
+        def initialize(*)
+          @records_seen = Hash.new {|h, k| h[k] = 0}
+          super
+        end
+
+        def handle_valid_record(record, data_type)
+          @records_seen[data_type] += 1
+        end
+      end
+
+      @handler = Handler.new
+    end
+
+    it 'calls handler once for each line of output' do
+      test_runner('bot-with-transformer').run
+
+      runner = test_runner('bot-with-transformer',
+        :record_handler => @handler
+      )
+
+      runner.process_output
+      expect(@handler.records_seen['primary data']).to eq(10)
+      expect(@handler.records_seen['simple-licence']).to eq(10)
+    end
+
+    it 'can cope when scraper has failed immediately' do
+      test_runner('bot-that-crashes-immediately').run
+
+      runner = test_runner('bot-with-transformer',
+        :record_handler => @handler
+      )
+
+      runner.process_output
+    end
+  end
+
+  describe '#set_up_output_directory' do
+    before do
+      @runner = test_runner('bot-with-transformer')
+    end
+
+    it 'clears existing output' do
+      path = File.join(@runner.base_directory, 'output', 'scraper.out')
+      FileUtils.touch(path)
+      @runner.set_up_output_directory
+      expect(File.exist?(path)).to be(false)
+    end
+
+    it 'does not clear existing files that are not output files' do
+      path = File.join(@runner.base_directory, 'output', 'stdout')
+      FileUtils.touch(path)
+      @runner.set_up_output_directory
+      expect(File.exist?(path)).to be(true)
+    end
+  end
+end
+
+
+RSpec::Matchers.define :have_output do |expected|
+  match do |actual|
+    runner, script = actual
+
+    expected_path = File.join('spec', 'outputs', expected)
+    expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
+    actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
+    actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
+    expect(expected_output).to eq(actual_output)
+  end
+end
+
+
+RSpec::Matchers.define :have_error_output_matching do |expected|
+  match do |actual|
+    runner, script = actual
+
+    actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
+    actual_output = File.read(actual_path)
+    expect(actual_output).to match(expected)
+  end
+end
+
+def test_runner(name, opts={})
+  test_bot_location = File.join(File.dirname(__FILE__), '../bots', name)
+  TurbotRunner::Runner.new(test_bot_location, opts)
+end