turbot-runner-morph 0.0.1

data/spec/lib/utils_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe TurbotRunner::Utils do
+  specify '.flatten' do
+    hash = {
+      'a' => {
+        'b' => {
+          'c' => '123',
+          'd' => '124',
+        },
+        'e' => {
+          'f' => '156',
+        }
+      }
+    }
+
+    expect(TurbotRunner::Utils.flatten(hash)).to eq({
+      'a.b.c' => '123',
+      'a.b.d' => '124',
+      'a.e.f' => '156',
+    })
+  end
+end

data/spec/lib/validator_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe TurbotRunner::Validator do
+  describe '.validate' do
+    specify 'with valid record' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expect(record).to be_valid
+    end
+
+    specify 'with record missing required field' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'number' => 123
+      }
+      expected_error = 'Missing required property: source_url'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record missing all identifying fields' do
+      record = {
+        'sample_date' => '2014-06-01',
+        'source_url' => 'http://example.com/123'
+      }
+      expected_error = 'There were no values provided for any of the identifying fields: number'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record with empty sample_date' do
+      record = {
+        'sample_date' => '',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    specify 'with record with invalid sample_date' do
+      record = {
+        'sample_date' => '2014-06-00',
+        'source_url' => 'http://example.com/123',
+        'number' => 123
+      }
+      expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
+      expect(record).to fail_validation_with(expected_error)
+    end
+
+    context 'with nested identifying fields' do
+      specify 'with record missing all identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {}},
+          'four' => {}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq('There were no values provided for any of the identifying fields: one.two.three, four.five.six')
+      end
+
+      specify 'with record missing some identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {'three' => 123}}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq(nil)
+      end
+
+      specify 'with record missing no identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {'three' => 123}},
+          'four' => {'five' => {'six' => 456}}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq(nil)
+      end
+    end
+  end
+end

data/spec/manual_spec.rb

@@ -0,0 +1,57 @@
+# These specs test behaviour that is most easily exercised or verified by hand.
+
+$:.unshift('lib')
+require 'turbot_runner'
+
+puts
+puts '-' * 80
+
+puts <<eos
+This tests whether stderr is directed to the console.
+When the scraper is run, you should see the following two lines in the console:
+
+doing...
+done
+
+Press <enter> to run the test.
+eos
+
+gets
+
+bot_location = File.join(File.dirname(__FILE__), 'bots/logging-bot')
+runner = TurbotRunner::Runner.new(bot_location).run
+
+puts
+puts 'Did you see the expected lines? [y]/n'
+
+exit(1) unless ['Y', 'y', ''].include?(gets.chomp)
+
+puts
+puts '-' * 80
+
+puts <<eos
+This tests whether hitting Ctrl-C interrupts a running scraper correctly.  When
+the scraper is run, it will pause after producing five lines of output, and
+instruct you to interrupt it.  You will have ten seconds to do so.
+
+Press <enter> to run the test.
+eos
+
+gets
+
+bot_location = File.join(File.dirname(__FILE__), 'bots/bot-with-pause')
+runner = TurbotRunner::Runner.new(bot_location).run
+
+expected_output = File.readlines('spec/outputs/truncated-scraper.out').map {|line| JSON.parse(line)}
+actual_output = File.readlines('spec/bots/bot-with-pause/output/scraper.out').map {|line| JSON.parse(line)}
+
+if expected_output == actual_output
+  puts 'Bot produced expected output'
+else
+  puts 'Bot did not produce expected output'
+  exit(1)
+end
+
+puts
+puts '-' * 80
+puts 'All tests passed!'

data/spec/outputs/full-scraper.out

@@ -0,0 +1,10 @@
+{"licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ5","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ6","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ7","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ8","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ9","source_url":"http://example.com","sample_date":"2014-06-01"}

data/spec/outputs/full-transformer.out

@@ -0,0 +1,10 @@
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ5","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ6","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ7","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ8","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"company_name":"Foo Widgets","company_jurisdiction":"gb","licence_number":"XYZ9","source_url":"http://example.com","sample_date":"2014-06-01"}

data/spec/outputs/truncated-scraper.out

@@ -0,0 +1,5 @@
+{"licence_number":"XYZ0","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ1","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ2","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ3","source_url":"http://example.com","sample_date":"2014-06-01"}
+{"licence_number":"XYZ4","source_url":"http://example.com","sample_date":"2014-06-01"}

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'turbot_runner'
+
+RSpec::Matchers.define(:fail_validation_with) do |expected_error|
+  match do |record|
+    identifying_fields = ['number']
+    @error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+    expect(@error).to eq(expected_error)
+  end
+
+  failure_message do |actual|
+    "Expected error to be #{expected_error}, but was #{@error}"
+  end
+end
+
+RSpec::Matchers.define(:be_valid) do
+  match do |record|
+    identifying_fields = ['number']
+    expect(TurbotRunner::Validator.validate('primary-data', record, identifying_fields)).to eq(nil)
+  end
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: turbot-runner-morph
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- OpenCorporates
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: openc-json_schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.13
+description: 
+email: bots@opencorporates.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/rspec
+- lib/turbot_runner.rb
+- lib/turbot_runner/base_handler.rb
+- lib/turbot_runner/exceptions.rb
+- lib/turbot_runner/prerun.rb
+- lib/turbot_runner/processor.rb
+- lib/turbot_runner/runner.rb
+- lib/turbot_runner/script_runner.rb
+- lib/turbot_runner/utils.rb
+- lib/turbot_runner/validator.rb
+- lib/turbot_runner/version.rb
+- schema/schemas/company-schema.json
+- schema/schemas/financial-payment-schema.json
+- schema/schemas/includes/address.json
+- schema/schemas/includes/alternative_name.json
+- schema/schemas/includes/company-for-nesting.json
+- schema/schemas/includes/company.json
+- schema/schemas/includes/entity.json
+- schema/schemas/includes/filing.json
+- schema/schemas/includes/financial-payment-data-object.json
+- schema/schemas/includes/identifier.json
+- schema/schemas/includes/industry_code.json
+- schema/schemas/includes/licence-data-object.json
+- schema/schemas/includes/officer.json
+- schema/schemas/includes/organisation.json
+- schema/schemas/includes/permission.json
+- schema/schemas/includes/person.json
+- schema/schemas/includes/person_name.json
+- schema/schemas/includes/previous_name.json
+- schema/schemas/includes/share-parcel-data.json
+- schema/schemas/includes/share-parcel.json
+- schema/schemas/includes/subsidiary-relationship-data.json
+- schema/schemas/includes/total-shares.json
+- schema/schemas/includes/unknown_entity_type.json
+- schema/schemas/licence-schema.json
+- schema/schemas/primary-data-schema.json
+- schema/schemas/share-parcel-schema.json
+- schema/schemas/simple-financial-payment-schema.json
+- schema/schemas/simple-licence-schema.json
+- schema/schemas/simple-subsidiary-schema.json
+- schema/schemas/subsidiary-relationship-schema.json
+- spec/bots/bot-that-crashes-immediately/manifest.json
+- spec/bots/bot-that-crashes-immediately/scraper.rb
+- spec/bots/bot-that-crashes-immediately/transformer1.rb
+- spec/bots/bot-that-crashes-in-scraper/manifest.json
+- spec/bots/bot-that-crashes-in-scraper/scraper.rb
+- spec/bots/bot-that-crashes-in-scraper/transformer1.rb
+- spec/bots/bot-that-crashes-in-transformer/manifest.json
+- spec/bots/bot-that-crashes-in-transformer/scraper.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer1.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
+- spec/bots/bot-that-emits-run-ended/manifest.json
+- spec/bots/bot-that-emits-run-ended/scraper.rb
+- spec/bots/bot-that-expects-file/manifest.json
+- spec/bots/bot-that-expects-file/scraper.rb
+- spec/bots/bot-that-expects-file/something.txt
+- spec/bots/bot-with-invalid-data-type/manifest.json
+- spec/bots/bot-with-invalid-data-type/scraper.rb
+- spec/bots/bot-with-invalid-sample-date/manifest.json
+- spec/bots/bot-with-invalid-sample-date/scraper.rb
+- spec/bots/bot-with-pause/manifest.json
+- spec/bots/bot-with-pause/scraper.rb
+- spec/bots/bot-with-transformer/manifest.json
+- spec/bots/bot-with-transformer/scraper.rb
+- spec/bots/bot-with-transformer/transformer.rb
+- spec/bots/bot-with-transformers/manifest.json
+- spec/bots/bot-with-transformers/scraper.rb
+- spec/bots/bot-with-transformers/transformer1.rb
+- spec/bots/bot-with-transformers/transformer2.rb
+- spec/bots/invalid-json-bot/manifest.json
+- spec/bots/invalid-json-bot/scraper.rb
+- spec/bots/invalid-record-bot/manifest.json
+- spec/bots/invalid-record-bot/scraper.rb
+- spec/bots/logging-bot/manifest.json
+- spec/bots/logging-bot/scraper.rb
+- spec/bots/python-bot/manifest.json
+- spec/bots/python-bot/scraper.py
+- spec/bots/ruby-bot/manifest.json
+- spec/bots/ruby-bot/scraper.rb
+- spec/bots/slow-bot/manifest.json
+- spec/bots/slow-bot/scraper.rb
+- spec/lib/processor_spec.rb
+- spec/lib/runner_spec.rb
+- spec/lib/utils_spec.rb
+- spec/lib/validator_spec.rb
+- spec/manual_spec.rb
+- spec/outputs/full-scraper.out
+- spec/outputs/full-transformer.out
+- spec/outputs/truncated-scraper.out
+- spec/spec_helper.rb
+homepage: http://turbot.opencorporates.com/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Utilities for running bots with Turbot
+test_files: []
+has_rdoc: