turbot-runner 0.2.32 → 0.2.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d41497aa58618792cf5f76e54e2d8a54bedadfb2
-  data.tar.gz: 41b301a1772f6fa01dbee1fa59fb7585f8368aaf
+SHA256:
+  metadata.gz: da8abedec4c778f73ad6ef717804a4a73349bdcbead7e9ac5be3e07fa8d5a7fa
+  data.tar.gz: 6f848dbec7fd83e678a99f7083ea9e9eb7009ac2962be7fc14bc87bb5f3a7cbe
 SHA512:
-  metadata.gz: da77da59c04ed6be51e206f9a76da556b571c90519a28cde2a628bc987efc12b6a665ea896db16413dd0bd6809761bccd6b7d01feddcc93017906ed509a54b29
-  data.tar.gz: a5c43adcef9d972f3a101cc335abe4338ae4a312c5ca11a59df1b4257ff54ec4c881cee747b9c665d4c4d304e3a97c267fef9553f18c9a76051994c9719b7abb
+  metadata.gz: 4815ac1f507e4c55349b8639bc854ec5e8eb6d1ac8efe5285fd7b439afab6bda4745e5760d165aa695cc91bb8ab31529fc8e3d59d2ab3a32a589ad52d5ee7e72
+  data.tar.gz: 8cb243bbfd0699a29ca20ee802a0e4184a23e45b95b0e4d0e147523aba1cc426bd4dff591e6dc4997ddcd6818445f7b9b7d4d6ea60b9db71c6797c775ec6a840

data/.travis.yml

@@ -8,7 +8,7 @@ rvm:
   - 2.2
   - 2.3
 before_install:
-  - gem update --system
-  - gem update bundler
+  - gem update --system 2.7.8
+  - gem install bundler -v 1.17.3
 script:
-  bundle exec rspec
+  bin/rspec

data/bin/bundle

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'bundle' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "rubygems"
+
+m = Module.new do
+    module_function
+
+  def invoked_as_script?
+    File.expand_path($0) == File.expand_path(__FILE__)
+  end
+
+  def env_var_version
+    ENV["BUNDLER_VERSION"]
+  end
+
+  def cli_arg_version
+    return unless invoked_as_script? # don't want to hijack other binstubs
+    return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update`
+    bundler_version = nil
+    update_index = nil
+    ARGV.each_with_index do |a, i|
+      if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN
+        bundler_version = a
+      end
+      next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/
+      bundler_version = $1 || ">= 0.a"
+      update_index = i
+    end
+    bundler_version
+  end
+
+  def gemfile
+    gemfile = ENV["BUNDLE_GEMFILE"]
+    return gemfile if gemfile && !gemfile.empty?
+
+    File.expand_path("../../Gemfile", __FILE__)
+  end
+
+  def lockfile
+    lockfile =
+      case File.basename(gemfile)
+      when "gems.rb" then gemfile.sub(/\.rb$/, gemfile)
+      else "#{gemfile}.lock"
+      end
+    File.expand_path(lockfile)
+  end
+
+  def lockfile_version
+    return unless File.file?(lockfile)
+    lockfile_contents = File.read(lockfile)
+    return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/
+    Regexp.last_match(1)
+  end
+
+  def bundler_version
+    @bundler_version ||= begin
+      env_var_version || cli_arg_version ||
+        lockfile_version || "#{Gem::Requirement.default}.a"
+    end
+  end
+
+  def load_bundler!
+    ENV["BUNDLE_GEMFILE"] ||= gemfile
+
+    # must dup string for RG < 1.8 compatibility
+    activate_bundler(bundler_version.dup)
+  end
+
+  def activate_bundler(bundler_version)
+    if Gem::Version.correct?(bundler_version) && Gem::Version.new(bundler_version).release < Gem::Version.new("2.0")
+      bundler_version = "< 2"
+    end
+    gem_error = activation_error_handling do
+      gem "bundler", bundler_version
+    end
+    return if gem_error.nil?
+    require_error = activation_error_handling do
+      require "bundler/version"
+    end
+    return if require_error.nil? && Gem::Requirement.new(bundler_version).satisfied_by?(Gem::Version.new(Bundler::VERSION))
+    warn "Activating bundler (#{bundler_version}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_version}'`"
+    exit 42
+  end
+
+  def activation_error_handling
+    yield
+    nil
+  rescue StandardError, LoadError => e
+    e
+  end
+end
+
+m.load_bundler!
+
+if m.invoked_as_script?
+  load Gem.bin_path("bundler", "bundle")
+end

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/lib/turbot_runner/runner.rb

@@ -124,15 +124,13 @@ module TurbotRunner
       # record handling.
       processor = Processor.new(nil, script_config, @record_handler)
       file = output_file(script_config[:file])
-      File.open(file) do |f|
-        f.each_line do |line|
-          processor.process(line, opts)
+      if File.exist?(file)
+        File.open(file) do |f|
+          f.each_line do |line|
+            processor.process(line, opts)
+          end
         end
       end
-    rescue Errno::ENOENT => e
-      # We only want to catch ENOENT if the output file doesn't exist, and not
-      # if, for instance, a schema file is missing.
-      raise unless e.message == "No such file or directory - #{output_file(script_config[:file])}"
     end
 
     def build_command(script, input_file=nil)

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.2.32'
+  VERSION = '0.2.33'
 end

data/schema/schemas/alternate-registration-schema.json

@@ -0,0 +1,88 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "A representation of multiple registrations for the same entity",
+  "type": "object",
+  "properties": {
+    "data_type": {
+      "enum": [
+        "alternate-registration"
+      ]
+    },
+    "alternate_entities": {
+      "description": "array of entities that are linked via alternate registrations",
+      "type": "array",
+      "items": {
+        "$ref": "includes/entity.json",
+        "minLength": 2
+      }
+    },
+    "alternate_registration_start_date": {
+      "description": "date when alternate registration started",
+      "type": "string",
+      "format": "date"
+    },
+    "alternate_registration_end_date": {
+      "description": "date when alternate registration ended",
+      "type": "string",
+      "format": "date"
+    },
+    "publication_date": {
+      "description": "the publication date of the filing/notice that gives details of the alternate registration",
+      "type": "string",
+      "format": "date"
+    },
+    "start_date": {
+      "description": "date when alternate registration was valid from",
+      "type": "string",
+      "format": "date"
+    },
+    "start_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
+    "sample_date": {
+      "description": "date when alternate registration was sampled",
+      "type": "string",
+      "format": "date"
+    },
+    "retrieved_at": {
+      "description": "date when alternate registration was retrieved",
+      "type": "string",
+      "format": "date"
+    },
+    "source_url": {
+      "description": "URL of the source of the data (e.g. download URL), or if there is not persistent URL the page from which it can be found (e.g. search page)",
+      "type": "string"
+    },
+    "confidence": {
+      "description": "Confidence in accuracy of data",
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "data_type",
+    "alternate_entities",
+    "retrieved_at",
+    "confidence"
+  ],
+  "anyOf": [
+    {
+      "required": [
+        "start_date"
+      ]
+    },
+    {
+      "required": [
+        "sample_date"
+      ]
+    }
+  ]
+}

data/schema/schemas/company-schema.json

@@ -132,6 +132,18 @@
         "$ref": "includes/alternative-name.json"
       }
     },
+    "subsequent_registrations": {
+      "type": "array",
+      "items": {
+        "$ref": "subsequent-registration-schema.json"
+      }
+    },
+    "alternate_registrations": {
+      "type": "array",
+      "items": {
+        "$ref": "alternate-registration-schema.json"
+      }
+    },
     "branch": {
       "type": [
         "string",
@@ -258,6 +270,7 @@
   "required": [
     "company_number",
     "name",
-    "jurisdiction_code"
+    "jurisdiction_code",
+    "retrieved_at"
   ]
 }

data/schema/schemas/control-statement-schema.json

@@ -76,11 +76,25 @@
       "type": "string",
       "format": "date"
     },
+    "start_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
     "end_date": {
       "description": "Date on which control ended",
       "type": "string",
       "format": "date"
     },
+    "end_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
     "retrieved_at": {
       "description": "Date-time this was retrieved from the source",
       "type": "string",

data/schema/schemas/subsequent-registration-schema.json

@@ -0,0 +1,89 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "A representation of entity registration changing over time",
+  "type": "object",
+  "properties": {
+    "data_type": {
+      "enum": [
+        "subsequent-registration"
+      ]
+    },
+    "previous_entity": {
+      "description": "the previous entity",
+      "$ref": "includes/entity.json"
+    },
+    "subsequent_entity": {
+      "description": "the subsequent entity",
+      "$ref": "includes/entity.json"
+    },
+    "subsequent_registration_start_date": {
+      "description": "date when subsequent registration started",
+      "type": "string",
+      "format": "date"
+    },
+    "previous_registration_end_date": {
+      "description": "date when previous registration ended",
+      "type": "string",
+      "format": "date"
+    },
+    "publication_date": {
+      "description": "the publication date of the filing/notice that gives details of the alternate registration",
+      "type": "string",
+      "format": "date"
+    },
+    "start_date": {
+      "description": "date when subsequent registration was valid from",
+      "type": "string",
+      "format": "date"
+    },
+    "start_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
+    "sample_date": {
+      "description": "date when subsequent registration was sampled",
+      "type": "string",
+      "format": "date"
+    },
+    "retrieved_at": {
+      "description": "date when subsequent registration was retrieved",
+      "type": "string",
+      "format": "date"
+    },
+    "source_url": {
+      "description": "URL of the source of the data (e.g. download URL), or if there is not persistent URL the page from which it can be found (e.g. search page)",
+      "type": "string"
+    },
+    "confidence": {
+      "description": "Confidence in accuracy of data",
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "data_type",
+    "previous_entity",
+    "subsequent_entity",
+    "retrieved_at",
+    "confidence"
+  ],
+  "anyOf": [
+    {
+      "required": [
+        "start_date"
+      ]
+    },
+    {
+      "required": [
+        "sample_date"
+      ]
+    }
+  ]
+}

data/schema/schemas/supplier-relationship-schema.json

@@ -0,0 +1,82 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "data_type": {
+      "enum": [
+        "supplier-relationship"
+      ]
+    },
+    "customer": {
+      "$ref": "includes/entity.json"
+    },
+    "supplier": {
+      "$ref": "includes/entity.json"
+    },
+    "start_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "start_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
+    "end_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "end_date_type": {
+      "enum": [
+        "at",
+        "before",
+        "after"
+      ]
+    },
+    "sample_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "retrieved_at": {
+      "type": "string",
+      "format": "date"
+    },
+    "source_url": {
+      "type": "string"
+    },
+    "confidence": {
+      "enum": [
+        "HIGH",
+        "MEDIUM",
+        "LOW"
+      ]
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "data_type",
+    "supplier",
+    "customer",
+    "retrieved_at",
+    "confidence"
+  ],
+  "anyOf": [
+    {
+      "required": [
+        "start_date"
+      ]
+    },
+    {
+      "required": [
+        "sample_date"
+      ]
+    },
+    {
+      "required": [
+        "end_date"
+      ]
+    }
+  ]
+}

data/spec/lib/runner_spec.rb

@@ -7,6 +7,10 @@ describe TurbotRunner::Runner do
     puts 'If all specs passed, you should now run `ruby spec/manual.rb`'
   end
 
+  after do
+    FileUtils.rm_rf(File.join(@runner.base_directory, "output")) if @runner
+  end
+
   describe '#run' do
     context 'with a bot written in ruby' do
       before do
@@ -69,11 +73,14 @@ describe TurbotRunner::Runner do
 
     context 'with a bot that logs' do
       context 'when logging to file enabled' do
+        before do
+          @runner = test_runner('logging-bot', :log_to_file => true)
+        end
+
         it 'logs to file' do
           expected_log = "doing...\ndone\n"
-          runner = test_runner('logging-bot', :log_to_file => true)
-          runner.run
-          expect(runner).to have_error_output_matching('scraper', expected_log)
+          @runner.run
+          expect(@runner).to have_error_output_matching('scraper', expected_log)
         end
       end
 
@@ -201,15 +208,21 @@ describe TurbotRunner::Runner do
     end
 
     context 'with a scraper that produces an invalid record' do
-      it 'returns false' do
+      before do
         @runner = test_runner('invalid-record-bot')
+      end
+
+      it 'returns false' do
         expect(@runner).to fail_in_scraper
       end
     end
 
     context 'with a scraper that produces invalid JSON' do
-      it 'returns false' do
+      before do
         @runner = test_runner('invalid-json-bot')
+      end
+
+      it 'returns false' do
         expect(@runner).to fail_in_scraper
       end
     end
@@ -220,11 +233,14 @@ describe TurbotRunner::Runner do
       # output file is created; however, the way we're redirecting
       # stdout using the shell means the file doesn't get created
       # until
-      it 'returns false' do
+      before do
         @runner = test_runner('bot-with-pause',
           :timeout => 1,
           :log_to_file => true
         )
+      end
+
+      it 'returns false' do
         expect(@runner).to fail_in_scraper
       end
     end
@@ -315,51 +331,67 @@ describe TurbotRunner::Runner do
       @handler = Handler.new
     end
 
-    it 'calls handler once for each line of output' do
-      test_runner('bot-with-transformer').run
+    context 'with a bot that runs correctly' do
+      before do
+        @runner = test_runner('bot-with-transformer')
+        @runner.run
+      end
 
-      runner = test_runner('bot-with-transformer',
-        :record_handler => @handler
-      )
+      it 'calls handler once for each line of output' do
+        runner = test_runner('bot-with-transformer',
+          :record_handler => @handler
+        )
 
-      runner.process_output
-      expect(@handler.records_seen['primary data']).to eq(10)
-      expect(@handler.records_seen['simple-licence']).to eq(10)
-    end
+        runner.process_output
+        expect(@handler.records_seen['primary data']).to eq(10)
+        expect(@handler.records_seen['simple-licence']).to eq(10)
+      end
 
-    it 'passes opts to processor.process' do
-      test_runner('bot-with-transformer').run
-      runner = test_runner('bot-with-transformer',
-        :record_handler => @handler
-      )
-      opts = {frob: 5}
-      processor = double('processor')
-      allow(TurbotRunner::Processor).to receive(:new).and_return(processor)
-      expect(processor).to receive(:process).with(anything, opts).at_least(:once)
-      runner.process_output(opts)
+      it 'passes opts to processor.process' do
+        runner = test_runner('bot-with-transformer',
+          :record_handler => @handler
+        )
+        opts = {frob: 5}
+        processor = double('processor')
+        allow(TurbotRunner::Processor).to receive(:new).and_return(processor)
+        expect(processor).to receive(:process).with(anything, opts).at_least(:once)
+        runner.process_output(opts)
+      end
+
+      context 'when skip_data_types is set' do
+        it 'skips the data type' do
+          runner = test_runner('bot-with-transformer',
+            :record_handler => @handler
+          )
+
+          runner.process_output(skip_data_types: ['primary data'])
+          expect(@handler.records_seen['primary data']).to eq(0)
+          expect(@handler.records_seen['simple-licence']).to eq(10)
+        end
+      end
     end
 
-    it 'can cope when scraper has failed immediately' do
-      test_runner('bot-that-crashes-immediately').run
+    context 'with a bot that crashes immediately' do
+      before do
+        @runner = test_runner('bot-that-crashes-immediately')
+        @runner.run
+      end
 
-      runner = test_runner('bot-that-crashes-immediately',
-        :record_handler => @handler
-      )
+      it 'can cope with the empty files' do
+        runner = test_runner('bot-that-crashes-immediately',
+          :record_handler => @handler
+        )
 
-      runner.process_output
+        runner.process_output
+      end
     end
 
-    context 'when skip_data_types is set' do
-      it 'skips the data type' do
-        test_runner('bot-with-transformer').run
-
+    context 'when no bot has run' do
+      it 'proceeds without errors' do
         runner = test_runner('bot-with-transformer',
           :record_handler => @handler
         )
-
-        runner.process_output(skip_data_types: ['primary data'])
-        expect(@handler.records_seen['primary data']).to eq(0)
-        expect(@handler.records_seen['simple-licence']).to eq(10)
+        runner.process_output
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.2.32
+  version: 0.2.33
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-06 00:00:00.000000000 Z
+date: 2019-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -68,7 +68,9 @@ dependencies:
         version: 3.4.0
 description: 
 email: bots@opencorporates.com
-executables: []
+executables:
+- bundle
+- rspec
 extensions: []
 extra_rdoc_files: []
 files:
@@ -81,6 +83,8 @@ files:
 - README.md
 - Rakefile
 - appveyor.yml
+- bin/bundle
+- bin/rspec
 - lib/turbot_runner.rb
 - lib/turbot_runner/base_handler.rb
 - lib/turbot_runner/exceptions.rb
@@ -92,6 +96,7 @@ files:
 - lib/turbot_runner/validator.rb
 - lib/turbot_runner/version.rb
 - schema/schemas/accounts-statement-schema.json
+- schema/schemas/alternate-registration-schema.json
 - schema/schemas/company-schema.json
 - schema/schemas/control-statement-schema.json
 - schema/schemas/filing-schema.json
@@ -135,7 +140,9 @@ files:
 - schema/schemas/simple-financial-payment-schema.json
 - schema/schemas/simple-licence-schema.json
 - schema/schemas/simple-subsidiary-schema.json
+- schema/schemas/subsequent-registration-schema.json
 - schema/schemas/subsidiary-relationship-schema.json
+- schema/schemas/supplier-relationship-schema.json
 - schema/schemas/trademark-registration-schema.json
 - spec/bots/bot-that-crashes-immediately/manifest.json
 - spec/bots/bot-that-crashes-immediately/scraper.rb
@@ -215,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Utilities for running bots with Turbot