turbot-runner 0.2.14 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a93e7b12e6d13b95affabf91eb2bcf3914260948
-  data.tar.gz: f129aea4ce1e9f857fa1e9a587dc5fd69a732c81
+  metadata.gz: f9da65f51ab5d12fe470d847e617ee7f65f39217
+  data.tar.gz: 692ce7bfe967e01b6c7485f301cc8697d6bb2cb1
 SHA512:
-  metadata.gz: 21329ceb668293687d5c0822ae29192a4c30ba3cc263cff784e0373cf94a8e9033d05305eafe53c5508622002c93db493571667a7c20149ac375750f24301134
-  data.tar.gz: f48480dacda4fdf1c3e3d4b21e01ac86851a6d91b463449c2670749fac9079fcdce794b833a9f7315b4dd1405afe525becb19b0e5ce4cb1cd1c3340a3d14b1d2
+  metadata.gz: 4fab8ca2464231881299ff8ad68f0549ac3aa077ca637965413846880f09f3d547c66efb13ef73704eb0dd72faffcf9e295b83f092e3f333406351fd6930b126
+  data.tar.gz: 937e02cc9213c18c484cc9262c2691802c437f7f47e92d0f58f1c08a37d4925b570033394aec46751a9e6e58673e9426f83547b47ebb175a0dca9d3839f31740

data/.gitignore

@@ -1,12 +1,15 @@
-/.bundle
+*.gem
+.bundle
+.yardoc
+coverage
+Gemfile.lock
+doc/*
+pkg/*
 /.rvmrc
-/coverage
-/pkg
-/rdoc
 /tags
 /vendor
 /.rbenv-version
+/spec/bots/*/output
 *~
 .#*
 #*#
-/spec/bots/*/output

data/.gitmodules

@@ -1,3 +1,3 @@
 [submodule "schema"]
 	path = schema
-	url = git@github.com:openc/openc-schema.git
+	url = https://github.com/openc/openc-schema.git

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--order random

data/.travis.yml

@@ -0,0 +1,11 @@
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.2.0
+  - 2.3.0
+script:
+  bundle exec rspec --exclude-pattern "spec/manual_spec.rb"

data/README.md

@@ -1,5 +1,11 @@
 # turbot-runner
 
+[![Gem Version](https://badge.fury.io/rb/turbot-runner.svg)](https://badge.fury.io/rb/turbot-runner)
+[![Build Status](https://secure.travis-ci.org/openc/turbot-runner.png)](https://travis-ci.org/openc/turbot-runner)
+[![Dependency Status](https://gemnasium.com/openc/turbot-runner.png)](https://gemnasium.com/openc/turbot-runner)
+[![Coverage Status](https://coveralls.io/repos/openc/turbot-runner/badge.png)](https://coveralls.io/r/openc/turbot-runner)
+[![Code Climate](https://codeclimate.com/github/openc/turbot-runner.png)](https://codeclimate.com/github/openc/turbot-runner)
+
 ## Getting started
 
     git submodule update --init
@@ -60,8 +66,4 @@ default, 24 hours) it interrupts itself, and marks the run as having failed.
 
 ## Running the tests
 
-Tests are run with rspec:
-
-`./bin/rspec`
-
 The first two specs to run require some manual input.

data/lib/turbot_runner/processor.rb

@@ -18,6 +18,7 @@ module TurbotRunner
         else
           record = Openc::JsonSchema.convert_dates(schema_path, JSON.parse(line))
 
+          # TODO Document why we aren't passing retrieved_at to the validator.
           record_to_validate = record.select {|k, v| k != 'retrieved_at'}
 
           error_message = Validator.validate(

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.2.14'
+  VERSION = '0.2.15'
 end

data/schema/schemas/accounts-statement-schema.json

@@ -117,7 +117,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "currency": {
       "type": "string",

data/schema/schemas/company-schema.json

@@ -29,7 +29,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "current_status": {
       "type": [

data/schema/schemas/filing-schema.json

@@ -64,9 +64,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "Date-time this was retrieved from the source",
+      "description": "The time or date at which the source URL was requested",
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "other_attributes": {
       "description": "Use for other attributes for which we don't yet have curated schema attributes",

data/schema/schemas/gazette-notice-schema.json

@@ -267,6 +267,13 @@
             "type": "string",
             "enum": ["other"]
           },
+          "classification": {
+            "description": "The type of judgment",
+            "type": "array",
+            "items": {
+              "$ref": "includes/classification.json"
+            }
+          },
           "body": {
             "description": "The unstructured prose content",
             "$ref": "#/definitions/body"
@@ -405,9 +412,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "The time at which the source URL was requested",
+      "description": "The time or date at which the source URL was requested",
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "confidence": {
       "description": "The scraper's author's confidence in the accuracy of the data",

data/schema/schemas/includes/company-for-nesting.json

@@ -33,7 +33,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "current_status": {
       "type": [
@@ -239,7 +239,16 @@
     }
   },
   "additionalProperties": false,
-  "required": [
-    "name"
+  "anyOf": [
+    {
+      "required": [
+        "name"
+      ]
+    },
+    {
+      "required": [
+        "company_number"
+      ]
+    }
   ]
-}
+}

data/schema/schemas/licence-schema.json

@@ -59,9 +59,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "Date-time this was retrieved from the source",
+      "description": "The time or date at which the source URL was requested",
       "type": "string",
-      "format": "date-time"
+      "format": "date"
     },
     "licence_url": {
       "type": "string",

data/spec/lib/runner_spec.rb

@@ -168,7 +168,7 @@ describe TurbotRunner::Runner do
     context 'with a handler that interrupts the runner' do
       before do
         class Handler < TurbotRunner::BaseHandler
-          def initialize(*)
+          def initialize
             @count = 0
             super
           end
@@ -266,11 +266,12 @@ describe TurbotRunner::Runner do
 
     context 'when the scraped data is provided' do
       before do
+        @runner = test_runner('bot-with-transformer', :scraper_provided => true)
+        @runner.set_up_output_directory
         FileUtils.cp(
           File.join('spec', 'outputs', 'full-scraper.out'),
           File.join(File.dirname(__FILE__), '../bots', 'bot-with-transformer', 'output', 'scraper.out')
         )
-        @runner = test_runner('bot-with-transformer', :scraper_provided => true)
       end
 
       it 'does not run scraper' do
@@ -296,8 +297,8 @@ describe TurbotRunner::Runner do
       class Handler < TurbotRunner::BaseHandler
         attr_reader :records_seen
 
-        def initialize(*)
-          @records_seen = Hash.new {|h, k| h[k] = 0}
+        def initialize
+          @records_seen = Hash.new(0)
           super
         end
 
@@ -324,7 +325,7 @@ describe TurbotRunner::Runner do
     it 'can cope when scraper has failed immediately' do
       test_runner('bot-that-crashes-immediately').run
 
-      runner = test_runner('bot-with-transformer',
+      runner = test_runner('bot-that-crashes-immediately',
         :record_handler => @handler
       )
 
@@ -338,15 +339,19 @@ describe TurbotRunner::Runner do
     end
 
     it 'clears existing output' do
+      @runner.set_up_output_directory
       path = File.join(@runner.base_directory, 'output', 'scraper.out')
       FileUtils.touch(path)
+
       @runner.set_up_output_directory
       expect(File.exist?(path)).to be(false)
     end
 
     it 'does not clear existing files that are not output files' do
+      @runner.set_up_output_directory
       path = File.join(@runner.base_directory, 'output', 'stdout')
       FileUtils.touch(path)
+
       @runner.set_up_output_directory
       expect(File.exist?(path)).to be(true)
     end

data/spec/spec_helper.rb

@@ -1,4 +1,15 @@
-require 'turbot_runner'
+require 'rubygems'
+
+require 'simplecov'
+require 'coveralls'
+SimpleCov.formatter = Coveralls::SimpleCov::Formatter
+SimpleCov.start do
+  add_filter 'spec'
+end
+
+require 'rspec'
+
+require File.dirname(__FILE__) + '/../lib/turbot_runner'
 
 RSpec::Matchers.define(:fail_validation_with) do |expected_error|
   match do |record|

data/turbot-runner.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "activesupport", '4.1.4'
   gem.add_dependency "openc-json_schema"
-  gem.add_development_dependency "pry"
+
+  gem.add_development_dependency "coveralls"
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "rspec", ">= 3.0"
-  gem.add_development_dependency "rspec-mocks", ">= 3.0"
+  gem.add_development_dependency "rspec", "~> 3.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.15
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-08 00:00:00.000000000 Z
+date: 2016-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -70,30 +70,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: rspec-mocks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.4.0
 description: 
 email: bots@opencorporates.com
 executables: []
@@ -102,6 +88,8 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitmodules"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - README.md
 - Rakefile
@@ -232,4 +220,60 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Utilities for running bots with Turbot
-test_files: []
+test_files:
+- spec/bots/bot-that-crashes-immediately/manifest.json
+- spec/bots/bot-that-crashes-immediately/scraper.rb
+- spec/bots/bot-that-crashes-immediately/transformer1.rb
+- spec/bots/bot-that-crashes-in-scraper/manifest.json
+- spec/bots/bot-that-crashes-in-scraper/scraper.rb
+- spec/bots/bot-that-crashes-in-scraper/transformer1.rb
+- spec/bots/bot-that-crashes-in-transformer/manifest.json
+- spec/bots/bot-that-crashes-in-transformer/scraper.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer1.rb
+- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
+- spec/bots/bot-that-emits-run-ended/manifest.json
+- spec/bots/bot-that-emits-run-ended/scraper.rb
+- spec/bots/bot-that-emits-snapshot-ended/manifest.json
+- spec/bots/bot-that-emits-snapshot-ended/scraper.rb
+- spec/bots/bot-that-expects-file/manifest.json
+- spec/bots/bot-that-expects-file/scraper.rb
+- spec/bots/bot-that-expects-file/something.txt
+- spec/bots/bot-that-is-allowed-to-produce-duplicates/manifest.json
+- spec/bots/bot-that-is-allowed-to-produce-duplicates/scraper.rb
+- spec/bots/bot-that-produces-duplicates/manifest.json
+- spec/bots/bot-that-produces-duplicates/scraper.rb
+- spec/bots/bot-with-invalid-data-type/manifest.json
+- spec/bots/bot-with-invalid-data-type/scraper.rb
+- spec/bots/bot-with-invalid-sample-date/manifest.json
+- spec/bots/bot-with-invalid-sample-date/scraper.rb
+- spec/bots/bot-with-pause/manifest.json
+- spec/bots/bot-with-pause/scraper.rb
+- spec/bots/bot-with-transformer/manifest.json
+- spec/bots/bot-with-transformer/scraper.rb
+- spec/bots/bot-with-transformer/transformer.rb
+- spec/bots/bot-with-transformers/manifest.json
+- spec/bots/bot-with-transformers/scraper.rb
+- spec/bots/bot-with-transformers/transformer1.rb
+- spec/bots/bot-with-transformers/transformer2.rb
+- spec/bots/invalid-json-bot/manifest.json
+- spec/bots/invalid-json-bot/scraper.rb
+- spec/bots/invalid-record-bot/manifest.json
+- spec/bots/invalid-record-bot/scraper.rb
+- spec/bots/logging-bot/manifest.json
+- spec/bots/logging-bot/scraper.rb
+- spec/bots/python-bot/manifest.json
+- spec/bots/python-bot/scraper.py
+- spec/bots/ruby-bot/manifest.json
+- spec/bots/ruby-bot/scraper.rb
+- spec/bots/slow-bot/manifest.json
+- spec/bots/slow-bot/scraper.rb
+- spec/lib/processor_spec.rb
+- spec/lib/runner_spec.rb
+- spec/lib/utils_spec.rb
+- spec/lib/validator_spec.rb
+- spec/manual_spec.rb
+- spec/outputs/full-scraper.out
+- spec/outputs/full-transformer.out
+- spec/outputs/truncated-scraper.out
+- spec/spec_helper.rb
+has_rdoc: