turbot-runner 0.2.13 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b717e95fb7ff52f87fb33c5cf95efe329fcc179b
-  data.tar.gz: 2ea03178c9f54114f59980588b67bd84e6e5e3db
+  metadata.gz: a93e7b12e6d13b95affabf91eb2bcf3914260948
+  data.tar.gz: f129aea4ce1e9f857fa1e9a587dc5fd69a732c81
 SHA512:
-  metadata.gz: 5d55bc8a5f4e8b35e0c5ac9b805dc6582774bd285b2da3c42870474af3e47815579fa0b9582a67738de964b83e4d755d501ed945e263015cd136ddeb5f78d96b
-  data.tar.gz: 2d87e2bebf9d7884997941941ba2cafc66aac41a5d7e903754688643859a0f8e74acee903265ab2e0dead577a9fe49b3da0a63b1bd2e5e523d622d32af29d727
+  metadata.gz: 21329ceb668293687d5c0822ae29192a4c30ba3cc263cff784e0373cf94a8e9033d05305eafe53c5508622002c93db493571667a7c20149ac375750f24301134
+  data.tar.gz: f48480dacda4fdf1c3e3d4b21e01ac86851a6d91b463449c2670749fac9079fcdce794b833a9f7315b4dd1405afe525becb19b0e5ce4cb1cd1c3340a3d14b1d2

data/README.md

@@ -18,3 +18,50 @@ Bump the version in `lib/turbot_runner/version.rb` according to the [Semantic Ve
     rake release # requires Rubygems credentials
 
 Finally, [rebuild the Docker image](https://github.com/openc/morph-docker-ruby#readme).
+
+## Rough outline of how it works
+
+TurbotRunner is responsible for running a scraper, transforming its data, and
+then validating and processing any output.
+
+Work is coordinated by an instance of `Runner`.  Most of the interesting work
+is done in `Runner#run_script`, which constructs a command like:
+
+    python transformer.py >transformer.out 2>transformer.err <scraper.out
+
+This command is then passed to an instance of `ScriptRunner` which runs the
+command via `system` in a new thread.  The main thread then monitors the output
+file, and processes each complete line of output.
+
+A line is processed by an instance of `Processor`, which checks that the line
+is valid JSON, and then passes it on to the instance of a subclass of
+`BaseHandler` that was passed to the `Runner` when it was created.
+
+The subclass of `BaseHandler` can implement any of the following methods:
+
+ * `handle_valid_record`
+ * `handle_invalid_record`
+ * `handle_invalid_json`
+ * `handle_snapshot_ended`
+
+If the `Processor` finds an invalid record, it interrupts the `ScriptRunner`,
+and marks the run as having failed.
+
+The `Processor` will catch an `InterruptRun` that's raised by
+`handler.handle_valid_record`, which will interrupt the `ScriptRunner`, but
+will not mark the run as having failed.
+
+When the `ScriptRunner` is interrupted, it will kill the running process, by
+sending SIGINT to all the processes in the current process group.  The current
+process is set up (via `trap('INT') {}` to ignore this.
+
+If the `ScriptRunner` reads no output from the command within a timeout (by
+default, 24 hours) it interrupts itself, and marks the run as having failed.
+
+## Running the tests
+
+Tests are run with rspec:
+
+`./bin/rspec`
+
+The first two specs to run require some manual input.

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.2.13'
+  VERSION = '0.2.14'
 end

data/schema/schemas/accounts-statement-schema.json

@@ -117,7 +117,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "currency": {
       "type": "string",

data/schema/schemas/company-schema.json

@@ -29,7 +29,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "current_status": {
       "type": [

data/schema/schemas/filing-schema.json

@@ -64,9 +64,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "The time or date at which the source URL was requested",
+      "description": "Date-time this was retrieved from the source",
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "other_attributes": {
       "description": "Use for other attributes for which we don't yet have curated schema attributes",

data/schema/schemas/gazette-notice-schema.json

@@ -267,13 +267,6 @@
             "type": "string",
             "enum": ["other"]
           },
-          "classification": {
-            "description": "The type of judgment",
-            "type": "array",
-            "items": {
-              "$ref": "includes/classification.json"
-            }
-          },
           "body": {
             "description": "The unstructured prose content",
             "$ref": "#/definitions/body"
@@ -412,9 +405,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "The time or date at which the source URL was requested",
+      "description": "The time at which the source URL was requested",
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "confidence": {
       "description": "The scraper's author's confidence in the accuracy of the data",

data/schema/schemas/includes/company-for-nesting.json

@@ -33,7 +33,7 @@
     },
     "retrieved_at": {
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "current_status": {
       "type": [
@@ -239,16 +239,7 @@
     }
   },
   "additionalProperties": false,
-  "anyOf": [
-    {
-      "required": [
-        "name"
-      ]
-    },
-    {
-      "required": [
-        "company_number"
-      ]
-    }
+  "required": [
+    "name"
   ]
-}
+}

data/schema/schemas/licence-schema.json

@@ -59,9 +59,9 @@
       "format": "date"
     },
     "retrieved_at": {
-      "description": "The time or date at which the source URL was requested",
+      "description": "Date-time this was retrieved from the source",
       "type": "string",
-      "format": "date"
+      "format": "date-time"
     },
     "licence_url": {
       "type": "string",

data/spec/lib/runner_spec.rb

@@ -2,11 +2,6 @@ require 'json'
 require 'turbot_runner'
 
 describe TurbotRunner::Runner do
-  after(:all) do
-    puts
-    puts 'If all specs passed, you should now run `ruby spec/manual_spec.rb`'
-  end
-
   describe '#run' do
     context 'with a bot written in ruby' do
       before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.14
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-03 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -96,8 +96,7 @@ dependencies:
         version: '3.0'
 description: 
 email: bots@opencorporates.com
-executables:
-- rspec
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -106,7 +105,6 @@ files:
 - Gemfile
 - README.md
 - Rakefile
-- bin/rspec
 - lib/turbot_runner.rb
 - lib/turbot_runner/base_handler.rb
 - lib/turbot_runner/exceptions.rb
@@ -234,60 +232,4 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Utilities for running bots with Turbot
-test_files:
-- spec/bots/bot-that-crashes-immediately/manifest.json
-- spec/bots/bot-that-crashes-immediately/scraper.rb
-- spec/bots/bot-that-crashes-immediately/transformer1.rb
-- spec/bots/bot-that-crashes-in-scraper/manifest.json
-- spec/bots/bot-that-crashes-in-scraper/scraper.rb
-- spec/bots/bot-that-crashes-in-scraper/transformer1.rb
-- spec/bots/bot-that-crashes-in-transformer/manifest.json
-- spec/bots/bot-that-crashes-in-transformer/scraper.rb
-- spec/bots/bot-that-crashes-in-transformer/transformer1.rb
-- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
-- spec/bots/bot-that-emits-run-ended/manifest.json
-- spec/bots/bot-that-emits-run-ended/scraper.rb
-- spec/bots/bot-that-emits-snapshot-ended/manifest.json
-- spec/bots/bot-that-emits-snapshot-ended/scraper.rb
-- spec/bots/bot-that-expects-file/manifest.json
-- spec/bots/bot-that-expects-file/scraper.rb
-- spec/bots/bot-that-expects-file/something.txt
-- spec/bots/bot-that-is-allowed-to-produce-duplicates/manifest.json
-- spec/bots/bot-that-is-allowed-to-produce-duplicates/scraper.rb
-- spec/bots/bot-that-produces-duplicates/manifest.json
-- spec/bots/bot-that-produces-duplicates/scraper.rb
-- spec/bots/bot-with-invalid-data-type/manifest.json
-- spec/bots/bot-with-invalid-data-type/scraper.rb
-- spec/bots/bot-with-invalid-sample-date/manifest.json
-- spec/bots/bot-with-invalid-sample-date/scraper.rb
-- spec/bots/bot-with-pause/manifest.json
-- spec/bots/bot-with-pause/scraper.rb
-- spec/bots/bot-with-transformer/manifest.json
-- spec/bots/bot-with-transformer/scraper.rb
-- spec/bots/bot-with-transformer/transformer.rb
-- spec/bots/bot-with-transformers/manifest.json
-- spec/bots/bot-with-transformers/scraper.rb
-- spec/bots/bot-with-transformers/transformer1.rb
-- spec/bots/bot-with-transformers/transformer2.rb
-- spec/bots/invalid-json-bot/manifest.json
-- spec/bots/invalid-json-bot/scraper.rb
-- spec/bots/invalid-record-bot/manifest.json
-- spec/bots/invalid-record-bot/scraper.rb
-- spec/bots/logging-bot/manifest.json
-- spec/bots/logging-bot/scraper.rb
-- spec/bots/python-bot/manifest.json
-- spec/bots/python-bot/scraper.py
-- spec/bots/ruby-bot/manifest.json
-- spec/bots/ruby-bot/scraper.rb
-- spec/bots/slow-bot/manifest.json
-- spec/bots/slow-bot/scraper.rb
-- spec/lib/processor_spec.rb
-- spec/lib/runner_spec.rb
-- spec/lib/utils_spec.rb
-- spec/lib/validator_spec.rb
-- spec/manual_spec.rb
-- spec/outputs/full-scraper.out
-- spec/outputs/full-transformer.out
-- spec/outputs/truncated-scraper.out
-- spec/spec_helper.rb
-has_rdoc: 
+test_files: []

data/bin/rspec

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-#
-# This file was generated by Bundler.
-#
-# The application 'rspec' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require 'pathname'
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-require 'rubygems'
-require 'bundler/setup'
-
-load Gem.bin_path('rspec-core', 'rspec')