turbot-runner 0.1.4 → 0.1.5

data/lib/turbot_runner/base_handler.rb

@@ -3,6 +3,9 @@ module TurbotRunner
     def handle_valid_record(record, data_type)
     end
 
+    def handle_run_ended
+    end
+
     def handle_invalid_record(record, data_type, line)
     end
 

data/lib/turbot_runner/processor.rb

@@ -12,18 +12,23 @@ module TurbotRunner
 
     def process(line)
       begin
-        record = JSON.parse(line)
-        errors = validate(record)
+        if line.strip == "RUN ENDED"
+          @record_handler.handle_run_ended
+          @runner.interrupt
+        else
+          record = JSON.parse(line)
+          errors = validate(record)
 
-        if errors.empty?
-          begin
-            @record_handler.handle_valid_record(record, @data_type)
-          rescue InterruptRun
-            @runner.interrupt
+          if errors.empty?
+            begin
+              @record_handler.handle_valid_record(record, @data_type)
+            rescue InterruptRun
+              @runner.interrupt
+            end
+          else
+            @record_handler.handle_invalid_record(record, @data_type, errors)
+            @runner.interrupt_and_mark_as_failed
           end
-        else
-          @record_handler.handle_invalid_record(record, @data_type, errors)
-          @runner.interrupt_and_mark_as_failed
         end
       rescue JSON::ParserError
         @record_handler.handle_invalid_json(line)

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

data/spec/bots/bot-that-emits-run-ended/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-that-emits-run-ended",
+    "description": "This is a bot that emits RUN ENDED",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-that-emits-run-ended/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(2) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end
+puts "RUN ENDED"

data/spec/lib/runner_spec.rb

@@ -91,6 +91,25 @@ describe TurbotRunner::Runner do
       end
     end
 
+    context 'with a bot that outputs RUN ENDED' do
+      before do
+        @runner = TurbotRunner::Runner.new(
+          'spec/bots/bot-that-emits-run-ended',
+          :log_to_file => true
+        )
+      end
+      it 'calls handle_run_ended on the handler' do
+        expect_any_instance_of(TurbotRunner::BaseHandler).to receive(:handle_run_ended)
+        @runner.run
+      end
+
+      it 'interrupts the run' do
+        expect_any_instance_of(TurbotRunner::ScriptRunner).to receive(:interrupt)
+        @runner.run
+      end
+    end
+
+
     context 'with a bot that crashes in scraper' do
       before do
         @runner = TurbotRunner::Runner.new(
@@ -209,7 +228,7 @@ describe TurbotRunner::Runner do
           @records_seen[data_type] += 1
         end
       end
-      
+
       handler = Handler.new
       runner = TurbotRunner::Runner.new(
         'spec/bots/bot-with-transformer',

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
+  prerelease: 
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-08 00:00:00.000000000 Z
+date: 2014-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -39,34 +42,14 @@ files:
 - lib/turbot_runner/runner.rb
 - lib/turbot_runner/script_runner.rb
 - lib/turbot_runner/version.rb
-- schema/schemas/company-schema.json
-- schema/schemas/financial-payment-schema.json
-- schema/schemas/includes/address.json
-- schema/schemas/includes/alternative_name.json
-- schema/schemas/includes/company.json
-- schema/schemas/includes/filing.json
-- schema/schemas/includes/financial-payment-data-object.json
-- schema/schemas/includes/industry_code.json
-- schema/schemas/includes/licence-data-object.json
-- schema/schemas/includes/officer.json
-- schema/schemas/includes/previous_name.json
-- schema/schemas/includes/share-parcel-data.json
-- schema/schemas/includes/share-parcel.json
-- schema/schemas/includes/subsidiary-relationship-data.json
-- schema/schemas/includes/total-shares.json
-- schema/schemas/licence-schema.json
-- schema/schemas/primary-data-schema.json
-- schema/schemas/share-parcel-schema.json
-- schema/schemas/simple-financial-payment-schema.json
-- schema/schemas/simple-licence-schema.json
-- schema/schemas/simple-subsidiary-schema.json
-- schema/schemas/subsidiary-relationship-schema.json
 - spec/bots/bot-that-crashes-in-scraper/manifest.json
 - spec/bots/bot-that-crashes-in-scraper/scraper.rb
 - spec/bots/bot-that-crashes-in-transformer/manifest.json
 - spec/bots/bot-that-crashes-in-transformer/scraper.rb
 - spec/bots/bot-that-crashes-in-transformer/transformer1.rb
 - spec/bots/bot-that-crashes-in-transformer/transformer2.rb
+- spec/bots/bot-that-emits-run-ended/manifest.json
+- spec/bots/bot-that-emits-run-ended/scraper.rb
 - spec/bots/bot-with-pause/manifest.json
 - spec/bots/bot-with-pause/scraper.rb
 - spec/bots/bot-with-transformer/manifest.json
@@ -94,29 +77,51 @@ files:
 - spec/outputs/full-scraper.out
 - spec/outputs/full-transformer.out
 - spec/outputs/truncated-scraper.out
+- schema/schemas/company-schema.json
+- schema/schemas/financial-payment-schema.json
+- schema/schemas/includes/address.json
+- schema/schemas/includes/alternative_name.json
+- schema/schemas/includes/company.json
+- schema/schemas/includes/filing.json
+- schema/schemas/includes/financial-payment-data-object.json
+- schema/schemas/includes/industry_code.json
+- schema/schemas/includes/licence-data-object.json
+- schema/schemas/includes/officer.json
+- schema/schemas/includes/previous_name.json
+- schema/schemas/includes/share-parcel-data.json
+- schema/schemas/includes/share-parcel.json
+- schema/schemas/includes/subsidiary-relationship-data.json
+- schema/schemas/includes/total-shares.json
+- schema/schemas/licence-schema.json
+- schema/schemas/primary-data-schema.json
+- schema/schemas/share-parcel-schema.json
+- schema/schemas/simple-financial-payment-schema.json
+- schema/schemas/simple-licence-schema.json
+- schema/schemas/simple-subsidiary-schema.json
+- schema/schemas/subsidiary-relationship-schema.json
 homepage: http://turbot.opencorporates.com/
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Utilities for running bots with Turbot
 test_files: []
-has_rdoc: 

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    M2Q4MGEyNmViMzEzODU2YWJkNTRkNjY2NzQ4N2JkODUzYmM3MTVhOQ==
-  data.tar.gz: !binary |-
-    MjJhZTJjZjMwYmZjNTY5Mjk3YWU4OTUwMTA1M2I4MjNiZWFiNzVlYw==
-SHA512:
-  metadata.gz: !binary |-
-    ZjYzMTZhODVkNmU3MWE3ZjkzNTNkMzRhNzZjMWUzODlhODc0YmFlODhlMWE3
-    YzUxMDU3Y2E5MTBhZTg2MTY3MjIyMDNkZmE2OTIwOWI1M2RmMTQ4MDQzODlm
-    MjE0MGMzNTc2NjZlZWI5MGI5NTgxZTQzZDRkYmE4YzUxZjAzOWE=
-  data.tar.gz: !binary |-
-    ZGE3NDZmZGRhYWI2NjNlZTM4MWExYzg3MmZiYzNiMzExMTFhZDlkYWVhOTIy
-    MTk3Mzk1NTU3Y2RhZDY2ZGQ1Yzc5ZTg0YTMxMmQyN2Y0YzVmMjA5ZjVlODk4
-    MDg1MDQ0ZWI4MDk1ZWZjZjk2ZmMyOTEwMDY5Mzc5NjFjZWRjZjg=