turbot-runner 0.0.17 → 0.0.18

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGMxNGNmYmMwNGQ4ZDQ2MDkzOGQ4NmQ3NTI5NzFjY2U4ZWQ4Y2ZlNQ==
+  data.tar.gz: !binary |-
+    MjJkOTI5NmU3MjViYWFhZjlhZjJlMmZmNzIzZGI2Mzc5YTA2NWZmNw==
+SHA512:
+  metadata.gz: !binary |-
+    NDNmZDcwMGVmMDk4ZjM2ZDM0OWQ5ODgxYzJlNjRkMTIxNTk5NDZlMDc4ZGFh
+    OGUwNGIwOWMwOGY3ODNjYmE4ODE2NGYzMjJlM2M0NmU4MDE1YmEyZTE5NDU2
+    MWM1M2FkN2Y0MmRiODgyNzMwYzBkMjczYzEwYjJmOWIxMjIyNDU=
+  data.tar.gz: !binary |-
+    NmQzMDIwNTI3YWU2M2NhMjJiN2NlYThiMWVlNTkyM2E1MWE0MGM3MTlmZDVk
+    ZDFjNDcwZTI3ZDllOTJjYWRmNjc0OTg2OWFkZmZjM2RjYjhhM2Y0ZDMzYzMw
+    M2U2MmU4Yzg1YzgwMWRmN2QxZjgyNjZmNWE4MmFmMTNlMDBhYWQ=

data/lib/turbot_runner.rb

@@ -1,5 +1,8 @@
+require 'active_support/core_ext/hash/slice'
+require 'active_support/core_ext/object/to_query'
 require 'json'
 require 'open3'
+require 'set'
 require 'timeout'
 
 module TurbotRunner
@@ -25,11 +28,11 @@ module TurbotRunner
       @status = :initialized
       @interrupted = false
       @schemas = {}
+      @seen_uids = Set.new
     end
 
     def run(opts={})
       @status = :running
-      validation_required = opts[:validate] || true
 
       command = "#{interpreter_for(scraper_file)} #{scraper_file}"
       data_type = @config['data_type']
@@ -67,6 +70,10 @@ module TurbotRunner
               runner.send_line(line)
               line1 = runner.get_next_line
 
+              # A transformer should output an empty line if it doesn't make
+              # sense to transform a record.
+              next if line1.strip.empty?
+
               begin
                 record1 = JSON.parse(line1)
               rescue JSON::ParserError
@@ -136,14 +143,17 @@ module TurbotRunner
 
       if messages.empty?
         identifying_fields = identifying_fields_for_data_type(data_type)
+        identifying_hash = record.slice(*identifying_fields)
 
-        hash = Hash.new
-        identifying_fields.each do |k|
-          hash[k] = record[k] if record.has_key?(k)
-        end
-
-        if hash.empty?
+        if identifying_hash.empty?
           messages << "Missing attributes for identifying fields: #{identifying_fields.join(', ')}"
+        else
+          record_uid = Digest::SHA1.hexdigest(identifying_hash.to_query)
+          if @seen_uids.include?(record_uid)
+            messages << "Values for identifying fields must be unique.  There has already been a record with: #{identifying_hash.to_json}"
+          else
+            @seen_uids << record_uid
+          end
         end
       end
 

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.0.17'
+  VERSION = '0.0.18'
 end

metadata

@@ -1,20 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.0.17
-  prerelease: 
+  version: 0.0.18
 platform: ruby
 authors:
 - OpenCorporates
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-15 00:00:00.000000000 Z
+date: 2014-07-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -22,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -37,13 +48,6 @@ files:
 - lib/prerun.rb
 - lib/turbot_runner.rb
 - lib/turbot_runner/version.rb
-- spec/dummy-bot-python/manifest.json
-- spec/dummy-bot-python/scraper.py
-- spec/dummy-bot-python/transformer.py
-- spec/dummy-bot-ruby/manifest.json
-- spec/dummy-bot-ruby/scraper.rb
-- spec/dummy-bot-ruby/transformer.rb
-- spec/turbot_runner_spec.rb
 - schema/schemas/company-schema.json
 - schema/schemas/financial-payment-schema.json
 - schema/schemas/includes/address.json
@@ -64,29 +68,36 @@ files:
 - schema/schemas/share-parcel-schema.json
 - schema/schemas/simple-licence-schema.json
 - schema/schemas/subsidiary-relationship-schema.json
+- spec/dummy-bot-python/manifest.json
+- spec/dummy-bot-python/scraper.py
+- spec/dummy-bot-python/transformer.py
+- spec/dummy-bot-ruby/manifest.json
+- spec/dummy-bot-ruby/scraper.rb
+- spec/dummy-bot-ruby/transformer.rb
+- spec/turbot_runner_spec.rb
 homepage: http://turbot.opencorporates.com/
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Utilities for running bots with Turbot
 test_files: []
+has_rdoc: