turbot-runner 0.1.26 → 0.1.27

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZmZhZTI1N2ZkMWFlODhiOGEwNjlhMGE4MzkwMjk1MmQ5OTc2YjcyNg==
+    ZWIwOWIxZjA3YTkxYWE4MjY2N2VhYzI2MzAxNGJkMGI2N2Y5NGM5Zg==
   data.tar.gz: !binary |-
-    NDU1MTY4Yzk1ZDJkMmEwYjM0N2YxMDI3N2IxYjVlYTg0MDE0MTU1Mw==
+    NGZjZmUxMzk1ZWJlN2MxZGY5MGUxM2M4NzkyMjE1YTA0ZTFkY2M3Zg==
 SHA512:
   metadata.gz: !binary |-
-    OGI4ZjgyOTJjZGI3ZTVhZWY4YWQ3YzE4Y2I0MjIwZjU3ZDZjZWZjYjk4OTYy
-    N2ViZWM2ZTlkMTcxYWUxZjQ1ZTBkNjBjMjc3MWRhYjhiZThkZWZkYTljMjlh
-    YTkwZjY1OTNiYzBjNWE3N2NlODcyNjFkMjkzNWM3NzI1YjFjYTQ=
+    MDE3NmM3MjY2ODhlMzM4NmU1ZDdkM2RkOGFhNmU5MzNhZjc3MTY3MDQyZjA4
+    ZTBlNDFmZTRkMDBhYWY3OWY3Mjg1Mjk0NjZlZjk4ODVjNGQzNWVlMDVmMGU3
+    MmUxZWZmZDBkNDdjYWZkY2U1Y2Q2ZmMyOWYxNGYyZWI0ZDU2MTE=
   data.tar.gz: !binary |-
-    ODdjNTI1YWNjOWUyOGEzY2ZlYWZhOWFmM2Y5MDNiMWE1Mzc4MDE3YmE2ZTU3
-    YTg1NDFiZGNmOWZkNDY3N2IyOWY5YTAwNzc5NDFjYWQ1Y2U4YWUyMDFmZWJm
-    ZmU5NWJmMjU0MzJlN2Q0MzQ0NGFkMzhiNDZkYWRlMTNlZjY4OTk=
+    MDlhNTM3ZWM2NjQ5OTk1MWY2Y2JjZTkyYTMwZTBlMGQ4ZDIxNDczNzZkM2Ex
+    YmZlZTcwMDgwM2ZlOTM1MzMwZTQ0ZWRhMmE3OTIxNjMxYjY1YzQ5MTRiZTk4
+    ODIzYjY4ZWUyNjQzMjQwYzJmN2MwN2Q2NmM2NzAxNjM4Mjg1Y2U=

data/lib/turbot_runner/utils.rb

@@ -5,5 +5,43 @@ module TurbotRunner
     def deep_copy(thing)
       Marshal.load(Marshal.dump(thing))
     end
+
+    # This turns a hash of the form:
+    #
+    # {
+    #   'a' => {
+    #     'b' => {
+    #       'c' => '123',
+    #       'd' => '124',
+    #     },
+    #     'e' => {
+    #       'f' => '156',
+    #     }
+    #   }
+    # }
+    #
+    # into a hash of the form:
+    #
+    # {
+    #   'a.b.c' => '123',
+    #   'a.b.d' => '124',
+    #   'a.e.f' => '156',
+    # }
+    def flatten(hash)
+      pairs = []
+
+      hash.each do |k, v|
+        case v
+        when Hash
+          flatten(v).each do |k1, v1|
+            pairs << ["#{k}.#{k1}", v1]
+          end
+        else
+          pairs << [k, v]
+        end
+      end
+
+      Hash[pairs]
+    end
   end
 end

data/lib/turbot_runner/validator.rb

@@ -9,7 +9,9 @@ module TurbotRunner
       message = nil
 
       if error.nil?
-        identifying_attributes = record.reject do |k, v|
+        flattened_record = TurbotRunner::Utils.flatten(record)
+
+        identifying_attributes = flattened_record.reject do |k, v|
           !identifying_fields.include?(k) || v.nil? || v == ''
         end
 

data/lib/turbot_runner/version.rb

@@ -1,3 +1,3 @@
 module TurbotRunner
-  VERSION = '0.1.26'
+  VERSION = '0.1.27'
 end

data/spec/lib/utils_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe TurbotRunner::Utils do
+  specify '.flatten' do
+    hash = {
+      'a' => {
+        'b' => {
+          'c' => '123',
+          'd' => '124',
+        },
+        'e' => {
+          'f' => '156',
+        }
+      }
+    }
+
+    expect(TurbotRunner::Utils.flatten(hash)).to eq({
+      'a.b.c' => '123',
+      'a.b.d' => '124',
+      'a.e.f' => '156',
+    })
+  end
+end

data/spec/lib/validator_spec.rb

@@ -48,5 +48,42 @@ describe TurbotRunner::Validator do
       expected_error = 'Property not of expected format: sample_date (must be of format yyyy-mm-dd)'
       expect(record).to fail_validation_with(expected_error)
     end
+
+    context 'with nested identifying fields' do
+      specify 'with record missing all identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {}},
+          'four' => {}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq('There were no values provided for any of the identifying fields: one.two.three, four.five.six')
+      end
+
+      specify 'with record missing some identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {'three' => 123}}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq(nil)
+      end
+
+      specify 'with record missing no identifying fields' do
+        record = {
+          'sample_date' => '2014-06-01',
+          'source_url' => 'http://example.com/123',
+          'one' => {'two' => {'three' => 123}},
+          'four' => {'five' => {'six' => 456}}
+        }
+        identifying_fields = ['one.two.three', 'four.five.six']
+        error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
+        expect(error).to eq(nil)
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: turbot-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.26
+  version: 0.1.27
 platform: ruby
 authors:
 - OpenCorporates
@@ -110,6 +110,7 @@ files:
 - spec/bots/slow-bot/scraper.rb
 - spec/lib/processor_spec.rb
 - spec/lib/runner_spec.rb
+- spec/lib/utils_spec.rb
 - spec/lib/validator_spec.rb
 - spec/manual_spec.rb
 - spec/outputs/full-scraper.out