scraperwiki-api 0.0.3 → 0.0.4

data/README.md

@@ -49,9 +49,13 @@ If your project uses a lot of scrapers – for example, [OpenCorporates](http:/
       it {should be_editable_by('frabcus')}
       it {should run(:daily)}
       it {should_not be_broken}
+      it {should have_a_row_count_of(42).on('swdata')}
+
+      # Check for missing keys:
       it {should have_at_least_the_keys(['name', 'email']).on('swdata')}
+
+      # Check for extra keys:
       it {should have_at_most_the_keys(['name', 'email', 'tel', 'fax']).on('swdata')}
-      it {should have_a_row_count_of(42).on('swdata')}
     end
 
     data = api.datastore_sqlite('example-scraper', 'SELECT * from `swdata`')
@@ -60,6 +64,9 @@ If your project uses a lot of scrapers – for example, [OpenCorporates](http:/
       include ScraperWiki::API::Matchers
       subject {data}
 
+      it {should set_any_of(['name', 'first_name', 'last_name'])}
+
+      # Validate the values of individual fields:
       it {should_not have_blank_values.in('name')}
       it {should have_unique_values.in('email')}
       it {should have_values_of(['M', 'F']).in('gender')}
@@ -67,7 +74,16 @@ If your project uses a lot of scrapers – for example, [OpenCorporates](http:/
       it {should have_values_starting_with('http://').in('url')}
       it {should have_values_ending_with('Inc.').in('company_name')}
       it {should have_integer_values.in('year')}
-      it {should set_any_of(['name', 'first_name', 'last_name'])}
+
+      # If you store a hash or an array of hashes in a field as a JSON string,
+      # you can validate the values of these subfields by chaining on an +at+:
+      it {should have_values_of(['M', 'F']).in('extra').at('gender')}
+
+      # Check for missing keys within subfields:
+      it {should have_values_with_at_least_the_keys(['subfield1', 'subfield2']).in('fieldA')}
+
+      # Check for extra keys within subfields:
+      it {should have_values_with_at_most_the_keys(['subfield1', 'subfield2', 'subfield3', 'subfield4']).in('fieldA')}
     end
 
 More documentation at [RubyDoc.info](http://rdoc.info/gems/scraperwiki-api/ScraperWiki/API/Matchers).

data/lib/scraperwiki-api/matchers.rb

@@ -16,9 +16,13 @@ module ScraperWiki
     #     it {should be_editable_by('frabcus')}
     #     it {should run(:daily)}
     #     it {should_not be_broken}
+    #     it {should have_a_row_count_of(42).on('swdata')}
+    #
+    #     # Check for missing keys:
     #     it {should have_at_least_the_keys(['name', 'email']).on('swdata')}
+    #
+    #     # Check for extra keys:
     #     it {should have_at_most_the_keys(['name', 'email', 'tel', 'fax']).on('swdata')}
-    #     it {should have_a_row_count_of(42).on('swdata')}
     #   end
     #
     #   data = api.datastore_sqlite('example-scraper', 'SELECT * from `swdata`')
@@ -27,6 +31,9 @@ module ScraperWiki
     #     include ScraperWiki::API::Matchers
     #     subject {data}
     #
+    #     it {should set_any_of(['name', 'first_name', 'last_name'])}
+    #
+    #     # Validate the values of individual fields:
     #     it {should_not have_blank_values.in('name')}
     #     it {should have_unique_values.in('email')}
     #     it {should have_values_of(['M', 'F']).in('gender')}
@@ -34,7 +41,16 @@ module ScraperWiki
     #     it {should have_values_starting_with('http://').in('url')}
     #     it {should have_values_ending_with('Inc.').in('company_name')}
     #     it {should have_integer_values.in('year')}
-    #     it {should set_any_of(['name', 'first_name', 'last_name'])}
+    #
+    #     # If you store a hash or an array of hashes in a field as a JSON string,
+    #     # you can validate the values of these subfields by chaining on an +at+:
+    #     it {should have_values_of(['M', 'F']).in('extra').at('gender')}
+    #
+    #     # Check for missing keys within subfields:
+    #     it {should have_values_with_at_least_the_keys(['subfield1', 'subfield2']).in('fieldA')}
+    #
+    #     # Check for extra keys within subfields:
+    #     it {should have_values_with_at_most_the_keys(['subfield1', 'subfield2', 'subfield3', 'subfield4']).in('fieldA')}
     #   end
     #
     # RSpec matchers for ScraperWiki scrapers.
@@ -54,7 +70,7 @@ module ScraperWiki
         end
 
         def failure_message
-          NotImplementerError
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
 
         def negative_failure_message
@@ -149,12 +165,16 @@ module ScraperWiki
           difference.empty?
         end
 
+        def failure_message
+          "#{@actual['short_name']} #{failure_predicate}: #{difference.join ', '}"
+        end
+
         def failure_predicate
-          raise NotImplementerError
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
 
-        def failure_message
-          "#{@actual['short_name']} #{failure_predicate}: #{difference.join ', '}"
+        def difference
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
       end
 
@@ -245,7 +265,7 @@ module ScraperWiki
               hash
             end
           else
-            raise NotImplementerError
+            raise NotImplementerError, "Can only handle jsondict or jsonlist formats"
           end
         end
 
@@ -262,11 +282,11 @@ module ScraperWiki
         end
 
         def matches
-          raise NotImplementerError
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
 
         def mismatches
-          raise NotImplementerError
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
 
         def failures
@@ -285,10 +305,6 @@ module ScraperWiki
           end
         end
 
-        def failure_description
-          raise NotImplementerError
-        end
-
         def failure_message
           "#{failure_size} of #{items.size} #{failure_description}\n#{failures.map(&:inspect).join "\n"}"
         end
@@ -296,6 +312,10 @@ module ScraperWiki
         def negative_failure_message
           failure_message
         end
+
+        def failure_description
+          raise NotImplementerError, 'Subclasses must implement this method'
+        end
       end
 
       class SetAnyOf < DatastoreMatcher
@@ -323,16 +343,47 @@ module ScraperWiki
           self
         end
 
-        def matches
-          items.select do |item|
-            match? item[@field]
+        def at(subfield)
+          @subfield = subfield
+          self
+        end
+
+        # @note +@subfield+ can be a hash or an array of hashes
+        def matcher(meth)
+          if @subfield
+            items.send(meth) do |item|
+              if blank? item[@field]
+                true
+              else
+                v = Yajl::Parser.parse item[@field]
+                if Hash === v
+                  v.has_key?(@subfield) && match?(v[@subfield])
+                elsif Array === v
+                  v.all? do |w|
+                    if Hash === w
+                      w.has_key?(@subfield) && match?(w[@subfield])
+                    else
+                      raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+                    end
+                  end
+                else
+                  raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+                end
+              end
+            end
+          else
+            items.send(meth) do |item|
+              match? item[@field]
+            end
           end
         end
 
+        def matches
+          matcher :select
+        end
+
         def mismatches
-          items.reject do |item|
-            match? item[@field]
-          end
+          matcher :reject
         end
 
         def blank?(v)
@@ -340,7 +391,15 @@ module ScraperWiki
         end
 
         def failure_description
-          "'#{@field}' values #{failure_predicate}"
+          if @subfield
+            "#{@field}:#{@subfield} values #{failure_predicate}"
+          else
+            "#{@field} values #{failure_predicate}"
+          end
+        end
+
+        def failure_predicate
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
       end
 
@@ -392,9 +451,34 @@ module ScraperWiki
       class HaveUniqueValues < FieldMatcher
         def mismatches
           counts = Hash.new 0
-          items.each_with_index do |item,index|
-            unless blank? item[@field]
-              counts[item[@field]] += 1
+          if @subfield
+            items.each do |item|
+              unless blank? item[@field]
+                v = Yajl::Parser.parse item[@field]
+                if Hash === v
+                  unless blank? v[@subfield]
+                    counts[v[@subfield]] += 1
+                  end
+                elsif Array === v
+                  v.each do |w|
+                    if Hash === w
+                      unless blank? w[@subfield]
+                        counts[w[@subfield]] += 1
+                      end
+                    else
+                      raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+                    end
+                  end
+                else
+                  raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+                end
+              end
+            end
+          else
+            items.each do |item|
+              unless blank? item[@field]
+                counts[item[@field]] += 1
+              end
             end
           end
           counts.select{|_,count| count > 1}.keys
@@ -454,6 +538,67 @@ module ScraperWiki
       def have_integer_values
         HaveIntegerValues.new nil
       end
+
+      class FieldKeyMatcher < FieldMatcher
+        def match?(v)
+          if blank? v
+            true
+          else
+            w = Yajl::Parser.parse v
+            if Hash === w
+              difference(w).empty?
+            elsif Array === w
+              w.all? do |x|
+                if Hash === x
+                  difference(x).empty?
+                else
+                  raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+                end
+              end
+            else
+              raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
+            end
+          end
+        end
+
+        def difference(v)
+          raise NotImplementerError, 'Subclasses must implement this method'
+        end
+
+        def failure_predicate
+          "#{predicate}: #{difference.join ', '}"
+        end
+      end
+
+      class HaveValuesWithAtLeastTheKeys < FieldKeyMatcher
+        def difference(v)
+          @expected - v.keys
+        end
+
+        def failure_predicate
+          'have missing keys'
+        end
+      end
+      # @example
+      #   it {should have_values_with_at_least_the_keys(['subfield1', 'subfield2']).in('fieldA')}
+      def have_values_with_at_least_the_keys(expected)
+        HaveValuesWithAtLeastTheKeys.new expected
+      end
+
+      class HaveValuesWithAtMostTheKeys < FieldKeyMatcher
+        def difference(v)
+          v.keys - @expected
+        end
+
+        def failure_predicate
+          'have extra keys'
+        end
+      end
+      # @example
+      #   it {should have_values_with_at_most_the_keys(['subfield1', 'subfield2', 'subfield3', 'subfield4']).in('fieldA')}
+      def have_values_with_at_most_the_keys(expected)
+        HaveValuesWithAtMostTheKeys.new expected
+      end
     end
   end
 end

data/lib/scraperwiki-api/version.rb

@@ -1,5 +1,5 @@
 module ScraperWiki
   class API
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/scraperwiki-api.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency('httparty', '~> 0.7.8')
-  s.add_development_dependency('rspec', '~> 2.10.0')
+  s.add_runtime_dependency('yajl-ruby', '~> 1.0')
+  s.add_runtime_dependency('httparty', '~> 0.8.0')
+  s.add_development_dependency('rspec', '~> 2.10')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraperwiki-api
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -11,28 +11,39 @@ bindir: bin
 cert_chain: []
 date: 2012-05-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &70366370555240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366370555240
 - !ruby/object:Gem::Dependency
   name: httparty
-  requirement: &70314147045360 !ruby/object:Gem::Requirement
+  requirement: &70366370554220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *70314147045360
+  version_requirements: *70366370554220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70314147044520 !ruby/object:Gem::Requirement
+  requirement: &70366370553000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.10.0
+        version: '2.10'
   type: :development
   prerelease: false
-  version_requirements: *70314147044520
+  version_requirements: *70366370553000
 description: A Ruby wrapper for the ScraperWiki API
 email:
 - info@opennorth.ca