scraperwiki-api 0.0.4 → 0.0.5

data/README.md

@@ -34,7 +34,9 @@ More documentation at [RubyDoc.info](http://rdoc.info/gems/scraperwiki-api/Scrap
 
 ## Scraper validations
 
-If your project uses a lot of scrapers – for example, [OpenCorporates](http://opencorporates.com/), which [scrapes company registries around the world](http://blog.opencorporates.com/2011/03/25/building-a-global-database-the-open-distributed-way/), or [Represent](http://represent.opennorth.ca/), which scrapes information on elected officials from government websites in Canada – you'll want to check that your scrapers behave the way you expect them to. This gem defines [RSpec](https://www.relishapp.com/rspec) matchers to do just that. For example:
+If your project uses a lot of scrapers – for example, [OpenCorporates](http://opencorporates.com/), which [scrapes company registries around the world](http://blog.opencorporates.com/2011/03/25/building-a-global-database-the-open-distributed-way/), or [Represent](http://represent.opennorth.ca/), which scrapes information on elected officials from government websites in Canada – you'll want to check that your scrapers behave the way you expect them to. This gem defines [RSpec](https://www.relishapp.com/rspec) matchers to do just that.
+
+You can validate a scraper's metadata (how often it runs, what fields it stores, etc.) like so:
 
     require 'scraperwiki-api'
     api = ScraperWiki::API.new
@@ -49,40 +51,48 @@ If your project uses a lot of scrapers – for example, [OpenCorporates](http:/
       it {should be_editable_by('frabcus')}
       it {should run(:daily)}
       it {should_not be_broken}
+
+      # Validate the properties of a SQLite table by chaining on a +on+.
       it {should have_a_row_count_of(42).on('swdata')}
 
-      # Check for missing keys:
+      # Ensure that the scraper sets required fields.
       it {should have_at_least_the_keys(['name', 'email']).on('swdata')}
 
-      # Check for extra keys:
+      # Ensure that the scraper doesn't set too many fields.
       it {should have_at_most_the_keys(['name', 'email', 'tel', 'fax']).on('swdata')}
     end
 
+And you can validate the scraped data like so:
+
+    require 'scraperwiki-api'
+    api = ScraperWiki::API.new
+
     data = api.datastore_sqlite('example-scraper', 'SELECT * from `swdata`')
 
     describe 'example-scraper' do
       include ScraperWiki::API::Matchers
       subject {data}
 
+      # If you need at least one of a set of fields to be set:
       it {should set_any_of(['name', 'first_name', 'last_name'])}
 
-      # Validate the values of individual fields:
+      # Validate the values of individual fields by chaining on an +in+.
       it {should_not have_blank_values.in('name')}
       it {should have_unique_values.in('email')}
       it {should have_values_of(['M', 'F']).in('gender')}
-      it {should have_values_matching(/\A[^@\s]+@[^a\s]+\z/).in('email')}
+      it {should have_values_matching(/\A[^@\s]+@[^@\s]+\z/).in('email')}
       it {should have_values_starting_with('http://').in('url')}
       it {should have_values_ending_with('Inc.').in('company_name')}
       it {should have_integer_values.in('year')}
 
       # If you store a hash or an array of hashes in a field as a JSON string,
-      # you can validate the values of these subfields by chaining on an +at+:
+      # you can validate the values of these subfields by chaining on an +at+.
       it {should have_values_of(['M', 'F']).in('extra').at('gender')}
 
-      # Check for missing keys within subfields:
+      # Check for missing keys within subfields.
       it {should have_values_with_at_least_the_keys(['subfield1', 'subfield2']).in('fieldA')}
 
-      # Check for extra keys within subfields:
+      # Check for extra keys within subfields.
       it {should have_values_with_at_most_the_keys(['subfield1', 'subfield2', 'subfield3', 'subfield4']).in('fieldA')}
     end
 

data/lib/scraperwiki-api/matchers.rb

@@ -37,7 +37,7 @@ module ScraperWiki
     #     it {should_not have_blank_values.in('name')}
     #     it {should have_unique_values.in('email')}
     #     it {should have_values_of(['M', 'F']).in('gender')}
-    #     it {should have_values_matching(/\A[^@\s]+@[^a\s]+\z/).in('email')}
+    #     it {should have_values_matching(/\A[^@\s]+@[^@\s]+\z/).in('email')}
     #     it {should have_values_starting_with('http://').in('url')}
     #     it {should have_values_ending_with('Inc.').in('company_name')}
     #     it {should have_integer_values.in('year')}
@@ -74,7 +74,7 @@ module ScraperWiki
         end
 
         def negative_failure_message
-          failure_message
+          raise NotImplementerError, 'Subclasses must implement this method'
         end
       end
 
@@ -92,6 +92,10 @@ module ScraperWiki
         def failure_message
           "expected #{@actual['short_name']} to be #{@expected}"
         end
+
+        def negative_failure_message
+          "expected #{@actual['short_name']} to not be #{@expected}"
+        end
       end
       # @example
       #   it {should be_public}
@@ -120,6 +124,10 @@ module ScraperWiki
         def failure_message
           "expected #{@actual['short_name']} to be editable by #{@expected}"
         end
+
+        def negative_failure_message
+          "expected #{@actual['short_name']} to not be editable by #{@expected}"
+        end
       end
       # @example
       #   it {should be_editable_by 'frabcus'}
@@ -140,6 +148,14 @@ module ScraperWiki
             "expected #{@actual['short_name']} to run #{@expected}"
           end
         end
+
+        def negative_failure_message
+          if @expected == -1
+            "expected #{@actual['short_name']} to run at some time"
+          else
+            "expected #{@actual['short_name']} to not run #{@expected}"
+          end
+        end
       end
       # @example
       #   it {should run(:daily)}
@@ -169,10 +185,18 @@ module ScraperWiki
           "#{@actual['short_name']} #{failure_predicate}: #{difference.join ', '}"
         end
 
+        def negative_failure_message
+          "#{@actual['short_name']} #{negative_failure_predicate}: #{difference.join ', '}"
+        end
+
         def failure_predicate
           raise NotImplementerError, 'Subclasses must implement this method'
         end
 
+        def negative_failure_message
+          raise NotImplementerError, 'Subclasses must implement this method'
+        end
+
         def difference
           raise NotImplementerError, 'Subclasses must implement this method'
         end
@@ -186,6 +210,10 @@ module ScraperWiki
         def failure_predicate
           'is missing keys'
         end
+
+        def negative_failure_predicate
+          "isn't missing keys"
+        end
       end
       # @example
       #   it {should have_at_least_the_keys(['fieldA', 'fieldB']).on('swdata')}
@@ -201,6 +229,10 @@ module ScraperWiki
         def failure_predicate
           'has extra keys'
         end
+
+        def negative_failure_predicate
+          'has no extra keys'
+        end
       end
       # @example
       #   it {should have_at_most_the_keys(['fieldA', 'fieldB', 'fieldC', 'fieldD']).on('swdata')}
@@ -217,6 +249,10 @@ module ScraperWiki
         def failure_message
           "expected #{@actual['short_name']} to have #{@expected} rows, not #{@actual['datasummary']['tables'][@table]['count']}"
         end
+
+        def negative_failure_message
+          "expected #{@actual['short_name']} to not have #{@expected} rows"
+        end
       end
       # @example
       #   it {should have_a_row_count_of(42).on('swdata')}
@@ -243,6 +279,10 @@ module ScraperWiki
         def failure_message
           "#{@actual['short_name']} is broken: #{exception_message}"
         end
+
+        def negative_failure_message
+          "#{@actual['short_name']} isn't broken: #{exception_message}"
+        end
       end
       # @example
       #   it {should_not be_broken}
@@ -310,12 +350,16 @@ module ScraperWiki
         end
 
         def negative_failure_message
-          failure_message
+          "#{failure_size} of #{items.size} #{negative_failure_description}\n#{failures.map(&:inspect).join "\n"}"
         end
 
         def failure_description
           raise NotImplementerError, 'Subclasses must implement this method'
         end
+
+        def negative_failure_description
+          raise NotImplementerError, 'Subclasses must implement this method'
+        end
       end
 
       class SetAnyOf < DatastoreMatcher
@@ -330,6 +374,10 @@ module ScraperWiki
         def failure_description
           "records didn't set any of #{@expected.join ','}"
         end
+
+        def negative_failure_description
+          "records set any of #{@expected.join ','}"
+        end
       end
       # @example
       #   it {should set_any_of(['name', 'first_name', 'last_name'])}
@@ -353,15 +401,23 @@ module ScraperWiki
           if @subfield
             items.send(meth) do |item|
               if blank? item[@field]
-                true
+                meth == :reject
               else
                 v = Yajl::Parser.parse item[@field]
                 if Hash === v
-                  v.has_key?(@subfield) && match?(v[@subfield])
+                  if blank? v[@subfield]
+                    meth == :reject
+                  else
+                    match? v[@subfield]
+                  end
                 elsif Array === v
                   v.all? do |w|
                     if Hash === w
-                      w.has_key?(@subfield) && match?(w[@subfield])
+                      if blank? w[@subfield]
+                        meth == :reject
+                      else
+                        match? w[@subfield]
+                      end
                     else
                       raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
                     end
@@ -373,7 +429,11 @@ module ScraperWiki
             end
           else
             items.send(meth) do |item|
-              match? item[@field]
+              if blank? item[@field]
+                meth == :reject
+              else
+                match? item[@field]
+              end
             end
           end
         end
@@ -398,9 +458,21 @@ module ScraperWiki
           end
         end
 
+        def negative_failure_description
+          if @subfield
+            "#{@field}:#{@subfield} values #{negative_failure_predicate}"
+          else
+            "#{@field} values #{negative_failure_predicate}"
+          end
+        end
+
         def failure_predicate
           raise NotImplementerError, 'Subclasses must implement this method'
         end
+
+        def negative_failure_predicate
+          raise NotImplementerError, 'Subclasses must implement this method'
+        end
       end
 
       class HaveBlankValues < FieldMatcher
@@ -411,6 +483,10 @@ module ScraperWiki
         def failure_predicate
           'are blank'
         end
+
+        def negative_failure_predicate
+          'are present'
+        end
       end
       # @example
       #   it {should_not have_blank_values.in('name')}
@@ -420,12 +496,16 @@ module ScraperWiki
 
       class HaveValuesOf < FieldMatcher
         def match?(v)
-          blank?(v) || @expected.include?(v)
+          @expected.include? v
         end
 
         def failure_predicate
           "aren't one of #{@expected.join ', '}"
         end
+
+        def negative_failure_predicate
+          "are one of #{@expected.join ', '}"
+        end
       end
       # @example
       #   it {should have_values_of(['M', 'F']).in('gender')}
@@ -435,12 +515,16 @@ module ScraperWiki
 
       class HaveValuesMatching < FieldMatcher
         def match?(v)
-          blank?(v) || v[@expected]
+          v[@expected]
         end
 
         def failure_predicate
           "don't match #{@expected.inspect}"
         end
+
+        def negative_failure_predicate
+          "match #{@expected.inspect}"
+        end
       end
       # @example
       #   it {should have_values_matching(/\A[^@\s]+@[^a\s]+\z/).in('email')}
@@ -485,7 +569,11 @@ module ScraperWiki
         end
 
         def failure_predicate
-          'are not unique'
+          "aren't unique"
+        end
+
+        def negative_failure_predicate
+          'are unique'
         end
       end
       # @example
@@ -496,12 +584,16 @@ module ScraperWiki
 
       class HaveValuesStartingWith < FieldMatcher
         def match?(v)
-          blank?(v) || v.start_with?(@expected)
+          v.start_with? @expected
         end
 
         def failure_predicate
           "don't start with #{@expected}"
         end
+
+        def negative_failure_predicate
+          "start with #{@expected}"
+        end
       end
       # @example
       #   it {should have_values_starting_with('http://').in('url')}
@@ -511,12 +603,16 @@ module ScraperWiki
 
       class HaveValuesEndingWith < FieldMatcher
         def match?(v)
-          blank?(v) || v.end_with?(@expected)
+          v.end_with? @expected
         end
 
         def failure_predicate
           "don't end with #{@expected}"
         end
+
+        def negative_failure_predicate
+          "end with #{@expected}"
+        end
       end
       # @example
       #   it {should have_values_ending_with('Inc.').in('company_name')}
@@ -526,12 +622,16 @@ module ScraperWiki
 
       class HaveIntegerValues < FieldMatcher
         def match?(v)
-          blank?(v) || (Integer(v) rescue false)
+          Integer(v) rescue false
         end
 
         def failure_predicate
           "aren't integers"
         end
+
+        def negative_failure_predicate
+          'are integers'
+        end
       end
       # @example
       #   it {should have_integer_values.in('year')}
@@ -541,23 +641,19 @@ module ScraperWiki
 
       class FieldKeyMatcher < FieldMatcher
         def match?(v)
-          if blank? v
-            true
-          else
-            w = Yajl::Parser.parse v
-            if Hash === w
-              difference(w).empty?
-            elsif Array === w
-              w.all? do |x|
-                if Hash === x
-                  difference(x).empty?
-                else
-                  raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
-                end
+          w = Yajl::Parser.parse v
+          if Hash === w
+            difference(w).empty?
+          elsif Array === w
+            w.all? do |x|
+              if Hash === x
+                difference(x).empty?
+              else
+                raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
               end
-            else
-              raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
             end
+          else
+            raise NotImplementerError, 'Can only handle subfields that are hashes or arrays of hashes'
           end
         end
 
@@ -568,6 +664,10 @@ module ScraperWiki
         def failure_predicate
           "#{predicate}: #{difference.join ', '}"
         end
+
+        def negative_failure_predicate
+          "#{negative_predicate}: #{difference.join ', '}"
+        end
       end
 
       class HaveValuesWithAtLeastTheKeys < FieldKeyMatcher
@@ -575,8 +675,12 @@ module ScraperWiki
           @expected - v.keys
         end
 
-        def failure_predicate
-          'have missing keys'
+        def predicate
+          'are missing keys'
+        end
+
+        def negative_predicate
+          "aren't missing keys"
         end
       end
       # @example
@@ -590,9 +694,13 @@ module ScraperWiki
           v.keys - @expected
         end
 
-        def failure_predicate
+        def predicate
           'have extra keys'
         end
+
+        def negative_predicate
+          'have no extra keys'
+        end
       end
       # @example
       #   it {should have_values_with_at_most_the_keys(['subfield1', 'subfield2', 'subfield3', 'subfield4']).in('fieldA')}

data/lib/scraperwiki-api/version.rb

@@ -1,5 +1,5 @@
 module ScraperWiki
   class API
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraperwiki-api
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-28 00:00:00.000000000 Z
+date: 2012-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70366370555240 !ruby/object:Gem::Requirement
+  requirement: &70265218891100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70366370555240
+  version_requirements: *70265218891100
 - !ruby/object:Gem::Dependency
   name: httparty
-  requirement: &70366370554220 !ruby/object:Gem::Requirement
+  requirement: &70265218882020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *70366370554220
+  version_requirements: *70265218882020
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70366370553000 !ruby/object:Gem::Requirement
+  requirement: &70265218880500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '2.10'
   type: :development
   prerelease: false
-  version_requirements: *70366370553000
+  version_requirements: *70265218880500
 description: A Ruby wrapper for the ScraperWiki API
 email:
 - info@opennorth.ca