spidey-mongo 0.0.3 → 0.0.4

data/README.md

@@ -46,12 +46,15 @@ With persistent storage of the URL-crawling queue, it's now possible to stop cra
 
 ### Recording Results
 
-By default, invocations of `record(data)` by the spider simply insert new documents into the result collection. If corresponding results may already exist in the collection and should instead be updated, use the `set_result_key` helper (with a proc or method symbol) to specify how to find the document to update:
+By default, invocations of `record(data)` by the spider simply insert new documents into the result collection. If corresponding results may already exist in the collection and should instead be updated, define a `result_key` method that returns a key by which to find the corresponding document. The method is called with a hash of the data being recorded:
 
     class EbaySpider < Spidey::AbstractSpider
       include Spidey::Strategies::Mongo
-      set_result_key ->(data) { data[:auction_id] }
       
+      def result_key(data)
+        data[:detail_url]
+      end
+
       # ...
     end
 

data/lib/spidey-mongo/version.rb

@@ -1,5 +1,5 @@
 module Spidey
   module Mongo
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/lib/spidey/strategies/mongo.rb

@@ -2,24 +2,6 @@ module Spidey::Strategies
   module Mongo
     attr_accessor :url_collection, :result_collection, :error_collection
   
-    module ClassMethods
-      def set_result_key(callback)
-        @result_key = callback
-      end
-    
-      def result_key(spider, data)
-        case @result_key
-        when Symbol then spider.send(@result_key, data)
-        when Proc then @result_key.call(data)
-        else nil
-        end
-      end
-    end
-  
-    def self.included(base)
-      base.extend ClassMethods
-    end
-  
     def initialize(attrs = {})
       self.url_collection = attrs.delete(:url_collection)
       self.result_collection = attrs.delete(:result_collection)
@@ -44,7 +26,7 @@ module Spidey::Strategies
   
     def record(data)
       $stderr.puts "Recording #{data.inspect.truncate(500)}" if verbose
-      if key = self.class.result_key(self, data)
+      if respond_to?(:result_key) && key = result_key(data)
         result_collection.update({'key' => key}, {'$set' => data}, upsert: true)
       else
         result_collection.insert data

data/spec/spidey/strategies/mongo_spec.rb

@@ -3,8 +3,11 @@ require 'spec_helper'
 describe Spidey::Strategies::Mongo do
   class TestSpider < Spidey::AbstractSpider
     include Spidey::Strategies::Mongo
-    set_result_key ->(data) { data[:detail_url] }
     handle "http://www.cnn.com", :process_home
+    
+    def result_key(data)
+      data[:detail_url]
+    end
   end
   
   before(:each) do
@@ -15,6 +18,10 @@ describe Spidey::Strategies::Mongo do
       error_collection: @db['errors'])
   end
   
+  after(:each) do
+    %w{ urls results errors }.each{ |col| @db[col].drop }
+  end
+  
   it "should add initial URLs to collection" do
     doc = @db['urls'].find_one(url: "http://www.cnn.com")
     doc['handler'].should == :process_home
@@ -33,6 +40,12 @@ describe Spidey::Strategies::Mongo do
     doc['foo'].should == 'bar'
   end
   
+  it "should update existing result" do
+    @db['results'].insert key: 'http://foo.bar', detail_url: 'http://foo.bar'
+    @spider.record detail_url: 'http://foo.bar', foo: 'bar'
+    @db['results'].count.should == 1
+  end
+  
   it "should add error" do
     @spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
     doc = @db['errors'].find_one

data/spidey-mongo.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
+  s.add_development_dependency "ruby-debug19"
   
   s.add_runtime_dependency "spidey"
   s.add_runtime_dependency "mongo"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spidey-mongo
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-06-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70227312727160 !ruby/object:Gem::Requirement
+  requirement: &70312341584000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70227312727160
+  version_requirements: *70312341584000
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70227312726140 !ruby/object:Gem::Requirement
+  requirement: &70312341583440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70227312726140
+  version_requirements: *70312341583440
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &70312341582900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70312341582900
 - !ruby/object:Gem::Dependency
   name: spidey
-  requirement: &70227312725220 !ruby/object:Gem::Requirement
+  requirement: &70312341582320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70227312725220
+  version_requirements: *70312341582320
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70227312724420 !ruby/object:Gem::Requirement
+  requirement: &70312341581640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70227312724420
+  version_requirements: *70312341581640
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70227312717760 !ruby/object:Gem::Requirement
+  requirement: &70312341580620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70227312717760
+  version_requirements: *70312341580620
 description: Implements a MongoDB back-end for Spidey, a framework for crawling and
   scraping web sites.
 email:
@@ -100,7 +111,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 128629480123059091
+      hash: -1659482461320296287
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -109,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 128629480123059091
+      hash: -1659482461320296287
 requirements: []
 rubyforge_project: spidey-mongo
 rubygems_version: 1.8.10