spidey-mongo 0.0.4 → 0.0.5

data/lib/spidey/strategies/mongo.rb

@@ -25,11 +25,12 @@ module Spidey::Strategies
     end
   
     def record(data)
-      $stderr.puts "Recording #{data.inspect.truncate(500)}" if verbose
-      if respond_to?(:result_key) && key = result_key(data)
-        result_collection.update({'key' => key}, {'$set' => data}, upsert: true)
+      doc = data.merge('spider' => self.class.name)
+      $stderr.puts "Recording #{doc.inspect.truncate(500)}" if verbose
+      if respond_to?(:result_key) && key = result_key(doc)
+        result_collection.update({'key' => key}, {'$set' => doc}, upsert: true)
       else
-        result_collection.insert data
+        result_collection.insert doc
       end
     end
   
@@ -43,7 +44,8 @@ module Spidey::Strategies
   
     def add_error(attrs)
       error = attrs.delete(:error)
-      error_collection.insert attrs.merge(created_at: Time.now, error: error.class.name, message: error.message)
+      doc = attrs.merge(created_at: Time.now, error: error.class.name, message: error.message, spider: self.class.name)
+      error_collection.insert doc
       $stderr.puts "Error on #{attrs[:url]}. #{error.class}: #{error.message}" if verbose
     end
   

data/lib/spidey-mongo/version.rb

@@ -1,5 +1,5 @@
 module Spidey
   module Mongo
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/spec/spidey/strategies/mongo_spec.rb

@@ -25,6 +25,7 @@ describe Spidey::Strategies::Mongo do
   it "should add initial URLs to collection" do
     doc = @db['urls'].find_one(url: "http://www.cnn.com")
     doc['handler'].should == :process_home
+    doc['spider'].should == 'TestSpider'
   end
   
   it "should not add duplicate URLs" do
@@ -38,6 +39,7 @@ describe Spidey::Strategies::Mongo do
     doc = @db['results'].find_one
     doc['detail_url'].should == 'http://www.cnn.com'
     doc['foo'].should == 'bar'
+    doc['spider'].should == 'TestSpider'
   end
   
   it "should update existing result" do
@@ -53,6 +55,7 @@ describe Spidey::Strategies::Mongo do
     doc['url'].should == 'http://www.cnn.com'
     doc['handler'].should == :blah
     doc['message'].should == 'WTF'
+    doc['spider'].should == 'TestSpider'
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spidey-mongo
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-27 00:00:00.000000000Z
+date: 2012-08-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70312341584000 !ruby/object:Gem::Requirement
+  requirement: &70172417847540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70312341584000
+  version_requirements: *70172417847540
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70312341583440 !ruby/object:Gem::Requirement
+  requirement: &70172417846740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70312341583440
+  version_requirements: *70172417846740
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70312341582900 !ruby/object:Gem::Requirement
+  requirement: &70172417845560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70312341582900
+  version_requirements: *70172417845560
 - !ruby/object:Gem::Dependency
   name: spidey
-  requirement: &70312341582320 !ruby/object:Gem::Requirement
+  requirement: &70172417844480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70312341582320
+  version_requirements: *70172417844480
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70312341581640 !ruby/object:Gem::Requirement
+  requirement: &70172417830340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70312341581640
+  version_requirements: *70172417830340
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70312341580620 !ruby/object:Gem::Requirement
+  requirement: &70172417829000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70312341580620
+  version_requirements: *70172417829000
 description: Implements a MongoDB back-end for Spidey, a framework for crawling and
   scraping web sites.
 email:
@@ -111,7 +111,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1659482461320296287
+      hash: -2776115176941497831
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -120,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1659482461320296287
+      hash: -2776115176941497831
 requirements: []
 rubyforge_project: spidey-mongo
 rubygems_version: 1.8.10