spidey-mongo 0.1.0 → 0.2.0

data/README.md

@@ -5,7 +5,7 @@ This gem implements a [MongoDB](http://www.mongodb.org/) back-end for [Spidey](h
 
 See [Spidey](https://githubcom/joeyAghion/spidey)'s documentation for a basic example spider class.
 
-The default implementation stores the queue of URLs being crawled, any generated results, and errors as attributes on the spider instance (i.e., in memory). By including this gem's `Spidey::Strategies::Mongo` module, spider implementations can store them in a MongoDB database instead.
+The default implementation stores the queue of URLs being crawled, any generated results, and errors as attributes on the spider instance (i.e., in memory). By including this gem's module, spider implementations can store them in a MongoDB database instead.
 
 Usage
 -----
@@ -15,6 +15,16 @@ Usage
     gem install spidey-mongo
 
 
+### `mongo` versus `moped`
+
+Spidey-Mongo provides two strategies:
+
+* `Spidey::Strategies::Mongo`: Compatible with 10gen's [`mongo`](https://github.com/mongodb/mongo-ruby-driver) gem
+* `Spidey::Strategies::Moped`: Compatible with the [`moped`](https://github.com/mongoid/moped) gem, e.g., for use with Mongoid 3.x
+
+You can include either strategy in your classes, as appropriate. All the examples in this README assume `Spidey::Strategies::Mongo`.
+
+
 ### Example spider class
 
     class EbaySpider < Spidey::AbstractSpider
@@ -65,6 +75,15 @@ Testing
 
     bundle exec rspec
 
+Contributors
+------------
+
+[Joey Aghion](https://github.com/joeyAghion), [Frank Macreery](https://github.com/fancyremarker)
+
+To Do
+-----
+* Extract behaviors shared by `Mongo` and `Moped` strategies.
+
 Copyright
 ---------
-Copyright (c) 2012 Joey Aghion, Art.sy Inc. See [LICENSE.txt](LICENSE.txt) for further details.
+Copyright (c) 2012, 2013 Joey Aghion, Artsy Inc. See [LICENSE.txt](LICENSE.txt) for further details.

data/lib/spidey/strategies/mongo.rb

@@ -1,20 +1,20 @@
 module Spidey::Strategies
   module Mongo
     attr_accessor :url_collection, :result_collection, :error_collection
-  
+
     def initialize(attrs = {})
       self.url_collection = attrs.delete(:url_collection)
       self.result_collection = attrs.delete(:result_collection)
       self.error_collection = attrs.delete(:error_collection)
       super attrs
     end
-  
+
     def crawl(options = {})
       @crawl_started_at = Time.now
       @until = Time.now + options[:crawl_for] if options[:crawl_for]
       super options
     end
-  
+
     def handle(url, handler, default_data = {})
       Spidey.logger.info "Queueing #{url.inspect[0..200]}..."
       url_collection.update(
@@ -23,7 +23,7 @@ module Spidey::Strategies
         upsert: true
       )
     end
-  
+
     def record(data)
       doc = data.merge('spider' => self.class.name)
       Spidey.logger.info "Recording #{doc.inspect[0..500]}..."
@@ -33,7 +33,7 @@ module Spidey::Strategies
         result_collection.insert doc
       end
     end
-  
+
     def each_url(&block)
       while url = get_next_url
         break if url['last_crawled_at'] && url['last_crawled_at'] >= @crawl_started_at  # crawled already in this batch
@@ -41,14 +41,14 @@ module Spidey::Strategies
         yield url['url'], url['handler'], url['default_data'].symbolize_keys
       end
     end
-  
+
     def add_error(attrs)
       error = attrs.delete(:error)
       doc = attrs.merge(created_at: Time.now, error: error.class.name, message: error.message, spider: self.class.name)
       error_collection.insert doc
       Spidey.logger.error "Error on #{attrs[:url]}. #{error.class}: #{error.message}"
     end
-  
+
   private
 
     def get_next_url
@@ -57,6 +57,6 @@ module Spidey::Strategies
         sort: [[:last_crawled_at, ::Mongo::ASCENDING], [:_id, ::Mongo::ASCENDING]]
       })
     end
-  
+
   end
 end

data/lib/spidey/strategies/moped.rb

@@ -0,0 +1,62 @@
+module Spidey::Strategies
+  module Moped
+    attr_accessor :url_collection, :result_collection, :error_collection
+
+    def initialize(attrs = {})
+      self.url_collection = attrs.delete(:url_collection)
+      self.result_collection = attrs.delete(:result_collection)
+      self.error_collection = attrs.delete(:error_collection)
+      super attrs
+    end
+
+    def crawl(options = {})
+      @crawl_started_at = Time.now
+      @until = Time.now + options[:crawl_for] if options[:crawl_for]
+      super options
+    end
+
+    def handle(url, handler, default_data = {})
+      Spidey.logger.info "Queueing #{url.inspect[0..200]}..."
+      url_collection.find(
+        {'spider' => self.class.name, 'url' => url}
+      ).upsert(
+        {'$set' => {'handler' => handler, 'default_data' => default_data}}
+      )
+    end
+
+    def record(data)
+      doc = data.merge('spider' => self.class.name)
+      Spidey.logger.info "Recording #{doc.inspect[0..500]}..."
+      if respond_to?(:result_key) && key = result_key(doc)
+        result_collection.find({'key' => key}).upsert({'$set' => doc})
+      else
+        result_collection.insert doc
+      end
+    end
+
+    def each_url(&block)
+      while url = get_next_url
+        break if url['last_crawled_at'] && url['last_crawled_at'] >= @crawl_started_at  # crawled already in this batch
+        url_collection.find({'_id' => url['_id']}).update('$set' => {last_crawled_at: Time.now})
+        yield url['url'], url['handler'], url['default_data'].symbolize_keys
+      end
+    end
+
+    def add_error(attrs)
+      error = attrs.delete(:error)
+      doc = attrs.merge(created_at: Time.now, error: error.class.name, message: error.message, spider: self.class.name)
+      error_collection.insert doc
+      Spidey.logger.error "Error on #{attrs[:url]}. #{error.class}: #{error.message}"
+    end
+
+  private
+
+    def get_next_url
+      return nil if (@until && Time.now >= @until)  # exceeded time bound
+      url_collection.find({spider: self.class.name}).sort({
+        'last_crawled_at' => 1, '_id' => 1
+      }).first
+    end
+
+  end
+end

data/lib/spidey-mongo/version.rb

@@ -1,5 +1,5 @@
 module Spidey
   module Mongo
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/spidey-mongo.rb

@@ -1,5 +1,5 @@
 require 'spidey'
-require 'mongo'
 
 require 'spidey-mongo/version'
-require 'spidey/strategies/mongo'
+require 'spidey/strategies/mongo'
+require 'spidey/strategies/moped'

data/spec/spidey/strategies/mongo_spec.rb

@@ -1,53 +1,54 @@
 require 'spec_helper'
+require 'mongo'
 
 describe Spidey::Strategies::Mongo do
-  class TestSpider < Spidey::AbstractSpider
+  class TestMongoSpider < Spidey::AbstractSpider
     include Spidey::Strategies::Mongo
     handle "http://www.cnn.com", :process_home
-    
+
     def result_key(data)
       data[:detail_url]
     end
   end
-  
+
   before(:each) do
     @db = Mongo::Connection.new['spidey-mongo-test']
-    @spider = TestSpider.new(
+    @spider = TestMongoSpider.new(
       url_collection: @db['urls'],
       result_collection: @db['results'],
       error_collection: @db['errors'])
   end
-  
+
   after(:each) do
     %w{ urls results errors }.each{ |col| @db[col].drop }
   end
-  
+
   it "should add initial URLs to collection" do
     doc = @db['urls'].find_one(url: "http://www.cnn.com")
     doc['handler'].should == :process_home
-    doc['spider'].should == 'TestSpider'
+    doc['spider'].should == 'TestMongoSpider'
   end
-  
+
   it "should not add duplicate URLs" do
     @spider.send :handle, "http://www.cnn.com", :process_home
     @db['urls'].find(url: "http://www.cnn.com").count.should == 1
   end
-  
+
   it "should add results" do
     @spider.record detail_url: 'http://www.cnn.com', foo: 'bar'
     @db['results'].count.should == 1
     doc = @db['results'].find_one
     doc['detail_url'].should == 'http://www.cnn.com'
     doc['foo'].should == 'bar'
-    doc['spider'].should == 'TestSpider'
+    doc['spider'].should == 'TestMongoSpider'
   end
-  
+
   it "should update existing result" do
     @db['results'].insert key: 'http://foo.bar', detail_url: 'http://foo.bar'
     @spider.record detail_url: 'http://foo.bar', foo: 'bar'
     @db['results'].count.should == 1
   end
-  
+
   it "should add error" do
     @spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
     doc = @db['errors'].find_one
@@ -55,7 +56,7 @@ describe Spidey::Strategies::Mongo do
     doc['url'].should == 'http://www.cnn.com'
     doc['handler'].should == :blah
     doc['message'].should == 'WTF'
-    doc['spider'].should == 'TestSpider'
+    doc['spider'].should == 'TestMongoSpider'
   end
-  
+
 end

data/spec/spidey/strategies/moped_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'moped'
+
+describe Spidey::Strategies::Moped do
+  class TestMopedSpider < Spidey::AbstractSpider
+    include Spidey::Strategies::Moped
+    handle "http://www.cnn.com", :process_home
+
+    def result_key(data)
+      data[:detail_url]
+    end
+  end
+
+  before(:each) do
+    @db = Moped::Session.new(['127.0.0.1:27017'])
+    @db.use 'spidey-mongo-test'
+    @spider = TestMopedSpider.new(
+      url_collection: @db['urls'],
+      result_collection: @db['results'],
+      error_collection: @db['errors'])
+  end
+
+  after(:each) do
+    %w{ urls results errors }.each{ |col| @db[col].drop }
+  end
+
+  it "should add initial URLs to collection" do
+    doc = @db['urls'].find(url: "http://www.cnn.com").first
+    doc['handler'].should == :process_home
+    doc['spider'].should == 'TestMopedSpider'
+  end
+
+  it "should not add duplicate URLs" do
+    @spider.send :handle, "http://www.cnn.com", :process_home
+    @db['urls'].find(url: "http://www.cnn.com").count.should == 1
+  end
+
+  it "should add results" do
+    @spider.record detail_url: 'http://www.cnn.com', foo: 'bar'
+    @db['results'].find.count.should == 1
+    doc = @db['results'].find.first
+    doc['detail_url'].should == 'http://www.cnn.com'
+    doc['foo'].should == 'bar'
+    doc['spider'].should == 'TestMopedSpider'
+  end
+
+  it "should update existing result" do
+    @db['results'].insert key: 'http://foo.bar', detail_url: 'http://foo.bar'
+    @spider.record detail_url: 'http://foo.bar', foo: 'bar'
+    @db['results'].find.count.should == 1
+  end
+
+  it "should add error" do
+    @spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
+    doc = @db['errors'].find.first
+    doc['error'].should == 'Exception'
+    doc['url'].should == 'http://www.cnn.com'
+    doc['handler'].should == :blah
+    doc['message'].should == 'WTF'
+    doc['spider'].should == 'TestMopedSpider'
+  end
+
+end

data/spidey-mongo.gemspec

@@ -21,8 +21,9 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
-  
+  s.add_development_dependency "mongo"
+  s.add_development_dependency "bson_ext"
+  s.add_development_dependency "moped"
+
   s.add_runtime_dependency "spidey", ">= 0.1.0"
-  s.add_runtime_dependency "mongo"
-  s.add_runtime_dependency "bson_ext"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spidey-mongo
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-03 00:00:00.000000000 Z
+date: 2013-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,30 +44,30 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: spidey
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.1.0
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: bson_ext
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -76,14 +76,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bson_ext
+  name: moped
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -91,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: spidey
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: Implements a MongoDB back-end for Spidey, a framework for crawling and
   scraping web sites.
 email:
@@ -107,8 +123,10 @@ files:
 - lib/spidey-mongo.rb
 - lib/spidey-mongo/version.rb
 - lib/spidey/strategies/mongo.rb
+- lib/spidey/strategies/moped.rb
 - spec/spec_helper.rb
 - spec/spidey/strategies/mongo_spec.rb
+- spec/spidey/strategies/moped_spec.rb
 - spidey-mongo.gemspec
 homepage: https://github.com/joeyAghion/spidey-mongo
 licenses:
@@ -125,7 +143,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4205551958644275814
+      hash: 987129952958952365
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -134,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4205551958644275814
+      hash: 987129952958952365
 requirements: []
 rubyforge_project: spidey-mongo
 rubygems_version: 1.8.25
@@ -145,3 +163,4 @@ summary: Implements a MongoDB back-end for Spidey, a framework for crawling and
 test_files:
 - spec/spec_helper.rb
 - spec/spidey/strategies/mongo_spec.rb
+- spec/spidey/strategies/moped_spec.rb