spidey-mongo 0.0.1 → 0.0.3

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Joey Aghion, Art.sy Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+Spidey-Mongo
+============
+
+This gem implements a [MongoDB](http://www.mongodb.org/) back-end for [Spidey](https://github.com/joeyAghion/spidey), a very simple framework for crawling and scraping web sites.
+
+See [Spidey](https://githubcom/joeyAghion/spidey)'s documentation for a basic example spider class.
+
+The default implementation stores the queue of URLs being crawled, any generated results, and errors as attributes on the spider instance (i.e., in memory). By including this gem's `Spidey::Strategies::Mongo` module, spider implementations can store them in a MongoDB database instead.
+
+Usage
+-----
+
+### Install the gem
+
+    gem install spidey-mongo
+
+
+### Example spider class
+
+    class EbaySpider < Spidey::AbstractSpider
+      include Spidey::Strategies::Mongo
+      
+      handle "http://www.ebay.com", :process_home
+      
+      def process_home(page, default_data = {})
+        # ...
+      end
+    end
+
+### Invocation
+
+The spider's constructor accepts new parameters for each of the MongoDB collections to employ: `url_collection`, `result_collection`, and `error_collection`.
+
+    db = Mongo::Connection.new['example']
+    
+    spider = EbaySpider.new(
+      url_collection: db['urls'],
+      result_collection: db['results'],
+      error_collection: db['errors'])
+
+With persistent storage of the URL-crawling queue, it's now possible to stop crawling and resume at a later point. The `crawl` method accepts a new optional `crawl_for` parameter specifying the number of seconds after which to stop.
+
+    spider.crawl crawl_for: 600  # seconds, or more conveniently (w/ActiveSupport): 10.minutes
+
+(The base implementation's `max_urls` parameter is also useful for this purpose.)
+
+### Recording Results
+
+By default, invocations of `record(data)` by the spider simply insert new documents into the result collection. If corresponding results may already exist in the collection and should instead be updated, use the `set_result_key` helper (with a proc or method symbol) to specify how to find the document to update:
+
+    class EbaySpider < Spidey::AbstractSpider
+      include Spidey::Strategies::Mongo
+      set_result_key ->(data) { data[:auction_id] }
+      
+      # ...
+    end
+
+This performs an `upsert` instead of the usual `insert` (i.e., an update if a result document matching the key already exists, or insert otherwise).
+
+Testing
+-------
+
+    bundle exec rspec
+
+Copyright
+---------
+Copyright (c) 2012 Joey Aghion, Art.sy Inc. See [LICENSE.txt](LICENSE.txt) for further details.

data/lib/spidey-mongo/version.rb

@@ -1,5 +1,5 @@
 module Spidey
   module Mongo
-    VERSION = "0.0.1"
+    VERSION = "0.0.3"
   end
 end

data/spidey-mongo.gemspec

@@ -10,6 +10,7 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/joeyAghion/spidey-mongo"
   s.summary     = %q{Implements a MongoDB back-end for Spidey, a framework for crawling and scraping web sites.}
   s.description = %q{Implements a MongoDB back-end for Spidey, a framework for crawling and scraping web sites.}
+  s.license     = 'MIT'
 
   s.rubyforge_project = "spidey-mongo"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spidey-mongo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-06-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70361603997680 !ruby/object:Gem::Requirement
+  requirement: &70227312727160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70361603997680
+  version_requirements: *70227312727160
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70361603997220 !ruby/object:Gem::Requirement
+  requirement: &70227312726140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70361603997220
+  version_requirements: *70227312726140
 - !ruby/object:Gem::Dependency
   name: spidey
-  requirement: &70361603996320 !ruby/object:Gem::Requirement
+  requirement: &70227312725220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70361603996320
+  version_requirements: *70227312725220
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70361603995160 !ruby/object:Gem::Requirement
+  requirement: &70227312724420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70361603995160
+  version_requirements: *70227312724420
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70361603994540 !ruby/object:Gem::Requirement
+  requirement: &70227312717760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70361603994540
+  version_requirements: *70227312717760
 description: Implements a MongoDB back-end for Spidey, a framework for crawling and
   scraping web sites.
 email:
@@ -76,6 +76,8 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- LICENSE.txt
+- README.md
 - Rakefile
 - lib/spidey-mongo.rb
 - lib/spidey-mongo/version.rb
@@ -84,7 +86,8 @@ files:
 - spec/spidey/strategies/mongo_spec.rb
 - spidey-mongo.gemspec
 homepage: https://github.com/joeyAghion/spidey-mongo
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -97,7 +100,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3377333768066102144
+      hash: 128629480123059091
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -106,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3377333768066102144
+      hash: 128629480123059091
 requirements: []
 rubyforge_project: spidey-mongo
 rubygems_version: 1.8.10