spidey-mongo 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rspec

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in spidey-mongo.gemspec
+
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/spidey/strategies/mongo.rb

@@ -0,0 +1,78 @@
+module Spidey::Strategies
+  module Mongo
+    attr_accessor :url_collection, :result_collection, :error_collection
+  
+    module ClassMethods
+      def set_result_key(callback)
+        @result_key = callback
+      end
+    
+      def result_key(spider, data)
+        case @result_key
+        when Symbol then spider.send(@result_key, data)
+        when Proc then @result_key.call(data)
+        else nil
+        end
+      end
+    end
+  
+    def self.included(base)
+      base.extend ClassMethods
+    end
+  
+    def initialize(attrs = {})
+      self.url_collection = attrs.delete(:url_collection)
+      self.result_collection = attrs.delete(:result_collection)
+      self.error_collection = attrs.delete(:error_collection)
+      super attrs
+    end
+  
+    def crawl(options = {})
+      @crawl_started_at = Time.now
+      @until = Time.now + options[:crawl_for] if options[:crawl_for]
+      super options
+    end
+  
+    def handle(url, handler, default_data = {})
+      $stderr.puts "Queueing #{url.inspect.truncate(500)}" if verbose
+      url_collection.update(
+        {'spider' => self.class.name, 'url' => url},
+        {'$set' => {'handler' => handler, 'default_data' => default_data}},
+        upsert: true
+      )
+    end
+  
+    def record(data)
+      $stderr.puts "Recording #{data.inspect.truncate(500)}" if verbose
+      if key = self.class.result_key(self, data)
+        result_collection.update({'key' => key}, {'$set' => data}, upsert: true)
+      else
+        result_collection.insert data
+      end
+    end
+  
+    def each_url(&block)
+      while url = get_next_url
+        break if url['last_crawled_at'] && url['last_crawled_at'] >= @crawl_started_at  # crawled already in this batch
+        url_collection.update({'_id' => url['_id']}, '$set' => {last_crawled_at: Time.now})
+        yield url['url'], url['handler'], url['default_data'].symbolize_keys
+      end
+    end
+  
+    def add_error(attrs)
+      error = attrs.delete(:error)
+      error_collection.insert attrs.merge(created_at: Time.now, error: error.class.name, message: error.message)
+      $stderr.puts "Error on #{attrs[:url]}. #{error.class}: #{error.message}" if verbose
+    end
+  
+  private
+
+    def get_next_url
+      return nil if (@until && Time.now >= @until)  # exceeded time bound
+      url_collection.find_one({spider: self.class.name}, {
+        sort: [[:last_crawled_at, ::Mongo::ASCENDING], [:_id, ::Mongo::ASCENDING]]
+      })
+    end
+  
+  end
+end

data/lib/spidey-mongo/version.rb

@@ -0,0 +1,5 @@
+module Spidey
+  module Mongo
+    VERSION = "0.0.1"
+  end
+end

data/lib/spidey-mongo.rb

@@ -0,0 +1,5 @@
+require 'spidey'
+require 'mongo'
+
+require 'spidey-mongo/version'
+require 'spidey/strategies/mongo'

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'spidey-mongo'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end

data/spec/spidey/strategies/mongo_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe Spidey::Strategies::Mongo do
+  class TestSpider < Spidey::AbstractSpider
+    include Spidey::Strategies::Mongo
+    set_result_key ->(data) { data[:detail_url] }
+    handle "http://www.cnn.com", :process_home
+  end
+  
+  before(:each) do
+    @db = Mongo::Connection.new['spidey-mongo-test']
+    @spider = TestSpider.new(
+      url_collection: @db['urls'],
+      result_collection: @db['results'],
+      error_collection: @db['errors'])
+  end
+  
+  it "should add initial URLs to collection" do
+    doc = @db['urls'].find_one(url: "http://www.cnn.com")
+    doc['handler'].should == :process_home
+  end
+  
+  it "should not add duplicate URLs" do
+    @spider.send :handle, "http://www.cnn.com", :process_home
+    @db['urls'].find(url: "http://www.cnn.com").count.should == 1
+  end
+  
+  it "should add results" do
+    @spider.record detail_url: 'http://www.cnn.com', foo: 'bar'
+    @db['results'].count.should == 1
+    doc = @db['results'].find_one
+    doc['detail_url'].should == 'http://www.cnn.com'
+    doc['foo'].should == 'bar'
+  end
+  
+  it "should add error" do
+    @spider.add_error error: Exception.new("WTF"), url: "http://www.cnn.com", handler: :blah
+    doc = @db['errors'].find_one
+    doc['error'].should == 'Exception'
+    doc['url'].should == 'http://www.cnn.com'
+    doc['handler'].should == :blah
+    doc['message'].should == 'WTF'
+  end
+  
+end

data/spidey-mongo.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "spidey-mongo/version"
+
+Gem::Specification.new do |s|
+  s.name        = "spidey-mongo"
+  s.version     = Spidey::Mongo::VERSION
+  s.authors     = ["Joey Aghion"]
+  s.email       = ["joey@aghion.com"]
+  s.homepage    = "https://github.com/joeyAghion/spidey-mongo"
+  s.summary     = %q{Implements a MongoDB back-end for Spidey, a framework for crawling and scraping web sites.}
+  s.description = %q{Implements a MongoDB back-end for Spidey, a framework for crawling and scraping web sites.}
+
+  s.rubyforge_project = "spidey-mongo"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  
+  s.add_runtime_dependency "spidey"
+  s.add_runtime_dependency "mongo"
+  s.add_runtime_dependency "bson_ext"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: spidey-mongo
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Joey Aghion
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70361603997680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70361603997680
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70361603997220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70361603997220
+- !ruby/object:Gem::Dependency
+  name: spidey
+  requirement: &70361603996320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70361603996320
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: &70361603995160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70361603995160
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: &70361603994540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70361603994540
+description: Implements a MongoDB back-end for Spidey, a framework for crawling and
+  scraping web sites.
+email:
+- joey@aghion.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/spidey-mongo.rb
+- lib/spidey-mongo/version.rb
+- lib/spidey/strategies/mongo.rb
+- spec/spec_helper.rb
+- spec/spidey/strategies/mongo_spec.rb
+- spidey-mongo.gemspec
+homepage: https://github.com/joeyAghion/spidey-mongo
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3377333768066102144
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3377333768066102144
+requirements: []
+rubyforge_project: spidey-mongo
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Implements a MongoDB back-end for Spidey, a framework for crawling and scraping
+  web sites.
+test_files:
+- spec/spec_helper.rb
+- spec/spidey/strategies/mongo_spec.rb