spidey 0.0.2

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rspec

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in spidey.gemspec
+gemspec

data/README.md

@@ -0,0 +1,27 @@
+Spidey
+======
+
+
+
+Example
+-------
+
+
+Strategies
+----------
+
+### In-Memory
+
+### Mongo
+
+
+Contributing
+------------
+
+To Do
+-----
+* Add examples
+
+
+License
+-------

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/spidey.rb

@@ -0,0 +1,2 @@
+require "spidey/version"
+require "spidey/abstract_spider"

data/lib/spidey/abstract_spider.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+require 'mechanize'
+
+module Spidey
+  class AbstractSpider
+    attr_accessor :urls, :handlers, :results, :request_interval, :verbose, :errors
+
+    DEFAULT_REQUEST_INTERVAL = 3  # seconds
+
+    def self.handle(url, handler, default_data = {})
+      start_urls << url
+      handlers[url] = [handler, default_data]
+    end
+
+    # Accepts:
+    #   request_interval: number of seconds to wait between requests (default: 3)
+    #   verbose: prints debugging and progress information if true
+    def initialize(attrs = {})
+      @urls = []
+      @handlers = {}
+      @results = []
+      self.class.start_urls.each { |url| handle url, *self.class.handlers[url] }
+      @request_interval = attrs[:request_interval] || DEFAULT_REQUEST_INTERVAL
+      @verbose = !!attrs[:verbose]
+    end
+
+    # Iterates through URLs queued for handling, including any that are added in the course of crawling. Accepts:
+    #   max_urls: maximum number of URLs to crawl before returning (optional)
+    def crawl(options = {})
+      @errors = []
+      i = 0
+      each_url do |url, handler, default_data|
+        break if options[:max_urls] && i >= options[:max_urls]
+        begin
+          page = agent.get(url)
+          $stderr.puts "Handling #{url.inspect}" if verbose
+          send handler, page, default_data
+        rescue => ex
+          add_error url: url, handler: handler, error: ex
+        end
+        sleep request_interval if request_interval > 0
+        i += 1
+      end
+    end
+
+  protected
+
+    # Override this for custom queueing of crawled URLs.
+    def handle(url, handler, default_data = {})
+      unless @handlers[url]
+        @urls << url
+        @handlers[url] = [handler, default_data]
+      end
+    end
+
+    # Override this for custom storage or prioritization of crawled URLs.
+    # Iterates through URL queue, yielding the URL, handler, and default data.
+    def each_url(&block)
+      urls.each do |url|
+        yield url, handlers[url].first, handlers[url].last
+      end
+    end
+    
+    # Override this for custom result storage.
+    def record(data)
+      results << data
+      $stderr.puts "Recording #{data.inspect}" if verbose
+    end
+    
+    # Override this for custom error-handling.
+    def add_error(attrs)
+      @errors << attrs
+      $stderr.puts "Error on #{attrs[:url]}. #{attrs[:error].class}: #{attrs[:error].message}" if verbose
+    end
+
+    def resolve_url(href, page)
+      agent.agent.resolve(href, page).to_s
+    end
+
+    # Strips ASCII/Unicode whitespace from ends and substitutes ASCII for Unicode internal spaces.
+    def clean(str)
+      return nil unless str
+      str.gsub(/\p{Space}/, ' ').strip.squeeze(' ')
+    end
+
+  private
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    def self.start_urls
+      @start_urls ||= []
+    end
+
+    def self.handlers
+      @handlers ||= {}
+    end
+
+  end
+  
+end

data/lib/spidey/version.rb

@@ -0,0 +1,3 @@
+module Spidey
+  VERSION = "0.0.2"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'spidey'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end

data/spec/spidey/abstract_spider_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Spidey::AbstractSpider do
+  describe '#crawl' do
+    before(:each) do
+      @agent = double('agent')
+      @page = double('page')
+      @agent.stub(get: @page)
+      Mechanize.stub(new: @agent)
+    end
+  
+    describe "with handle declaration" do
+      class TestSpider < Spidey::AbstractSpider
+        handle "http://www.yahoo.com", :process_home
+      end
+    
+      it "invokes handler" do
+        @agent.should_receive(:get).with("http://www.yahoo.com").and_return(@page)
+        spider = TestSpider.new request_interval: 0
+        spider.should_receive(:process_home).with(@page, {})
+        spider.crawl
+      end
+    
+      it "records errors" do
+        spider = TestSpider.new request_interval: 0
+        spider.should_receive(:process_home).and_raise("Whoops - some error")
+        spider.crawl
+        spider.errors.size.should == 1
+        spider.errors.last[:url].should == "http://www.yahoo.com"
+        spider.errors.last[:handler].should == :process_home
+        spider.errors.last[:error].message.should == "Whoops - some error"
+      end
+    
+      describe "with follow-up URL handlers" do
+        class TestSpider < Spidey::AbstractSpider
+          def process_home(page, default_data = {})
+            handle "http://www.yahoo.com/deep_page.html", :process_deep_page
+          end
+        end
+
+        it "invokes configured handlers on follow-up URLs" do
+          spider = TestSpider.new request_interval: 0
+          page2 = double('page')
+          @agent.should_receive(:get).with("http://www.yahoo.com/deep_page.html").and_return(page2)
+          spider.should_receive(:process_deep_page).with(page2, {})
+          spider.crawl
+        end
+      end
+    end
+  
+    describe "with default_data" do
+      class TestSpiderWithData < Spidey::AbstractSpider
+        handle "http://www.yahoo.com", :process_home, a: 1, b: 2
+      end
+    
+      it "passes default data through" do
+        spider = TestSpiderWithData.new request_interval: 0
+        spider.should_receive(:process_home).with(@page, a: 1, b: 2)
+        spider.crawl
+      end
+    end
+  end
+
+  describe '#clean' do
+    {
+      "Untitled, " => "Untitled,",
+      " Pahk the Cah" => "Pahk the Cah",
+      "	Untitled	1999 " => "Untitled 1999",
+      nil => nil
+    }.each do |original, cleaned|
+      it "replaces '#{original}' with '#{cleaned}'" do
+        Spidey::AbstractSpider.new(request_interval: 0).send(:clean, original).should == cleaned
+      end
+    end
+  end
+end

data/spidey.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "spidey/version"
+
+Gem::Specification.new do |s|
+  s.name        = "spidey"
+  s.version     = Spidey::VERSION
+  s.authors     = ["Joey Aghion"]
+  s.email       = ["joey@aghion.com"]
+  s.homepage    = "https://github.com/joeyAghion/spidey"
+  s.summary     = %q{A loose framework for crawling and scraping web sites.}
+  s.description = %q{A loose framework for crawling and scraping web sites.}
+  
+  s.rubyforge_project = "spidey"
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  
+  s.add_runtime_dependency "mechanize"
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: spidey
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Joey Aghion
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-26 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70136044260380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70136044260380
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70136044256960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70136044256960
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: &70136044255140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70136044255140
+description: A loose framework for crawling and scraping web sites.
+email:
+- joey@aghion.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- lib/spidey.rb
+- lib/spidey/abstract_spider.rb
+- lib/spidey/version.rb
+- spec/spec_helper.rb
+- spec/spidey/abstract_spider_spec.rb
+- spidey.gemspec
+homepage: https://github.com/joeyAghion/spidey
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3586547064482976006
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3586547064482976006
+requirements: []
+rubyforge_project: spidey
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A loose framework for crawling and scraping web sites.
+test_files:
+- spec/spec_helper.rb
+- spec/spidey/abstract_spider_spec.rb