spidey 0.0.4 → 0.1.0

data/.gitignore

@@ -3,3 +3,4 @@
 Gemfile.lock
 pkg/*
 .rspec
+.rbenv-version

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - "2.0.0"
+  - rbx-19mode
+matrix:
+  allow_failures:
+    - rvm: rbx-19mode

data/README.md

@@ -1,4 +1,4 @@
-Spidey
+Spidey [![Build Status](https://travis-ci.org/joeyAghion/spidey.png?branch=master)](https://travis-ci.org/joeyAghion/spidey)
 ======
 
 Spidey provides a bare-bones framework for crawling and scraping web sites. Its goal is to keep boilerplate scraping logic out of your code.
@@ -55,6 +55,12 @@ By default, the lists of URLs being crawled, results scraped, and errors encount
 Add the [spidey-mongo](https://github.com/joeyAghion/spidey-mongo) gem and include `Spidey::Strategies::Mongo` in your spider to instead use MongoDB to persist these data. [See the docs](https://github.com/joeyAghion/spidey-mongo) for more information. Or, you can implement your own strategy by overriding the appropriate methods from `AbstractSpider`.
 
 
+Logging
+-------
+
+You may set `Spidey.logger` to a logger of your choosing. When used in a Rails environment, the logger defaults to the Rails logger. Otherwise, it's directed to STDOUT.
+
+
 Contributing
 ------------
 
@@ -68,4 +74,4 @@ To Do
 
 Copyright
 ---------
-Copyright (c) 2012 Joey Aghion, Art.sy Inc. See [LICENSE.txt](LICENSE.txt) for further details.
+Copyright (c) 2012 [Joey Aghion](http://halfamind.aghion.com), [Artsy Inc](http://artsy.net). See [LICENSE.txt](LICENSE.txt) for further details.

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+task :test => :spec
+task :default => :spec

data/lib/spidey.rb

@@ -1,2 +1,12 @@
+require "logger"
 require "spidey/version"
 require "spidey/abstract_spider"
+
+module Spidey
+  extend self
+  attr_accessor :logger
+end
+
+Spidey.logger = Logger.new(STDOUT)
+
+require 'spidey/railtie' if defined?(::Rails)

data/lib/spidey/abstract_spider.rb

@@ -3,7 +3,7 @@ require 'mechanize'
 
 module Spidey
   class AbstractSpider
-    attr_accessor :urls, :handlers, :results, :request_interval, :verbose, :errors
+    attr_accessor :urls, :handlers, :results, :request_interval, :errors
 
     DEFAULT_REQUEST_INTERVAL = 3  # seconds
 
@@ -14,14 +14,12 @@ module Spidey
 
     # Accepts:
     #   request_interval: number of seconds to wait between requests (default: 3)
-    #   verbose: prints debugging and progress information if true
     def initialize(attrs = {})
       @urls = []
       @handlers = {}
       @results = []
       self.class.start_urls.each { |url| handle url, *self.class.handlers[url] }
       @request_interval = attrs[:request_interval] || DEFAULT_REQUEST_INTERVAL
-      @verbose = !!attrs[:verbose]
     end
 
     # Iterates through URLs queued for handling, including any that are added in the course of crawling. Accepts:
@@ -33,7 +31,7 @@ module Spidey
         break if options[:max_urls] && i >= options[:max_urls]
         begin
           page = agent.get(url)
-          $stderr.puts "Handling #{url.inspect}" if verbose
+          Spidey.logger.info "Handling #{url.inspect}"
           send handler, page, default_data
         rescue => ex
           add_error url: url, handler: handler, error: ex
@@ -60,17 +58,17 @@ module Spidey
         yield url, handlers[url].first, handlers[url].last
       end
     end
-    
+
     # Override this for custom result storage.
     def record(data)
       results << data
-      $stderr.puts "Recording #{data.inspect}" if verbose
+      Spidey.logger.info "Recording #{data.inspect}"
     end
-    
+
     # Override this for custom error-handling.
     def add_error(attrs)
       @errors << attrs
-      $stderr.puts "Error on #{attrs[:url]}. #{attrs[:error].class}: #{attrs[:error].message}" if verbose
+      Spidey.logger.error "Error on #{attrs[:url]}. #{attrs[:error].class}: #{attrs[:error].message}"
     end
 
     def resolve_url(href, page)
@@ -98,5 +96,5 @@ module Spidey
     end
 
   end
-  
+
 end

data/lib/spidey/railtie.rb

@@ -0,0 +1,7 @@
+module Spidey
+  class Railtie < ::Rails::Railtie
+    initializer 'spidey.configure_rails_logger' do
+      Spidey.logger = ::Rails.logger
+    end
+  end
+end

data/lib/spidey/version.rb

@@ -1,3 +1,3 @@
 module Spidey
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/spidey.gemspec

@@ -11,17 +11,16 @@ Gem::Specification.new do |s|
   s.summary     = %q{A loose framework for crawling and scraping web sites.}
   s.description = %q{A loose framework for crawling and scraping web sites.}
   s.license     = 'MIT'
-  
+
   s.rubyforge_project = "spidey"
-  
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  
+
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
-  s.add_development_dependency "ruby-debug19"
-  
+
   s.add_runtime_dependency "mechanize"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spidey
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-21 00:00:00.000000000 Z
+date: 2013-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,22 +43,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: ruby-debug19
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -83,6 +67,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -90,6 +75,7 @@ files:
 - examples/ebay_pet_supplies_spider.rb
 - lib/spidey.rb
 - lib/spidey/abstract_spider.rb
+- lib/spidey/railtie.rb
 - lib/spidey/version.rb
 - spec/spec_helper.rb
 - spec/spidey/abstract_spider_spec.rb
@@ -109,7 +95,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3162500508741796001
+      hash: 101937807007286306
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -118,10 +104,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3162500508741796001
+      hash: 101937807007286306
 requirements: []
 rubyforge_project: spidey
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: A loose framework for crawling and scraping web sites.