scrapey 0.0.7 → 0.0.8

data/lib/scrapey/browser.rb

@@ -0,0 +1,11 @@
+require 'selenium-webdriver'
+require 'capybara/dsl'
+require 'capybara-webkit'
+
+Capybara.run_server = false
+Capybara.current_driver = :webkit
+
+class Browser
+  include Capybara::DSL
+end
+@browser = Browser.new

data/lib/scrapey/cache/disk.rb

@@ -13,7 +13,7 @@ module Scrapey
     return nil unless File::exists?(filename)
     debug "Loading #{filename} from cache"
     begin
-      Nokogiri::HTML Marshal.load(File.read(filename))
+      Nokogiri::HTML Marshal.load(File.open(filename, "rb"){|f| f.read})
     rescue Exception => e
       puts e.message
     end

data/lib/scrapey/constants.rb

@@ -1,5 +1,5 @@
 module Scrapey
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
 end

data/lib/scrapey/database.rb

@@ -22,7 +22,15 @@ module Scrapey
   end
 
   def init_db
-    ['active_record', @config['database']['adapter'], 'tzinfo', 'active_support/all', 'active_support/multibyte/chars'].each{|lib| require lib}
+    [
+    'active_record',
+    'active_record/schema',
+    'active_record/connection_adapters/abstract/schema_definitions',
+    @config['database']['adapter'],
+    'tzinfo',
+    'active_support/all',
+    'active_support/multibyte/chars'
+    ].each{|lib| require lib}
   	ActiveRecord::Base.establish_connection(@config['database']) 
   end
 end

data/lib/scrapey/multi.rb

@@ -1,18 +1,23 @@
 require 'httpclient'
 
+# monkey patch to remove annoying httpclient warnings
+class HTTPClient; def warn str; end; end
+
 module Scrapey
   def multi_get_or_post method, all_urls, options = {}
     all_urls.reject!{|url| is_cached? url} if @use_cache
     return unless all_urls.size > 0
 
-    threads    = options[:threads]    || [10, all_urls.size].min
+    threads    = options[:threads]    || 20
     on_success = options[:on_success] || :on_success
     on_error   = options[:on_error]   || :on_error
     user_agent = options[:user_agent] || "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
     proxy      = options[:proxy]      || nil
+    timeout    = options[:timeout]    || 1000
 
     @lock ||= Mutex.new
-    @http_clients ||= threads.times.map{HTTPClient.new(options[:proxies] ? options[:proxies].rotate!.first : proxy, user_agent).tap{|c| c.ssl_config.verify_mode, c.receive_timeout = OpenSSL::SSL::VERIFY_NONE, 10000}}
+    @http_clients ||= threads.times.map{HTTPClient.new(options[:proxies] ? options[:proxies].rotate!.first : proxy, user_agent).tap{|c| c.ssl_config.verify_mode, c.receive_timeout, c.ssl_config.verify_callback = OpenSSL::SSL::VERIFY_NONE, timeout, proc{true}}}
+
     debug 'starting multi'
 
     all_urls.each_slice(threads) do |urls|

data/lib/scrapey/scrapey.rb

@@ -52,7 +52,7 @@ module Scrapey
     end
     case
       when item.is_a?(Array) then @csv << item
-      when item.is_a?(Hash)
+      when item.is_a?(Hash) || item.is_a?(CSV::Row)
         raise 'No fields defined!' unless @fields
         @csv << @fields.map{|f| item[f]}
       else raise "unsupported type: #{item.class}"

data/lib/scrapey/tee.rb

@@ -0,0 +1,20 @@
+module Scrapey
+  class Tee
+    def initialize *targets
+       @targets = targets
+    end
+
+    def write *args
+      @targets.each {|t| t.write(*args)}
+    end
+
+    def flush *args
+      @targets.each {|t| t.flush(*args)}
+    end
+
+    def close
+      @targets.each(&:close)
+    end
+  end
+end
+

data/lib/scrapey.rb

@@ -8,6 +8,7 @@ require "scrapey/constants"
 require "scrapey/cache"
 require "scrapey/database"
 require "scrapey/multi"
+require "scrapey/tee"
 
 include Scrapey
 
@@ -23,3 +24,5 @@ config_file = "#{BASEDIR}/config/config.yml"
 @config = File.exists?(config_file) ? YAML::load(File.open(config_file)) : {}
 
 init_db if @config['database']
+
+$stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "a"))

data/scrapey.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Scrapey::VERSION
   gem.add_dependency(%q<mechanize>)
+  gem.add_dependency(%q<httpclient>)
   gem.add_dependency(%q<json>, ["~> 1.7.0"])
 end
 

data/template/src/template.rb

@@ -11,7 +11,7 @@ require 'pry'
 def scrape div
   a = div.at('a')
   url = URI.join(@url, a[:href]).to_s
-  return unless visited? url
+  return if visited? url
   item = {}
   
   save item  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-27 00:00:00.000000000 Z
+date: 2012-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -62,6 +78,7 @@ files:
 - examples/output.csv
 - examples/redis.rb
 - examples/status_check.rb
+- lib/scrapey/browser.rb
 - lib/scrapey/cache/disk.rb
 - lib/scrapey/cache/redis.rb
 - lib/scrapey/cache.rb
@@ -69,9 +86,9 @@ files:
 - lib/scrapey/database.rb
 - lib/scrapey/multi.rb
 - lib/scrapey/scrapey.rb
+- lib/scrapey/tee.rb
 - lib/scrapey/template.rb
 - lib/scrapey.rb
-- output.csv
 - scrapey.gemspec
 - template/config/config.yml
 - template/Gemfile

data/output.csv

@@ -1,5 +0,0 @@
-url,status
-http://www.bing.com/,200
-http://www.bing.com/404.html,404
-http://locahlost2/foo,getaddrinfo: No such host is known.  (http://locahlost2:80)
-http://www.google.com/,302