scrapey 0.0.8 → 0.0.11

data/lib/scrapey.rb

@@ -17,7 +17,7 @@ include Scrapey
 @agent.user_agent = "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
 @agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
 # default output file
-@output = 'output.csv'
+@output = File.join BASEDIR, 'output.csv'
 
 # read config file
 config_file = "#{BASEDIR}/config/config.yml"
@@ -25,4 +25,4 @@ config_file = "#{BASEDIR}/config/config.yml"
 
 init_db if @config['database']
 
-$stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "a"))
+$stderr = Scrapey::Tee.new(STDERR, File.open("#{BASEDIR}/errors.log", "w"))

data/lib/scrapey/cache.rb

@@ -11,4 +11,10 @@ module Scrapey
     end
   end
 
+  def disable_cache
+    @use_cache = false
+    yield
+    @use_cache = true
+  end
+
 end

data/lib/scrapey/cache/disk.rb

@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Scrapey
 
   def cache_filename url
@@ -22,4 +24,9 @@ module Scrapey
   def save_cache url, doc, options = {}
     File.open(cache_filename(url), "wb") {|f| f << Marshal.dump(doc) }
   end
+
+  def delete_cache url
+    FileUtils.rm cache_filename(url)
+  end
+
 end

data/lib/scrapey/constants.rb

@@ -1,5 +1,5 @@
 module Scrapey
-  VERSION = "0.0.8"
+  VERSION = "0.0.11"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
 end

data/lib/scrapey/multi.rb

@@ -5,18 +5,25 @@ class HTTPClient; def warn str; end; end
 
 module Scrapey
   def multi_get_or_post method, all_urls, options = {}
-    all_urls.reject!{|url| is_cached? url} if @use_cache
-    return unless all_urls.size > 0
 
-    threads    = options[:threads]    || 20
-    on_success = options[:on_success] || :on_success
-    on_error   = options[:on_error]   || :on_error
-    user_agent = options[:user_agent] || "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
-    proxy      = options[:proxy]      || nil
-    timeout    = options[:timeout]    || 1000
+    # some sensible defaults
+    threads         = options[:threads]         || 20
+    on_success      = options[:on_success]      || :on_success
+    on_error        = options[:on_error]        || :on_error
+    user_agent      = options[:user_agent]      || "Scrapey v#{Scrapey::VERSION} - #{Scrapey::URL}"
+    proxy           = options[:proxy]           || nil
+    timeout         = options[:timeout]         || 1000
+    follow_redirect = options[:follow_redirect] || true
 
     @lock ||= Mutex.new
-    @http_clients ||= threads.times.map{HTTPClient.new(options[:proxies] ? options[:proxies].rotate!.first : proxy, user_agent).tap{|c| c.ssl_config.verify_mode, c.receive_timeout, c.ssl_config.verify_callback = OpenSSL::SSL::VERIFY_NONE, timeout, proc{true}}}
+
+    @http_clients ||= threads.times.map do
+      c = HTTPClient.new proxy, user_agent
+      c.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      c.receive_timeout =  timeout
+      yield c if block_given?
+      c
+    end
 
     debug 'starting multi'
 
@@ -24,7 +31,7 @@ module Scrapey
       urls.each_with_index.map do |url, i|
         Thread.new do
           begin
-            response = @http_clients[i].send method, url, options[:query], options[:headers]
+            response = @http_clients[i].send method, url, options[:query], options[:headers], :follow_redirect => follow_redirect
           rescue Exception => e
             error = e
           end

data/lib/scrapey/scrapey.rb

@@ -1,7 +1,7 @@
 module Scrapey
 
   def get_or_post method, url, options={}, *args
-    agent = method == 'goto' ? @browser : @agent
+    agent = ['goto', 'visit'].include?(method) ? @browser : @agent
     _retries = options.delete :retries
     _sleep = options.delete :sleep
     begin
@@ -15,7 +15,8 @@ module Scrapey
       return doc if doc
 
       page = agent.send *new_args
-      save_cache(url, page.root.to_s) if @use_cache
+      str = page.respond_to?('root') ? page.root.to_s : page.body
+      save_cache(url, str) if @use_cache
 
       #exit if Object.const_defined? :Ocra
       page
@@ -36,6 +37,7 @@ module Scrapey
   def post *args; get_or_post 'post', *args; end
   def head *args; get_or_post 'head', *args; end
   def goto *args; get_or_post 'goto', *args; end
+  def visit *args; get_or_post 'visit', *args; end
 
   def set_proxy *args
     @agent.set_proxy *args

data/lib/scrapey/tee.rb

@@ -5,7 +5,7 @@ module Scrapey
     end
 
     def write *args
-      @targets.each {|t| t.write(*args)}
+      @targets.each {|t| t.write(*args); t.flush}
     end
 
     def flush *args

data/lib/scrapey/tor.rb

@@ -0,0 +1,15 @@
+require 'net/telnet'
+
+module Scrapey
+  def use_tor
+    set_proxy('localhost', 8118)
+  end
+
+  def change_identity
+    debug "changing identity..."
+    localhost = Net::Telnet::new("Host" => "localhost", "Port" => "9051", "Timeout" => 10, "Prompt" => /250 OK\n/)
+    localhost.cmd('AUTHENTICATE ""') {|c| raise "Cannot authenticate to Tor" unless "250 OK\n" == c}
+    localhost.cmd('signal NEWNYM') {|c| raise "Cannot switch Tor to new route" unless "250 OK\n" == c}
+    localhost.close
+  end
+end

data/template/src/template.rb

@@ -1,9 +1,6 @@
 require 'scrapey'
 require 'pry'
 
-# some skeleton code that I like to start with
-# require 'scrapey/multi' #=> requires em-http-request
-
 # sample customizations...
 # @agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5'
 # @output = Time.now.strftime("#{BASEDIR}/Output/output_%Y_%m_%d_%H_%M_%S.csv")
@@ -28,3 +25,6 @@ fields 'name', 'address', 'zip'
 
 page = get @url
 scrape page.at('div')
+
+#@csv.close
+#%x{call #{@output}}

data/template/template.iss

@@ -3,10 +3,22 @@ AppName=Template Scraper
 AppVersion=1.0
 DefaultDirName={localappdata}\Template Scraper
 DefaultGroupName=Template Scraper
+// ChangesAssociations=yes
 
 [Files]
 Source: "config\*"; DestDir: "{app}\config"; 
 Source: "src\*"; DestDir: "{app}\src"; 
+Source: "icon.ico"; DestDir: "{app}"; 
 
 [Icons]
-Name: "{group}\Template Scraper"; Filename: "{app}\template.exe"
+Name: "{group}\Template Scraper"; Filename: "{app}\template.exe"
+// Name: "{group}\Edit Config"; Filename: "{win}\notepad.exe"; Parameters: "{app}\config\config.yml"; Comment: "Config"; WorkingDir: "{app}\config"
+// Name: "{group}\View Output File"; Filename: "{app}\output.csv"; WorkingDir: "{app}"
+Name: "{group}\Uninstall Template Scraper"; Filename: "{uninstallexe}"
+
+[Registry]
+// Root: HKCR; Subkey: SystemFileAssociations\.csv\shell\Template Scraper; ValueType: string; Flags: uninsdeletekey deletekey; ValueName: Icon; ValueData: """{app}\icon.ico"""
+// Root: HKCR; Subkey: SystemFileAssociations\.csv\shell\Template Scraper\command; ValueType: string; ValueData: """{app}\template.exe"" ""%1"""; Flags: uninsdeletekey deletekey
+
+[Run]
+//Filename: schtasks.exe; Parameters:" /CREATE /TN ""Scheduled Template"" /TR ""{app}\template.exe"" /SC DAILY /ST 02:30"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.11
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-30 00:00:00.000000000 Z
+date: 2012-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -88,6 +88,7 @@ files:
 - lib/scrapey/scrapey.rb
 - lib/scrapey/tee.rb
 - lib/scrapey/template.rb
+- lib/scrapey/tor.rb
 - lib/scrapey.rb
 - scrapey.gemspec
 - template/config/config.yml