scrapey 0.0.17 → 0.0.19

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWYyNGI3OGE3MTA2ZmQxMGE0MzE1MmE2ZjA5YTFhYTliOTY1OTY5ZQ==
+    ZWMwZWY5N2ExNjliMDVhNmQzNDEyNjJmZjExNjZkNjMyM2VkMGFhZA==
   data.tar.gz: !binary |-
-    YjRjOTVjOTkzNmEwOGE2NmMzYTVkNmNjMGRkODRjZjQ2OWM3OWNhNw==
+    YmNjZDFkMTg0MDZjZmZmMDM4ZjRmMzNiYjVjZWE3NTk3Y2YwYWY1Mg==
 SHA512:
   metadata.gz: !binary |-
-    NzkzOWNhMTA2MGQ3MDYwYjA2ZjQ2M2Y1OTQ4YTczMDljMWQ2YjRhYjcyMTk0
-    Yzc4ZjczNjU5MTBjN2MyOTczM2Y0NDZkNzY0MDdhOGU4MDQ1ODA3ODMwZTJi
-    MzMyZGFlNDc4N2MxMmViYjM5MjE2N2Y1MjFiNDY0ODJiNGM3ZDE=
+    NmEwY2RjZTAxNDBlM2JmNmI0MGExZDg0MGI3OWJkZjlmZTI0YTQ2ZDkyNTI4
+    ZDEyOGVhNDAxYjIxYWVkOTkyZTUwNDM3ZDY2MzJiZTEwYjZkN2M2ZGExYjNh
+    NWYwOWJmNzFiMjYwNjAyYzRlNmRiNDA2MjUzNGJlOTNmNWMzNDc=
   data.tar.gz: !binary |-
-    MWQyZjQ1NTA4NjA4ZGVmNjNjYmQ0MDY2ZDJhZWZlMGJhYWI1NDIyOTcwNzhi
-    MDg0YWU0NmIyMmNhY2E1MTE3NGY3ODE0NDhmNDE3NTc4OGVhNjg0NjA4OWRk
-    MTVmNmVmNDUyZGU5ZmFiMjg0N2Y5ZmVhM2UyMWRmYjM2MmQwMDE=
+    ZjkzYmMzYjJlZTFkNmU0NWVjMTQ3YjBhZGE5NzY5ZTdmN2Q2N2U0NGI1YmEx
+    NTVmN2MzYTVkN2FjYTA2YWRkNGY1Y2RjNDY5MjE4YjIxOGNjMmVmNGQ4MzA3
+    MjE2MTRkNzkwMDc3MGYwMjE0M2Y3YmI2M2RhZWUzZDJlMGVhNWY=

data/lib/scrapey.rb

@@ -31,7 +31,7 @@ unless defined? Rails
   @output = File.join BASEDIR, 'output.csv'
 
   # read config file
-  config_file = "#{BASEDIR}/config/config.yml"
+  config_file = @config_file_path || "#{BASEDIR}/config/config.yml"
   @config = File.exists?(config_file) ? YAML::load(File.open(config_file)) : {}
 
   init_db if @config['database']
@@ -50,11 +50,11 @@ if defined?(Ocra)
     'active_record',
     'active_record/schema',
     'active_record/connection_adapters/abstract/schema_definitions',
-    @config['database'] ? @config['database']['adapter'] : 'mysql',
+    @config['database'] ? @config['database']['adapter'] : 'mysql2',
     'tzinfo',
     'active_support/all',
     'active_support/multibyte/chars'
-    ].each{|lib| require lib}
+    ].each{|lib| puts lib; require lib}
   end
 end
 

data/lib/scrapey/constants.rb

@@ -1,5 +1,5 @@
 module Scrapey
-  VERSION = "0.0.17"
+  VERSION = "0.0.19"
   BASEDIR = File.expand_path(File.dirname($0)).gsub(/\/src$/,'')
   URL = "https://github.com/monkeysuffrage/scrapey"
 end

data/lib/scrapey/scrapey.rb

@@ -1,3 +1,5 @@
+# require 'phantom_mechanize'
+
 module Scrapey
 
   def self.init b
@@ -12,8 +14,6 @@ module Scrapey
 
   def get_or_post method, url, options={}, *args
     agent = ['goto', 'visit'].include?(method) ? @browser : @agent
-    _retries = options.delete :retries
-    _sleep = options.delete :sleep
     begin
       new_args = method, url
       unless options.empty? && args.empty? 
@@ -21,34 +21,26 @@ module Scrapey
         args.each{|arg| new_args << arg}
       end
       
-      doc = load_cache(url) if @use_cache
+      key = method == 'post' ? url + options.to_s : url
+      doc = load_cache(key) if @use_cache
       return doc if doc
 
       page = agent.send *new_args
       # str = page.respond_to?('root') ? page.root.to_s : page.body
       # save_cache(url, str) if @use_cache
-      save_cache(url, page.body) if @use_cache
+      save_cache(key, page.body) if @use_cache
 
       #exit if Object.const_defined? :Ocra
       page
     rescue Exception => e
-      case
-        when defined? on_error
-          return on_error e, method, url, options, *args
-        when _retries && _retries > 0
-          puts "Error. Retries remaining: #{options[:retries]}"
-          sleep _sleep if _sleep
-          get_or_post method, url, options.merge({:retries => _retries - 1, :sleep => _sleep}), *args
-        else raise e
-      end
+      puts e.message
+      raise e
     end
   end
 
   def get *args; get_or_post 'get', *args; end
   def post *args; get_or_post 'post', *args; end
-  def head *args; get_or_post 'head', *args; end
-  def goto *args; get_or_post 'goto', *args; end
-  def visit *args; get_or_post 'visit', *args; end
+  def phget *args; get_or_post 'phget', *args; end
 
   def set_proxy *args
     @agent.set_proxy *args

data/scrapey.gemspec

@@ -16,7 +16,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Scrapey::VERSION
   gem.add_dependency(%q<mechanize>)
-  gem.add_dependency(%q<httpclient>)
-  gem.add_dependency(%q<json>, ["~> 1.7.0"])
 end
 

data/template/src/downloader.rb

@@ -1,25 +1,168 @@
 require 'scrapey'
+require 'watir-webdriver'
+require 'pry'
+require "socksify"
+require 'socksify/http'
+require 'net/https' 
+
+# Mechanize: call @agent.set_socks(addr, port) before using
+# any of it's methods; it might be working in other cases,
+# but I just didn't tried :)
+class Mechanize::HTTP::Agent
+public
+  def set_socks addr, port
+    set_http unless @http
+    class << @http
+      attr_accessor :socks_addr, :socks_port
+
+      def http_class
+        Net::HTTP.SOCKSProxy(socks_addr, socks_port)
+      end
+    end
+    @http.socks_addr = addr
+    @http.socks_port = port
+    @http.open_timeout = 100 
+    @http.read_timeout = 100
+  end
+end
+
+at_exit do
+  Process.kill 9, Process.pid
+  @threads.each do |t| 
+    Thread.kill t
+    print 'k'
+  end
+end
 
 use_cache
 
+@failures = {}
+@max_failures = 5
+@max_threads = 50
+
+if arg = ARGV.find{|x| x[/--retries=(\d+)/]}
+  @max_failures = $1.to_i
+  ARGV.delete arg
+end
+
+if arg = ARGV.find{|x| x[/--threads=(\d+)/]}
+  @max_threads = $1.to_i
+  ARGV.delete arg
+end
+
+@socks = false
+if arg = ARGV.find{|x| x[/socks/]}
+  @socks = true
+  ARGV.delete arg
+end
+
+
 # File.open("#{BASEDIR}/config/urls.txt", 'w'){|f| f<< (0..100).map{|i| "http://www.example.com/id=#{i}"} * "\n"}
-@queue = File.read("#{BASEDIR}/config/urls.txt").split("\n").reject{|url| is_cached?(url)}.shuffle
-@proxies ||= File.read("#{BASEDIR}/config/proxies.txt").scan(/[\w.]+:\d+/)
+@queue ||= File.read("#{BASEDIR}/config/urls.txt").split(/[[:space:]]+/).reject{|url| is_cached?(url)}.shuffle
+
+if arg = ARGV.find{|x| x[/nopattern/]}
+  @queue.reject!{|x| x[/google|facebook|twitter|findthebest|linkedin|yellowpages|bizapedia|dandb|manta|indeed|hoovers|cortera|yelp|yellowpages|whitepages|angieslist/i]}
+  ARGV.delete arg
+end
+
+
+if @socks
+  @proxies = File.read("#{BASEDIR}/config/socks.txt").scan(/[\w.]+:\d+/).shuffle
+else
+  @proxies = File.read("#{BASEDIR}/config/proxies.txt").scan(/[\w.]+:\d+/).shuffle
+end
+
+if @pattern = ARGV[0]
+  @queue = @queue.select{|x| x[/#{@pattern}/]}
+end
 
-def response_ok? page
-  page.body[/pub-9059175907567062/] && !page.body[/IP address/i]
+# binding.pry
+
+def response_ok? page, url = nil
+  if $0[/get_emails/]
+    return !page.body[/zscaler|captcha/i]
+  end
+
+  return false if page.body[/Welcome To Zscaler/]
+
+  case url
+    when /google.com\/search/
+      return page.body[/ - Google Search/i]
+    when /facebook/
+      return page.body[/akamai/i] && !page.body[/Security Check Required/i]
+    when /twitter/
+      return page.body[/tweets/i]
+    when /findthebest/
+      return page.body[/findthebest/i] && !page.body[/Captcha/i]
+    when /linkedin/
+      return page.body[/linkedin/i] && !page.body[/Captcha/i]
+    when /yellowpages/
+      return page.body[/yellowpages/i] && !page.body[/Captcha|IP Address/i]
+    when /bizapedia.com/
+      return page.body[/bizapedia/i] && !page.body[/Captcha|IP Address/i]
+    when /dandb.com/
+      return page.body[/dandb/i] && !page.body[/Captcha/i]
+    when /topdrz.com/
+      return page.body[/topdrz/i] && !page.body[/Captcha/i]
+    when /businessfinder\.[a-z]{2}\.com/
+      return page.body[/DC.title/i]
+    when /hipaaspace.com/
+      return page.body[/Fax/i]
+    when /manta.com/
+      if page.body[/(Zscaler|Captcha|IP Address|distil_ident_block)/i]
+        puts $1
+        return false
+      end
+      return page.body[/UA-10299948/]
+    when /indeed.com\/cmp.*$(?<!review)/
+      return page.body[/indeed/i] && !page.body[/Captcha|IP Address/i]
+    when /hoovers.com\/company-information/
+      return page.body[/hoovers/i] && !page.body[/Captcha|IP Address/i]
+    when /cortera.com/
+      return page.body[/cortera/i] && !page.body[/Captcha|IP Address/i]
+    when /yelp.com/
+      return !!((page.title[/Yelp/i] && !page.title[/Captcha/i]) || page.body['yelp-biz-id'])
+    when /yellowpages.com.au/
+      return !!page.body['listing-name']
+    when /whitepages.com\/business/
+      return !!page.body['app-id=287734809']
+    when /angieslist.com.*\d.htm/
+      return !!page.title['Angies List']
+    when /addresssearch/
+      return page.body['g-plusone']
+
+
+
+  end
+  return false if page.body[/exceeded your daily request/]
+  begin
+    result = JSON.parse(page.body)['results'][0]
+    return true if result['address_components'].find{|x|x['types'].include?('country')}['short_name'] == 'US'
+  rescue
+  end
+  return !page.body[/zscaler|captcha/i]
+  puts "no match: #{url}"
+  page.body[/UA-10299948/i] && !page.body[/Authentication Required/i]
 end
 
 def clean str
   str.gsub(/[[:space:]]+/, ' ').strip
 end
 
+def check browser
+  html = browser.html.to_s
+  return true if html[/Pardon Our Interruption|Zscaler|captcha/i]
+  return true if browser.html.length > 5000
+  false
+end
+
 def download
   loop do
     Mechanize.start do |agent|
-      agent.read_timeout = agent.open_timeout = 30
+      agent.read_timeout = agent.open_timeout = agent.idle_timeout = 10000
+      keep_alive = false
       agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      agent.user_agent = [
+      ua = agent.user_agent = [
       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36',
       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36',
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1',
@@ -40,26 +183,58 @@ def download
       end
       unless proxy = @proxies.shift
         puts "no more proxies"
-        return
+        exit
       end
       @proxies.push proxy
       host, port = proxy.split(':')
-      agent.set_proxy host, port.to_i
+      if @socks
+        agent.agent.set_socks host, port.to_i
+      else
+        agent.set_proxy host, port.to_i, 'user', 'pass'
+      end
       begin
-        page = agent.get url
-        unless response_ok?(page)
-          page.search('script,style').remove
-          puts clean(page.body)
+        agent.request_headers = {'Referer' => 'http://www.google.com/search'}
+        page = nil
+        if url[/manta/]
+          html = `phantomjs --proxy=#{proxy} #{BASEDIR}/src/cookies.js #{url}`
+          page = Mechanize::Page.new URI.parse(url), [], html, nil, Mechanize.new
+        else
+          page = agent.get url
+        end
+
+        unless response_ok?(page, url)
+          # binding.pry if url[/manta/] && !page.body[/timed out|blocked|forbidden/i]
+          if page.title
+            puts page.title.strip
+          else
+            raise "no title for: #{url}"
+          end
           raise 'str'
         end
         save_cache url, page.body
 
         @good += 1
-        puts url
+        puts "- [#{@queue.length + @threads.select(&:alive?).length}/#{@proxies.length}] #{url}"
       rescue StandardError => e
-        puts e.message[0..99]
-        @queue.push url
-        @proxies -= [proxy]
+        @failures[url] ||= 0
+        @failures[url] += 1
+        unless @failures[url] >= @max_failures 
+          @queue.push(url) # unless e.message[/no title for/]
+        end
+        # binding.pry
+        if e.message[/execurtion exeprrred/]
+          print 'r'
+        elsif e.message[/403/] && !@pattern
+          if (rand * 3).to_i == 0
+            @proxies -= [proxy]
+            print '!'
+          end
+        else
+          @proxies -= [proxy]
+          print '!'
+        end
+        puts "! - #{@failures[url]} - #{e.message[0..99]}"
+
         agent.cookie_jar.clear!
       end
     end
@@ -67,18 +242,28 @@ def download
 
 end
 
-threads = []
-@deficit = 0
+def run
+  puts @queue.length
+  @num_threads = [@max_threads, @queue.length].min
+  puts "#{@proxies.length} proxies, #{@queue.length} urls, #{@num_threads} threads"
 
-until @queue.empty?
-  @good = 0
-  start_time = Time.now
+  @banned_for = []
 
-  @proxies.shuffle!
+  @threads = []
+  @deficit = 0
 
-  10.times do
-    threads << Thread.new { download }
-  end
-  threads.each { |t| t.join }
+  until @queue.empty? || @proxies.empty?
+    @good = 0
+    start_time = Time.now
+
+    @proxies.shuffle!
+
+    @num_threads.times do
+      @threads << Thread.new { download }
+    end
+    @threads.each { |t| t.join }
 
+  end
 end
+run
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrapey
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.19
 platform: ruby
 authors:
 - P Guardiario
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-29 00:00:00.000000000 Z
+date: 2016-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -24,34 +24,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: httpclient
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.0
 description: A simple scraping framework
 email:
 - pguardiario@gmail.com