email_crawler 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de057cc74d307b12298c221ec86e0c6e646f3bc1
-  data.tar.gz: 1b7be472da61eec61943e249486d8ecc5cd01c56
+  metadata.gz: a260b02f463c94ff01de5957eb1ec7ea95e8e150
+  data.tar.gz: aac88c2198ed95902045c7ee2fc874bf6d5b65f4
 SHA512:
-  metadata.gz: 4077de2915db17beaa66786fc9ef978ea6bab0ea86076a644dd2f3d6bd16f12e63df12238c5cd91a62116c7327617eeccd172dd54201457c4b874da32ede00af
-  data.tar.gz: 81996e8daad656dafde6899ceb33f55680ce7dacb4e276f3cb8f96a0f11e3cacb973ab199f2d36fe02569c90dfef2f09007ce138d5688e602b0f1e8458e17fda
+  metadata.gz: 00f1003d1e385527d0bcceae8fdddd042e06b074f4f1d3447b1f79506fcc173aa8d7ed750a5698738a5c59a28c0e8306e4f9ea4859bc318c5697f36d6ce0b3c1
+  data.tar.gz: c53d9f2e6cede921ec98c1ce42986d628e4394124514cca1aaf0b0a2fc174d966e20abed152368545714b4e11da724e12b294f491b10c48b8d1fa5dd1d930e21

data/README.md

@@ -10,37 +10,43 @@ Email crawler: crawls the top ten Google search results looking for email addres
 
 ## Usage
 
-1. Ask for help
+* Ask for help
 
 ```bash
 email-crawler --help
 ```
 
-2. Simplest Google search
+* Simplest Google search
 
 ```bash
 email-crawler --query "berlin walks"
 ```
 
-3. Select which Google website to use (defaults to google.com.br)
+* Select which Google website to use (defaults to google.com.br)
 
 ```bash
 email-crawler --query "berlin walks" --google-website google.de
 ```
 
-4. Specify how many search results URLs to collect (defaults to 100)
+* Specify how many search results URLs to collect (defaults to 100)
 
 ```bash
 email-crawler --query "berlin walks" --max-results 250
 ```
 
-5. Specify how many internal links are to be scanned for email addresses (defaults to 100)
+* Specify how many internal links are to be scanned for email addresses (defaults to 100)
 
 ```bash
 email-crawler --query "berlin walks" --max-links 250
 ```
 
-6. Redirect output to a file
+* Specify how many threads to use when searching for links and email addresses (defaults to 10)
+
+```bash
+email-crawler --query "berlin walks" --concurrency 25
+```
+
+* Redirect output to a file
 
 ```bash
 email-crawler --query "berlin walks" > ~/Desktop/belin-walks-emails.csv

data/bin/email-crawler

@@ -11,6 +11,7 @@ class OptionsParser
     options.google_website = "google.com.br"
     options.max_results = ::EmailCrawler::Scraper::MAX_RESULTS
     options.max_links = ::EmailCrawler::PageLinks::MAX_LINKS
+    options.max_concurrency = ::EmailCrawler::Runner::MAX_CONCURRENCY
 
     opt_parser = OptionParser.new do |opts|
       opts.banner = "Usage: email-crawler [options]"
@@ -38,6 +39,12 @@ class OptionsParser
               "  (per search result, defaults to 100)") do |max_links|
         options.max_links = max_links.to_i
       end
+
+      opts.on("-c", "--concurrency 25",
+              "Max # of threads to use to look for links and email addresses",
+              "  (defaults to 10)") do |max_concurrency|
+        options.max_concurrency = max_concurrency.to_i
+      end
     end
 
     opt_parser.parse!(args)
@@ -50,8 +57,11 @@ if options.q.empty?
   print "The -q switch is mandatory\n"
   exit(1)
 else
-  csv = EmailCrawler::Runner.new(options.google_website).run(options.q,
-                                                             options.max_results,
-                                                             options.max_links)
+  runner = EmailCrawler::Runner.new(options.google_website) do |runner|
+    runner.max_results = options.max_results
+    runner.max_links = options.max_links
+    runner.max_concurrency = options.max_concurrency
+  end
+  csv = runner.run(options.q)
   $stdout << "#{csv}\n"
 end

data/email_crawler.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "mechanize"
   spec.add_runtime_dependency "dotenv"
+  spec.add_runtime_dependency "thread_safe"
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"

data/lib/email_crawler/email_scanner.rb

@@ -4,6 +4,7 @@ module EmailCrawler
   class EmailScanner
     EMAIL_REGEXP = /\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}\b/i
     SLEEP_TIME = 0.5
+    UTF_8 = "UTF-8".freeze
 
     def initialize(url)
       @url = url
@@ -17,6 +18,7 @@ module EmailCrawler
 
       links.each do |link|
         @logger.info "searching for emails on '#{link}'.."
+        retried = false
 
         html = begin
           open(link).read
@@ -31,7 +33,18 @@ module EmailCrawler
         end
         next unless html
 
-        emails = html.scan(EMAIL_REGEXP)
+        begin
+          emails = html.scan(EMAIL_REGEXP)
+        rescue ArgumentError => err
+          if retried
+            emails = []
+          else
+            @logger.warn err.inspect
+            html.encode!(UTF_8, UTF_8, invalid: :replace, undef: :replace, replace: "")
+            retried = true
+            retry
+          end
+        end
         emails_by_link[link] = Set.new(emails) unless emails.empty?
         sleep(SLEEP_TIME)
       end

data/lib/email_crawler/page_links.rb

@@ -2,6 +2,7 @@ module EmailCrawler
   class PageLinks
     MAX_LINKS  = 100
     SLEEP_TIME = 0.5
+    MAX_RETRIES = 5
 
     include MechanizeHelper
 
@@ -25,11 +26,32 @@ module EmailCrawler
 
     def fetch_links(max_links = MAX_LINKS)
       queue, links = Set.new([@url]), Set.new([@url])
+      retries = 0
 
       until queue.empty?
         current_link = queue.first
         @logger.info "current_link: #{current_link}"
-        page = get(current_link)
+
+        begin
+          page = get(current_link)
+        rescue Net::HTTP::Persistent::Error => err
+          @logger.warn err.inspect
+          page = nil
+
+          if retries < MAX_RETRIES
+            retries += 1
+            @logger.debug "Retry ##{retries}"
+            agent.shutdown
+            Thread.current[:agent] = nil
+            sleep(SLEEP_TIME)
+            retry
+          else
+            @logger.error "Giving up grabbing link for '#{@url}' after #{retries} retries"
+            break
+          end
+        else
+          retries = 0
+        end
 
         if page
           new_links = page.links_with(href: @domain).map(&:href)

data/lib/email_crawler/scraper.rb

@@ -40,7 +40,8 @@ module EmailCrawler
     end
 
     def agent
-      @agent ||= new_agent { |agent| agent.set_proxy(Proxy.random, "8888") }
+      @agent ||= new_agent
+      # @agent ||= new_agent { |agent| agent.set_proxy(Proxy.random, "8888") }
     end
   end
 end

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/email_crawler.rb

@@ -2,6 +2,7 @@ require "thread"
 require "logger"
 require "csv"
 require "set"
+require "thread_safe"
 
 require_relative "email_crawler/version"
 require_relative "email_crawler/mechanize_helper"
@@ -11,53 +12,87 @@ require_relative "email_crawler/email_scanner"
 
 module EmailCrawler
   class Runner
+    MAX_CONCURRENCY = 10
+
+    attr_writer :max_results, :max_links, :max_concurrency
+
     def initialize(google_website)
       @google_website = google_website
 
+      # @logger = ::Logger.new(STDOUT).tap do |logger|
       log_file = File.join(ENV["HOME"], "email-crawler.log")
       file = File.open(log_file, File::WRONLY | File::APPEND | File::CREAT)
       @logger = ::Logger.new(file).tap do |logger|
         logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
       end
+
+      yield(self)
+
+      @logger.info "max_results: #{@max_results}"
+      @logger.info "max_links: #{@max_links}"
+      @logger.info "max_concurrency: #{@max_concurrency}"
     end
 
-    def run(q, max_results = Scraper::MAX_RESULTS, max_links = PageLinks::MAX_LINKS)
-      urls = Scraper.new(@google_website, max_results).search_result_urls_for(q)
-      urls.each { |url, links| @logger.info "#{url}" }
+    def run(q)
+      urls = Scraper.new(@google_website, @max_results).search_result_urls_for(q)
+      urls.each { |url| @logger.info "#{url}" }
+      queue = Queue.new
+      urls.each { |url| queue.push(url) }
+      links_by_url = ThreadSafe::Array.new
 
-      threads = (1..urls.length).map do |i|
-        Thread.new(i, urls[i-1]) do |i, url|
-          @logger.info "[Thread ##{i}] grabbing page links for '#{url}'.."
-          Thread.current[:url] = url
-          Thread.current[:links] = PageLinks.for(url, max_links)
+      threads = (1..[urls.length, @max_concurrency].min).map do |i|
+        Thread.new(i) do |i|
+          url = begin
+                  queue.pop(true)
+                rescue ThreadError; end
+
+          while url
+            @logger.info "[Thread ##{i}] grabbing page links for '#{url}'.."
+            links = PageLinks.for(url, @max_links)
+            links_by_url << [url, links]
+
+            url = begin
+                    queue.pop(true)
+                  rescue ThreadError; end
+          end
         end
       end
-
       threads.each(&:join)
-      threads.each { |thread| @logger.info "#{thread[:url]} (#{thread[:links].length} links)" }
-      links_by_url = Hash[threads.map { |thread| [thread[:url], thread[:links]] }]
-
-      threads = (links_by_url).map.with_index do |arr, i|
-        Thread.new(i+1, arr.first, arr.last) do |i, url, links|
-          @logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
-          Thread.current[:url] = url
-          Thread.current[:emails] = EmailScanner.new(url).scan(links)
+      @logger.debug "links_by_url: #{links_by_url.inspect}"
+
+      links_by_url.each { |arr| queue.push(arr) }
+      emails_by_url = ThreadSafe::Hash.new
+      threads = (1..[links_by_url.length, MAX_CONCURRENCY].min).map do |i|
+        Thread.new(i) do |i|
+          arr = begin
+                  queue.pop(true)
+                rescue ThreadError; end
+
+          while arr
+            url, links = arr
+            @logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
+            emails = EmailScanner.new(url).scan(links)
+            emails_by_url[url] = emails
+
+            arr = begin
+                    queue.pop(true)
+                  rescue ThreadError; end
+          end
         end
       end
-
       threads.each(&:join)
+      @logger.debug "emails_by_url: #{emails_by_url.inspect}"
 
       read_emails = Set.new
       CSV.generate do |csv|
         csv << %w(Email Domain URL)
         csv << []
 
-        threads.each do |thread|
-          email_count = thread[:emails].inject(0) { |sum, arr| sum += arr.last.length }
-          @logger.info "#{thread[:url]} (#{email_count} emails)"
+        emails_by_url.each do |url, emails_by_link|
+          email_count = emails_by_link.inject(0) { |sum, arr| sum += arr.last.length }
+          @logger.info "#{url} (#{email_count} emails)"
 
-          url = thread[:url]
-          thread[:emails].each do |link, emails|
+          emails_by_link.each do |link, emails|
             emails.each do |email|
               csv << [email, url, link] if read_emails.add?(email)
             end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2014-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thread_safe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement