email_crawler 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2de69de34df458d935de9deaed59993e9b66e26b
-  data.tar.gz: ac1e1063e63f708ed663183b4bb3b6331b686520
+  metadata.gz: de057cc74d307b12298c221ec86e0c6e646f3bc1
+  data.tar.gz: 1b7be472da61eec61943e249486d8ecc5cd01c56
 SHA512:
-  metadata.gz: de5977cd80e45368403850c964e4e50e56c4efef0f17d16ecb5cf942f612aff49d00542ae462f4f6b477b8d9d91201c8364e0995cbed88e696a32c1dea230304
-  data.tar.gz: 4d5b2008c4d23886f45ebeae9d46391d3b22ac6e5eab74a4b93b9c6aee43af7898978e225482818bb81b6d822b16caeb53bb686433b09af79e560eb5a65655dc
+  metadata.gz: 4077de2915db17beaa66786fc9ef978ea6bab0ea86076a644dd2f3d6bd16f12e63df12238c5cd91a62116c7327617eeccd172dd54201457c4b874da32ede00af
+  data.tar.gz: 81996e8daad656dafde6899ceb33f55680ce7dacb4e276f3cb8f96a0f11e3cacb973ab199f2d36fe02569c90dfef2f09007ce138d5688e602b0f1e8458e17fda

data/README.md

@@ -19,25 +19,31 @@ email-crawler --help
 2. Simplest Google search
 
 ```bash
-email-crawler -q "berlin walks"
+email-crawler --query "berlin walks"
 ```
 
 3. Select which Google website to use (defaults to google.com.br)
 
 ```bash
-email-crawler -q "berlin walks" -g google.de
+email-crawler --query "berlin walks" --google-website google.de
 ```
 
-4. Specify how many internal links are to be scanned for email addresses (defaults to 100)
+4. Specify how many search results URLs to collect (defaults to 100)
 
 ```bash
-email-crawler -q "berlin walks" -g google.de -m 250
+email-crawler --query "berlin walks" --max-results 250
 ```
 
-5. Redirect output to a file
+5. Specify how many internal links are to be scanned for email addresses (defaults to 100)
 
 ```bash
-email-crawler -q "berlin walks" -g google.de -m 250 > ~/Desktop/belin-walks-emails.csv
+email-crawler --query "berlin walks" --max-links 250
+```
+
+6. Redirect output to a file
+
+```bash
+email-crawler --query "berlin walks" > ~/Desktop/belin-walks-emails.csv
 ```
 
 ## Contributing

data/bin/email-crawler

@@ -3,11 +3,14 @@
 require 'optparse'
 require 'ostruct'
 
+require_relative "../lib/email_crawler"
+
 class OptionsParser
   def self.parse(args)
     options = OpenStruct.new
     options.google_website = "google.com.br"
-    options.max_links = 100
+    options.max_results = ::EmailCrawler::Scraper::MAX_RESULTS
+    options.max_links = ::EmailCrawler::PageLinks::MAX_LINKS
 
     opt_parser = OptionParser.new do |opts|
       opts.banner = "Usage: email-crawler [options]"
@@ -24,7 +27,13 @@ class OptionsParser
         options.google_website = google_website
       end
 
-      opts.on("-m", "--max-links 250",
+      opts.on("-r", "--max-results 250",
+              "Max # of search result URLs to collect before crawling each one for email addresses",
+              "  (defaults to 100)") do |max_results|
+        options.max_results = max_results.to_i
+      end
+
+      opts.on("-l", "--max-links 250",
               "Max # of internal links to visit searching for emails",
               "  (per search result, defaults to 100)") do |max_links|
         options.max_links = max_links.to_i
@@ -41,7 +50,8 @@ if options.q.empty?
   print "The -q switch is mandatory\n"
   exit(1)
 else
-  require_relative "../lib/email_crawler"
-  csv = EmailCrawler::Runner.new(options.google_website).run(options.q, options.max_links)
+  csv = EmailCrawler::Runner.new(options.google_website).run(options.q,
+                                                             options.max_results,
+                                                             options.max_links)
   $stdout << "#{csv}\n"
 end

data/lib/email_crawler.rb

@@ -21,8 +21,8 @@ module EmailCrawler
       end
     end
 
-    def run(q, max_links = PageLinks::MAX_LINKS)
-      urls = Scraper.new(@google_website).top_ten_urls_for(q)
+    def run(q, max_results = Scraper::MAX_RESULTS, max_links = PageLinks::MAX_LINKS)
+      urls = Scraper.new(@google_website, max_results).search_result_urls_for(q)
       urls.each { |url, links| @logger.info "#{url}" }
 
       threads = (1..urls.length).map do |i|

data/lib/email_crawler/scraper.rb

@@ -2,30 +2,45 @@ require_relative "proxy"
 
 module EmailCrawler
   class Scraper
-    MAX_URLS = 10
+    MAX_RESULTS = 100
 
     include MechanizeHelper
 
-    def initialize(google_website)
+    def initialize(google_website, max_results = MAX_RESULTS)
       @google_website = "https://www.#{google_website}/"
+      @max_results = max_results
     end
 
-    def top_ten_urls_for(q)
+    def search_result_urls_for(q)
       search_page = agent.get(@google_website)
       search_form = search_page.form_with(action: "/search")
       search_form.field_with(name: "q").value = q
       search_results_page = agent.submit(search_form)
-      search_results_page.search("#search ol li h3.r a").
-        map { |a| a["href"].downcase }.
-        reject { |url| url =~ %r(\A/search[?]q=) }.
-        first(MAX_URLS)
+      urls = search_results_on(search_results_page)
+
+      page = 1
+      while urls.size < @max_results
+        next_page_link = search_results_page.link_with(href: /start=#{page*10}/)
+        return urls unless next_page_link
+
+        next_search_results_page = next_page_link.click
+        urls.concat(search_results_on(next_search_results_page)).uniq!
+        page += 1
+      end
+
+      urls.first(@max_results)
     end
 
   private
 
+    def search_results_on(page)
+      page.search("#search ol li h3.r a").
+        map { |a| a["href"].downcase }.
+        reject { |url| url =~ %r(\A/search[?]q=) }
+    end
+
     def agent
       @agent ||= new_agent { |agent| agent.set_proxy(Proxy.random, "8888") }
-      # @agent ||= new_agent
     end
   end
 end

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-05 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize