email_crawler 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d6abc9f4ac1c4c0323addb2123d8795aa268cb2
-  data.tar.gz: e835dcd5d5b52bc30ec55831503dfa2f354a4d76
+  metadata.gz: b158c0d7a48cac6a56c44b905b833934e3e88ee2
+  data.tar.gz: 9b13281878621e1679de17c30e5b13d4ea04bd98
 SHA512:
-  metadata.gz: 12e7baf2369874c8f759f47d54fbebf3bdcea67c2d4065dda0e461b2f052e9a97982323d3abbc2e0d06c7c28ac1f486854c64193e0cacb12d19074821dcd03aa
-  data.tar.gz: b89ff4b54670fb2ef27dadefbb25476aaa6fe27b5f9374cadaf466e619e130e0c7270a6aa7ebf41dcfaac4774536f7edf18ddf8d4d39fb1caf2aaa086703ee1a
+  metadata.gz: 589043168d0c1a9ad72dcf3e4d7f2d32194004e76cff744e9fd9ec7187cb8e4101d22f2122f6d236a586626cdd86588ee9a676cc4afe3e91e8624fca0a557d19
+  data.tar.gz: 1955a721beb4336cb87067460a923826e801701c7d40695f96fb73b41843ecbf58592efe7b4dbcf1053dd76ca61dfefd3a141e897e282db296323945e181c070

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 .rbenv-gemsets
 .env
+bin/*

data/.ruby-version

@@ -1 +1 @@
-2.1.0
+2.2.3

data/README.md

@@ -44,6 +44,12 @@ email-crawler --query "berlin walks" --max-links 250
 email-crawler --query "berlin walks" --concurrency 25
 ```
 
+* Exclude certain domains from pages scanned for email addresses
+
+```bash
+email-crawler --query "berlin walks" --blacklist berlin.de --blacklist berlin.com
+```
+
 * Redirect output to a file
 
 ```bash

data/bin/email-crawler

@@ -70,5 +70,5 @@ else
     runner.blacklisted_domains = options.blacklisted_domains
   end
   csv = runner.run(options.q)
-  $stdout << "#{csv}\n"
+  STDOUT << "#{csv}\n"
 end

data/email_crawler.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "dotenv"
   spec.add_runtime_dependency "thread_safe"
 
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest", "~> 5.2.3"
+  spec.add_development_dependency "minitest", "~> 5.8"
 end

data/lib/email_crawler.rb

@@ -1,4 +1,3 @@
-require "thread"
 require "logger"
 require "csv"
 require "set"
@@ -14,7 +13,8 @@ module EmailCrawler
   class Runner
     MAX_CONCURRENCY = 50
 
-    attr_writer :max_results, :max_links, :max_concurrency, :logger, :blacklisted_domains
+    attr_writer :max_results, :max_links, :max_concurrency, :logger,
+                :blacklisted_domains
 
     def initialize(google_website)
       @google_website = google_website
@@ -27,7 +27,7 @@ module EmailCrawler
                          blacklisted_domains: @blacklisted_domains).
                      search_result_urls_for(q)
       urls.each { |url| logger.info "#{url}" }
-      queue = Queue.new
+      queue = Thread::Queue.new
       urls.each { |url| queue.push(url) }
       links_by_url = ThreadSafe::Array.new
 
@@ -62,7 +62,7 @@ module EmailCrawler
           while arr
             url, links = arr
             logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
-            emails = EmailScanner.new(url, logger).scan(links)
+            emails = EmailScanner.new(logger).scan(links)
             emails_by_url[url] = emails
 
             arr = begin
@@ -92,11 +92,11 @@ module EmailCrawler
       end
     end
 
-  private
+    private
 
     def logger
       @logger ||= begin
-        path = File.join(ENV["HOME"], "email_crawler.log")
+        path = File.join(ENV["HOME"], "email-crawler.log")
         file = File.open(path, File::WRONLY | File::APPEND | File::CREAT)
         logger = ::Logger.new(file).tap do |logger|
           logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR

data/lib/email_crawler/email_scanner.rb

@@ -3,31 +3,28 @@ require "open-uri"
 module EmailCrawler
   class EmailScanner
     EMAIL_REGEXP = /\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}\b/i
-    SLEEP_TIME = 0.5
     UTF_8 = "UTF-8".freeze
 
-    def initialize(url, logger = Logger.new("/dev/null"))
-      @url, @logger = url, logger
+    def initialize(logger = Logger.new("/dev/null"))
+      @logger = logger
     end
 
     def scan(links)
-      emails_by_link = {}
-
-      links.each do |link|
+      links.each_with_object({}) do |link, h|
         @logger.info "searching for emails on '#{link}'.."
         retried = false
 
         html = begin
-          open(link).read
-        rescue OpenURI::HTTPError => err
-          @logger.warn(err)
-          nil
-        rescue => err
-          if err.message =~ /redirection forbidden/
-            link = err.message.split(" ").last
-            retry
-          end
-        end
+                 open(link).read
+               rescue OpenURI::HTTPError => err
+                 @logger.warn(err)
+                 nil
+               rescue => err
+                 if err.message =~ /redirection forbidden/
+                   link = err.message.split(" ").last
+                   retry
+                 end
+               end
         next unless html
 
         begin
@@ -42,11 +39,9 @@ module EmailCrawler
             retry
           end
         end
-        emails_by_link[link] = Set.new(emails) unless emails.empty?
-        sleep(SLEEP_TIME)
-      end
 
-      emails_by_link
+        h[link] = Set.new(emails) unless emails.empty?
+      end
     end
   end
 end

data/lib/email_crawler/mechanize_helper.rb

@@ -6,7 +6,7 @@ module EmailCrawler
 
     def new_agent
       Thread.current[:agent] ||= Mechanize.new do |agent|
-        agent.user_agent_alias = "Mac Safari"
+        agent.user_agent_alias = "Windows Mozilla"
         agent.open_timeout = agent.read_timeout = READ_TIMEOUT
         agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
         agent.history.max_size = 1

data/lib/email_crawler/page_links.rb

@@ -73,7 +73,7 @@ module EmailCrawler
       links.to_a
     end
 
-  private
+    private
 
     def agent
       @agent ||= new_agent

data/lib/email_crawler/scraper.rb

@@ -1,3 +1,5 @@
+require "set"
+require_relative "mechanize_helper"
 require_relative "url_helper"
 
 module EmailCrawler
@@ -8,41 +10,53 @@ module EmailCrawler
     include URLHelper
 
     def initialize(google_website, max_results: MAX_RESULTS, blacklisted_domains: [])
-      @google_website = "https://www.#{google_website}/"
+      @search_url = "https://www.#{google_website}/search?q="
       @max_results = max_results
       @blacklisted_domains = blacklisted_domains.map { |domain| /#{domain}\z/ }
     end
 
     def search_result_urls_for(q)
-      search_page = agent.get(@google_website)
-      search_form = search_page.form_with(action: "/search")
-      search_form.field_with(name: "q").value = q
-      search_results_page = agent.submit(search_form)
-      urls = search_results_on(search_results_page)
+      search_results_page = agent.get(@search_url + CGI.escape(q))
+      urls = Set.new(search_results_on(search_results_page))
 
       page = 1
       while urls.size < @max_results
         next_page_link = search_results_page.link_with(href: /start=#{page*10}/)
-        return urls unless next_page_link
+        break unless next_page_link
 
         next_search_results_page = next_page_link.click
-        urls.concat(search_results_on(next_search_results_page)).uniq!
+        search_results_on(next_search_results_page).each do |url|
+          urls << url
+        end
+
         page += 1
       end
 
-      urls.first(@max_results)
+      urls.to_a.first(@max_results)
     end
 
-  private
+    private
 
     def search_results_on(page)
-      page.search("#search ol li h3.r a").
-        map { |a| a["href"].downcase }.
-        reject { |url| url =~ %r(\A/search[?]q=) }.
-        reject do |url|
+      urls = page.search("#search ol li.g h3.r a").map do |a|
+        href = a[:href]
+        url = href =~ %r(/url\?q=) && $POSTMATCH
+
+        if url
+          url = url =~ /&sa=/ && $PREMATCH
+          CGI.unescape(url) if url
+        end
+      end
+      urls.compact!
+
+      unless @blacklisted_domains.empty?
+        urls.delete_if do |url|
           domain = extract_domain_from(url)
           @blacklisted_domains.any? { |blacklisted_domain| domain =~ blacklisted_domain }
         end
+      end
+
+      urls
     end
 
     def agent

data/lib/email_crawler/url_helper.rb

@@ -11,6 +11,7 @@ module URLHelper
             return
           end
     host = uri.host || url[DOMAIN_REGEXP, 1].to_s
+
     if www || host !~ WWW_REGEXP
       host.downcase
     else

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

data/spec/lib/email_crawler/email_scanner_spec.rb

@@ -4,9 +4,9 @@ require File.expand_path("lib/email_crawler")
 
 module EmailCrawler
   describe EmailScanner do
-    subject { EmailScanner.new("google.com") }
+    subject { EmailScanner.new }
 
-    let(:link) { "http://www.kitaylaw.com/contact.php" }
+    let(:link) { "https://www.mrosupply.com/page/plain/contact-us/" }
 
     it "scans links for email addresses" do
       emails_by_link = subject.scan([link])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.1.0
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2015-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -56,16 +56,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.3
+        version: '5.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.3
+        version: '5.8'
 description: 
 email:
 - cristianrasch@fastmail.fm
@@ -102,7 +102,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".env.example"
 - ".gitignore"
 - ".ruby-version"
 - Gemfile
@@ -142,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: 'Email crawler: crawls the top ten Google search results looking for email

data/.env.example

@@ -1,2 +0,0 @@
-DO_CLIENT_ID=top
-DO_API_KEY=secret