email_crawler 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 62e4b0ccd69dee8523ebdbb12e3a14c6820b7592
+  data.tar.gz: dc1c7941ed58d5de961ad7bcad9f95af16fb164f
+SHA512:
+  metadata.gz: a71df8fb24765c8acafef1da841b1dfc5c4b7ffdb1f429fc0ffddc375f2ef28107fe353eb4deb352b697d84584c666adfab4370d635487305d38d64a1f5cb1d3
+  data.tar.gz: 43de8cb5b8fa2456ffed8a3e08c04e580079170eb47593587e5abee57103ba6c0bbde70fc62bd0d86a90cce5fb2b11c76ba7c5a2c7357b1a29f9fd6ba479a770

data/.env.example

@@ -0,0 +1,2 @@
+DO_CLIENT_ID=top
+DO_API_KEY=secret

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rbenv-gemsets
+.env

data/.ruby-version

@@ -0,0 +1 @@
+2.1.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in email_crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Cristian Rasch
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# EmailCrawler
+
+Email crawler: crawls the top ten Google search results looking for email addresses and exports them to CSV.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'email_crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install email_crawler
+
+## Usage
+
+1. Ask for help
+
+```bash
+email-crawler --help
+```
+
+2. Simplest Google search
+
+```bash
+email-crawler -q "berlin walks"
+```
+
+3. Select which Google website to use (defaults to google.com.br)
+
+```bash
+email-crawler -q "berlin walks" -g google.de
+```
+
+4. Specify how many internal links are to be scanned for email addresses (defaults to 100)
+
+```bash
+email-crawler -q "berlin walks" -g google.de -m 250
+```
+
+5. Redirect output to a file
+
+```bash
+email-crawler -q "berlin walks" -g google.de -m 250 > ~/Desktop/belin-walks-emails.csv
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/cristianrasch/email_crawler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.pattern = "spec/**/*_spec.rb"
+  t.verbose = true
+end
+
+task default: :test

data/bin/email-crawler

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+
+class OptionsParser
+  def self.parse(args)
+    options = OpenStruct.new
+    options.google_website = "google.com.br"
+    options.max_links = 100
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = "Usage: email-crawler [options]"
+      opts.separator ""
+
+      opts.on("-q", '--query "SEARCH TERM/EXPRESSION"',
+              "The term/expression you want to search for") do |q|
+        options.q = q
+      end
+
+      opts.on("-g", "--google-website google.com.au",
+              "An alternative Google website",
+              "  (defaults to Google Brazil)") do |google_website|
+        options.google_website = google_website
+      end
+
+      opts.on("-m", "--max-links 250",
+              "Max # of internal links to visit searching for emails",
+              "  (per search result, defaults to 100)") do |max_links|
+        options.max_links = max_links.to_i
+      end
+    end
+
+    opt_parser.parse!(args)
+    options
+  end
+end
+
+options = OptionsParser.parse(ARGV)
+if options.q.empty?
+  print "The -q switch is mandatory\n"
+  exit(1)
+else
+  require_relative "../lib/email_crawler"
+  csv = EmailCrawler::Runner.new(options.google_website).run(options.q, options.max_links)
+  $stdout << "#{csv}\n"
+end

data/email_crawler.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'email_crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "email_crawler"
+  spec.version       = EmailCrawler::VERSION
+  spec.authors       = ["Cristian Rasch"]
+  spec.email         = ["cristianrasch@fastmail.fm"]
+  spec.summary       = %q{Email crawler: crawls the top ten Google search results looking for email addresses and exports them to CSV.}
+  spec.homepage      = "https://github.com/cristianrasch/email_crawler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "mechanize"
+  spec.add_runtime_dependency "dotenv"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest", "~> 5.2.3"
+end

data/lib/email_crawler/email_scanner.rb

@@ -0,0 +1,42 @@
+require "open-uri"
+
+module EmailCrawler
+  class EmailScanner
+    EMAIL_REGEXP = /\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}\b/i
+    SLEEP_TIME = 0.5
+
+    def initialize(url)
+      @url = url
+      @logger = ::Logger.new(STDOUT).tap do |logger|
+        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
+      end
+    end
+
+    def scan(links)
+      emails_by_link = {}
+
+      links.each do |link|
+        @logger.info "searching for emails on '#{link}'.."
+
+        html = begin
+          open(link).read
+        rescue OpenURI::HTTPError => err
+          @logger.warn(err)
+          nil
+        rescue => err
+          if err.message =~ /redirection forbidden/
+            link = err.message.split(" ").last
+            retry
+          end
+        end
+        next unless html
+
+        emails = html.scan(EMAIL_REGEXP)
+        emails_by_link[link] = Set.new(emails) unless emails.empty?
+        sleep(SLEEP_TIME)
+      end
+
+      emails_by_link
+    end
+  end
+end

data/lib/email_crawler/mechanize_helper.rb

@@ -0,0 +1,22 @@
+require "mechanize"
+
+module EmailCrawler
+  module MechanizeHelper
+    def new_agent
+      Thread.current[:agent] ||= Mechanize.new do |agent|
+        agent.user_agent_alias = "Mac Safari"
+        agent.open_timeout = agent.read_timeout = 30
+        agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        agent.history.max_size = 1
+        yield(agent) if block_given?
+      end
+    end
+
+    def get(url)
+      begin
+        page = agent.get(url)
+        page if page.is_a?(Mechanize::Page)
+      rescue Mechanize::Error, Net::OpenTimeout; end
+    end
+  end
+end

data/lib/email_crawler/page_links.rb

@@ -0,0 +1,60 @@
+module EmailCrawler
+  class PageLinks
+    MAX_LINKS  = 100
+    SLEEP_TIME = 0.5
+
+    include MechanizeHelper
+
+    def initialize(url)
+      @url = url
+      uri = URI(url)
+      scheme_and_host = if uri.host
+                          "#{uri.scheme}://#{uri.host}"
+                        else
+                          url[%r(\A(https?://([^/]+))), 1]
+                        end
+      @domain = Regexp.new("#{scheme_and_host}/", true)
+      @logger = ::Logger.new(STDOUT).tap do |logger|
+        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
+      end
+    end
+
+    def self.for(url, max_links = MAX_LINKS)
+      new(url).fetch_links(max_links)
+    end
+
+    def fetch_links(max_links = MAX_LINKS)
+      queue, links = Set.new([@url]), Set.new([@url])
+
+      until queue.empty?
+        current_link = queue.first
+        @logger.info "current_link: #{current_link}"
+        page = get(current_link)
+
+        if page
+          new_links = page.links_with(href: @domain).map(&:href)
+          new_links.reject! { |link| links.include?(link) }
+          @logger.debug "found: #{new_links.length} new link(s)"
+          new_links.each { |link| queue << link }
+          links << current_link
+
+          if links.length == max_links
+            break
+          else
+            sleep(SLEEP_TIME)
+          end
+        end
+
+        queue.delete(current_link)
+      end
+
+      links.to_a
+    end
+
+  private
+
+    def agent
+      @agent ||= new_agent
+    end
+  end
+end

data/lib/email_crawler/proxy.rb

@@ -0,0 +1,28 @@
+require "open-uri"
+require "json"
+require "dotenv"
+
+module EmailCrawler
+  class Proxy
+    class << self
+      def random
+        all.sample
+      end
+
+    private
+
+      def all
+        @all ||= begin
+          Dotenv.load
+
+          json = JSON.parse(open("https://api.digitalocean.com/droplets/?client_id=#{ENV['DO_CLIENT_ID']}&api_key=#{ENV['DO_API_KEY']}").read)
+          json["droplets"].
+            select{ |droplet| droplet["name"] =~ /proxy\d+/ }.
+            map { |droplet| droplet["ip_address"] }
+        end
+      end
+    end
+  end
+end
+
+

data/lib/email_crawler/scraper.rb

@@ -0,0 +1,31 @@
+require_relative "proxy"
+
+module EmailCrawler
+  class Scraper
+    MAX_URLS = 10
+
+    include MechanizeHelper
+
+    def initialize(google_website)
+      @google_website = "https://www.#{google_website}/"
+    end
+
+    def top_ten_urls_for(q)
+      search_page = agent.get(@google_website)
+      search_form = search_page.form_with(action: "/search")
+      search_form.field_with(name: "q").value = q
+      search_results_page = agent.submit(search_form)
+      search_results_page.search("#search ol li h3.r a").
+        map { |a| a["href"].downcase }.
+        reject { |url| url =~ %r(\A/search[?]q=) }.
+        first(MAX_URLS)
+    end
+
+  private
+
+    def agent
+      @agent ||= new_agent { |agent| agent.set_proxy(Proxy.random, "8888") }
+      # @agent ||= new_agent
+    end
+  end
+end

data/lib/email_crawler/version.rb

@@ -0,0 +1,3 @@
+module EmailCrawler
+  VERSION = "0.0.2"
+end

data/lib/email_crawler.rb

@@ -0,0 +1,69 @@
+require "thread"
+require "logger"
+require "csv"
+require "set"
+
+require_relative "email_crawler/version"
+require_relative "email_crawler/mechanize_helper"
+require_relative "email_crawler/scraper"
+require_relative "email_crawler/page_links"
+require_relative "email_crawler/email_scanner"
+
+module EmailCrawler
+  class Runner
+    def initialize(google_website)
+      @google_website = google_website
+
+      log_file = File.join(ENV["HOME"], "email-crawler.log")
+      file = File.open(log_file, File::WRONLY | File::APPEND | File::CREAT)
+      @logger = ::Logger.new(file).tap do |logger|
+        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
+      end
+    end
+
+    def run(q, max_links = PageLinks::MAX_LINKS)
+      urls = Scraper.new(@google_website).top_ten_urls_for(q)
+      urls.each { |url, links| @logger.info "#{url}" }
+
+      threads = (1..urls.length).map do |i|
+        Thread.new(i, urls[i-1]) do |i, url|
+          @logger.info "[Thread ##{i}] grabbing page links for '#{url}'.."
+          Thread.current[:url] = url
+          Thread.current[:links] = PageLinks.for(url, max_links)
+        end
+      end
+
+      threads.each(&:join)
+      threads.each { |thread| @logger.info "#{thread[:url]} (#{thread[:links].length} links)" }
+      links_by_url = Hash[threads.map { |thread| [thread[:url], thread[:links]] }]
+
+      threads = (links_by_url).map.with_index do |arr, i|
+        Thread.new(i+1, arr.first, arr.last) do |i, url, links|
+          @logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
+          Thread.current[:url] = url
+          Thread.current[:emails] = EmailScanner.new(url).scan(links)
+        end
+      end
+
+      threads.each(&:join)
+
+      read_emails = Set.new
+      CSV.generate do |csv|
+        csv << %w(Email Domain URL)
+        csv << []
+
+        threads.each do |thread|
+          email_count = thread[:emails].inject(0) { |sum, arr| sum += arr.last.length }
+          @logger.info "#{thread[:url]} (#{email_count} emails)"
+
+          url = thread[:url]
+          thread[:emails].each do |link, emails|
+            emails.each do |email|
+              csv << [email, url, link] if read_emails.add?(email)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/lib/email_crawler/email_scanner_spec.rb

@@ -0,0 +1,16 @@
+require_relative "../../spec_helper"
+
+require File.expand_path("lib/email_crawler")
+
+module EmailCrawler
+  describe EmailScanner do
+    subject { EmailScanner.new("google.com") }
+
+    let(:link) { "http://www.kitaylaw.com/contact.php" }
+
+    it "scans links for email addresses" do
+      emails_by_link = subject.scan([link])
+      emails_by_link[link].wont_be_empty
+    end
+  end
+end

data/spec/lib/email_crawler/page_links_spec.rb

@@ -0,0 +1,13 @@
+require_relative "../../spec_helper"
+
+require File.expand_path("lib/email_crawler")
+
+module EmailCrawler
+  describe PageLinks do
+    let(:max_links) { 25 }
+
+    it "returns the first N internal links" do
+      PageLinks.for("http://www.visitberlin.de/en", max_links).length.must_equal max_links
+    end
+  end
+end

data/spec/lib/email_crawler/scraper_spec.rb

@@ -0,0 +1,13 @@
+require_relative "../../spec_helper"
+
+require File.expand_path("lib/email_crawler")
+
+module EmailCrawler
+  describe Scraper do
+    subject { Scraper.new("google.de") }
+
+    it "returns the top 10 URLs for a given search term/expression" do
+      subject.top_ten_urls_for("berlin tours").length.must_equal 10
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require "minitest/autorun"
+require "minitest/pride"

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: email_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Cristian Rasch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+description: 
+email:
+- cristianrasch@fastmail.fm
+executables:
+- email-crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env.example"
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/email-crawler
+- email_crawler.gemspec
+- lib/email_crawler.rb
+- lib/email_crawler/email_scanner.rb
+- lib/email_crawler/mechanize_helper.rb
+- lib/email_crawler/page_links.rb
+- lib/email_crawler/proxy.rb
+- lib/email_crawler/scraper.rb
+- lib/email_crawler/version.rb
+- spec/lib/email_crawler/email_scanner_spec.rb
+- spec/lib/email_crawler/page_links_spec.rb
+- spec/lib/email_crawler/scraper_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/cristianrasch/email_crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: 'Email crawler: crawls the top ten Google search results looking for email
+  addresses and exports them to CSV.'
+test_files:
+- spec/lib/email_crawler/email_scanner_spec.rb
+- spec/lib/email_crawler/page_links_spec.rb
+- spec/lib/email_crawler/scraper_spec.rb
+- spec/spec_helper.rb