email_crawler 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49d12206392f47b3417cebc922a65e38030aab31
-  data.tar.gz: 953cd1f8ab44e9fafc361b06495d9278832b72e5
+  metadata.gz: 6c7248b8e8688fb03bc939e0690a87920bd43f74
+  data.tar.gz: d125e3a9ad7b554e7e98d994c9b158ecfaf93b6b
 SHA512:
-  metadata.gz: 21531db699176ddca60be2c5215129fc601574674b43f463da2d83b175c46693cd8d90c9fc76a246cf988224afd82cbc1ed6dcb450c938535a859376de44da68
-  data.tar.gz: 2b80fb41432a1a549b92fc7342e995e20fbf0753db51488546dddf634f4adb50e67bd2248a2efc6aa17e60d38b74e60318937d499e1651b1275330ca81982add
+  metadata.gz: e238ffdaabe400c7d70ee323a6769aece2b8b141c530aacbbc7c822b6952f44f6379d77a744b45c2e2e1061e8ee3f4f06def65a34299442d149055b591f7f665
+  data.tar.gz: 79a2d7d3e0c7f60312e81f72a6b9dcd7b1d2b5b8de672ec3b3fdf823021b57514d77577f01f48cc7609a4650b7bcf1646d312900553d8f9ca55c716668b183ea

data/lib/email_crawler/scraper.rb

@@ -1,14 +1,16 @@
-require_relative "proxy"
+require_relative "url_helper"
 
 module EmailCrawler
   class Scraper
     MAX_RESULTS = 100
 
     include MechanizeHelper
+    include URLHelper
 
-    def initialize(google_website, max_results = MAX_RESULTS)
+    def initialize(google_website, max_results: MAX_RESULTS, blacklisted_domains: [])
       @google_website = "https://www.#{google_website}/"
       @max_results = max_results
+      @blacklisted_domains = blacklisted_domains.map { |domain| /#{domain}\z/ }
     end
 
     def search_result_urls_for(q)
@@ -36,12 +38,15 @@ module EmailCrawler
     def search_results_on(page)
       page.search("#search ol li h3.r a").
         map { |a| a["href"].downcase }.
-        reject { |url| url =~ %r(\A/search[?]q=) }
+        reject { |url| url =~ %r(\A/search[?]q=) }.
+        reject do |url|
+          domain = extract_domain_from(url)
+          @blacklisted_domains.any? { |blacklisted_domain| domain =~ blacklisted_domain }
+        end
     end
 
     def agent
       @agent ||= new_agent
-      # @agent ||= new_agent { |agent| agent.set_proxy(Proxy.random, "8888") }
     end
   end
 end

data/lib/email_crawler/url_helper.rb

@@ -0,0 +1,20 @@
+require "English"
+
+module URLHelper
+  DOMAIN_REGEXP = %r(https://([^/]+))i
+  WWW_REGEXP = /\Awww[.]/i
+
+  def extract_domain_from(url, www = false)
+    uri = begin
+            URI(url)
+          rescue URI::InvalidURIError
+            return
+          end
+    host = uri.host || url[DOMAIN_REGEXP, 1].to_s
+    if www || host !~ WWW_REGEXP
+      host.downcase
+    else
+      $POSTMATCH.downcase
+    end
+  end
+end

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

data/lib/email_crawler.rb

@@ -22,7 +22,7 @@ module EmailCrawler
     end
 
     def run(q)
-      urls = Scraper.new(@google_website, @max_results).search_result_urls_for(q)
+      urls = Scraper.new(@google_website, max_results: @max_results).search_result_urls_for(q)
       urls.each { |url| logger.info "#{url}" }
       queue = Queue.new
       urls.each { |url| queue.push(url) }

data/spec/lib/email_crawler/scraper_spec.rb

@@ -6,7 +6,7 @@ module EmailCrawler
   describe Scraper do
     let(:max_results) { 10 }
 
-    subject { Scraper.new("google.de", max_results) }
+    subject { Scraper.new("google.de", max_results: max_results) }
 
     it "returns the top 10 URLs for a given search term/expression" do
       subject.search_result_urls_for("berlin tours").length.must_equal max_results

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Cristian Rasch
@@ -115,8 +115,8 @@ files:
 - lib/email_crawler/email_scanner.rb
 - lib/email_crawler/mechanize_helper.rb
 - lib/email_crawler/page_links.rb
-- lib/email_crawler/proxy.rb
 - lib/email_crawler/scraper.rb
+- lib/email_crawler/url_helper.rb
 - lib/email_crawler/version.rb
 - spec/lib/email_crawler/email_scanner_spec.rb
 - spec/lib/email_crawler/page_links_spec.rb

data/lib/email_crawler/proxy.rb

@@ -1,30 +0,0 @@
-require "openssl"
-require "open-uri"
-require "json"
-require "dotenv"
-
-module EmailCrawler
-  class Proxy
-    class << self
-      def random
-        all.sample
-      end
-
-    private
-
-      def all
-        @all ||= begin
-          Dotenv.load
-
-          json = JSON.parse(open("https://api.digitalocean.com/droplets/?client_id=#{ENV['DO_CLIENT_ID']}&api_key=#{ENV['DO_API_KEY']}",
-                                 ssl_verify_mode: ::OpenSSL::SSL::VERIFY_NONE).read)
-          json["droplets"].
-            select{ |droplet| droplet["name"] =~ /proxy\d+/ }.
-            map { |droplet| droplet["ip_address"] }
-        end
-      end
-    end
-  end
-end
-
-