email_crawler 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aadbe92dd2c7670e25d389a5008badaae8334fc0
-  data.tar.gz: 265adbfb7bb28397ff93af922ddd670044b2f2a6
+  metadata.gz: 49d12206392f47b3417cebc922a65e38030aab31
+  data.tar.gz: 953cd1f8ab44e9fafc361b06495d9278832b72e5
 SHA512:
-  metadata.gz: 0c2eb98801a94251434c1c009a357552813c0f3734d999f5b29c740f5419756561d97ef5b30c22472c1eaacf0cc37bfde835e47502b516d4a45232cdff8a846b
-  data.tar.gz: 607749a5ce4831fcbd929721f09191f3e2d4f10d93485fc41c92577b5250c2d480b1b1138493ce18519e30969a91ab0e101c7300f31a59ef0482b598589b7935
+  metadata.gz: 21531db699176ddca60be2c5215129fc601574674b43f463da2d83b175c46693cd8d90c9fc76a246cf988224afd82cbc1ed6dcb450c938535a859376de44da68
+  data.tar.gz: 2b80fb41432a1a549b92fc7342e995e20fbf0753db51488546dddf634f4adb50e67bd2248a2efc6aa17e60d38b74e60318937d499e1651b1275330ca81982add

data/lib/email_crawler/email_scanner.rb

@@ -6,11 +6,8 @@ module EmailCrawler
     SLEEP_TIME = 0.5
     UTF_8 = "UTF-8".freeze
 
-    def initialize(url)
-      @url = url
-      @logger = ::Logger.new(STDOUT).tap do |logger|
-        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
-      end
+    def initialize(url, logger = Logger.new("/dev/null"))
+      @url, @logger = url, logger
     end
 
     def scan(links)

data/lib/email_crawler/page_links.rb

@@ -6,7 +6,7 @@ module EmailCrawler
 
     include MechanizeHelper
 
-    def initialize(url)
+    def initialize(url, logger = Logger.new("/dev/null"))
       @url = url
       uri = URI(url)
       scheme_and_host = if uri.host
@@ -15,13 +15,11 @@ module EmailCrawler
                           url[%r(\A(https?://([^/]+))), 1]
                         end
       @domain = Regexp.new("#{scheme_and_host}/", true)
-      @logger = ::Logger.new(STDOUT).tap do |logger|
-        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
-      end
+      @logger = logger
     end
 
-    def self.for(url, max_links = MAX_LINKS)
-      new(url).fetch_links(max_links)
+    def self.for(url, max_links: MAX_LINKS, logger: Logger.new("/dev/null"))
+      new(url, logger).fetch_links(max_links)
     end
 
     def fetch_links(max_links = MAX_LINKS)

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/email_crawler.rb

@@ -14,28 +14,16 @@ module EmailCrawler
   class Runner
     MAX_CONCURRENCY = 50
 
-    attr_writer :max_results, :max_links, :max_concurrency
+    attr_writer :max_results, :max_links, :max_concurrency, :logger
 
     def initialize(google_website)
       @google_website = google_website
-
-      # @logger = ::Logger.new(STDOUT).tap do |logger|
-      log_file = File.join(ENV["HOME"], "email-crawler.log")
-      file = File.open(log_file, File::WRONLY | File::APPEND | File::CREAT)
-      @logger = ::Logger.new(file).tap do |logger|
-        logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
-      end
-
       yield(self)
-
-      @logger.info "max_results: #{@max_results}"
-      @logger.info "max_links: #{@max_links}"
-      @logger.info "max_concurrency: #{@max_concurrency}"
     end
 
     def run(q)
       urls = Scraper.new(@google_website, @max_results).search_result_urls_for(q)
-      urls.each { |url| @logger.info "#{url}" }
+      urls.each { |url| logger.info "#{url}" }
       queue = Queue.new
       urls.each { |url| queue.push(url) }
       links_by_url = ThreadSafe::Array.new
@@ -47,8 +35,8 @@ module EmailCrawler
                 rescue ThreadError; end
 
           while url
-            @logger.info "[Thread ##{i}] grabbing page links for '#{url}'.."
-            links = PageLinks.for(url, @max_links)
+            logger.info "[Thread ##{i}] grabbing page links for '#{url}'.."
+            links = PageLinks.for(url, max_links: @max_links, logger: logger)
             links_by_url << [url, links]
 
             url = begin
@@ -58,7 +46,7 @@ module EmailCrawler
         end
       end
       threads.each(&:join)
-      @logger.debug "links_by_url: #{links_by_url.inspect}"
+      logger.debug "links_by_url: #{links_by_url.inspect}"
 
       links_by_url.each { |arr| queue.push(arr) }
       emails_by_url = ThreadSafe::Hash.new
@@ -70,8 +58,8 @@ module EmailCrawler
 
           while arr
             url, links = arr
-            @logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
-            emails = EmailScanner.new(url).scan(links)
+            logger.info "[Thread ##{i}] scanning for emails on page '#{url}' (#{links.length} links)"
+            emails = EmailScanner.new(url, logger).scan(links)
             emails_by_url[url] = emails
 
             arr = begin
@@ -81,7 +69,7 @@ module EmailCrawler
         end
       end
       threads.each(&:join)
-      @logger.debug "emails_by_url: #{emails_by_url.inspect}"
+      logger.debug "emails_by_url: #{emails_by_url.inspect}"
 
       read_emails = Set.new
       CSV.generate do |csv|
@@ -90,7 +78,7 @@ module EmailCrawler
 
         emails_by_url.each do |url, emails_by_link|
           email_count = emails_by_link.inject(0) { |sum, arr| sum += arr.last.length }
-          @logger.info "#{url} (#{email_count} emails)"
+          logger.info "#{url} (#{email_count} emails)"
 
           emails_by_link.each do |link, emails|
             emails.each do |email|
@@ -100,5 +88,17 @@ module EmailCrawler
         end
       end
     end
+
+  private
+
+    def logger
+      @logger ||= begin
+        path = File.join(ENV["HOME"], "email_crawler.log")
+        file = File.open(path, File::WRONLY | File::APPEND | File::CREAT)
+        logger = ::Logger.new(file).tap do |logger|
+          logger.level = ENV["DEBUG"] ? Logger::INFO : Logger::ERROR
+        end
+      end
+    end
   end
 end

data/spec/lib/email_crawler/page_links_spec.rb

@@ -7,7 +7,7 @@ module EmailCrawler
     let(:max_links) { 25 }
 
     it "returns the first N internal links" do
-      PageLinks.for("http://www.visitberlin.de/en", max_links).length.must_equal max_links
+      PageLinks.for("http://www.visitberlin.de/en", max_links: max_links).length.must_equal max_links
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Cristian Rasch