email_crawler 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b158c0d7a48cac6a56c44b905b833934e3e88ee2
-  data.tar.gz: 9b13281878621e1679de17c30e5b13d4ea04bd98
+  metadata.gz: 45594bfd727c7a3e1e5125755c5b4b136cee5638
+  data.tar.gz: 7af69b7e472e31f47f32f44ebbf975241d1041d2
 SHA512:
-  metadata.gz: 589043168d0c1a9ad72dcf3e4d7f2d32194004e76cff744e9fd9ec7187cb8e4101d22f2122f6d236a586626cdd86588ee9a676cc4afe3e91e8624fca0a557d19
-  data.tar.gz: 1955a721beb4336cb87067460a923826e801701c7d40695f96fb73b41843ecbf58592efe7b4dbcf1053dd76ca61dfefd3a141e897e282db296323945e181c070
+  metadata.gz: 574179cf97e5a5131db4889b02d2ee939bcdcf303e0904439bf4d80370fc9a255816698f5883739f18166efcc71b088efc61a357253112d1ddacd1819a0bcc46
+  data.tar.gz: 404d06b126c8e6c55c0e81708d0f12a1dd63dec4069802e7d8899efbb05c0213f1f8701a19037eed1247b6345e4a965bcee0411411bddcd3adbdd13d33fab0db

data/lib/email_crawler.rb

@@ -31,7 +31,7 @@ module EmailCrawler
       urls.each { |url| queue.push(url) }
       links_by_url = ThreadSafe::Array.new
 
-      threads = (1..[urls.length, @max_concurrency].min).map do |i|
+      threads = (1..[queue.size, @max_concurrency].min).map do |i|
         Thread.new(i) do |i|
           url = begin
                   queue.pop(true)

data/lib/email_crawler/email_scanner.rb

@@ -1,10 +1,10 @@
-require "open-uri"
-
 module EmailCrawler
   class EmailScanner
     EMAIL_REGEXP = /\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,4}\b/i
     UTF_8 = "UTF-8".freeze
 
+    include MechanizeHelper
+
     def initialize(logger = Logger.new("/dev/null"))
       @logger = logger
     end
@@ -14,17 +14,12 @@ module EmailCrawler
         @logger.info "searching for emails on '#{link}'.."
         retried = false
 
-        html = begin
-                 open(link).read
-               rescue OpenURI::HTTPError => err
-                 @logger.warn(err)
-                 nil
-               rescue => err
-                 if err.message =~ /redirection forbidden/
-                   link = err.message.split(" ").last
-                   retry
-                 end
-               end
+        begin
+          html = get(link).body
+        rescue => err
+          @logger.warn err.inspect
+          nil
+        end
         next unless html
 
         begin

data/lib/email_crawler/mechanize_helper.rb

@@ -37,5 +37,11 @@ module EmailCrawler
         end
       end
     end
+
+    private
+
+    def agent
+      @agent ||= new_agent
+    end
   end
 end

data/lib/email_crawler/page_links.rb

@@ -8,8 +8,10 @@ module EmailCrawler
 
     def initialize(url, logger = Logger.new("/dev/null"))
       @url = url
-      uri = URI(url)
-      scheme_and_host = if uri.host
+      uri = begin
+              URI(url)
+            rescue; end
+      scheme_and_host = if uri && uri.host
                           "#{uri.scheme}://#{uri.host}"
                         else
                           url[%r(\A(https?://([^/]+))), 1]
@@ -47,7 +49,7 @@ module EmailCrawler
             @logger.error "Giving up grabbing link for '#{@url}' after #{retries} retries"
             break
           end
-        rescue URI::InvalidComponentError => err
+        rescue => err
           @logger.warn err.inspect
         else
           retries = 0
@@ -72,11 +74,5 @@ module EmailCrawler
 
       links.to_a
     end
-
-    private
-
-    def agent
-      @agent ||= new_agent
-    end
   end
 end

data/lib/email_crawler/scraper.rb

@@ -58,9 +58,5 @@ module EmailCrawler
 
       urls
     end
-
-    def agent
-      @agent ||= new_agent
-    end
   end
 end

data/lib/email_crawler/version.rb

@@ -1,3 +1,3 @@
 module EmailCrawler
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: email_crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Cristian Rasch