title_grabber 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7532ff11d949765bb08b4ae1deeefc1aea1c70e8b794139cf747cf10c948e433
-  data.tar.gz: f59192b55720a85dab8767b03346fc335e825d5696d20f7ac46e0880a081b96d
+  metadata.gz: 6a541f189b34294558e51d72b88dc447e55e7b1d2fb802463ccd9bf27c2b4e19
+  data.tar.gz: a1ab49973e029e4bdcbcae37fefbafe6ae2e5042dc653aa3b3e7dbff42c2d804
 SHA512:
-  metadata.gz: 1102126705a068b7e7e92721c93cc283bf9b4bef28a49c7799aeb3f03825548751ebb108861e73952ba79de57620d4e9390a6a95df32b9a04cc2e7a75df704c5
-  data.tar.gz: 24f10e575e1f5b706877e952e7fefdcba17a589a3a0f0282c697d6b1882e79f40066bac01509e5d3ed43d50f2102e68ff3f2baae5c5bd9c3eecd295e86f5af03
+  metadata.gz: f6c4ecdb7a780af13b926146a76d337fd433def78e4326ee7e674fb8617a3902d5942ef61c3a14c3d7a3522920daf7a66c2fdc2e85a402c81f076a2589692eab
+  data.tar.gz: 2d921af22310ede97c1035c858abb1c87da7125df09587124b1a6d36a1aab6fa0cc745d560a83cce74b9aca2a63ae96244c74a05429cd124ec6342fdb64598ac

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.2.4)
+    title_grabber (0.3.0)
       http (~> 4.1)
       oga (~> 2.15)
 

data/lib/http_helper.rb

@@ -5,19 +5,20 @@ require "http"
 require_relative "text_helper"
 
 module HTTPHelper
-  WRITE_TO = Integer(ENV.fetch("WRITE_TIMEOUT", 5))
-  CONN_TO = Integer(ENV.fetch("CONNECT_TIMEOUT", 10))
+  WRITE_TO = Integer(ENV.fetch("WRITE_TIMEOUT", 15))
+  CONN_TO = Integer(ENV.fetch("CONNECT_TIMEOUT", 15))
   READ_TO = Integer(ENV.fetch("READ_TIMEOUT", 15))
   MAX_HOPS = 5
   MAX_RETRIES = 3
   INVALID_BYTE_SEQ = "invalid byte sequence".freeze
   CONNECTION_ERRORS = ["SSL_connect", "Connection reset"].freeze
+  REST_INTERVAL = 0.5..1
 
   include TextHelper
 
   def open_w_timeout(url, write_to: WRITE_TO, connect_to: CONN_TO,
                           read_to: READ_TO)
-    logger.info "GET #{url}"
+    logger.info "[Thread: ##{Thread.current[:id]}] GET #{url}"
     retries = 0
 
     begin
@@ -35,27 +36,29 @@ module HTTPHelper
         retries += 1
 
         if retries <= MAX_RETRIES
-          logger.warn "URL: #{url} [#{msg}] - Retry ##{retries}"
+          rest_time = rand(REST_INTERVAL)
+          logger.warn "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{msg}] - Going to sleep for #{rest_time} secs - Retry ##{retries}"
+          sleep(rest_time)
           retry
         else
-          logger.error "URL: #{url} [#{msg}]"
+          logger.error "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{msg}]"
           nil
         end
       end
     rescue => err
-      logger.error "URL: #{url} [#{err.message}]"
+      logger.error "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{err.message}]"
       nil
     else
       utf8_encode(body)
     end
   end
 
-  private
-
   def logger
     @logger ||= Logger.new(STDOUT)
   end
 
+  private
+
   def ssl_ctx
     @ssl_ctx ||= begin
                   ctx = OpenSSL::SSL::SSLContext.new

data/lib/text_helper.rb

@@ -1,4 +1,6 @@
 module TextHelper
+  SINGLE_SPACE = -" "
+
   def utf8_encode(text = nil)
     begin
       String(text).encode(-"UTF-8", invalid: :replace, undef: :replace,
@@ -11,8 +13,9 @@ module TextHelper
   # document.querySelector('title').textContent.trim().replace(/\n/g, ' ').replace(/\s{2,}/g, ' ')
   def clean_up_whitespace(text)
     text.strip!
-    text.gsub!("\n", " ")
-    text.gsub(/\s{2,}/, ' ')
+    text.gsub!("\n", SINGLE_SPACE)
+    text.gsub(/\s{2,}/, SINGLE_SPACE)
+    text.delete!(-"\u0000") # get rid of nasty null bytes
     text
   end
 end

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

data/lib/title_grabber.rb

@@ -51,23 +51,34 @@ module TitleGrabber
         end
         lines = nil
 
-        thr_cnt = [Etc.nprocessors * 2, queue.size].min
-        threads = 1.upto(thr_cnt).map {
-          Thread.new do
+        thr_cnt = [Integer(ENV.fetch("MAX_THREADS", Etc.nprocessors)),
+                   queue.size].min
+        threads = 1.upto(thr_cnt).map.with_index { |_, i|
+          Thread.new(i) do |j|
+            Thread.current[:id] = i + 1
+
             url = begin
                     queue.pop(true)
                   rescue ThreadError; end
 
             while url
               if html = open_w_timeout(url)
-                doc = Oga.parse_html(html)
-                page_title = doc.at_css('title')&.text || -""
-                clean_up_whitespace(page_title) unless page_title.empty?
-                article_title = doc.at_css('article h1')&.text
-                article_title ||= doc.at_css('h1')&.text || -""
-                clean_up_whitespace(article_title) unless article_title.empty?
-
-                csv << [url, page_title, article_title]
+                doc = begin
+                        Oga.parse_html(html)
+                      rescue LL::ParserError => err
+                        logger.error "[Thread: ##{Thread.current[:id]}] Unable to parse HTML from URL '#{url}' - #{err.message}"
+                        nil
+                      end
+
+                if doc
+                  page_title = doc.at_css('title')&.text || -""
+                  clean_up_whitespace(page_title) unless page_title.empty?
+                  article_title = doc.at_css('article h1')&.text
+                  article_title ||= doc.at_css('h1')&.text || -""
+                  clean_up_whitespace(article_title) unless article_title.empty?
+
+                  csv << [url, page_title, article_title]
+                end
               end
 
               url = begin

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-03 00:00:00.000000000 Z
+date: 2019-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http