title_grabber 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a541f189b34294558e51d72b88dc447e55e7b1d2fb802463ccd9bf27c2b4e19
-  data.tar.gz: a1ab49973e029e4bdcbcae37fefbafe6ae2e5042dc653aa3b3e7dbff42c2d804
+  metadata.gz: f341d404fbe5f16ef1eb904c75fbdd9331a76de7cb08c969d59526dbae9d95d0
+  data.tar.gz: 586a09fdca43a48db702b69e4e6e5d123e123d5aa6631a8df8c7b9eb46dd8820
 SHA512:
-  metadata.gz: f6c4ecdb7a780af13b926146a76d337fd433def78e4326ee7e674fb8617a3902d5942ef61c3a14c3d7a3522920daf7a66c2fdc2e85a402c81f076a2589692eab
-  data.tar.gz: 2d921af22310ede97c1035c858abb1c87da7125df09587124b1a6d36a1aab6fa0cc745d560a83cce74b9aca2a63ae96244c74a05429cd124ec6342fdb64598ac
+  metadata.gz: 14ceff1f56aea4cd1b9185b46e99af361ae243f809d50571933a96de2a757aa0589b4ecda34dfdeb9ff647c42412fed95acd1a1ee03fb875ed05229f53a71505
+  data.tar.gz: fc2924e719dab7d72b8adf119fe11d29909433a84732a19b878a2faf437dfb5e4e595c696e63d525864133499816bb9722c76af375dea8000851e455908705ae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.3.0)
+    title_grabber (0.3.1)
       http (~> 4.1)
       oga (~> 2.15)
 

data/lib/http_helper.rb

@@ -1,5 +1,3 @@
-require "logger"
-
 require "http"
 
 require_relative "text_helper"
@@ -37,7 +35,7 @@ module HTTPHelper
 
         if retries <= MAX_RETRIES
           rest_time = rand(REST_INTERVAL)
-          logger.warn "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{msg}] - Going to sleep for #{rest_time} secs - Retry ##{retries}"
+          logger.warn "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{msg}] - Going to sleep for #{rest_time.round(1)} secs - Retry ##{retries}"
           sleep(rest_time)
           retry
         else

data/lib/text_helper.rb

@@ -2,11 +2,16 @@ module TextHelper
   SINGLE_SPACE = -" "
 
   def utf8_encode(text = nil)
+    txt = String(text)
+
     begin
-      String(text).encode(-"UTF-8", invalid: :replace, undef: :replace,
+      txt.encode!(-"UTF-8", invalid: :replace, undef: :replace,
                                     replace: -"")
     rescue EncodingError
       -""
+    else
+      txt.delete!(-"\u0000") # get rid of nasty null bytes
+      txt
     end
   end
 
@@ -15,7 +20,6 @@ module TextHelper
     text.strip!
     text.gsub!("\n", SINGLE_SPACE)
     text.gsub(/\s{2,}/, SINGLE_SPACE)
-    text.delete!(-"\u0000") # get rid of nasty null bytes
     text
   end
 end

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/title_grabber.rb

@@ -1,6 +1,8 @@
 require "csv"
 require "etc"
 require "fileutils"
+require "logger"
+require "pathname"
 
 require "bundler/setup"
 require "oga"
@@ -24,16 +26,27 @@ module TitleGrabber
     include HTTPHelper
     include TextHelper
 
-    attr_reader :lines, :out_path
+    attr_reader :lines, :out_path, :tmp_path, :logger
 
     def initialize(lines, options)
       @lines = lines
       @out_path = options[:output]
+      @tmp_path = @out_path.sub_ext(".tmp#{@out_path.extname}")
+
+      logging_target = if ENV["DEBUG"]
+                         STDOUT
+                       else
+                         log_file = Pathname(__FILE__).sub_ext(".log").
+                                                       basename.
+                                                       open("w")
+                       end
+      @logger = Logger.new(logging_target)
     end
 
     def call
+      install_at_exit_handler
+
       queue = Queue.new
-      tmp_path = out_path.sub_ext(".tmp#{out_path.extname}")
       CSV.open(tmp_path, "w", force_quotes: true) do |csv|
         csv << HEADERS
 
@@ -62,10 +75,10 @@ module TitleGrabber
                   rescue ThreadError; end
 
             while url
-              if html = open_w_timeout(url)
+              if (html = open_w_timeout(url)) && !html.empty?
                 doc = begin
                         Oga.parse_html(html)
-                      rescue LL::ParserError => err
+                      rescue ArgumentError, LL::ParserError => err
                         logger.error "[Thread: ##{Thread.current[:id]}] Unable to parse HTML from URL '#{url}' - #{err.message}"
                         nil
                       end
@@ -88,12 +101,20 @@ module TitleGrabber
           end
         }.each(&:join)
       end
-
-      FileUtils.mv(tmp_path, out_path)
     end
 
     private
 
+    def install_at_exit_handler
+      at_exit do
+        if tmp_path.size?
+          FileUtils.mv(tmp_path, out_path)
+        else
+          tmp_path.unlink if tmp_path.exist?
+        end
+      end
+    end
+
     def processed_urls
       @processed_urls ||= begin
                             urls = {}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Cristian Rasch