title_grabber 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f341d404fbe5f16ef1eb904c75fbdd9331a76de7cb08c969d59526dbae9d95d0
-  data.tar.gz: 586a09fdca43a48db702b69e4e6e5d123e123d5aa6631a8df8c7b9eb46dd8820
+  metadata.gz: 8c1d79385bd87e3116d10dc56c14b69929dccb1eb2597e99ed2166c683146f59
+  data.tar.gz: 1ea1fc865f6570c0ecb0f0f39c023585a9fac0c11dca43efbfe23a1d0aa1f075
 SHA512:
-  metadata.gz: 14ceff1f56aea4cd1b9185b46e99af361ae243f809d50571933a96de2a757aa0589b4ecda34dfdeb9ff647c42412fed95acd1a1ee03fb875ed05229f53a71505
-  data.tar.gz: fc2924e719dab7d72b8adf119fe11d29909433a84732a19b878a2faf437dfb5e4e595c696e63d525864133499816bb9722c76af375dea8000851e455908705ae
+  metadata.gz: 3b2fe97ff3c4a8723b6bfa634f3bf00f8096375a9066c217359e6cae0e5ba657560093ebf647a4c042f82b482030a842c0896a94ceb71fd943143bd0f9fa03b3
+  data.tar.gz: 936b43e66159e54246a57f59570e5dca550d4afbf87f7fae4bf1be8166af8fe0b1cab8d000409eb68293d4b5046ef40791574567010347e00ed20c359b9f5593

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.3.1)
+    title_grabber (0.3.2)
       http (~> 4.1)
       oga (~> 2.15)
 

data/README.md

@@ -33,6 +33,22 @@ Data is either recorded to out.csv in the CWD or the file specified using the
 title-grabber -o ~/output.csv /abs/path/2/file1.txt rel/path/2/file2.txt
 ```
 
+### Environment Variables
+
+```
+DEBUG - when set it logs to STDOUT instead of to its default target, title_grabber.log
+```
+
+```
+MAX_THREADS - max. # of threads to use, defaults to the # of CPU cores in the machine
+```
+
+```
+CONNECT_TIMEOUT (in seconds) - defaults to 15
+READ_TIMEOUT (in seconds) - defaults to 15
+WRITE_TIMEOUT (in seconds) - defaults to 15
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/exe/title-grabber

@@ -6,15 +6,38 @@ require "pathname"
 require_relative '../lib/title_grabber'
 
 script_path = Pathname(__FILE__)
-def_out_path = Pathname('out.csv')
 
-arguments = { output: def_out_path }
+arguments = {}
 OptionParser.new do |args|
   args.banner = "Usage: #{script_path.basename} [options]"
 
-  args.on("-o", "--output FILE", "Output file (defaults to #{def_out_path.basename})") do |out|
+  args.on("-o", "--output FILE", "Output file. Defaults to #{TitleGrabber::DEF_OUT_PATH.basename}") do |out|
     arguments[:output] = Pathname(out)
   end
+
+  args.on("--connect-timeout TIMEOUT", Integer, "HTTP Connect timeout. Defaults to the value of the CONNECT_TIMEOUT env var or #{TitleGrabber::CONNECT_TO}") do |timeout|
+    arguments[:connect_to] = timeout
+  end
+
+  args.on("--read-timeout TIMEOUT", Integer, "HTTP Read timeout. Defaults to the value of the READ_TIMEOUT env var or #{TitleGrabber::READ_TO}") do |timeout|
+    arguments[:read_to] = timeout
+  end
+
+  args.on("--write-timeout TIMEOUT", Integer, "HTTP Write timeout. Defaults to the value of the WRITE_TIMEOUT env var or #{TitleGrabber::WRITE_TO}") do |timeout|
+    arguments[:write_to] = timeout
+  end
+
+  args.on("-r", "--max-retries RETRIES", Integer, "Max. # of times to retry failed HTTP reqs. Defaults to the value of the MAX_RETRIES env var or #{TitleGrabber::MAX_RETRIES}") do |retries|
+    arguments[:max_retries] = retries
+  end
+
+  args.on("-t", "--max-threads THREADS", Integer, "Max. # of threads to use. Defaults to the value of the MAX_THREADS env var or the # of logical processors in the system") do |threads|
+    arguments[:max_threads] = threads
+  end
+
+  args.on("-d", "--debug", "Log to STDOUT instead of to a file in the CWD") do |debug|
+    arguments[:debug] = true
+  end
 end.parse!
 
 if ARGV.empty?

data/lib/http_helper.rb

@@ -3,19 +3,14 @@ require "http"
 require_relative "text_helper"
 
 module HTTPHelper
-  WRITE_TO = Integer(ENV.fetch("WRITE_TIMEOUT", 15))
-  CONN_TO = Integer(ENV.fetch("CONNECT_TIMEOUT", 15))
-  READ_TO = Integer(ENV.fetch("READ_TIMEOUT", 15))
   MAX_HOPS = 5
-  MAX_RETRIES = 3
   INVALID_BYTE_SEQ = "invalid byte sequence".freeze
   CONNECTION_ERRORS = ["SSL_connect", "Connection reset"].freeze
   REST_INTERVAL = 0.5..1
 
   include TextHelper
 
-  def open_w_timeout(url, write_to: WRITE_TO, connect_to: CONN_TO,
-                          read_to: READ_TO)
+  def open_w_timeout(url, write_to:, connect_to:, read_to:, max_retries:)
     logger.info "[Thread: ##{Thread.current[:id]}] GET #{url}"
     retries = 0
 
@@ -33,7 +28,7 @@ module HTTPHelper
            CONNECTION_ERRORS.any? { |e| msg.start_with?(e) }
         retries += 1
 
-        if retries <= MAX_RETRIES
+        if retries <= max_retries
           rest_time = rand(REST_INTERVAL)
           logger.warn "[Thread: ##{Thread.current[:id]}] URL: #{url} [#{msg}] - Going to sleep for #{rest_time.round(1)} secs - Retry ##{retries}"
           sleep(rest_time)

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/lib/title_grabber.rb

@@ -1,5 +1,5 @@
-require "csv"
 require "etc"
+require "csv"
 require "fileutils"
 require "logger"
 require "pathname"
@@ -12,6 +12,12 @@ require_relative "http_helper"
 require_relative "text_helper"
 
 module TitleGrabber
+  DEF_OUT_PATH = Pathname('out.csv')
+  CONNECT_TO = 15
+  READ_TO = 15
+  WRITE_TO = 15
+  MAX_RETRIES = 5
+  MAX_THREADS = Etc.nprocessors
   URL_RE = %r(https?://\S+)i
   URL_HEADER = -"url"
   PAGE_TIT_HEAD = -"page_title"
@@ -26,14 +32,22 @@ module TitleGrabber
     include HTTPHelper
     include TextHelper
 
-    attr_reader :lines, :out_path, :tmp_path, :logger
+    attr_reader :lines, :out_path, :tmp_path, :connect_to, :read_to, :write_to,
+                :max_retries, :max_threads, :logger
 
     def initialize(lines, options)
       @lines = lines
-      @out_path = options[:output]
+
+      @out_path = options.fetch(:output, DEF_OUT_PATH)
       @tmp_path = @out_path.sub_ext(".tmp#{@out_path.extname}")
 
-      logging_target = if ENV["DEBUG"]
+      @connect_to = options.fetch(:connect_to, CONNECT_TO)
+      @read_to = options.fetch(:read_to, READ_TO)
+      @write_to = options.fetch(:write_to, WRITE_TO)
+      @max_retries = options.fetch(:max_retries, MAX_RETRIES)
+      @max_threads = options.fetch(:max_th, Etc.nprocessors)
+
+      logging_target = if options[:debug]
                          STDOUT
                        else
                          log_file = Pathname(__FILE__).sub_ext(".log").
@@ -64,8 +78,7 @@ module TitleGrabber
         end
         lines = nil
 
-        thr_cnt = [Integer(ENV.fetch("MAX_THREADS", Etc.nprocessors)),
-                   queue.size].min
+        thr_cnt = [max_threads, queue.size].min
         threads = 1.upto(thr_cnt).map.with_index { |_, i|
           Thread.new(i) do |j|
             Thread.current[:id] = i + 1
@@ -75,7 +88,7 @@ module TitleGrabber
                   rescue ThreadError; end
 
             while url
-              if (html = open_w_timeout(url)) && !html.empty?
+              if (html = open_w_timeout(url, **http_opts)) && !html.empty?
                 doc = begin
                         Oga.parse_html(html)
                       rescue ArgumentError, LL::ParserError => err
@@ -134,5 +147,10 @@ module TitleGrabber
                             urls
                           end
     end
+
+    def http_opts
+      @http_opts ||= { connect_to: connect_to, read_to: read_to,
+                       write_to: write_to, max_retries: max_retries }
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Cristian Rasch