title_grabber 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b27aa7cf58b7c372dad1d8fb3939440b19b9c47aa2930d8a2036444fc3bb4d6e
-  data.tar.gz: 9e31dba7e53e44c4bebd500bbf24b2cae21417e54d142f73967b8c3c7febda53
+  metadata.gz: 6642eaa211c58d0debf01ff8b0129832ef2c83d7e280f048bab0656ae7f0aec1
+  data.tar.gz: 3d9bc77e04fd081dd9c3792c9cc378c6eac60a818c8cdd9efdfed134f52ec57d
 SHA512:
-  metadata.gz: eeac69b3fd15620ac1f3597c273f8325041a0221465546f7fc6172c0fa5407ab1c4b9494fc8f3b661e9334e6514b2be4e88e2b04e6eaace6694e24db6613e06c
-  data.tar.gz: 2b7fb101126d9b5deb79a53920c614601b0472576f11fa02febf95b8749b0d38974adc79ee5dff75ff0a13e4cee5ae38c08d9ce2278622d7b1c4b7f768df4791
+  metadata.gz: e5455e96ff7c4ad6654c0c3e57fff4dc60627c55fe586d7bc72f9492c591765451101bfaee1a3537a881713f469600ffc7343081ff9fcc73d56aa733af42e4e1
+  data.tar.gz: 5474dcf7991beb7ea7c48cfe95fa98ab02cd976d068da9a1246ad518eb4963b5c701f46f54c41735b2d77f5309391dcf8f82692b803b401c1b70ef1e75d3147c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.3.6)
+    title_grabber (0.3.7)
       http (~> 4.1)
       oga (~> 2.15)
 

data/exe/title-grabber

@@ -32,6 +32,10 @@ OptionParser.new do |args|
     arguments[:write_to] = timeout
   end
 
+  args.on("--max-redirects REDIRECTS", Integer, "Max. # of HTTP redirects to follow. Defaults to the value of the MAX_REDIRECTS env var or #{TitleGrabber::MAX_REDIRECTS}") do |redirects|
+    arguments[:max_redirects] = redirects
+  end
+
   args.on("-r", "--max-retries RETRIES", Integer, "Max. # of times to retry failed HTTP reqs. Defaults to the value of the MAX_RETRIES env var or #{TitleGrabber::MAX_RETRIES}") do |retries|
     arguments[:max_retries] = retries
   end

data/lib/http_helper.rb

@@ -3,7 +3,6 @@ require "http"
 require_relative "text_helper"
 
 module HTTPHelper
-  MAX_HOPS = 5
   INVALID_BYTE_SEQ = "invalid byte sequence".freeze
   CONNECTION_ERRORS = ["SSL_connect", "Connection reset"].freeze
   REST_INTERVAL = 0.5..1
@@ -17,9 +16,11 @@ module HTTPHelper
     begin
       res = Timeout.timeout(read_to) {
               HTTP.timeout(write: write_to, connect: connect_to, read: read_to).
-                   follow(max_hops: MAX_HOPS).
+                   follow(max_hops: max_redirects).
                    get(url, ssl_context: ssl_ctx)
             }
+    rescue HTTP::Redirector::TooManyRedirectsError
+      logger.warn "[#{Thread.current.name}] GET #{url} resulted in more than #{max_redirects} redirect#{'s' unless max_redirects == 1}"
     rescue HTTP::Error, Timeout::Error => err
       msg = err.message
 

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.3.6"
+  VERSION = "0.3.7"
 end

data/lib/title_grabber.rb

@@ -16,6 +16,7 @@ module TitleGrabber
   CONNECT_TO = 15
   READ_TO = 15
   WRITE_TO = 15
+  MAX_REDIRECTS = 5
   MAX_RETRIES = 5
   MAX_THREADS = Etc.nprocessors
   URL_RE = %r(https?://\S+)i
@@ -35,7 +36,7 @@ module TitleGrabber
     include TextHelper
 
     attr_reader :lines, :out_path, :tmp_path, :connect_to, :read_to, :write_to,
-                :max_retries, :max_threads, :logger
+                :max_redirects, :max_retries, :max_threads, :logger
 
     def initialize(lines, options)
       @lines = lines
@@ -46,6 +47,7 @@ module TitleGrabber
       @connect_to = options.fetch(:connect_to, CONNECT_TO)
       @read_to = options.fetch(:read_to, READ_TO)
       @write_to = options.fetch(:write_to, WRITE_TO)
+      @max_redirects = options.fetch(:max_redirects, MAX_REDIRECTS)
       @max_retries = options.fetch(:max_retries, MAX_RETRIES)
       @max_threads = options.fetch(:max_th, Etc.nprocessors)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Cristian Rasch