title_grabber 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58ceec17ac17673c12eee2cdd5fbef23cdc561b89b5820f652e592a91047a399
-  data.tar.gz: 3f4be7dbff89b096c51fb28839d0bd2f4a85873507ee6600b7d2e481c16ea766
+  metadata.gz: '028ecf2bea074495354ac08494517b7f254e8a27d782e2a3ff1eec7bd313474b'
+  data.tar.gz: 20adeb673a80980e1c3e8ee04285451eb0aa4d8e98c4c17cad5b72f9549f04ee
 SHA512:
-  metadata.gz: 2e87da24fb755d6869b68af2fd201a19e7f9ea6754b1dd26263f58124055847a9594cd186a88c39b2b91f59507f969a35b47bbf4924ca0c35c8f2776e4533836
-  data.tar.gz: '0957bf0dc17b6f02dff00b185ae0fd9ba163ce9610655063dab0ce8e16bd8c761db1a7cd8d283a685a408ed2470bef1ac7bcb42167f995e2857feeeb94e7647c'
+  metadata.gz: 2a6268347f464325956f4281f0132b9f6c03d945eefab10754fb93387b0bd0db384cac05899238532f42c62869f58e6282e1e8eadf326bde9026b3d65ba97261
+  data.tar.gz: e1388a778ea78d29f9c791db0e6eaaa0fa379ba912f6869a0b90f087c1fb7e366c61b08f47dce25f003abbc8f6909dde3b3c99a1636fdcf1777eb2fcbb61c72e

data/Gemfile.lock

@@ -1,42 +1,24 @@
 PATH
   remote: .
   specs:
-    title_grabber (0.4.0)
-      http (~> 4.1)
+    title_grabber (0.4.1)
       oga (~> 2.15)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
     ansi (1.5.0)
     ast (2.4.0)
-    domain_name (0.5.20180417)
-      unf (>= 0.0.5, < 1.0.0)
-    http (4.1.1)
-      addressable (~> 2.3)
-      http-cookie (~> 1.0)
-      http-form_data (~> 2.0)
-      http_parser.rb (~> 0.6.0)
-    http-cookie (1.0.3)
-      domain_name (~> 0.5)
-    http-form_data (2.1.1)
-    http_parser.rb (0.6.0)
     minitest (5.11.3)
     minitest-line (0.6.5)
       minitest (~> 5.0)
     oga (2.15)
       ast
       ruby-ll (~> 2.1)
-    public_suffix (3.0.3)
     rake (10.5.0)
     ruby-ll (2.1.2)
       ansi
       ast
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.5)
 
 PLATFORMS
   ruby

data/README.md

@@ -39,6 +39,18 @@ See all available CLI switches and env vars
 title-grabber -h
 ```
 
+    Usage: title-grabber [options]
+        -V, --version                    Print program version and exit
+        -f, --files /f1.txt,f2.txt       1 or more comma-separated paths to text files containing 1 URL per line
+        -o, --output FILE                Output file. Defaults to out.csv
+            --connect-timeout TIMEOUT    HTTP Connect timeout. Defaults to the value of the CONNECT_TIMEOUT env var or 15
+            --read-timeout TIMEOUT       HTTP Read timeout. Defaults to the value of the READ_TIMEOUT env var or 15
+            --write-timeout TIMEOUT      HTTP Write timeout. Defaults to the value of the WRITE_TIMEOUT env var or 15
+            --max-redirects REDIRECTS    Max. # of HTTP redirects to follow. Defaults to the value of the MAX_REDIRECTS env var or 5
+        -r, --max-retries RETRIES        Max. # of times to retry failed HTTP reqs. Defaults to the value of the MAX_RETRIES env var or 5
+        -t, --max-threads THREADS        Max. # of threads to use. Defaults to the value of the MAX_THREADS env var or the # of logical processors in the system
+        -d, --debug                      Log to STDOUT instead of to a file in the CWD
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/exe/title-grabber

@@ -16,7 +16,7 @@ OptionParser.new do |args|
     exit
   end
 
-  args.on("-f", "--files /abs/f1,rel/f2", Array, "1 or more comma-separated paths to text files containing 1 URL per line ") do |files|
+  args.on("-f", "--files /f1.txt,f2.txt", Array, "1 or more comma-separated paths to text files containing 1 URL per line ") do |files|
     arguments[:file_paths] = files.map { |f| Pathname(f).expand_path }.
                                    select { |f| f.file? && f.exist? }
   end
@@ -33,14 +33,6 @@ OptionParser.new do |args|
     arguments[:read_to] = timeout
   end
 
-  args.on("--write-timeout TIMEOUT", Integer, "HTTP Write timeout. Defaults to the value of the WRITE_TIMEOUT env var or #{TitleGrabber::WRITE_TO}") do |timeout|
-    arguments[:write_to] = timeout
-  end
-
-  args.on("--max-redirects REDIRECTS", Integer, "Max. # of HTTP redirects to follow. Defaults to the value of the MAX_REDIRECTS env var or #{TitleGrabber::MAX_REDIRECTS}") do |redirects|
-    arguments[:max_redirects] = redirects
-  end
-
   args.on("-r", "--max-retries RETRIES", Integer, "Max. # of times to retry failed HTTP reqs. Defaults to the value of the MAX_RETRIES env var or #{TitleGrabber::MAX_RETRIES}") do |retries|
     arguments[:max_retries] = retries
   end

data/lib/http_helper.rb

@@ -1,35 +1,47 @@
-require "http"
+require "openssl"
+require "open-uri"
+require "timeout"
 
 require_relative "text_helper"
 
 module HTTPHelper
   INVALID_BYTE_SEQ = "invalid byte sequence".freeze
-  CONNECTION_ERRORS = ["SSL_connect", "Connection reset"].freeze
+  CONNECTION_ERRORS = ["SSL_connect", "Connection reset",
+                       "execution expired"].freeze
   REST_INTERVAL = 0.5..1
+  # redirection forbidden: https://t.co/blui5zKJjD -> http://bit.ly/2HXRYGw (RuntimeError)
+  REDIR_FORBIDDEN = /redirection forbidden/
+  HTTP_REDIR = %r(-> (http://\S+))
+  SRV_UNAVAILABLE = 503
 
   include TextHelper
 
-  def open_w_timeout(url, write_to:, connect_to:, read_to:, max_retries:)
+  def open_w_timeout(url, connect_to:, read_to:, max_retries:)
     logger.info "[#{Thread.current.name}] GET #{url}"
     retries = 0
 
     begin
       res = Timeout.timeout(read_to) {
-              HTTP.timeout(write: write_to, connect: connect_to, read: read_to).
-                   follow(max_hops: max_redirects).
-                   get(url, ssl_context: ssl_ctx)
+              open(url, ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE,
+                        open_timeout: connect_to, read_timeout: read_to)
             }
-    rescue HTTP::Redirector::TooManyRedirectsError
-      logger.warn "[#{Thread.current.name}] GET #{url} resulted in more than #{max_redirects} redirect#{'s' unless max_redirects == 1}"
-      nil
     rescue => err
       msg = err.message
+      if msg =~ REDIR_FORBIDDEN
+        url = msg[HTTP_REDIR, 1]
+        url ? retry : return
+      end
 
-      if err.kind_of?(HTTP::Error) || err.kind_of?(Timeout::Error) ||
+      if err.is_a?(OpenURI::HTTPError) || err.is_a?(Timeout::Error) ||
            CONNECTION_ERRORS.any? { |e| msg.start_with?(e) }
         retries += 1
 
-        if retries <= max_retries
+        will_retry = retries <= max_retries
+        if will_retry && err.is_a?(OpenURI::HTTPError)
+          will_retry = Integer(err.io.status.first) == SRV_UNAVAILABLE
+        end
+
+        if will_retry
           rest_time = rand(REST_INTERVAL)
           logger.warn "[#{Thread.current.name}] URL: #{url} [#{msg}] - Going to sleep for #{rest_time.round(1)} secs - Retry ##{retries}"
           sleep(rest_time)
@@ -45,20 +57,10 @@ module HTTPHelper
     end
   end
 
-  def read_w_timeout(url, write_to:, connect_to:, read_to:, max_retries:)
-    if res = open_w_timeout(url, write_to: write_to, connect_to: connect_to,
-                                 read_to: read_to, max_retries: max_retries)
-      [res.uri.to_s, utf8_encode(res.to_s)]
+  def read_w_timeout(url, connect_to:, read_to:, max_retries:)
+    if res = open_w_timeout(url, connect_to: connect_to, read_to: read_to,
+                                 max_retries: max_retries)
+      [res.base_uri.to_s, utf8_encode(res.read)]
     end
   end
-
-  private
-
-  def ssl_ctx
-    @ssl_ctx ||= begin
-                  ctx = OpenSSL::SSL::SSLContext.new
-                  ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
-                  ctx
-                  end
-  end
 end

data/lib/title_grabber/version.rb

@@ -1,3 +1,3 @@
 module TitleGrabber
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/title_grabber.rb

@@ -13,11 +13,9 @@ require_relative "text_helper"
 
 module TitleGrabber
   DEF_OUT_PATH = Pathname('out.csv')
-  CONNECT_TO = 15
-  READ_TO = 15
-  WRITE_TO = 15
-  MAX_REDIRECTS = 5
-  MAX_RETRIES = 5
+  CONNECT_TO = 30
+  READ_TO = 30
+  MAX_RETRIES = 3
   MAX_THREADS = Etc.nprocessors
   URL_RE = %r(https?://\S+)i
   URL_HEADER = -"url"
@@ -41,8 +39,8 @@ module TitleGrabber
     include HTTPHelper
     include TextHelper
 
-    attr_reader :file_paths, :out_path, :tmp_path, :connect_to, :read_to, :write_to,
-                :max_redirects, :max_retries, :max_threads, :logger
+    attr_reader :file_paths, :out_path, :tmp_path, :connect_to, :read_to,
+                :max_retries, :max_threads, :logger
 
     def initialize(options)
       @file_paths = options[:file_paths]
@@ -52,10 +50,8 @@ module TitleGrabber
 
       @connect_to = options.fetch(:connect_to, CONNECT_TO)
       @read_to = options.fetch(:read_to, READ_TO)
-      @write_to = options.fetch(:write_to, WRITE_TO)
-      @max_redirects = options.fetch(:max_redirects, MAX_REDIRECTS)
       @max_retries = options.fetch(:max_retries, MAX_RETRIES)
-      @max_threads = options.fetch(:max_th, Etc.nprocessors)
+      @max_threads = options.fetch(:max_threads, Etc.nprocessors)
 
       logging_target = if options[:debug]
                          STDOUT
@@ -118,7 +114,7 @@ module TitleGrabber
                   tweet_urls.uniq!
                   tweet_urls.map! do |url|
                     if url.match?(URL_RE) && (res = open_w_timeout(url, **http_opts))
-                      uri = res.uri
+                      uri = res.base_uri
                       uri.host == TWITTER_HOST && !uri.to_s.match?(TWITTER_STATUS_RE) ? nil : uri.to_s
                     else
                       url
@@ -192,7 +188,7 @@ module TitleGrabber
 
     def http_opts
       @http_opts ||= { connect_to: connect_to, read_to: read_to,
-                       write_to: write_to, max_retries: max_retries }
+                       max_retries: max_retries }
     end
   end
 end

data/title_grabber.gemspec

@@ -38,7 +38,6 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = "~> 2.3"
 
-  spec.add_runtime_dependency "http", "~> 4.1"
   spec.add_runtime_dependency "oga", "~> 2.15"
 
   spec.add_development_dependency "bundler", "~> 1.17"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: title_grabber
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Cristian Rasch
@@ -10,20 +10,6 @@ bindir: exe
 cert_chain: []
 date: 2019-04-18 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: http
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement