nethttputils 0.4.1.0 → 0.4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff2b86fcc4b55f5f9ea7818eb61b47458da62fa0
-  data.tar.gz: 610d74457c21418c24b092a65ba47abfd5571b37
+  metadata.gz: 6b310f69be9a582878a60741eb385b2cbc61884b
+  data.tar.gz: fee129582c51e0ec02abc63e318a681534edc391
 SHA512:
-  metadata.gz: 819dc2799c568c4b4aa4d199d14e3f5a0b70253bc0eb6831205482274abb2bbe90f7fb8504d15f6e4561dfa2d1e7d464a4534943a2d6ebd188d7e345752cfc39
-  data.tar.gz: cde0231d2c7779642ecf68d052cc117178b9905ba8a1bf3427d7c901050f05b75d290624fda7c5a2e2a2f4ed5fc7114390aad00bcdcbc45bab8783d2ee844ff0
+  metadata.gz: d7f628d6558dbe9aa0d08125c5a294c957235858829013ac4dd655b0b28fc63e38bed1827ce35fbb9de1d66ca6acde828fa3a5d77ea90a3e08238751cab9b0ea
+  data.tar.gz: ea174acf8e166ef034d6d151b171babdb19fa3736cd23fbdb006c460deb28236f1ce33c1bbc72194d30695498ccc1292108063b6319b54eace9631a5b3e68bc7

data/lib/nethttputils.rb

@@ -33,7 +33,8 @@ module NetHTTPUtils
           gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
     end
 
-    def start_http url, max_start_http_retry_delay = 3600, timeout = 30
+    def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
+      timeout ||= 30
       uri = url
       uri = URI.parse begin
         URI url
@@ -45,6 +46,7 @@ module NetHTTPUtils
       begin
         Net::HTTP.start(
           uri.host, uri.port,
+          *(proxy.split ?: if proxy),
           use_ssl: uri.scheme == "https",
           verify_mode: OpenSSL::SSL::VERIFY_NONE,
           **({open_timeout: timeout}), #  if timeout
@@ -66,7 +68,7 @@ module NetHTTPUtils
           end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
           http
         end
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
           raise
@@ -74,10 +76,6 @@ module NetHTTPUtils
         logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
         sleep delay
         retry
-      rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
-        logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
-        sleep 5
-        retry
       rescue SocketError => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
@@ -103,14 +101,17 @@ module NetHTTPUtils
     end
 
     private
-    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_read_retry_delay: 3600, patch_request: nil, &block
-          logger = NetHTTPUtils.logger
+    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, force_post: false, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
+      timeout ||= 30
+      logger = NetHTTPUtils.logger
+      logger.info [mtd, http].inspect
 
           uri = http.instance_variable_get :@uri
-          logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query && !form.empty?
+      if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty?  # not .upcase! because it's not defined for Symbol
+        logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query
           # we can't just merge because URI fails to parse such queries as "/?1"
-
-          uri.query = URI.encode_www_form form if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty?
+        uri.query = URI.encode_www_form form
+      end
           cookies = {}
           prepare_request = lambda do |uri|
             case mtd.upcase
@@ -129,15 +130,20 @@ module NetHTTPUtils
               request.basic_auth *auth if auth
               if (mtd == :POST || mtd == :PATCH) && !form.empty?
                 case type
-                  when :json ; request.body = JSON.dump form
-                               request.content_type = "application/json"
-                  when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
+                  when :json
+                                    request.body = JSON.dump form
+                                    request.content_type = "application/json"
+                  when :multipart
+                    request.set_form form, "multipart/form-data"
+                  when :form
+                               if form.any?{ |k, v| v.respond_to? :to_path }
                                  request.set_form form, "multipart/form-data"
                                else
                                  request.set_form_data form
                                  request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
                                end
-                  else       ; raise "unknown content-type '#{type}'"
+                  else
+                    raise "unknown content-type '#{type}'"
                 end
               end
               header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
@@ -172,7 +178,7 @@ module NetHTTPUtils
             delay = 5
             response = begin
               http.request request, &block
-            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
+            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
               raise if max_read_retry_delay < delay *= 2
               logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
               sleep delay
@@ -180,12 +186,13 @@ module NetHTTPUtils
             rescue EOFError => e
               raise unless e.backtrace.empty?
               # https://bugs.ruby-lang.org/issues/13018
-              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html?spref=reddit
+              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
               raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
             end
             # response.instance_variable_set "@nethttputils_close", http.method(:finish)
             # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
 
+            now = Time.now
             remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
               logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
               [
@@ -197,7 +204,13 @@ module NetHTTPUtils
               [
                 response.fetch("x-rate-limit-remaining").to_i,
                 response.fetch("x-rate-limit-reset").to_i,
-                Time.now.to_i,
+                now.to_i,
+              ]
+            elsif response.key? "x-ratelimit-remaining"
+              [
+                response.fetch("x-ratelimit-remaining").to_i,
+                now + response.fetch("x-ratelimit-reset").to_i,
+                now.to_i,
               ]
             end
             if remaining
@@ -222,8 +235,12 @@ module NetHTTPUtils
               response.add_field "Set-Cookie", "#{k}=#{v}"
             end
 
+            logger.info "response.code = #{response.code}"
             case response.code
+            when /\A20/
+              response
             when /\A30\d\z/
+              next response if no_redirect
               logger.info "redirect: #{response["location"]}"
               require "addressable"
               new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
@@ -234,10 +251,10 @@ module NetHTTPUtils
                  http.use_ssl? != (new_uri.scheme == "https")
                 logger.debug "changing host from '#{http.address}' to '#{new_host}'"
                 # http.finish   # why commented out?
-                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
+                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
               end
-              if request.method == "POST"
-                logger.info "POST redirects to GET (RFC)"
+              if !force_post && request.method == "POST"
+                logger.info "POST redirects to GET (RFC)"   # TODO: do it only on code 307; note that some servers still do 302
                 mtd = :GET
               end
               do_request.call prepare_request[new_uri]
@@ -270,11 +287,10 @@ module NetHTTPUtils
                 end
               }"
               response
-            when /\A20/
-              response
             else
-              logger.warn "code #{response.code} at #{request.method} #{request.uri} from #{
-                [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
+              logger.warn "code #{response.code} from #{request.method} #{request.uri} at #{
+                caller_path, caller_locs = caller_locations.chunk(&:path).first
+                [caller_path, caller_locs.map(&:lineno).chunk(&:itself).map(&:first)].join ":"
               }"
               logger.debug "< body: #{
                 response.body.tap do |body|
@@ -292,11 +308,13 @@ module NetHTTPUtils
 
     require "set"
     @@_405 ||= Set.new
-    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
+    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false,
+        timeout: nil, no_redirect: false,
         max_start_http_retry_delay: 3600,
         max_read_retry_delay: 3600,
         patch_request: nil, &block
-      http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
+      timeout ||= 30
+      http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
       path = http.instance_variable_get(:@uri).path
 
       check_code = lambda do |body|
@@ -327,7 +345,8 @@ module NetHTTPUtils
           check_code.call body
         end
       end
-      body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
+      body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
+        timeout: timeout, no_redirect: no_redirect,
         max_read_retry_delay: max_read_retry_delay,
         patch_request: patch_request, &block
       check_code.call body
@@ -337,7 +356,6 @@ module NetHTTPUtils
         Zlib::GzipReader.new(StringIO.new(body)).read
       else
         body
-      end.tap do |string|
       end
     # ensure
     #   response.instance_variable_get("@nethttputils_close").call if response
@@ -479,13 +497,13 @@ if $0 == __FILE__
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
   [
-    ["https://imgur.com/a/cccccc"],
-    ["https://imgur.com/mM4Dh7Z"],
+    # ["https://imgur.com/a/oacI3gl"],  # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
+    # ["https://imgur.com/mM4Dh7Z"],    # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
     ["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
   ].each do |url, expectation|
     begin
       puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
-      fail
+      fail url
     rescue NetHTTPUtils::Error => e
       raise e.code.inspect unless e.code == 404
       raise e.to_s if e.to_s != expectation if expectation

data/nethttputils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "nethttputils"
-  spec.version      = "0.4.1.0"
+  spec.version      = "0.4.2.0"
   spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
   spec.description = <<-EOF
     Back in 2015 I was a guy automating things at my job and two scripts had a common need --
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
 
   spec.require_path = "lib"
-  spec.files        = `git ls-files -z`.split(?\0) - spec.test_files
+  spec.files        = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
 
   spec.add_dependency "addressable"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nethttputils
 version: !ruby/object:Gem::Version
-  version: 0.4.1.0
+  version: 0.4.2.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-23 00:00:00.000000000 Z
+date: 2021-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -40,10 +40,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".travis.yml"
-- Gemfile
 - LICENSE
-- Rakefile
 - lib/nethttputils.rb
 - nethttputils.gemspec
 homepage: https://github.com/nakilon/nethttputils

data/.travis.yml

@@ -1,29 +0,0 @@
----
-language: ruby
-
-script: "bundle install && bundle exec ruby lib/nethttputils.rb"
-
-os:
-  - linux
-  - osx
-rvm:
-  - ruby-head
-  - 2.5
-  - 2.4
-  - 2.3
-  - 2.2
-  - 2.1
-  - 2.0
-  - jruby-head
-matrix:
-  allow_failures:
-  # something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
-  - rvm: jruby-head
-  - rvm: 2.3
-    os: osx
-  - rvm: 2.2
-    os: osx
-  - rvm: 2.1
-    os: osx
-  - rvm: 2.0
-    os: osx

data/Gemfile

@@ -1,3 +0,0 @@
-source "https://rubygems.org"
-gem "byebug"
-gemspec

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"