nethttputils 0.4.1.1 → 0.4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c68a49af2dbbca3cacf81cd5518df2ce6f2cb72
-  data.tar.gz: bae1db040743d7f9ea9ac17d7aeafba778f227c5
+  metadata.gz: ecb7a8e00f832f9c1fa1f79610772f8f955a177f
+  data.tar.gz: ee617bf653fca7736e95e257929ebb7a597872a1
 SHA512:
-  metadata.gz: 8ba54aa8a5fe007911b8af3117061b1a279182eb27cbdcd2be0a6293428d4408356b2b8238c7342284e93da9ee20f62d169b79346f1f30019677c24c0ca5c9af
-  data.tar.gz: 23e5657489821b849a611a3692d14335b46c1f367dfff1c130d862a1390eb5234742110cdda9d69bb06f9206ae98ed0fdf6dfa6d66fc7056bf45dacd62a0ca13
+  metadata.gz: e3632e1d20d3d853f2b7542364785ce5ba4ca8b4ff25aa2304856061bbe133cdd145ac18843a1bd52af275939e9f8354223c0d37dac6d64df5791dc2f21fc9db
+  data.tar.gz: 90e9b8a75802bd2d48cb727cf5c28ac4aa1c481e4af0afdc90d4e5b0f251e36293867819ef788774db550b92c0796482024913d313b7475e82737440c2685197

data/lib/nethttputils.rb

@@ -33,7 +33,8 @@ module NetHTTPUtils
           gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
     end
 
-    def start_http url, max_start_http_retry_delay = 3600, timeout = 30, proxy = nil
+    def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
+      timeout ||= 30
       uri = url
       uri = URI.parse begin
         URI url
@@ -100,14 +101,17 @@ module NetHTTPUtils
     end
 
     private
-    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_read_retry_delay: 3600, patch_request: nil, &block
-          logger = NetHTTPUtils.logger
+    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, force_post: false, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
+      timeout ||= 30
+      logger = NetHTTPUtils.logger
+      logger.info [mtd, http].inspect
 
           uri = http.instance_variable_get :@uri
-          logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query && !form.empty?
+      if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty?  # not .upcase! because it's not defined for Symbol
+        logger.debug "Warning: query params included in `url` argument are discarded because `:form` isn't empty" if uri.query
           # we can't just merge because URI fails to parse such queries as "/?1"
-
-          uri.query = URI.encode_www_form form if %i{ HEAD GET }.include?(mtd = mtd.upcase) && !form.empty?
+        uri.query = URI.encode_www_form form
+      end
           cookies = {}
           prepare_request = lambda do |uri|
             case mtd.upcase
@@ -126,15 +130,20 @@ module NetHTTPUtils
               request.basic_auth *auth if auth
               if (mtd == :POST || mtd == :PATCH) && !form.empty?
                 case type
-                  when :json ; request.body = JSON.dump form
-                               request.content_type = "application/json"
-                  when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
+                  when :json
+                                    request.body = JSON.dump form
+                                    request.content_type = "application/json"
+                  when :multipart
+                    request.set_form form, "multipart/form-data"
+                  when :form
+                               if form.any?{ |k, v| v.respond_to? :to_path }
                                  request.set_form form, "multipart/form-data"
                                else
                                  request.set_form_data form
                                  request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
                                end
-                  else       ; raise "unknown content-type '#{type}'"
+                  else
+                    raise "unknown content-type '#{type}'"
                 end
               end
               header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
@@ -177,7 +186,7 @@ module NetHTTPUtils
             rescue EOFError => e
               raise unless e.backtrace.empty?
               # https://bugs.ruby-lang.org/issues/13018
-              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html?spref=reddit
+              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
               raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
             end
             # response.instance_variable_set "@nethttputils_close", http.method(:finish)
@@ -226,8 +235,12 @@ module NetHTTPUtils
               response.add_field "Set-Cookie", "#{k}=#{v}"
             end
 
+            logger.info "response.code = #{response.code}"
             case response.code
+            when /\A20/
+              response
             when /\A30\d\z/
+              next response if no_redirect
               logger.info "redirect: #{response["location"]}"
               require "addressable"
               new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
@@ -238,10 +251,10 @@ module NetHTTPUtils
                  http.use_ssl? != (new_uri.scheme == "https")
                 logger.debug "changing host from '#{http.address}' to '#{new_host}'"
                 # http.finish   # why commented out?
-                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
+                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
               end
-              if request.method == "POST"
-                logger.info "POST redirects to GET (RFC)"
+              if !force_post && request.method == "POST"
+                logger.info "POST redirects to GET (RFC)"   # TODO: do it only on code 307; note that some servers still do 302
                 mtd = :GET
               end
               do_request.call prepare_request[new_uri]
@@ -274,11 +287,10 @@ module NetHTTPUtils
                 end
               }"
               response
-            when /\A20/
-              response
             else
-              logger.warn "code #{response.code} at #{request.method} #{request.uri} from #{
-                [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
+              logger.warn "code #{response.code} from #{request.method} #{request.uri} at #{
+                caller_path, caller_locs = caller_locations.chunk(&:path).first
+                [caller_path, caller_locs.map(&:lineno).chunk(&:itself).map(&:first)].join ":"
               }"
               logger.debug "< body: #{
                 response.body.tap do |body|
@@ -296,12 +308,11 @@ module NetHTTPUtils
 
     require "set"
     @@_405 ||= Set.new
-    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil,
-        timeout: 30,
-        max_start_http_retry_delay: 3600,
-        max_read_retry_delay: 3600,
+    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil, force_post: false, no_redirect: false, head: false,
+        timeout: nil, max_start_http_retry_delay: 3600, max_read_retry_delay: 3600,
         patch_request: nil, &block
-      http = start_http http, max_start_http_retry_delay, timeout, *proxy unless http.is_a? Net::HTTP
+      timeout ||= 30
+      http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
       path = http.instance_variable_get(:@uri).path
 
       check_code = lambda do |body|
@@ -318,7 +329,7 @@ module NetHTTPUtils
             )
         end
       end
-      if mtd == :GET && !@@_405.include?(http.address)
+      if head && mtd == :GET && !@@_405.include?(http.address)
         body = begin
           request_data http, :HEAD, form: form, header: header, auth: auth,
             max_start_http_retry_delay: max_start_http_retry_delay,
@@ -332,7 +343,8 @@ module NetHTTPUtils
           check_code.call body
         end
       end
-      body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
+      body = read http, mtd, type, form: form, header: header, auth: auth, force_post: force_post,
+        timeout: timeout, no_redirect: no_redirect,
         max_read_retry_delay: max_read_retry_delay,
         patch_request: patch_request, &block
       check_code.call body
@@ -342,7 +354,6 @@ module NetHTTPUtils
         Zlib::GzipReader.new(StringIO.new(body)).read
       else
         body
-      end.tap do |string|
       end
     # ensure
     #   response.instance_variable_get("@nethttputils_close").call if response
@@ -406,6 +417,7 @@ if $0 == __FILE__
   server.shutdown
   t.join
 
+  # HEAD should raise on 404 but not in two other cases
   [
     [WEBrick::HTTPStatus::NotFound, 404],
     [WEBrick::HTTPStatus::BadRequest],
@@ -417,7 +429,7 @@ if $0 == __FILE__
     end
     t = Thread.new{ server.start }
     begin
-      NetHTTPUtils.request_data "http://localhost:8000/"
+      NetHTTPUtils.request_data "http://localhost:8000/", head: true
       NetHTTPUtils.class_variable_get(:@@_405).clear
       fail if should_raise
     rescue NetHTTPUtils::Error => e
@@ -442,7 +454,7 @@ if $0 == __FILE__
   NetHTTPUtils.request_data("http://localhost:8000/", :head)
   fail stack.inspect unless stack == %w{ HEAD }
   stack.clear
-  NetHTTPUtils.request_data("http://localhost:8000/")
+  NetHTTPUtils.request_data("http://localhost:8000/", head: true)
   fail stack.inspect unless stack == %w{ HEAD GET }
   server.shutdown
   t.join
@@ -509,7 +521,7 @@ if $0 == __FILE__
     http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg
   }.each do |url|
     begin
-      NetHTTPUtils.request_data url, timeout: 5, max_read_retry_delay: -1
+      NetHTTPUtils.request_data url, timeout: 5, max_read_retry_delay: -1, head: true
       fail
     rescue Net::ReadTimeout
     end

data/nethttputils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "nethttputils"
-  spec.version      = "0.4.1.1"
+  spec.version      = "0.4.3.0"
   spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
   spec.description = <<-EOF
     Back in 2015 I was a guy automating things at my job and two scripts had a common need --

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nethttputils
 version: !ruby/object:Gem::Version
-  version: 0.4.1.1
+  version: 0.4.3.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-07 00:00:00.000000000 Z
+date: 2021-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable