nethttputils 0.3.3.0 → 0.4.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9afa291d86c40f37b207744ca23c7e12c330e055
-  data.tar.gz: 3afb6030d76642efed329bdf322c68dbb3eb2d00
+  metadata.gz: aa8c5d5ae78a7dbad64fe4d0e624666cfcbe1642
+  data.tar.gz: 8d09dcf215b50515dc6033f5ef6b6760f14c844c
 SHA512:
-  metadata.gz: a91b3cf20cbb611ff5e77d8efef83b37c01cabe413431dfb7edf7df9fe4c5eae4d92cf177891b04fd2e4a96b399871fcdc9b9afc2d2dd9926101b2321da5f28c
-  data.tar.gz: 137f44eee308529b23e5c6d9c592801950ddca31e156a6e0010b0a98f34cd8f758d6810143d34c0994b3ff17e42f1cc8a6d7608cdba9b7638c143781822732fa
+  metadata.gz: c315180c06335286428ffb1aff4c41ba6b1648644dc2c41b3f199c8fe2102238e0323359590bb74db31b487e4499b27c5d9823c0a300bae1eafe9f114a62cc59
+  data.tar.gz: 07b47969d1d87ffd4bdd409b038aef316b770f5d3b4d00d7ad0faa419ac559dfa362548c5f942df3300629a6ba9f5ef630370412e73a80afaef785dfde938bb3

data/lib/nethttputils.rb

@@ -18,19 +18,23 @@ module NetHTTPUtils
     attr_reader :code
     def initialize body, code = nil
       @code = code
+      body = body[0...997] + "..." if body.size > 1000
       super "HTTP error ##{code.inspect} #{body}"
     end
   end
+  class EOFError_from_rbuf_fill < StandardError
+  end
 
   class << self
 
     def remove_tags str
-      str.gsub(/<script( [a-z]+="[^"]*")*>.*?<\/script>/m, "").
-          gsub(/<style( [a-z]+="[^"]*")*>.*?<\/style>/m, "").
+      str.gsub(/<script( [a-z-]+="[^"]*")*>.*?<\/script>/m, "").
+          gsub(/<style( [a-z-]+="[^"]*")*>.*?<\/style>/m, "").
           gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
     end
 
-    def start_http url, max_start_http_retry_delay = 3600, timeout = 30
+    def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
+      timeout ||= 30
       uri = url
       uri = URI.parse begin
         URI url
@@ -42,6 +46,7 @@ module NetHTTPUtils
       begin
         Net::HTTP.start(
           uri.host, uri.port,
+          *(proxy.split ?: if proxy),
           use_ssl: uri.scheme == "https",
           verify_mode: OpenSSL::SSL::VERIFY_NONE,
           **({open_timeout: timeout}), #  if timeout
@@ -54,7 +59,7 @@ module NetHTTPUtils
               def << msg
                 @@buffer ||= "[Net::HTTP debug] "
                 @@buffer.concat msg
-                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 500
+                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 1000
                 return unless @@buffer.end_with? ?\n
                 NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
                 @@buffer = nil
@@ -63,7 +68,7 @@ module NetHTTPUtils
           end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
           http
         end
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
           raise
@@ -71,10 +76,6 @@ module NetHTTPUtils
         logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
         sleep delay
         retry
-      rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
-        logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
-        sleep 5
-        retry
       rescue SocketError => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
@@ -100,7 +101,8 @@ module NetHTTPUtils
     end
 
     private
-    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_read_retry_delay: 3600, patch_request: nil, &block
+    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
+      timeout ||= 30
           logger = NetHTTPUtils.logger
 
           uri = http.instance_variable_get :@uri
@@ -126,15 +128,20 @@ module NetHTTPUtils
               request.basic_auth *auth if auth
               if (mtd == :POST || mtd == :PATCH) && !form.empty?
                 case type
-                  when :json ; request.body = JSON.dump form
-                               request.content_type = "application/json"
-                  when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
+                  when :json
+                                    request.body = JSON.dump form
+                                    request.content_type = "application/json"
+                  when :multipart
+                    request.set_form form, "multipart/form-data"
+                  when :form
+                               if form.any?{ |k, v| v.respond_to? :to_path }
                                  request.set_form form, "multipart/form-data"
                                else
                                  request.set_form_data form
                                  request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
                                end
-                  else       ; raise "unknown content-type '#{type}'"
+                  else
+                    raise "unknown content-type '#{type}'"
                 end
               end
               header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
@@ -155,7 +162,7 @@ module NetHTTPUtils
                 request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join
               }#{curl_form}'#{uri.scheme}://#{uri.host}#{uri.path}#{"?#{uri.query}" if uri.query && !uri.query.empty?}'"
               logger.debug "> header: #{request.each_header.to_a}"
-              logger.debug "> body: #{request.body.inspect.tap{ |body| body[997..-1] = "..." if body.size > 500 }}"
+              logger.debug "> body: #{request.body.inspect.tap{ |body| body.replace body[0...997] + "..." if body.size > 1000 }}"
               # TODO this is buggy -- mixes lines from different files into one line
               stack = caller.reverse.map do |level|
                 /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
@@ -169,15 +176,21 @@ module NetHTTPUtils
             delay = 5
             response = begin
               http.request request, &block
-            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
               raise if max_read_retry_delay < delay *= 2
               logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
               sleep delay
               retry
+            rescue EOFError => e
+              raise unless e.backtrace.empty?
+              # https://bugs.ruby-lang.org/issues/13018
+              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
+              raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
             end
             # response.instance_variable_set "@nethttputils_close", http.method(:finish)
             # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
 
+            now = Time.now
             remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
               logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
               [
@@ -189,7 +202,13 @@ module NetHTTPUtils
               [
                 response.fetch("x-rate-limit-remaining").to_i,
                 response.fetch("x-rate-limit-reset").to_i,
-                Time.now.to_i,
+                now.to_i,
+              ]
+            elsif response.key? "x-ratelimit-remaining"
+              [
+                response.fetch("x-ratelimit-remaining").to_i,
+                now + response.fetch("x-ratelimit-reset").to_i,
+                now.to_i,
               ]
             end
             if remaining
@@ -215,9 +234,13 @@ module NetHTTPUtils
             end
 
             case response.code
+            when /\A20/
+              response
             when /\A30\d\z/
+              next response if no_redirect
               logger.info "redirect: #{response["location"]}"
-              new_uri = URI.join request.uri, response["location"]
+              require "addressable"
+              new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
               new_host = new_uri.host
               raise Error.new "redirected in place" if new_uri == http.instance_variable_get(:@uri)
               if http.address != new_host ||
@@ -225,7 +248,7 @@ module NetHTTPUtils
                  http.use_ssl? != (new_uri.scheme == "https")
                 logger.debug "changing host from '#{http.address}' to '#{new_host}'"
                 # http.finish   # why commented out?
-                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
+                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
               end
               if request.method == "POST"
                 logger.info "POST redirects to GET (RFC)"
@@ -261,8 +284,6 @@ module NetHTTPUtils
                 end
               }"
               response
-            when /\A20/
-              response
             else
               logger.warn "code #{response.code} at #{request.method} #{request.uri} from #{
                 [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
@@ -283,11 +304,13 @@ module NetHTTPUtils
 
     require "set"
     @@_405 ||= Set.new
-    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
+    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil,
+        timeout: nil, no_redirect: false,
         max_start_http_retry_delay: 3600,
         max_read_retry_delay: 3600,
         patch_request: nil, &block
-      http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
+      timeout ||= 30
+      http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
       path = http.instance_variable_get(:@uri).path
 
       check_code = lambda do |body|
@@ -318,7 +341,8 @@ module NetHTTPUtils
           check_code.call body
         end
       end
-      body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
+      body = read http, mtd, type, form: form, header: header, auth: auth,
+        timeout: timeout, no_redirect: no_redirect,
         max_read_retry_delay: max_read_retry_delay,
         patch_request: patch_request, &block
       check_code.call body
@@ -467,16 +491,16 @@ if $0 == __FILE__
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/400")) == "400 Bad Request"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/404")) == "404 Not Found"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/500")) == "500 Internal Server Error"
-  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")) == "502 Bad Gateway"
+  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
   [
-    ["https://imgur.com/a/cccccc"],
-    ["https://imgur.com/mM4Dh7Z"],
+    # ["https://imgur.com/a/oacI3gl"],  # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
+    # ["https://imgur.com/mM4Dh7Z"],    # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
     ["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
   ].each do |url, expectation|
     begin
       puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
-      fail
+      fail url
     rescue NetHTTPUtils::Error => e
       raise e.code.inspect unless e.code == 404
       raise e.to_s if e.to_s != expectation if expectation

data/nethttputils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "nethttputils"
-  spec.version      = "0.3.3.0"
+  spec.version      = "0.4.1.3"
   spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
   spec.description = <<-EOF
     Back in 2015 I was a guy automating things at my job and two scripts had a common need --
@@ -21,5 +21,7 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
 
   spec.require_path = "lib"
-  spec.files        = `git ls-files -z`.split(?\0) - spec.test_files
+  spec.files        = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
+
+  spec.add_dependency "addressable"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: nethttputils
 version: !ruby/object:Gem::Version
-  version: 0.3.3.0
+  version: 0.4.1.3
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
-dependencies: []
+date: 2021-06-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |2
       Back in 2015 I was a guy automating things at my job and two scripts had a common need --
       they both had to pass the same credentials to Jenkins (via query params, I guess).
@@ -26,9 +40,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".travis.yml"
 - LICENSE
-- Rakefile
 - lib/nethttputils.rb
 - nethttputils.gemspec
 homepage: https://github.com/nakilon/nethttputils

data/.travis.yml

@@ -1,21 +0,0 @@
----
-language: ruby
-
-script: "ruby lib/nethttputils.rb"
-
-os:
-  - linux
-  - osx
-rvm:
-  - ruby-head
-  - 2.5
-  - 2.4
-  - 2.3
-  - 2.2
-  - 2.1
-  - 2.0
-  - jruby-head
-matrix:
-  allow_failures:
-  # something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
-  - rvm: jruby-head

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"