nethttputils 0.3.2.11 → 0.4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bc1f0212b03ef427e313966fff8e8431f35b0fd
-  data.tar.gz: 1751888f47d8d2252a52549ec96126c768ebcbba
+  metadata.gz: 3a982e237dae69aa1f18645e6e63d231bb22d5e3
+  data.tar.gz: 71f6c6e79050d656e9bb8c290caa11e013acf386
 SHA512:
-  metadata.gz: a047da71bab1fdfa59c4dc1281654ca95914327ebf248080e884d414fdbce39ab0820c84199a359152c29c7f8a749993e9ddcad68238c44bea8a186b29ebf945
-  data.tar.gz: 12a74c116dd99b6d0e9c98f5cc4a2cb6f7c33b31cb939478b1e7c4763e8bf52108405d94200da74bdff672bc94870da1bba6d8d81186220e2b9e673a19ecbdec
+  metadata.gz: e14f163d07b052595d502d07a3d104bcee3c6fbefc6169495f778e589dc80562817912f7fec3183741fa56379a7c6d6bd9fbd0ab3b8b05cb1bae9b2b671eacc5
+  data.tar.gz: 6d8df37fb3fedc5ca757fe8345338421a55ff15283ad5f2456a4a1559de9bc6fe98551aabc56549fc27006a014e1d1702295f1bc19f8984d5eed1b7d82324579

data/lib/nethttputils.rb

@@ -18,19 +18,23 @@ module NetHTTPUtils
     attr_reader :code
     def initialize body, code = nil
       @code = code
+      body = body[0...997] + "..." if body.size > 1000
       super "HTTP error ##{code.inspect} #{body}"
     end
   end
+  class EOFError_from_rbuf_fill < StandardError
+  end
 
   class << self
 
     def remove_tags str
-      str.gsub(/<script( [a-z]+="[^"]*")*>.*?<\/script>/m, "").
-          gsub(/<style( [a-z]+="[^"]*")*>.*?<\/style>/m, "").
+      str.gsub(/<script( [a-z-]+="[^"]*")*>.*?<\/script>/m, "").
+          gsub(/<style( [a-z-]+="[^"]*")*>.*?<\/style>/m, "").
           gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
     end
 
-    def start_http url, max_start_http_retry_delay = 3600, timeout = 30
+    def start_http url, max_start_http_retry_delay = 3600, timeout = nil, no_redirect = false, proxy = nil
+      timeout ||= 30
       uri = url
       uri = URI.parse begin
         URI url
@@ -42,6 +46,7 @@ module NetHTTPUtils
       begin
         Net::HTTP.start(
           uri.host, uri.port,
+          *(proxy.split ?: if proxy),
           use_ssl: uri.scheme == "https",
           verify_mode: OpenSSL::SSL::VERIFY_NONE,
           **({open_timeout: timeout}), #  if timeout
@@ -54,7 +59,7 @@ module NetHTTPUtils
               def << msg
                 @@buffer ||= "[Net::HTTP debug] "
                 @@buffer.concat msg
-                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 500
+                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 1000
                 return unless @@buffer.end_with? ?\n
                 NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
                 @@buffer = nil
@@ -63,7 +68,7 @@ module NetHTTPUtils
           end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
           http
         end
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
           raise
@@ -71,10 +76,6 @@ module NetHTTPUtils
         logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
         sleep delay
         retry
-      rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
-        logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
-        sleep 5
-        retry
       rescue SocketError => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
@@ -100,7 +101,8 @@ module NetHTTPUtils
     end
 
     private
-    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_read_retry_delay: 3600, patch_request: nil, &block
+    def read http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: nil, no_redirect: false, max_read_retry_delay: 3600, patch_request: nil, &block
+      timeout ||= 30
           logger = NetHTTPUtils.logger
 
           uri = http.instance_variable_get :@uri
@@ -126,15 +128,20 @@ module NetHTTPUtils
               request.basic_auth *auth if auth
               if (mtd == :POST || mtd == :PATCH) && !form.empty?
                 case type
-                  when :json ; request.body = JSON.dump form
-                               request.content_type = "application/json"
-                  when :form ; if form.any?{ |k, v| v.respond_to? :to_path }
+                  when :json
+                                    request.body = JSON.dump form
+                                    request.content_type = "application/json"
+                  when :multipart
+                    request.set_form form, "multipart/form-data"
+                  when :form
+                               if form.any?{ |k, v| v.respond_to? :to_path }
                                  request.set_form form, "multipart/form-data"
                                else
                                  request.set_form_data form
                                  request.content_type = "application/x-www-form-urlencoded;charset=UTF-8"
                                end
-                  else       ; raise "unknown content-type '#{type}'"
+                  else
+                    raise "unknown content-type '#{type}'"
                 end
               end
               header.each{ |k, v| request[k.to_s] = v.is_a?(Array) ? v.first : v }
@@ -155,7 +162,7 @@ module NetHTTPUtils
                 request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join
               }#{curl_form}'#{uri.scheme}://#{uri.host}#{uri.path}#{"?#{uri.query}" if uri.query && !uri.query.empty?}'"
               logger.debug "> header: #{request.each_header.to_a}"
-              logger.debug "> body: #{request.body.inspect.tap{ |body| body[997..-1] = "..." if body.size > 500 }}"
+              logger.debug "> body: #{request.body.inspect.tap{ |body| body.replace body[0...997] + "..." if body.size > 1000 }}"
               # TODO this is buggy -- mixes lines from different files into one line
               stack = caller.reverse.map do |level|
                 /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
@@ -169,15 +176,21 @@ module NetHTTPUtils
             delay = 5
             response = begin
               http.request request, &block
-            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
               raise if max_read_retry_delay < delay *= 2
               logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
               sleep delay
               retry
+            rescue EOFError => e
+              raise unless e.backtrace.empty?
+              # https://bugs.ruby-lang.org/issues/13018
+              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html
+              raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
             end
             # response.instance_variable_set "@nethttputils_close", http.method(:finish)
             # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
 
+            now = Time.now
             remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
               logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
               [
@@ -189,7 +202,13 @@ module NetHTTPUtils
               [
                 response.fetch("x-rate-limit-remaining").to_i,
                 response.fetch("x-rate-limit-reset").to_i,
-                Time.now.to_i,
+                now.to_i,
+              ]
+            elsif response.key? "x-ratelimit-remaining"
+              [
+                response.fetch("x-ratelimit-remaining").to_i,
+                now + response.fetch("x-ratelimit-reset").to_i,
+                now.to_i,
               ]
             end
             if remaining
@@ -215,9 +234,13 @@ module NetHTTPUtils
             end
 
             case response.code
+            when /\A20/
+              response
             when /\A30\d\z/
+              next response if no_redirect
               logger.info "redirect: #{response["location"]}"
-              new_uri = URI.join request.uri, URI.escape(response["location"])
+              require "addressable"
+              new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
               new_host = new_uri.host
               raise Error.new "redirected in place" if new_uri == http.instance_variable_get(:@uri)
               if http.address != new_host ||
@@ -225,7 +248,7 @@ module NetHTTPUtils
                  http.use_ssl? != (new_uri.scheme == "https")
                 logger.debug "changing host from '#{http.address}' to '#{new_host}'"
                 # http.finish   # why commented out?
-                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
+                http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout, no_redirect
               end
               if request.method == "POST"
                 logger.info "POST redirects to GET (RFC)"
@@ -261,8 +284,6 @@ module NetHTTPUtils
                 end
               }"
               response
-            when /\A20/
-              response
             else
               logger.warn "code #{response.code} at #{request.method} #{request.uri} from #{
                 [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
@@ -283,11 +304,13 @@ module NetHTTPUtils
 
     require "set"
     @@_405 ||= Set.new
-    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
+    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil,
+        timeout: nil, no_redirect: no_redirect,
         max_start_http_retry_delay: 3600,
         max_read_retry_delay: 3600,
         patch_request: nil, &block
-      http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
+      timeout ||= 30
+      http = start_http http, max_start_http_retry_delay, timeout, no_redirect, *proxy unless http.is_a? Net::HTTP
       path = http.instance_variable_get(:@uri).path
 
       check_code = lambda do |body|
@@ -318,7 +341,8 @@ module NetHTTPUtils
           check_code.call body
         end
       end
-      body = read http, mtd, type, form: form, header: header, auth: auth, timeout: timeout,
+      body = read http, mtd, type, form: form, header: header, auth: auth,
+        timeout: timeout, no_redirect: no_redirect,
         max_read_retry_delay: max_read_retry_delay,
         patch_request: patch_request, &block
       check_code.call body
@@ -371,20 +395,20 @@ if $0 == __FILE__
   server = WEBrick::HTTPServer.new Port: 8000
   server.mount_proc "/1" do |req, res|
     next unless "GET" == req.request_method
-    res.cookies.push WEBrick::Cookie.new("1", "2")
-    res.cookies.push WEBrick::Cookie.new("3", "4")
-    res.cookies.push WEBrick::Cookie.new("8", "9")
-    res.cookies.push WEBrick::Cookie.new("a", "b")
-    res.cookies.push WEBrick::Cookie.new("1", "5")
-    res.cookies.push WEBrick::Cookie.new("f", "g h")
+    res.cookies.push WEBrick::Cookie.new "1", "2"
+    res.cookies.push WEBrick::Cookie.new "3", "4"
+    res.cookies.push WEBrick::Cookie.new "8", "9"
+    res.cookies.push WEBrick::Cookie.new "a", "b"
+    res.cookies.push WEBrick::Cookie.new "1", "5"
+    res.cookies.push WEBrick::Cookie.new "f", "g h"
     res.status = 300
     res["location"] = "/2"
   end
   server.mount_proc "/2" do |req, res|
-    res.cookies.push WEBrick::Cookie.new("3", "6=c")
-    res.cookies.push WEBrick::Cookie.new("a", "d e")
-    res.cookies.push WEBrick::Cookie.new("8", "")
-    res.cookies.push WEBrick::Cookie.new("4", "7")
+    res.cookies.push WEBrick::Cookie.new "3", "6=c"
+    res.cookies.push WEBrick::Cookie.new "a", "d e"
+    res.cookies.push WEBrick::Cookie.new "8", ""
+    res.cookies.push WEBrick::Cookie.new "4", "7"
   end
   t = Thread.new{ server.start }
   fail unless %w{ 3=6=c a=d\ e 8= 4=7 1=5 a=b } == p(NetHTTPUtils.request_data("http://localhost:8000/1").
@@ -467,16 +491,16 @@ if $0 == __FILE__
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/400")) == "400 Bad Request"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/404")) == "404 Not Found"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/500")) == "500 Internal Server Error"
-  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")) == "502 Bad Gateway"
+  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
   [
-    ["https://imgur.com/a/cccccc"],
-    ["https://imgur.com/mM4Dh7Z"],
+    # ["https://imgur.com/a/oacI3gl"],  # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
+    # ["https://imgur.com/mM4Dh7Z"],    # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
     ["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
   ].each do |url, expectation|
     begin
       puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
-      fail
+      fail url
     rescue NetHTTPUtils::Error => e
       raise e.code.inspect unless e.code == 404
       raise e.to_s if e.to_s != expectation if expectation
@@ -501,10 +525,6 @@ if $0 == __FILE__
     end
   end
 
-  begin
-    fail NetHTTPUtils.request_data "https://oi64.tinypic.com/29z7oxs.jpg?", timeout: 5, max_start_http_retry_delay: -1
-  rescue Net::OpenTimeout => e
-  end
   ## this stopped failing on High Sierra
   # begin
   #   # https://www.virtualself.co/?

data/nethttputils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "nethttputils"
-  spec.version      = "0.3.2.11"
+  spec.version      = "0.4.1.2"
   spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
   spec.description = <<-EOF
     Back in 2015 I was a guy automating things at my job and two scripts had a common need --
@@ -21,5 +21,7 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
 
   spec.require_path = "lib"
-  spec.files        = `git ls-files -z`.split(?\0) - spec.test_files
+  spec.files        = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
+
+  spec.add_dependency "addressable"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: nethttputils
 version: !ruby/object:Gem::Version
-  version: 0.3.2.11
+  version: 0.4.1.2
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-08 00:00:00.000000000 Z
-dependencies: []
+date: 2021-06-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |2
       Back in 2015 I was a guy automating things at my job and two scripts had a common need --
       they both had to pass the same credentials to Jenkins (via query params, I guess).
@@ -26,9 +40,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".travis.yml"
 - LICENSE
-- Rakefile
 - lib/nethttputils.rb
 - nethttputils.gemspec
 homepage: https://github.com/nakilon/nethttputils

data/.travis.yml

@@ -1,21 +0,0 @@
----
-language: ruby
-
-script: "ruby lib/nethttputils.rb"
-
-os:
-  - linux
-  - osx
-rvm:
-  - ruby-head
-  - 2.5
-  - 2.4
-  - 2.3
-  - 2.2
-  - 2.1
-  - 2.0
-  - jruby-head
-matrix:
-  allow_failures:
-  # something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
-  - rvm: jruby-head

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"