nethttputils 0.3.2.10 → 0.4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee1c0a46857d320778ba8a0816cd7e3cb7018fe9
-  data.tar.gz: 1ade1c1f815b7fd75924cdcbed5ce4b732a51570
+  metadata.gz: 3c68a49af2dbbca3cacf81cd5518df2ce6f2cb72
+  data.tar.gz: bae1db040743d7f9ea9ac17d7aeafba778f227c5
 SHA512:
-  metadata.gz: 7598d560362cee4112042c59a2710b5d757b29fd3eb4153093f2058ba8c7d7802b9f0ac72c9a9a11c1d2947ae104907c3b0dafb9a2def12d47a5662a59605fa6
-  data.tar.gz: cc0d7b860a63b7076ee405ded44f0b3c6c0183534b40cc1d6a1ac9030f23f6b06be9d1063a9c4da96325360a9f7fe298b4e336fb2b8295420ace73b4cef5dbf4
+  metadata.gz: 8ba54aa8a5fe007911b8af3117061b1a279182eb27cbdcd2be0a6293428d4408356b2b8238c7342284e93da9ee20f62d169b79346f1f30019677c24c0ca5c9af
+  data.tar.gz: 23e5657489821b849a611a3692d14335b46c1f367dfff1c130d862a1390eb5234742110cdda9d69bb06f9206ae98ed0fdf6dfa6d66fc7056bf45dacd62a0ca13

data/lib/nethttputils.rb

@@ -18,19 +18,22 @@ module NetHTTPUtils
     attr_reader :code
     def initialize body, code = nil
       @code = code
-      super "HTTP error ##{code} #{body}"
+      body = body[0...997] + "..." if body.size > 1000
+      super "HTTP error ##{code.inspect} #{body}"
     end
   end
+  class EOFError_from_rbuf_fill < StandardError
+  end
 
   class << self
 
     def remove_tags str
-      str.gsub(/<script( [a-z]+="[^"]*")*>.*?<\/script>/m, "").
-          gsub(/<style( [a-z]+="[^"]*")*>.*?<\/style>/m, "").
+      str.gsub(/<script( [a-z-]+="[^"]*")*>.*?<\/script>/m, "").
+          gsub(/<style( [a-z-]+="[^"]*")*>.*?<\/style>/m, "").
           gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
     end
 
-    def start_http url, max_start_http_retry_delay = 3600, timeout = 30
+    def start_http url, max_start_http_retry_delay = 3600, timeout = 30, proxy = nil
       uri = url
       uri = URI.parse begin
         URI url
@@ -42,6 +45,7 @@ module NetHTTPUtils
       begin
         Net::HTTP.start(
           uri.host, uri.port,
+          *(proxy.split ?: if proxy),
           use_ssl: uri.scheme == "https",
           verify_mode: OpenSSL::SSL::VERIFY_NONE,
           **({open_timeout: timeout}), #  if timeout
@@ -54,7 +58,7 @@ module NetHTTPUtils
               def << msg
                 @@buffer ||= "[Net::HTTP debug] "
                 @@buffer.concat msg
-                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 500
+                @@buffer = @@buffer[0...997] + "..." if @@buffer.size > 1000
                 return unless @@buffer.end_with? ?\n
                 NetHTTPUtils.logger.debug @@buffer.sub ?\n, "  "
                 @@buffer = nil
@@ -63,7 +67,7 @@ module NetHTTPUtils
           end ) if logger.level == Logger::DEBUG # use `logger.debug?`?
           http
         end
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
           raise
@@ -71,10 +75,6 @@ module NetHTTPUtils
         logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
         sleep delay
         retry
-      rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET => e
-        logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
-        sleep 5
-        retry
       rescue SocketError => e
         if max_start_http_retry_delay < delay *= 2
           e.message.concat " to #{uri}"
@@ -155,7 +155,7 @@ module NetHTTPUtils
                 request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join
               }#{curl_form}'#{uri.scheme}://#{uri.host}#{uri.path}#{"?#{uri.query}" if uri.query && !uri.query.empty?}'"
               logger.debug "> header: #{request.each_header.to_a}"
-              logger.debug "> body: #{request.body.inspect.tap{ |body| body[997..-1] = "..." if body.size > 500 }}"
+              logger.debug "> body: #{request.body.inspect.tap{ |body| body.replace body[0...997] + "..." if body.size > 1000 }}"
               # TODO this is buggy -- mixes lines from different files into one line
               stack = caller.reverse.map do |level|
                 /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
@@ -169,15 +169,21 @@ module NetHTTPUtils
             delay = 5
             response = begin
               http.request request, &block
-            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT, Errno::ENETUNREACH => e
               raise if max_read_retry_delay < delay *= 2
               logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
               sleep delay
               retry
+            rescue EOFError => e
+              raise unless e.backtrace.empty?
+              # https://bugs.ruby-lang.org/issues/13018
+              # https://blog.kalina.tech/2019/04/exception-without-backtrace-in-ruby.html?spref=reddit
+              raise EOFError_from_rbuf_fill.new "probably the old Ruby empty backtrace EOFError exception from net/protocol.rb"
             end
             # response.instance_variable_set "@nethttputils_close", http.method(:finish)
             # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
 
+            now = Time.now
             remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
               logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
               [
@@ -189,7 +195,13 @@ module NetHTTPUtils
               [
                 response.fetch("x-rate-limit-remaining").to_i,
                 response.fetch("x-rate-limit-reset").to_i,
-                Time.now.to_i,
+                now.to_i,
+              ]
+            elsif response.key? "x-ratelimit-remaining"
+              [
+                response.fetch("x-ratelimit-remaining").to_i,
+                now + response.fetch("x-ratelimit-reset").to_i,
+                now.to_i,
               ]
             end
             if remaining
@@ -217,13 +229,15 @@ module NetHTTPUtils
             case response.code
             when /\A30\d\z/
               logger.info "redirect: #{response["location"]}"
-              new_uri = URI.join request.uri, URI.escape(response["location"])
+              require "addressable"
+              new_uri = URI.join request.uri.to_s, Addressable::URI.escape(response["location"])
               new_host = new_uri.host
+              raise Error.new "redirected in place" if new_uri == http.instance_variable_get(:@uri)
               if http.address != new_host ||
                  http.port != new_uri.port ||
                  http.use_ssl? != (new_uri.scheme == "https")
                 logger.debug "changing host from '#{http.address}' to '#{new_host}'"
-                # http.finish
+                # http.finish   # why commented out?
                 http = NetHTTPUtils.start_http new_uri, http.instance_variable_get(:@max_start_http_retry_delay), timeout
               end
               if request.method == "POST"
@@ -282,11 +296,12 @@ module NetHTTPUtils
 
     require "set"
     @@_405 ||= Set.new
-    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30,
+    def request_data http, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, proxy: nil,
+        timeout: 30,
         max_start_http_retry_delay: 3600,
         max_read_retry_delay: 3600,
         patch_request: nil, &block
-      http = start_http http, max_start_http_retry_delay, timeout unless http.is_a? Net::HTTP
+      http = start_http http, max_start_http_retry_delay, timeout, *proxy unless http.is_a? Net::HTTP
       path = http.instance_variable_get(:@uri).path
 
       check_code = lambda do |body|
@@ -349,23 +364,41 @@ if $0 == __FILE__
   require "json"
   Thread.abort_on_exception = true
 
+  server = WEBrick::HTTPServer.new Port: 8000
+  tt = false
+  server.mount_proc "/" do |req, res|
+    next unless "HEAD" == req.request_method
+    fail if tt
+    tt = true
+    res.status = 300
+    res["location"] = "/"
+  end
+  t = Thread.new{ server.start }
+  begin
+    NetHTTPUtils.request_data "http://localhost:8000/"
+  rescue NetHTTPUtils::Error => e
+    raise if e.code
+  end
+  server.shutdown
+  t.join
+
   server = WEBrick::HTTPServer.new Port: 8000
   server.mount_proc "/1" do |req, res|
     next unless "GET" == req.request_method
-    res.cookies.push WEBrick::Cookie.new("1", "2")
-    res.cookies.push WEBrick::Cookie.new("3", "4")
-    res.cookies.push WEBrick::Cookie.new("8", "9")
-    res.cookies.push WEBrick::Cookie.new("a", "b")
-    res.cookies.push WEBrick::Cookie.new("1", "5")
-    res.cookies.push WEBrick::Cookie.new("f", "g h")
+    res.cookies.push WEBrick::Cookie.new "1", "2"
+    res.cookies.push WEBrick::Cookie.new "3", "4"
+    res.cookies.push WEBrick::Cookie.new "8", "9"
+    res.cookies.push WEBrick::Cookie.new "a", "b"
+    res.cookies.push WEBrick::Cookie.new "1", "5"
+    res.cookies.push WEBrick::Cookie.new "f", "g h"
     res.status = 300
     res["location"] = "/2"
   end
   server.mount_proc "/2" do |req, res|
-    res.cookies.push WEBrick::Cookie.new("3", "6=c")
-    res.cookies.push WEBrick::Cookie.new("a", "d e")
-    res.cookies.push WEBrick::Cookie.new("8", "")
-    res.cookies.push WEBrick::Cookie.new("4", "7")
+    res.cookies.push WEBrick::Cookie.new "3", "6=c"
+    res.cookies.push WEBrick::Cookie.new "a", "d e"
+    res.cookies.push WEBrick::Cookie.new "8", ""
+    res.cookies.push WEBrick::Cookie.new "4", "7"
   end
   t = Thread.new{ server.start }
   fail unless %w{ 3=6=c a=d\ e 8= 4=7 1=5 a=b } == p(NetHTTPUtils.request_data("http://localhost:8000/1").
@@ -448,16 +481,16 @@ if $0 == __FILE__
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/400")) == "400 Bad Request"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/404")) == "404 Not Found"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/500")) == "500 Internal Server Error"
-  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")) == "502 Bad Gateway"
+  fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/502")).start_with? "httpstat.us | 502: Bad gateway\nError\n502\n"
   fail unless NetHTTPUtils.method(:read).call(NetHTTPUtils.start_http("http://httpstat.us/503")) == "503 Service Unavailable"
   [
-    ["https://imgur.com/a/cccccc"],
-    ["https://imgur.com/mM4Dh7Z"],
+    # ["https://imgur.com/a/oacI3gl"],  # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
+    # ["https://imgur.com/mM4Dh7Z"],    # TODO: Imgur now hangs on these pages, I guess they had to be some 404 error page
     ["https://i.redd.it/si758zk7r5xz.jpg", "HTTP error #404 <image/png>"],
   ].each do |url, expectation|
     begin
       puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
-      fail
+      fail url
     rescue NetHTTPUtils::Error => e
       raise e.code.inspect unless e.code == 404
       raise e.to_s if e.to_s != expectation if expectation
@@ -482,10 +515,6 @@ if $0 == __FILE__
     end
   end
 
-  begin
-    fail NetHTTPUtils.request_data "https://oi64.tinypic.com/29z7oxs.jpg?", timeout: 5, max_start_http_retry_delay: -1
-  rescue Net::OpenTimeout => e
-  end
   ## this stopped failing on High Sierra
   # begin
   #   # https://www.virtualself.co/?

data/nethttputils.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "nethttputils"
-  spec.version      = "0.3.2.10"
+  spec.version      = "0.4.1.1"
   spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
   spec.description = <<-EOF
     Back in 2015 I was a guy automating things at my job and two scripts had a common need --
@@ -21,5 +21,7 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
 
   spec.require_path = "lib"
-  spec.files        = `git ls-files -z`.split(?\0) - spec.test_files
+  spec.files        = %w{ LICENSE nethttputils.gemspec lib/nethttputils.rb }
+
+  spec.add_dependency "addressable"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: nethttputils
 version: !ruby/object:Gem::Version
-  version: 0.3.2.10
+  version: 0.4.1.1
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-18 00:00:00.000000000 Z
-dependencies: []
+date: 2020-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |2
       Back in 2015 I was a guy automating things at my job and two scripts had a common need --
       they both had to pass the same credentials to Jenkins (via query params, I guess).
@@ -26,9 +40,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".travis.yml"
 - LICENSE
-- Rakefile
 - lib/nethttputils.rb
 - nethttputils.gemspec
 homepage: https://github.com/nakilon/nethttputils
@@ -51,7 +63,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: this tool is like a pet that I adopted young and now I depend on, sorry

data/.travis.yml

@@ -1,21 +0,0 @@
----
-language: ruby
-
-script: "ruby lib/nethttputils.rb"
-
-os:
-  - linux
-  - osx
-rvm:
-  - ruby-head
-  - 2.5
-  - 2.4
-  - 2.3
-  - 2.2
-  - 2.1
-  - 2.0
-  - jruby-head
-matrix:
-  allow_failures:
-  # something with `NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})` test
-  - rvm: jruby-head

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"