nethttputils 0.2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 71be1aac2cdebb5316afe367b7b7bd2eb8b5b60d
+  data.tar.gz: 4026ab5f4299bc880b3f8c13881114f4e49c67fd
+SHA512:
+  metadata.gz: 7fe70ebe65005756f847deb19f318d77ae8c46298420f869caa5d4a2cb75de826b0634f758048f8190188b7704469fa73a84435572dd585e325d62b30e26290e
+  data.tar.gz: 58debad4370c8c99abbbd149fcb02c553e674d79dcd89cd294d306f59429c7ede3e58518c4b20127107dfeb7fed5476aa25a5e8ffadc4eeabd54d9870ed44233

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Victor Maslov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/nethttputils.rb

@@ -0,0 +1,288 @@
+require "net/http"
+require "openssl"
+
+require "logger"
+
+
+module NetHTTPUtils
+  class << self
+    attr_accessor :logger
+  end
+  self.logger = Logger.new STDOUT
+  self.logger.level = ENV["LOGLEVEL_#{name}"] ? Logger.const_get(ENV["LOGLEVEL_#{name}"]) : Logger::WARN
+  self.logger.formatter = lambda do |severity, datetime, progname, msg|
+    "#{severity.to_s[0]} #{datetime.strftime "%y%m%d %H%M%S"} : #{name} : #{msg}\n"
+  end
+
+  class Error < RuntimeError
+    attr_reader :code
+    def initialize body, code = nil
+      @code = code
+      super "HTTP error ##{code} #{body}"
+    end
+  end
+
+  class << self
+
+    def remove_tags str
+      str.gsub(/<script( type="text\/javascript"| src="[^"]+")?>.*?<\/script>/m, "").gsub(/<[^>]*>/, "").strip
+    end
+
+    # TODO: make it private?
+    def get_response url, mtd = :GET, type = :form, form: {}, header: {}, auth: nil, timeout: 30, max_timeout_retry_delay: 3600, max_sslerror_retry_delay: 3600, max_read_retry_delay: 3600, max_econnrefused_retry_delay: 3600, patch_request: nil, &block
+      uri = URI.parse URI.escape url
+
+      logger.warn "Warning: query params included `url` are discarded because `:form` isn't empty" if uri.query && !form.empty?
+      # we can't just merge because URI fails to parse such queries as "/?1"
+
+      uri.query = URI.encode_www_form form if :GET == (mtd = mtd.upcase) && !form.empty?
+      cookies = {}
+      prepare_request = lambda do |uri|
+        case mtd.upcase
+          when :GET    ; Net::HTTP::Get
+          when :POST   ; Net::HTTP::Post
+          when :PUT    ; Net::HTTP::Put
+          when :DELETE ; Net::HTTP::Delete
+          when :PATCH  ; Net::HTTP::Patch
+          else         ; raise "unknown method '#{mtd}'"
+        end.new(uri).tap do |request| # somehow Get eats even raw url, not URI object
+          patch_request.call uri, form, request if patch_request
+          request.basic_auth *auth if auth
+          request["cookie"] = [*request["cookie"], cookies.map{ |k, v| "#{k}=#{v}" }].join "; " unless cookies.empty?
+          request.set_form_data form if !form.empty? && mtd == :POST
+          if mtd == :POST || mtd == :PATCH
+            request["Content-Type"] = case type
+              when :form ; "application/x-www-form-urlencoded;charset=UTF-8"
+              when :json ; request.body = JSON.dump form     # yes this overwrites form data set few lines higher
+                           "application/json"
+              else       ; raise "unknown content-type '#{type}'"
+            end
+          end
+          header.each{ |k, v| request[k.to_s] = v }
+
+          logger.info "> #{request} #{request.path}"
+          next unless logger.debug?
+          logger.debug "curl -s -D - #{request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join}#{url.gsub "&", "\\\\&"}"
+          logger.debug "> header: #{request.each_header.to_a}"
+          logger.debug "> body: #{request.body.inspect.tap{ |body| body[100..-1] = "..." if body.size > 100 }}"
+          stack = caller.reverse.map do |level|
+            /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
+          end.chunk(&:first).map do |file, group|
+            "#{file}:#{group.map(&:last).chunk{|_|_}.map(&:first).join(",")}"
+          end
+          logger.debug stack.join " -> "
+        end
+      end
+      start_http = lambda do |uri|
+        delay = 5
+        begin
+          Net::HTTP.start(
+            uri.host, uri.port,
+            use_ssl: uri.scheme == "https",
+            verify_mode: OpenSSL::SSL::VERIFY_NONE,
+            **({open_timeout: timeout}), #  if timeout
+            **({read_timeout: timeout}), #  if timeout
+          ) do |http|
+            # http.open_timeout = timeout   # seems like when opening hangs, this line in unreachable
+            # http.read_timeout = timeout
+            http.set_debug_output STDERR if logger.level == Logger::DEBUG # use `logger.debug?`?
+            http
+          end
+        rescue Errno::ECONNREFUSED => e
+          e.message.concat " to #{uri}"
+          raise if max_econnrefused_retry_delay < delay *= 2
+          logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}'"
+          sleep delay
+          retry
+        rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET, SocketError => e
+          if e.is_a?(SocketError) && e.message["getaddrinfo: "]
+            e.message.concat ": #{uri.host}"
+            raise e
+            # logger.warn "retrying in 60 seconds because of #{e.class} '#{e.message}'"
+            # sleep 60
+            # retry
+          end
+          logger.warn "retrying in 5 seconds because of #{e.class} '#{e.message}'"
+          sleep 5
+          retry
+        rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
+          raise if max_timeout_retry_delay < delay *= 2
+          logger.warn "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
+          sleep delay
+          retry
+        rescue OpenSSL::SSL::SSLError => e
+          raise if max_sslerror_retry_delay < delay *= 2
+          logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{uri}"
+          sleep delay
+          retry
+        end
+      end
+      http = start_http[uri]
+      do_request = lambda do |request|
+        delay = 1
+        response = begin
+          http.request request, &block
+        rescue Errno::ECONNREFUSED, Net::ReadTimeout, Net::OpenTimeout, Zlib::BufError, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+          raise if max_read_retry_delay < delay *= 2
+          logger.error "retrying in #{delay} seconds because of #{e.class} '#{e.message}' at: #{request.uri}"
+          sleep delay
+          retry
+        end
+        # response.instance_variable_set "@nethttputils_close", http.method(:finish)
+        # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
+
+        if response.key? "x-ratelimit-userremaining"
+          c = response.fetch("x-ratelimit-userremaining").to_i
+          logger.debug "x-ratelimit-userremaining: #{c}"
+          t = response.fetch("x-ratelimit-clientremaining").to_i
+          logger.debug "x-ratelimit-clientremaining: #{t}"
+          unless 100 < c
+            a = response.fetch("x-timer")[/\d+/].to_i
+            b = response.fetch("x-ratelimit-userreset").to_i
+            t = (b - a + 1).fdiv c
+            logger.warn "x-ratelimit sleep #{t} seconds"
+            sleep t
+          end
+        end
+
+        response.to_hash.fetch("set-cookie", []).each{ |c| k, v = c.split(?=); cookies[k] = v[/[^;]+/] }
+        case response.code
+        when /\A3\d\d\z/
+          logger.info "redirect: #{response["location"]}"
+          new_uri = URI.join request.uri, URI.escape(response["location"])
+          new_host = new_uri.host
+          if http.address != new_host ||
+             http.port != new_uri.port ||
+             http.use_ssl? != (new_uri.scheme == "https")
+            logger.debug "changing host from '#{http.address}' to '#{new_host}'"
+            # http.finish
+            http = start_http[new_uri]
+          end
+          do_request.call prepare_request[new_uri]
+        when "404"
+          logger.error "404 at #{request.method} #{request.uri} with body: #{
+            response.body.is_a?(Net::ReadAdapter) ? "impossible to reread Net::ReadAdapter -- check the IO you've used in block form" : response.body.tap do |body|
+              body.replace remove_tags body if body[/<html[> ]/]
+            end.inspect
+          }"
+          response
+        when "429"
+          logger.error "429 at #{request.method} #{request.uri} with body: #{response.body.inspect}"
+          response
+        when /\A50\d\z/
+          logger.error "#{response.code} at #{request.method} #{request.uri} with body: #{
+            response.body.tap do |body|
+              body.replace remove_tags body if body[/<html[> ]/]
+            end.inspect
+          }"
+          response
+        when /\A20/
+          response
+        else
+          logger.warn "code #{response.code} at #{request.method} #{request.uri}#{
+            " and so #{url}" if request.uri.to_s != url
+          } from #{
+            [__FILE__, caller.map{ |i| i[/(?<=:)\d+/] }].join ?:
+          }"
+          logger.debug "< header: #{response.to_hash}"
+          logger.debug "< body: #{
+            response.body.tap do |body|
+              body.replace remove_tags body if body[/<html[> ]/]
+            end.inspect
+          }"
+          response
+        end
+      end
+      do_request[prepare_request[uri]].tap do |response|
+        cookies.each{ |k, v| response.add_field "Set-Cookie", "#{k}=#{v};" }
+        logger.debug response.to_hash
+      end
+    end
+
+    def request_data *args, &block
+      response = get_response *args, &block
+      raise Error.new response.body, response.code.to_i unless response.code[/\A(20\d|3\d\d)\z/]
+      if response["content-encoding"] == "gzip"
+        Zlib::GzipReader.new(StringIO.new(response.body)).read
+      else
+        response.body
+      end.tap do |string|
+        string.instance_variable_set :@uri_path, response.uri.path
+      end
+    # ensure
+    #   response.instance_variable_get("@nethttputils_close").call if response
+    end
+
+  end
+end
+
+
+if $0 == __FILE__
+  STDOUT.sync = true
+  print "self testing... "
+  require "pp"
+
+  require "webrick"
+  require "json"
+  server = WEBrick::HTTPServer.new Port: 8000
+  server.mount_proc ?/ do |req, res|
+    # pp req.dup.tap{ |_| _.instance_variable_set "@config", nil }
+    # res.status = WEBrick::HTTPStatus::RC_ACCEPTED
+    res.body = JSON.dump [req.unparsed_uri, req.header.keys]
+  end
+  Thread.abort_on_exception = true
+  Thread.new{ server.start }
+  fail unless JSON.dump(["/", %w{ accept-encoding accept user-agent host connection }]) == NetHTTPUtils.request_data("http://localhost:8000/")
+  fail unless JSON.dump(["/?1", %w{ accept-encoding accept user-agent host connection }]) == NetHTTPUtils.request_data("http://localhost:8000/?1")
+  fail unless JSON.dump(["/?1=2", %w{ accept-encoding accept user-agent host connection }]) == NetHTTPUtils.request_data("http://localhost:8000/?1=2")
+  fail unless JSON.dump(["/?1=3", %w{ accept-encoding accept user-agent host connection }]) == NetHTTPUtils.request_data("http://localhost:8000/?1=2&3=4", form: {1=>3})
+  fail unless JSON.dump(["/", %w{ accept-encoding accept user-agent host content-type connection content-length }]) == NetHTTPUtils.request_data("http://localhost:8000/", :post, form: {1=>2})
+  server.shutdown
+
+  fail unless NetHTTPUtils.request_data("http://httpstat.us/200") == "200 OK"
+  [400, 404, 500, 503].each do |code|
+    begin
+      fail NetHTTPUtils.request_data "http://httpstat.us/#{code}"
+    rescue NetHTTPUtils::Error => e
+      raise unless e.code == code
+    end
+  end
+  fail unless NetHTTPUtils.get_response("http://httpstat.us/400").body == "400 Bad Request"
+  fail unless NetHTTPUtils.get_response("http://httpstat.us/404").body == "404 Not Found"
+  fail unless NetHTTPUtils.get_response("http://httpstat.us/500").body == "500 Internal Server Error"
+  fail unless NetHTTPUtils.get_response("http://httpstat.us/503").body == "503 Service Unavailable"
+  %w{
+    https://imgur.com/a/cccccc
+    https://imgur.com/mM4Dh7Z
+  }.each do |url|
+    begin
+      puts NetHTTPUtils.remove_tags NetHTTPUtils.request_data url
+      fail
+    rescue NetHTTPUtils::Error => e
+      raise unless e.code == 404
+    end
+  end
+  %w{
+    http://minus.com/lkP3hgRJd9npi
+    http://www.cutehalloweencostumeideas.org/wp-content/uploads/2017/10/Niagara-Falls_04.jpg
+  }.each do |url|
+    begin
+      fail NetHTTPUtils.request_data url
+    rescue SocketError => e
+      raise unless e.message["getaddrinfo: "]
+    end
+  end
+
+  begin
+    fail NetHTTPUtils.request_data "https://oi64.tinypic.com/29z7oxs.jpg?", timeout: 5, max_timeout_retry_delay: -1
+  rescue Net::OpenTimeout => e
+  end
+  ## this stopped failing on High Sierra
+  # begin
+  #   # https://www.virtualself.co/?
+  #   fail NetHTTPUtils.request_data "https://bulletinxp.com/curiosity/strange-weather/?", max_sslerror_retry_delay: -1
+  # rescue OpenSSL::SSL::SSLError => e
+  # end
+
+  puts "OK #{__FILE__}"
+end

data/nethttputils.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |spec|
+  spec.name         = "nethttputils"
+  spec.version      = "0.2.0.0"
+  spec.summary      = "this tool is like a pet that I adopted young and now I depend on, sorry"
+  spec.description = <<-EOF
+    Back in 2015 I was a guy automating things at my job and two scripts had a common need --
+    they both had to pass the same credentials to Jenkins (via query params, I guess).
+
+    That common tool with a single method was a Net::HTTP wrapper -- that's where the name from.
+    Then when the third script appeared two of them had to pass the Basic Auth.
+    The verb POST was added and common logging format, and relatively complex retry logic.
+    Then some website had redirects and I had to store cookies, then GZIP and API rate limits...
+
+    I was not going to gemify this monster but it is now a dependency in many other gems,
+    and since Gemfile does not support Github dependencies I have to finally gemify it.
+  EOF
+
+  spec.homepage     = "https://github.com/nakilon/nethttputils"
+  spec.author       = "Victor Maslov aka Nakilon"
+  spec.email        = "nakilon@gmail.com"
+  spec.license      = "MIT"
+
+  spec.require_path = "lib"
+  spec.files        = `git ls-files -z`.split(?\0) - spec.test_files
+end

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification
+name: nethttputils
+version: !ruby/object:Gem::Version
+  version: 0.2.0.0
+platform: ruby
+authors:
+- Victor Maslov aka Nakilon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-03 00:00:00.000000000 Z
+dependencies: []
+description: |2
+      Back in 2015 I was a guy automating things at my job and two scripts had a common need --
+      they both had to pass the same credentials to Jenkins (via query params, I guess).
+
+      That common tool with a single method was a Net::HTTP wrapper -- that's where the name from.
+      Then when the third script appeared two of them had to pass the Basic Auth.
+      The verb POST was added and common logging format, and relatively complex retry logic.
+      Then some website had redirects and I had to store cookies, then GZIP and API rate limits...
+
+      I was not going to gemify this monster but it is now a dependency in many other gems,
+      and since Gemfile does not support Github dependencies I have to finally gemify it.
+email: nakilon@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- Rakefile
+- lib/nethttputils.rb
+- nethttputils.gemspec
+homepage: https://github.com/nakilon/nethttputils
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: this tool is like a pet that I adopted young and now I depend on, sorry
+test_files: []