httpx 0.16.1 → 0.18.2

data/lib/httpx/plugins/aws_sigv4.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-require "aws-sdk-s3"
-
 module HTTPX
   module Plugins
     #
@@ -75,16 +72,16 @@ module HTTPX
 
           # canonical request
           creq = "#{request.verb.to_s.upcase}" \
-            "\n#{request.canonical_path}" \
-            "\n#{request.canonical_query}" \
-            "\n#{canonical_headers}" \
-            "\n#{signed_headers}" \
-            "\n#{content_hashed}"
+                 "\n#{request.canonical_path}" \
+                 "\n#{request.canonical_query}" \
+                 "\n#{canonical_headers}" \
+                 "\n#{signed_headers}" \
+                 "\n#{content_hashed}"
 
           credential_scope = "#{date}" \
-            "/#{@region}" \
-            "/#{@service}" \
-            "/#{lower_provider_prefix}_request"
+                             "/#{@region}" \
+                             "/#{@service}" \
+                             "/#{lower_provider_prefix}_request"
 
           algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
           # string to sign
@@ -142,6 +139,7 @@ module HTTPX
 
       class << self
         def load_dependencies(*)
+          require "set"
           require "digest/sha2"
           require "openssl"
         end

data/lib/httpx/plugins/compression.rb

@@ -72,6 +72,8 @@ module HTTPX
       end
 
       module ResponseBodyMethods
+        using ArrayExtensions
+
         attr_reader :encodings
 
         def initialize(*)
@@ -90,7 +92,7 @@ module HTTPX
             Float::INFINITY
           end
 
-          @_inflaters = @headers.get("content-encoding").map do |encoding|
+          @_inflaters = @headers.get("content-encoding").filter_map do |encoding|
             next if encoding == "identity"
 
             inflater = @options.encodings.registry(encoding).inflater(compressed_length)
@@ -100,7 +102,7 @@ module HTTPX
 
             @encodings << encoding
             inflater
-          end.compact
+          end
 
           # this can happen if the only declared encoding is "identity"
           remove_instance_variable(:@_inflaters) if @_inflaters.empty?
@@ -134,7 +136,7 @@ module HTTPX
         end
 
         def each(&blk)
-          return enum_for(__method__) unless block_given?
+          return enum_for(__method__) unless blk
 
           return deflate(&blk) if @buffer.size.zero?
 

data/lib/httpx/plugins/cookies/jar.rb

@@ -55,7 +55,7 @@ module HTTPX
       end
 
       def each(uri = nil, &blk)
-        return enum_for(__method__, uri) unless block_given?
+        return enum_for(__method__, uri) unless blk
 
         return @cookies.each(&blk) unless uri
 

data/lib/httpx/plugins/digest_authentication.rb

@@ -40,12 +40,12 @@ module HTTPX
 
         alias_method :digest_auth, :digest_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             digest = request.options.digest
 
             if digest
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if digest && !probe_response.is_a?(ErrorResponse) &&
                  probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
@@ -56,12 +56,12 @@ module HTTPX
                 token = digest.generate_header(request, probe_response)
                 request.headers["authorization"] = "Digest #{token}"
 
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/expect.rb

@@ -69,9 +69,14 @@ module HTTPX
       end
 
       module ConnectionMethods
-        def send(request)
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["expect"] == "100-continue"
+
           request.once(:expect) do
-            @timers.after(@options.expect_timeout) do
+            @timers.after(request.options.expect_timeout) do
+              # expect timeout expired
               if request.state == :expect && !request.expects?
                 Expect.no_expect_store << request.origin
                 request.headers.delete("expect")
@@ -79,7 +84,6 @@ module HTTPX
               end
             end
           end
-          super
         end
       end
 

data/lib/httpx/plugins/grpc/message.rb

@@ -17,7 +17,7 @@ module HTTPX
 
         # lazy decodes a grpc stream response
         def stream(response, &block)
-          return enum_for(__method__, response) unless block_given?
+          return enum_for(__method__, response) unless block
 
           response.each do |frame|
             decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
@@ -57,7 +57,7 @@ module HTTPX
 
             yield data
 
-            message = message.byteslice(5 + size..-1)
+            message = message.byteslice((5 + size)..-1)
           end
         end
 

data/lib/httpx/plugins/grpc.rb

@@ -143,9 +143,9 @@ module HTTPX
 
           session_class = Class.new(self.class) do
             class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-              def #{rpc_name}(input, **opts)
-                rpc_execute("#{rpc_name}", input, **opts)
-              end
+              def #{rpc_name}(input, **opts)              # def grpc_action(input, **opts)
+                rpc_execute("#{rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+              end                                         # end
             OUT
           end
 

data/lib/httpx/plugins/h2c.rb

@@ -24,15 +24,19 @@ module HTTPX
         def call(connection, request, response)
           connection.upgrade_to_h2c(request, response)
         end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
       end
 
       module InstanceMethods
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           upgrade_request, *remainder = requests
 
           return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
 
-          connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
+          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
           return super if connection && connection.upgrade_protocol == :h2c
 
@@ -42,7 +46,7 @@ module HTTPX
           upgrade_request.headers["upgrade"] = "h2c"
           upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
 
-          super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
+          super(upgrade_request, *remainder)
         end
       end
 

data/lib/httpx/plugins/internal_telemetry.rb

@@ -44,6 +44,11 @@ module HTTPX
           meter_elapsed_time("Session: initialized!!!")
         end
 
+        def close(*)
+          super
+          meter_elapsed_time("Session -> close")
+        end
+
         private
 
         def build_requests(*)
@@ -55,11 +60,6 @@ module HTTPX
           meter_elapsed_time("Session -> response") if response
           response
         end
-
-        def close(*)
-          super
-          meter_elapsed_time("Session -> close")
-        end
       end
 
       module RequestMethods
@@ -69,9 +69,9 @@ module HTTPX
         end
 
         def transition(nextstate)
-          state = @state
+          prev_state = @state
           super
-          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{prev_state}] -> #{@state}") if prev_state != @state
         end
       end
 
@@ -84,7 +84,7 @@ module HTTPX
         def transition(nextstate)
           state = @state
           super
-          meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
         end
       end
     end

data/lib/httpx/plugins/multipart/decoder.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX::Plugins
+  module Multipart
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    CRLF = "\r\n"
+
+    class FilePart < SimpleDelegator
+      attr_reader :original_filename, :content_type
+
+      def initialize(filename, content_type)
+        @original_filename = filename
+        @content_type = content_type
+        @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+        super(@file)
+      end
+    end
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    CONDISP = /Content-Disposition:\s*#{TOKEN}\s*/i.freeze
+    BROKEN_QUOTED = /^#{CONDISP}.*;\s*filename="(.*?)"(?:\s*$|\s*;\s*#{TOKEN}=)/i.freeze
+    BROKEN_UNQUOTED = /^#{CONDISP}.*;\s*filename=(#{TOKEN})/i.freeze
+    MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+    MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+    # Updated definitions from RFC 2231
+    ATTRIBUTE_CHAR = %r{[^ \t\v\n\r)(><@,;:\\"/\[\]?='*%]}.freeze
+    ATTRIBUTE = /#{ATTRIBUTE_CHAR}+/.freeze
+    SECTION = /\*[0-9]+/.freeze
+    REGULAR_PARAMETER_NAME = /#{ATTRIBUTE}#{SECTION}?/.freeze
+    REGULAR_PARAMETER = /(#{REGULAR_PARAMETER_NAME})=(#{VALUE})/.freeze
+    EXTENDED_OTHER_NAME = /#{ATTRIBUTE}\*[1-9][0-9]*\*/.freeze
+    EXTENDED_OTHER_VALUE = /%[0-9a-fA-F]{2}|#{ATTRIBUTE_CHAR}/.freeze
+    EXTENDED_OTHER_PARAMETER = /(#{EXTENDED_OTHER_NAME})=(#{EXTENDED_OTHER_VALUE}*)/.freeze
+    EXTENDED_INITIAL_NAME = /#{ATTRIBUTE}(?:\*0)?\*/.freeze
+    EXTENDED_INITIAL_VALUE = /[a-zA-Z0-9\-]*'[a-zA-Z0-9\-]*'#{EXTENDED_OTHER_VALUE}*/.freeze
+    EXTENDED_INITIAL_PARAMETER = /(#{EXTENDED_INITIAL_NAME})=(#{EXTENDED_INITIAL_VALUE})/.freeze
+    EXTENDED_PARAMETER = /#{EXTENDED_INITIAL_PARAMETER}|#{EXTENDED_OTHER_PARAMETER}/.freeze
+    DISPPARM = /;\s*(?:#{REGULAR_PARAMETER}|#{EXTENDED_PARAMETER})\s*/.freeze
+    RFC2183 = /^#{CONDISP}(#{DISPPARM})+$/i.freeze
+
+    class Decoder
+      BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+      WINDOW_SIZE = 2 << 14
+
+      def initialize(response)
+        @boundary = begin
+          m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+          raise Error, "no boundary declared in content-type header" unless m
+
+          m.strip
+        end
+        @buffer = "".b
+        @parts = {}
+        @intermediate_boundary = "--#{@boundary}"
+        @state = :idle
+      end
+
+      def call(response, _)
+        response.body.each do |chunk|
+          @buffer << chunk
+
+          parse
+        end
+
+        raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+        @parts
+      end
+
+      private
+
+      def parse
+        case @state
+        when :idle
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+          @state = :part_header
+        when :part_header
+          idx = @buffer.index("#{CRLF}#{CRLF}")
+
+          # raise Error, "couldn't parse part headers" unless idx
+          return unless idx
+
+          head = @buffer.byteslice(0..idx + 4 - 1)
+
+          @buffer = @buffer.byteslice(head.bytesize..-1)
+
+          content_type = head[MULTIPART_CONTENT_TYPE, 1]
+          if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+            name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+            name.gsub!(/\\(.)/, "\\1")
+            name
+          else
+            name = head[MULTIPART_CONTENT_ID, 1]
+          end
+
+          filename = get_filename(head)
+
+          name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+          @current = name
+
+          @parts[name] = if filename
+            FilePart.new(filename, content_type)
+          else
+            "".b
+          end
+
+          @state = :part_body
+        when :part_body
+          part = @parts[@current]
+
+          body_separator = if part.is_a?(FilePart)
+            "#{CRLF}#{CRLF}"
+          else
+            CRLF
+          end
+          idx = @buffer.index(body_separator)
+
+          if idx
+            payload = @buffer.byteslice(0..idx - 1)
+            @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+            part << payload
+            part.rewind if part.respond_to?(:rewind)
+            @state = :parse_boundary
+          else
+            part << @buffer
+            @buffer.clear
+          end
+        when :parse_boundary
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+          if @buffer == "--"
+            @buffer.clear
+            @state = :done
+            return
+          elsif @buffer.start_with?(CRLF)
+            @buffer = @buffer.byteslice(2..-1)
+            @state = :part_header
+          else
+            return
+          end
+        when :done
+          raise Error, "parsing should have been over by now"
+        end until @buffer.empty?
+      end
+
+      def get_filename(head)
+        filename = nil
+        case head
+        when RFC2183
+          params = Hash[*head.scan(DISPPARM).flat_map(&:compact)]
+
+          if (filename = params["filename"])
+            filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+          elsif (filename = params["filename*"])
+            encoding, _, filename = filename.split("'", 3)
+          end
+        when BROKEN_QUOTED, BROKEN_UNQUOTED
+          filename = Regexp.last_match(1)
+        end
+
+        return unless filename
+
+        filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+        filename.scrub!
+
+        filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+        filename.force_encoding ::Encoding.find(encoding) if encoding
+
+        filename
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -17,7 +17,7 @@ module HTTPX
 
       elsif defined?(MimeMagic)
 
-        def call(file, *)
+        def call(file, _)
           mime = MimeMagic.by_magic(file)
           mime.type if mime
         end
@@ -25,7 +25,7 @@ module HTTPX
       elsif system("which file", out: File::NULL)
         require "open3"
 
-        def call(file, *)
+        def call(file, _)
           return if file.eof? # file command returns "application/x-empty" for empty files
 
           Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
@@ -56,7 +56,7 @@ module HTTPX
 
       else
 
-        def call(*); end
+        def call(_, _); end
 
       end
     end

data/lib/httpx/plugins/multipart/part.rb

@@ -8,7 +8,7 @@ module HTTPX
       def call(value)
         # take out specialized objects of the way
         if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-          return [value, value.content_type, value.filename]
+          return value, value.content_type, value.filename
         end
 
         content_type = filename = nil
@@ -19,7 +19,7 @@ module HTTPX
           value = value[:body]
         end
 
-        value = value.open(:binmode => true) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
+        value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
         if value.is_a?(File)
           filename ||= File.basename(value.path)

data/lib/httpx/plugins/multipart.rb

@@ -29,13 +29,14 @@ module HTTPX
               # in order not to break legacy code, we'll keep loading http/form_data for them.
               require "http/form_data"
               warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
-                "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
-                "If you'd like to stop seeing this message, require 'http/form_data' yourself."
+                   "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
+                   "If you'd like to stop seeing this message, require 'http/form_data' yourself."
             end
           rescue LoadError
           end
           # :nocov:
           require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/decoder"
           require "httpx/plugins/multipart/part"
           require "httpx/plugins/multipart/mime_type_detector"
         end
@@ -56,6 +57,19 @@ module HTTPX
           end
         end
 
+        def decode(response)
+          content_type = response.content_type.mime_type
+
+          case content_type
+          when "application/x-www-form-urlencoded"
+            Transcoder::Form.decode(response)
+          when "multipart/form-data"
+            Decoder.new(response)
+          else
+            raise Error, "invalid form mime type (#{content_type})"
+          end
+        end
+
         def multipart?(data)
           data.any? do |_, v|
             MULTIPART_VALUE_COND.call(v) ||

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -34,13 +34,13 @@ module HTTPX
 
         alias_method :ntlm_auth, :ntlm_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             ntlm = request.options.ntlm
 
             if ntlm
               request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
                  probe_response.headers.key?("www-authenticate") &&
@@ -52,12 +52,12 @@ module HTTPX
                 request.transition(:idle)
 
                 request.headers["authorization"] = "NTLM #{ntlm_challenge}"
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/proxy/ssh.rb

@@ -19,10 +19,14 @@ module HTTPX
         end
 
         module InstanceMethods
-          private
+          def request(*args, **options)
+            raise ArgumentError, "must perform at least one request" if args.empty?
+
+            requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+            request = requests.first or return super
 
-          def send_requests(*requests, options)
-            request_options = @options.merge(options)
+            request_options = request.options
 
             return super unless request_options.proxy
 
@@ -37,18 +41,21 @@ module HTTPX
             if request_options.debug
               ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
             end
+
             request_uri = URI(requests.first.uri)
             @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)
-                super(*requests, options.merge(io: io))
+                super(*args, **options.merge(io: io))
               end
             ensure
               @_gateway.shutdown!
             end
           end
 
+          private
+
           def build_gateway_socket(port, request_uri, options)
             case request_uri.scheme
             when "https"

data/lib/httpx/plugins/proxy.rb

@@ -5,7 +5,8 @@ require "ipaddr"
 require "forwardable"
 
 module HTTPX
-  HTTPProxyError = Class.new(Error)
+  class HTTPProxyError < Error; end
+
   module Plugins
     #
     # This plugin adds support for proxies. It ships with support for:
@@ -136,10 +137,11 @@ module HTTPX
         def __proxy_error?(response)
           error = response.error
           case error
-          when ResolveError
+          when NativeResolveError
             # failed resolving proxy domain
-            proxy_uri = error.connection.options.proxy.uri
-            proxy_uri.to_s == @_proxy_uris.first
+            error.connection.origin.to_s == @_proxy_uris.first
+          when ResolveError
+            error.message.end_with?(@_proxy_uris.first)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true

data/lib/httpx/plugins/response_cache/store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module HTTPX::Plugins
+  module ResponseCache
+    class Store
+      extend Forwardable
+
+      def_delegator :@store, :clear
+
+      def initialize
+        @store = {}
+      end
+
+      def lookup(uri)
+        @store[uri]
+      end
+
+      def cached?(uri)
+        @store.key?(uri)
+      end
+
+      def cache(uri, response)
+        @store[uri] = response
+      end
+
+      def prepare(request)
+        cached_response = @store[request.uri]
+
+        return unless cached_response
+
+        original_request = cached_response.instance_variable_get(:@request)
+
+        if (vary = cached_response.headers["vary"])
+          if vary == "*"
+            return unless request.headers.same_headers?(original_request.headers)
+          else
+            return unless vary.split(/ *, */).all? do |cache_field|
+              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+            end
+          end
+        end
+
+        if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
+          request.headers.add("if-modified-since", last_modified)
+        end
+
+        if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          request.headers.add("if-none-match", etag)
+        end
+      end
+    end
+  end
+end