httpx 0.15.3 → 0.17.0

data/lib/httpx/plugins/h2c.rb

@@ -24,15 +24,19 @@ module HTTPX
         def call(connection, request, response)
           connection.upgrade_to_h2c(request, response)
         end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
       end
 
       module InstanceMethods
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           upgrade_request, *remainder = requests
 
           return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
 
-          connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
+          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
           return super if connection && connection.upgrade_protocol == :h2c
 
@@ -42,7 +46,7 @@ module HTTPX
           upgrade_request.headers["upgrade"] = "h2c"
           upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
 
-          super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
+          super(upgrade_request, *remainder)
         end
       end
 

data/lib/httpx/plugins/multipart/decoder.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX::Plugins
+  module Multipart
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    CRLF = "\r\n"
+
+    class FilePart < SimpleDelegator
+      attr_reader :original_filename, :content_type
+
+      def initialize(filename, content_type)
+        @original_filename = filename
+        @content_type = content_type
+        @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+        super(@file)
+      end
+    end
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    CONDISP = /Content-Disposition:\s*#{TOKEN}\s*/i.freeze
+    BROKEN_QUOTED = /^#{CONDISP}.*;\s*filename="(.*?)"(?:\s*$|\s*;\s*#{TOKEN}=)/i.freeze
+    BROKEN_UNQUOTED = /^#{CONDISP}.*;\s*filename=(#{TOKEN})/i.freeze
+    MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+    MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+    # Updated definitions from RFC 2231
+    ATTRIBUTE_CHAR = %r{[^ \t\v\n\r)(><@,;:\\"/\[\]?='*%]}.freeze
+    ATTRIBUTE = /#{ATTRIBUTE_CHAR}+/.freeze
+    SECTION = /\*[0-9]+/.freeze
+    REGULAR_PARAMETER_NAME = /#{ATTRIBUTE}#{SECTION}?/.freeze
+    REGULAR_PARAMETER = /(#{REGULAR_PARAMETER_NAME})=(#{VALUE})/.freeze
+    EXTENDED_OTHER_NAME = /#{ATTRIBUTE}\*[1-9][0-9]*\*/.freeze
+    EXTENDED_OTHER_VALUE = /%[0-9a-fA-F]{2}|#{ATTRIBUTE_CHAR}/.freeze
+    EXTENDED_OTHER_PARAMETER = /(#{EXTENDED_OTHER_NAME})=(#{EXTENDED_OTHER_VALUE}*)/.freeze
+    EXTENDED_INITIAL_NAME = /#{ATTRIBUTE}(?:\*0)?\*/.freeze
+    EXTENDED_INITIAL_VALUE = /[a-zA-Z0-9\-]*'[a-zA-Z0-9\-]*'#{EXTENDED_OTHER_VALUE}*/.freeze
+    EXTENDED_INITIAL_PARAMETER = /(#{EXTENDED_INITIAL_NAME})=(#{EXTENDED_INITIAL_VALUE})/.freeze
+    EXTENDED_PARAMETER = /#{EXTENDED_INITIAL_PARAMETER}|#{EXTENDED_OTHER_PARAMETER}/.freeze
+    DISPPARM = /;\s*(?:#{REGULAR_PARAMETER}|#{EXTENDED_PARAMETER})\s*/.freeze
+    RFC2183 = /^#{CONDISP}(#{DISPPARM})+$/i.freeze
+
+    class Decoder
+      BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+      WINDOW_SIZE = 2 << 14
+
+      def initialize(response)
+        @boundary = begin
+          m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+          raise Error, "no boundary declared in content-type header" unless m
+
+          m.strip
+        end
+        @buffer = "".b
+        @parts = {}
+        @intermediate_boundary = "--#{@boundary}"
+        @state = :idle
+      end
+
+      def call(response, _)
+        response.body.each do |chunk|
+          @buffer << chunk
+
+          parse
+        end
+
+        raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+        @parts
+      end
+
+      private
+
+      def parse
+        case @state
+        when :idle
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+          @state = :part_header
+        when :part_header
+          idx = @buffer.index("#{CRLF}#{CRLF}")
+
+          # raise Error, "couldn't parse part headers" unless idx
+          return unless idx
+
+          head = @buffer.byteslice(0..idx + 4 - 1)
+
+          @buffer = @buffer.byteslice(head.bytesize..-1)
+
+          content_type = head[MULTIPART_CONTENT_TYPE, 1]
+          if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+            name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+            name.gsub!(/\\(.)/, "\\1")
+            name
+          else
+            name = head[MULTIPART_CONTENT_ID, 1]
+          end
+
+          filename = get_filename(head)
+
+          name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+          @current = name
+
+          @parts[name] = if filename
+            FilePart.new(filename, content_type)
+          else
+            "".b
+          end
+
+          @state = :part_body
+        when :part_body
+          part = @parts[@current]
+
+          body_separator = if part.is_a?(FilePart)
+            "#{CRLF}#{CRLF}"
+          else
+            CRLF
+          end
+          idx = @buffer.index(body_separator)
+
+          if idx
+            payload = @buffer.byteslice(0..idx - 1)
+            @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+            part << payload
+            part.rewind if part.respond_to?(:rewind)
+            @state = :parse_boundary
+          else
+            part << @buffer
+            @buffer.clear
+          end
+        when :parse_boundary
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+          if @buffer == "--"
+            @buffer.clear
+            @state = :done
+            return
+          elsif @buffer.start_with?(CRLF)
+            @buffer = @buffer.byteslice(2..-1)
+            @state = :part_header
+          else
+            return
+          end
+        when :done
+          raise Error, "parsing should have been over by now"
+        end until @buffer.empty?
+      end
+
+      def get_filename(head)
+        filename = nil
+        case head
+        when RFC2183
+          params = Hash[*head.scan(DISPPARM).flat_map(&:compact)]
+
+          if (filename = params["filename"])
+            filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+          elsif (filename = params["filename*"])
+            encoding, _, filename = filename.split("'", 3)
+          end
+        when BROKEN_QUOTED, BROKEN_UNQUOTED
+          filename = Regexp.last_match(1)
+        end
+
+        return unless filename
+
+        filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+        filename.scrub!
+
+        filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+        filename.force_encoding ::Encoding.find(encoding) if encoding
+
+        filename
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -17,7 +17,7 @@ module HTTPX
 
       elsif defined?(MimeMagic)
 
-        def call(file, *)
+        def call(file, _)
           mime = MimeMagic.by_magic(file)
           mime.type if mime
         end
@@ -25,7 +25,7 @@ module HTTPX
       elsif system("which file", out: File::NULL)
         require "open3"
 
-        def call(file, *)
+        def call(file, _)
           return if file.eof? # file command returns "application/x-empty" for empty files
 
           Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
@@ -56,7 +56,7 @@ module HTTPX
 
       else
 
-        def call(*); end
+        def call(_, _); end
 
       end
     end

data/lib/httpx/plugins/multipart/part.rb

@@ -8,7 +8,7 @@ module HTTPX
       def call(value)
         # take out specialized objects of the way
         if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-          return [value, value.content_type, value.filename]
+          return value, value.content_type, value.filename
         end
 
         content_type = filename = nil
@@ -19,7 +19,7 @@ module HTTPX
           value = value[:body]
         end
 
-        value = value.open(:binmode => true) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
+        value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
         if value.is_a?(File)
           filename ||= File.basename(value.path)

data/lib/httpx/plugins/multipart.rb

@@ -36,6 +36,7 @@ module HTTPX
           end
           # :nocov:
           require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/decoder"
           require "httpx/plugins/multipart/part"
           require "httpx/plugins/multipart/mime_type_detector"
         end
@@ -56,6 +57,19 @@ module HTTPX
           end
         end
 
+        def decode(response)
+          content_type = response.content_type.mime_type
+
+          case content_type
+          when "application/x-www-form-urlencoded"
+            Transcoder::Form.decode(response)
+          when "multipart/form-data"
+            Decoder.new(response)
+          else
+            raise Error, "invalid form mime type (#{content_type})"
+          end
+        end
+
         def multipart?(data)
           data.any? do |_, v|
             MULTIPART_VALUE_COND.call(v) ||

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -15,13 +15,15 @@ module HTTPX
         end
 
         def extra_options(options)
-          Class.new(options.class) do
-            def_option(:ntlm, <<-OUT)
-              raise Error, ":ntlm must be a #{NTLMParams}" unless value.is_a?(#{NTLMParams})
+          options.merge(max_concurrent_requests: 1)
+        end
+      end
+
+      module OptionsMethods
+        def option_ntlm(value)
+          raise TypeError, ":ntlm must be a #{NTLMParams}" unless value.is_a?(NTLMParams)
 
-              value
-            OUT
-          end.new(options).merge(max_concurrent_requests: 1)
+          value
         end
       end
 
@@ -32,13 +34,13 @@ module HTTPX
 
         alias_method :ntlm_auth, :ntlm_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             ntlm = request.options.ntlm
 
             if ntlm
               request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
                  probe_response.headers.key?("www-authenticate") &&
@@ -50,12 +52,12 @@ module HTTPX
                 request.transition(:idle)
 
                 request.headers["authorization"] = "NTLM #{ntlm_challenge}"
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,8 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  Socks4Error = Class.new(Error)
+  class Socks4Error < Error; end
+
   module Plugins
     module Proxy
       module Socks4

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module HTTPX
-  Socks5Error = Class.new(Error)
+  class Socks5Error < Error; end
+
   module Plugins
     module Proxy
       module Socks5

data/lib/httpx/plugins/proxy/ssh.rb

@@ -6,23 +6,27 @@ module HTTPX
   module Plugins
     module Proxy
       module SSH
-        def self.load_dependencies(*)
-          require "net/ssh/gateway"
+        class << self
+          def load_dependencies(*)
+            require "net/ssh/gateway"
+          end
         end
 
-        def self.extra_options(options)
-          Class.new(options.class) do
-            def_option(:proxy, <<-OUT)
-              Hash[value]
-            OUT
-          end.new(options)
+        module OptionsMethods
+          def option_proxy(value)
+            Hash[value]
+          end
         end
 
         module InstanceMethods
-          private
+          def request(*args, **options)
+            raise ArgumentError, "must perform at least one request" if args.empty?
+
+            requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
 
-          def send_requests(*requests, options)
-            request_options = @options.merge(options)
+            request = requests.first or return super
+
+            request_options = request.options
 
             return super unless request_options.proxy
 
@@ -37,18 +41,21 @@ module HTTPX
             if request_options.debug
               ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
             end
+
             request_uri = URI(requests.first.uri)
             @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)
-                super(*requests, options.merge(io: io))
+                super(*args, **options.merge(io: io))
               end
             ensure
               @_gateway.shutdown!
             end
           end
 
+          private
+
           def build_gateway_socket(port, request_uri, options)
             case request_uri.scheme
             when "https"
@@ -65,7 +72,7 @@ module HTTPX
             when "http"
               TCPSocket.open("localhost", port)
             else
-              raise Error, "unexpected scheme: #{request_uri.scheme}"
+              raise TypeError, "unexpected scheme: #{request_uri.scheme}"
             end
           end
         end

data/lib/httpx/plugins/proxy.rb

@@ -5,7 +5,8 @@ require "ipaddr"
 require "forwardable"
 
 module HTTPX
-  HTTPProxyError = Class.new(Error)
+  class HTTPProxyError < Error; end
+
   module Plugins
     #
     # This plugin adds support for proxies. It ships with support for:
@@ -64,13 +65,11 @@ module HTTPX
           klass.plugin(:"proxy/socks4")
           klass.plugin(:"proxy/socks5")
         end
+      end
 
-        def extra_options(options)
-          Class.new(options.class) do
-            def_option(:proxy, <<-OUT)
-              value.is_a?(#{Parameters}) ? value : Hash[value]
-            OUT
-          end.new(options)
+      module OptionsMethods
+        def option_proxy(value)
+          value.is_a?(Parameters) ? value : Hash[value]
         end
       end
 
@@ -138,10 +137,11 @@ module HTTPX
         def __proxy_error?(response)
           error = response.error
           case error
-          when ResolveError
+          when NativeResolveError
             # failed resolving proxy domain
-            proxy_uri = error.connection.options.proxy.uri
-            proxy_uri.to_s == @_proxy_uris.first
+            error.connection.origin.to_s == @_proxy_uris.first
+          when ResolveError
+            error.message.end_with?(@_proxy_uris.first)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true