httpx 0.16.1 → 0.17.0

data/lib/httpx/plugins/multipart/decoder.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX::Plugins
+  module Multipart
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    CRLF = "\r\n"
+
+    class FilePart < SimpleDelegator
+      attr_reader :original_filename, :content_type
+
+      def initialize(filename, content_type)
+        @original_filename = filename
+        @content_type = content_type
+        @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+        super(@file)
+      end
+    end
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    CONDISP = /Content-Disposition:\s*#{TOKEN}\s*/i.freeze
+    BROKEN_QUOTED = /^#{CONDISP}.*;\s*filename="(.*?)"(?:\s*$|\s*;\s*#{TOKEN}=)/i.freeze
+    BROKEN_UNQUOTED = /^#{CONDISP}.*;\s*filename=(#{TOKEN})/i.freeze
+    MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+    MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+    # Updated definitions from RFC 2231
+    ATTRIBUTE_CHAR = %r{[^ \t\v\n\r)(><@,;:\\"/\[\]?='*%]}.freeze
+    ATTRIBUTE = /#{ATTRIBUTE_CHAR}+/.freeze
+    SECTION = /\*[0-9]+/.freeze
+    REGULAR_PARAMETER_NAME = /#{ATTRIBUTE}#{SECTION}?/.freeze
+    REGULAR_PARAMETER = /(#{REGULAR_PARAMETER_NAME})=(#{VALUE})/.freeze
+    EXTENDED_OTHER_NAME = /#{ATTRIBUTE}\*[1-9][0-9]*\*/.freeze
+    EXTENDED_OTHER_VALUE = /%[0-9a-fA-F]{2}|#{ATTRIBUTE_CHAR}/.freeze
+    EXTENDED_OTHER_PARAMETER = /(#{EXTENDED_OTHER_NAME})=(#{EXTENDED_OTHER_VALUE}*)/.freeze
+    EXTENDED_INITIAL_NAME = /#{ATTRIBUTE}(?:\*0)?\*/.freeze
+    EXTENDED_INITIAL_VALUE = /[a-zA-Z0-9\-]*'[a-zA-Z0-9\-]*'#{EXTENDED_OTHER_VALUE}*/.freeze
+    EXTENDED_INITIAL_PARAMETER = /(#{EXTENDED_INITIAL_NAME})=(#{EXTENDED_INITIAL_VALUE})/.freeze
+    EXTENDED_PARAMETER = /#{EXTENDED_INITIAL_PARAMETER}|#{EXTENDED_OTHER_PARAMETER}/.freeze
+    DISPPARM = /;\s*(?:#{REGULAR_PARAMETER}|#{EXTENDED_PARAMETER})\s*/.freeze
+    RFC2183 = /^#{CONDISP}(#{DISPPARM})+$/i.freeze
+
+    class Decoder
+      BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+      WINDOW_SIZE = 2 << 14
+
+      def initialize(response)
+        @boundary = begin
+          m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+          raise Error, "no boundary declared in content-type header" unless m
+
+          m.strip
+        end
+        @buffer = "".b
+        @parts = {}
+        @intermediate_boundary = "--#{@boundary}"
+        @state = :idle
+      end
+
+      def call(response, _)
+        response.body.each do |chunk|
+          @buffer << chunk
+
+          parse
+        end
+
+        raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+        @parts
+      end
+
+      private
+
+      def parse
+        case @state
+        when :idle
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+          @state = :part_header
+        when :part_header
+          idx = @buffer.index("#{CRLF}#{CRLF}")
+
+          # raise Error, "couldn't parse part headers" unless idx
+          return unless idx
+
+          head = @buffer.byteslice(0..idx + 4 - 1)
+
+          @buffer = @buffer.byteslice(head.bytesize..-1)
+
+          content_type = head[MULTIPART_CONTENT_TYPE, 1]
+          if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+            name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+            name.gsub!(/\\(.)/, "\\1")
+            name
+          else
+            name = head[MULTIPART_CONTENT_ID, 1]
+          end
+
+          filename = get_filename(head)
+
+          name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+          @current = name
+
+          @parts[name] = if filename
+            FilePart.new(filename, content_type)
+          else
+            "".b
+          end
+
+          @state = :part_body
+        when :part_body
+          part = @parts[@current]
+
+          body_separator = if part.is_a?(FilePart)
+            "#{CRLF}#{CRLF}"
+          else
+            CRLF
+          end
+          idx = @buffer.index(body_separator)
+
+          if idx
+            payload = @buffer.byteslice(0..idx - 1)
+            @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+            part << payload
+            part.rewind if part.respond_to?(:rewind)
+            @state = :parse_boundary
+          else
+            part << @buffer
+            @buffer.clear
+          end
+        when :parse_boundary
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+          if @buffer == "--"
+            @buffer.clear
+            @state = :done
+            return
+          elsif @buffer.start_with?(CRLF)
+            @buffer = @buffer.byteslice(2..-1)
+            @state = :part_header
+          else
+            return
+          end
+        when :done
+          raise Error, "parsing should have been over by now"
+        end until @buffer.empty?
+      end
+
+      def get_filename(head)
+        filename = nil
+        case head
+        when RFC2183
+          params = Hash[*head.scan(DISPPARM).flat_map(&:compact)]
+
+          if (filename = params["filename"])
+            filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+          elsif (filename = params["filename*"])
+            encoding, _, filename = filename.split("'", 3)
+          end
+        when BROKEN_QUOTED, BROKEN_UNQUOTED
+          filename = Regexp.last_match(1)
+        end
+
+        return unless filename
+
+        filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+        filename.scrub!
+
+        filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+        filename.force_encoding ::Encoding.find(encoding) if encoding
+
+        filename
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -17,7 +17,7 @@ module HTTPX
 
       elsif defined?(MimeMagic)
 
-        def call(file, *)
+        def call(file, _)
           mime = MimeMagic.by_magic(file)
           mime.type if mime
         end
@@ -25,7 +25,7 @@ module HTTPX
       elsif system("which file", out: File::NULL)
         require "open3"
 
-        def call(file, *)
+        def call(file, _)
           return if file.eof? # file command returns "application/x-empty" for empty files
 
           Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
@@ -56,7 +56,7 @@ module HTTPX
 
       else
 
-        def call(*); end
+        def call(_, _); end
 
       end
     end

data/lib/httpx/plugins/multipart/part.rb

@@ -8,7 +8,7 @@ module HTTPX
       def call(value)
         # take out specialized objects of the way
         if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-          return [value, value.content_type, value.filename]
+          return value, value.content_type, value.filename
         end
 
         content_type = filename = nil
@@ -19,7 +19,7 @@ module HTTPX
           value = value[:body]
         end
 
-        value = value.open(:binmode => true) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
+        value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
         if value.is_a?(File)
           filename ||= File.basename(value.path)

data/lib/httpx/plugins/multipart.rb

@@ -36,6 +36,7 @@ module HTTPX
           end
           # :nocov:
           require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/decoder"
           require "httpx/plugins/multipart/part"
           require "httpx/plugins/multipart/mime_type_detector"
         end
@@ -56,6 +57,19 @@ module HTTPX
           end
         end
 
+        def decode(response)
+          content_type = response.content_type.mime_type
+
+          case content_type
+          when "application/x-www-form-urlencoded"
+            Transcoder::Form.decode(response)
+          when "multipart/form-data"
+            Decoder.new(response)
+          else
+            raise Error, "invalid form mime type (#{content_type})"
+          end
+        end
+
         def multipart?(data)
           data.any? do |_, v|
             MULTIPART_VALUE_COND.call(v) ||

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -34,13 +34,13 @@ module HTTPX
 
         alias_method :ntlm_auth, :ntlm_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             ntlm = request.options.ntlm
 
             if ntlm
               request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
                  probe_response.headers.key?("www-authenticate") &&
@@ -52,12 +52,12 @@ module HTTPX
                 request.transition(:idle)
 
                 request.headers["authorization"] = "NTLM #{ntlm_challenge}"
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/proxy/ssh.rb

@@ -19,10 +19,14 @@ module HTTPX
         end
 
         module InstanceMethods
-          private
+          def request(*args, **options)
+            raise ArgumentError, "must perform at least one request" if args.empty?
+
+            requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+            request = requests.first or return super
 
-          def send_requests(*requests, options)
-            request_options = @options.merge(options)
+            request_options = request.options
 
             return super unless request_options.proxy
 
@@ -37,18 +41,21 @@ module HTTPX
             if request_options.debug
               ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
             end
+
             request_uri = URI(requests.first.uri)
             @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)
-                super(*requests, options.merge(io: io))
+                super(*args, **options.merge(io: io))
               end
             ensure
               @_gateway.shutdown!
             end
           end
 
+          private
+
           def build_gateway_socket(port, request_uri, options)
             case request_uri.scheme
             when "https"

data/lib/httpx/plugins/proxy.rb

@@ -5,7 +5,8 @@ require "ipaddr"
 require "forwardable"
 
 module HTTPX
-  HTTPProxyError = Class.new(Error)
+  class HTTPProxyError < Error; end
+
   module Plugins
     #
     # This plugin adds support for proxies. It ships with support for:
@@ -136,10 +137,11 @@ module HTTPX
         def __proxy_error?(response)
           error = response.error
           case error
-          when ResolveError
+          when NativeResolveError
             # failed resolving proxy domain
-            proxy_uri = error.connection.options.proxy.uri
-            proxy_uri.to_s == @_proxy_uris.first
+            error.connection.origin.to_s == @_proxy_uris.first
+          when ResolveError
+            error.message.end_with?(@_proxy_uris.first)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true

data/lib/httpx/plugins/stream.rb

@@ -6,7 +6,6 @@ module HTTPX
       @request = request
       @session = session
       @connections = connections
-      @options = @request.options
     end
 
     def each(&block)
@@ -72,7 +71,7 @@ module HTTPX
     private
 
     def response
-      @session.__send__(:receive_requests, [@request], @connections, @options) until @request.response
+      @session.__send__(:receive_requests, [@request], @connections) until @request.response
 
       @request.response
     end
@@ -106,7 +105,7 @@ module HTTPX
 
           request = requests.first
 
-          connections = _send_requests(requests, request.options)
+          connections = _send_requests(requests)
 
           StreamResponse.new(request, self, connections)
         end

data/lib/httpx/registry.rb

@@ -59,7 +59,7 @@ module HTTPX
         @registry ||= {}
         return @registry if tag.nil?
 
-        handler = @registry.fetch(tag)
+        handler = @registry[tag]
         raise(Error, "#{tag} is not registered in #{self}") unless handler
 
         handler

data/lib/httpx/request.rb

@@ -41,16 +41,15 @@ module HTTPX
 
     def_delegator :@body, :empty?
 
-    def_delegator :@body, :chunk!
-
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
       @options = Options.new(options)
       @uri     = Utils.to_uri(uri)
       if @uri.relative?
-        raise(Error, "invalid URI: #{@uri}") unless @options.origin
+        origin = @options.origin
+        raise(Error, "invalid URI: #{@uri}") unless origin
 
-        @uri = @options.origin.merge(@uri)
+        @uri = origin.merge(@uri)
       end
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -98,7 +97,7 @@ module HTTPX
     def response=(response)
       return unless response
 
-      if response.status == 100
+      if response.is_a?(Response) && response.status == 100
         @informational_status = response.status
         return
       end
@@ -158,7 +157,7 @@ module HTTPX
 
     class Body < SimpleDelegator
       class << self
-        def new(*, options)
+        def new(_, options)
           return options.body if options.body.is_a?(self)
 
           super
@@ -223,7 +222,7 @@ module HTTPX
       def unbounded_body?
         return @unbounded_body if defined?(@unbounded_body)
 
-        @unbounded_body = (chunked? || @body.bytesize == Float::INFINITY)
+        @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
       end
 
       def chunked?

data/lib/httpx/resolver/resolver_mixin.rb

@@ -9,7 +9,7 @@ module HTTPX
       include Callbacks
       include Loggable
 
-      CHECK_IF_IP = proc do |name|
+      CHECK_IF_IP = lambda do |name|
         begin
           IPAddr.new(name)
           true
@@ -55,6 +55,7 @@ module HTTPX
         return if ips.empty?
 
         ips.map { |ip| IPAddr.new(ip) }
+      rescue IOError
       end
 
       def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)