httpx 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e3727a5374ad3c2d6d7c5dbfd61040e3aa3ea6c74aa6ad12d662a5bd0b9a7af
-  data.tar.gz: 7b1a2aa15418b03fc276b81cfaf0565bbce59dd531c2bfc10f19ddc60506213b
+  metadata.gz: ed5ebccf22cf5c4719a81db15ee29a7d05e08fa43bd83b14c3b2682c083615be
+  data.tar.gz: 2befada7d0c17d0c20d0afce44fc0a9d3192ffb2bb311322ee171107d285d050
 SHA512:
-  metadata.gz: cda3ab9929396bdbad4ff38472dea4d7af34946e34cf995228ad5d52d05e37664b53f66e0a0ac8dd10193e55de52b4f7575f203bea8202b5474f0c3cca450d24
-  data.tar.gz: 025a985fc7c05abbb1bd66d23356d5165f7536d6565464e0729c618da624fc9f428e1046bba7c09dc5d7c23ada0587f73fb23e42128c96847857bcc1ccd6d408
+  metadata.gz: 9abffdd09d11f8c22c51f38064525d6d5250fe1e72a2e369f79c98e7d02ee6fe7a7442ac76758bf60cbc87eabe51a3faa30a675f4a30cb83a01e09aa98334d0a
+  data.tar.gz: 9c0706ac930dc0e082845795784a5007d0cd9bb25f4a81435be6d8e081ba29528a0df05fc03713dc07615329944ca586e4df8386766c6b5ff440c96d90dc472e

data/doc/release_notes/0_10_1.md

@@ -0,0 +1,39 @@
+# 0.10.1
+
+## Improvements
+
+### URL-encoded nested params
+
+url encoder now supports nested params, which is a standard of rack-based frameworks:
+
+```ruby
+HTTPX.post("https://httpbin.org/post", form: { a: { b: 1 }, c: [2, 3] })
+# a[b]=1&c[]=2&c[]=3
+```
+
+This encoding scheme is now the standard for URL-encoded request bodies, query params, and `:multipart` plugin requests.
+
+### Socks4 IPv6 addresses
+
+HTTPX supports IPv6 Socks4 proxies now. This support is restricted to rubies where `IPAddr#hton` is implemented though, so you are encouraged to upgrade.
+
+## More verbose HTTP Errors
+
+`HTTPX::Response#raise_for_status` was raising exceptions for client/server HTTP errors codes (4xx/5xx). However, only the status code was part of the message.
+
+From now on, both headers and the responnse payload will also appear, so expected more verbosity, but also more meaningful information.
+
+## Bugfixes
+
+* HTTP/2 and HTTP/1.1 exhausted connections now get properly migrated into a new connection;
+* HTTP/2 421 responses will now correctly migrate the connection and pendign requests to HTTP/1.1 (a hanging loop was being caused);
+* HTTP/2 connection failed with a GOAWAY settings timeout will now return error responses (instead of hanging indefinitely);
+* Non-IP proxy name-resolving errors will now move on to the next available proxy in the list (instead of hanging indefinitely);
+* Non-IP DNS resolve errors for `native` and `https` variannts will now return the appropriate error response (instead of hanging indefinitely);
+* 
+
+
+## Chore
+
+* `HTTPX.plugins` is now officially deprecated (use `HTTPX.plugin` instead);
+

data/lib/httpx/chainable.rb

@@ -42,12 +42,15 @@ module HTTPX
     end
 
     # deprecated
+    # :nocov:
     def plugins(*args, **opts)
+      warn ":#{__method__} is deprecated, use :plugin instead"
       klass = is_a?(Session) ? self.class : Session
       klass = Class.new(klass)
       klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
       klass.plugins(*args, **opts).new
     end
+    # :nocov:
 
     def with(options, &blk)
       branch(default_options.merge(options), &blk)
@@ -66,12 +69,10 @@ module HTTPX
     end
 
     def method_missing(meth, *args, **options)
-      if meth =~ /\Awith_(.+)/
-        option = Regexp.last_match(1).to_sym
-        with(option => (args.first || options))
-      else
-        super
-      end
+      return super unless meth =~ /\Awith_(.+)/
+
+      option = Regexp.last_match(1).to_sym
+      with(option => (args.first || options))
     end
 
     def respond_to_missing?(meth, *args)

data/lib/httpx/connection.rb

@@ -120,8 +120,8 @@ module HTTPX
       end
     end
 
-    def create_idle
-      self.class.new(@type, @origin, @options)
+    def create_idle(options = {})
+      self.class.new(@type, @origin, @options.merge(options))
     end
 
     def merge(connection)
@@ -131,17 +131,6 @@ module HTTPX
       end
     end
 
-    def unmerge(connection)
-      @origins -= connection.instance_variable_get(:@origins)
-      purge_pending do |request|
-        request.uri.origin == connection.origin && begin
-          request.transition(:idle)
-          connection.send(request)
-          true
-        end
-      end
-    end
-
     def purge_pending(&block)
       pendings = []
       if @parser
@@ -399,7 +388,7 @@ module HTTPX
       parser.on(:error) do |request, ex|
         case ex
         when MisdirectedRequestError
-          emit(:uncoalesce, request.uri)
+          emit(:misdirected, request)
         else
           response = ErrorResponse.new(request, ex, @options)
           request.emit(:response, response)
@@ -435,7 +424,7 @@ module HTTPX
           remove_instance_variable(:@total_timeout)
         end
 
-        @io.close
+        @io.close if @io
         @read_buffer.clear
         if @keep_alive_timer
           @keep_alive_timer.cancel

data/lib/httpx/connection/http1.rb

@@ -21,6 +21,7 @@ module HTTPX
       @version = [1, 1]
       @pending = []
       @requests = []
+      @handshake_completed = false
     end
 
     def interests
@@ -155,6 +156,7 @@ module HTTPX
       @parser.reset!
       @max_requests -= 1
       manage_connection(response)
+
       send(@pending.shift) unless @pending.empty?
     end
 
@@ -182,17 +184,28 @@ module HTTPX
       connection = response.headers["connection"]
       case connection
       when /keep-alive/i
+        if @handshake_completed
+          if @max_requests.zero?
+            @pending.concat(@requests)
+            @requests.clear
+            emit(:exhausted)
+          end
+          return
+        end
+
         keep_alive = response.headers["keep-alive"]
         return unless keep_alive
 
         parameters = Hash[keep_alive.split(/ *, */).map do |pair|
           pair.split(/ *= */)
         end]
-        @max_requests = parameters["max"].to_i if parameters.key?("max")
+        @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
+
         if parameters.key?("timeout")
           keep_alive_timeout = parameters["timeout"].to_i
           emit(:timeout, keep_alive_timeout)
         end
+        @handshake_completed = true
       when /close/i
         disable
       when nil

data/lib/httpx/connection/http2.rb

@@ -51,12 +51,8 @@ module HTTPX
       :rw
     end
 
-    def reset
-      init_connection
-    end
-
-    def close(*args)
-      @connection.goaway(*args) unless @connection.state == :closed
+    def close
+      @connection.goaway unless @connection.state == :closed
       emit(:close)
     end
 
@@ -163,6 +159,9 @@ module HTTPX
       @connection.send_connection_preface
     end
 
+    alias_method :reset, :init_connection
+    public :reset
+
     def handle_stream(stream, request)
       stream.on(:close, &method(:on_stream_close).curry[stream, request])
       stream.on(:half_close) do
@@ -270,16 +269,16 @@ module HTTPX
     end
 
     def on_close(_last_frame, error, _payload)
+      is_connection_closed = @connection.state == :closed
       if error && error != :no_error
+        @buffer.clear if is_connection_closed
         ex = Error.new(0, error)
         ex.set_backtrace(caller)
-        @streams.each_key do |request|
-          emit(:error, request, ex)
-        end
+        handle_error(ex)
       end
-      return unless @connection.state == :closed && @streams.size.zero?
+      return unless is_connection_closed && @streams.size.zero?
 
-      emit(:close)
+      emit(:close, is_connection_closed)
     end
 
     def on_frame_sent(frame)
@@ -317,7 +316,7 @@ module HTTPX
     end
 
     def on_pong(ping)
-      if !@pings.delete(ping)
+      if !@pings.delete(ping.to_s)
         close(:protocol_error, "ping payload did not match")
       else
         emit(:pong)

data/lib/httpx/errors.rb

@@ -41,7 +41,7 @@ module HTTPX
 
     def initialize(response)
       @response = response
-      super("HTTP Error: #{@response.status}")
+      super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
     end
 
     def status

data/lib/httpx/plugins/multipart.rb

@@ -23,12 +23,26 @@ module HTTPX
           def_delegator :@raw, :read
 
           def initialize(form)
-            @raw = HTTP::FormData.create(form)
+            @raw = if multipart?(form)
+              HTTP::FormData::Multipart.new(Hash[*form.map { |k, v| Transcoder.enum_for(:normalize_keys, k, v).to_a }])
+            else
+              HTTP::FormData::Urlencoded.new(form, :encoder => Transcoder::Form.method(:encode))
+            end
           end
 
           def bytesize
             @raw.content_length
           end
+
+          private
+
+          def multipart?(data)
+            data.any? do |_, v|
+              v.is_a?(HTTP::FormData::Part) ||
+                (v.respond_to?(:to_ary) && v.to_ary.any? { |e| e.is_a?(HTTP::FormData::Part) }) ||
+                (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| e.is_a?(HTTP::FormData::Part) })
+            end
+          end
         end
 
         def encode(form)

data/lib/httpx/plugins/proxy.rb

@@ -121,8 +121,7 @@ module HTTPX
         def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
-             # either it was a timeout error connecting, or it was a proxy error
-             PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
+             __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
 
           super
         end
+
+        def __proxy_error?(response)
+          error = response.error
+          case error
+          when ResolveError
+            # failed resolving proxy domain
+            proxy_uri = error.connection.options.proxy.uri
+            proxy_uri.to_s == @_proxy_uris.first
+          when *PROXY_ERRORS
+            # timeout errors connecting to proxy
+            true
+          else
+            false
+          end
+        end
       end
 
       module ConnectionMethods

data/lib/httpx/plugins/proxy/socks4.rb

@@ -10,7 +10,7 @@ module HTTPX
       module Socks4
         VERSION = 4
         CONNECT = 1
-        GRANTED = 90
+        GRANTED = 0x5A
         PROTOCOLS = %w[socks4 socks4a].freeze
 
         Error = Socks4Error
@@ -91,27 +91,25 @@ module HTTPX
         end
 
         module Packet
-          using(RegexpExtensions) unless Regexp.method_defined?(:match?)
-
           module_function
 
           def connect(parameters, uri)
             packet = [VERSION, CONNECT, uri.port].pack("CCn")
-            begin
-              ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
-
-              packet << [ip.to_i].pack("N")
-            rescue IPAddr::InvalidAddressError
-              if /^socks4a?$/.match?(parameters.uri.scheme)
-                # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
-                ip = IPAddr.new(Resolv.getaddress(uri.host))
-                packet << [ip.to_i].pack("N")
-              else
-                packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
+
+            case parameters.uri.scheme
+            when "socks4"
+              socks_host = uri.host
+              begin
+                ip = IPAddr.new(socks_host)
+                packet << ip.hton
+              rescue IPAddr::InvalidAddressError
+                socks_host = Resolv.getaddress(socks_host)
+                retry
               end
+              packet << [parameters.username].pack("Z*")
+            when "socks4a"
+              packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
             end
-            packet << [parameters.username].pack("Z*")
             packet
           end
         end

data/lib/httpx/pool.rb

@@ -108,10 +108,10 @@ module HTTPX
       end
     end
 
-    def on_resolver_error(ch, error)
-      ch.emit(:error, error)
+    def on_resolver_error(connection, error)
+      connection.emit(:error, error)
       # must remove connection by hand, hasn't been started yet
-      unregister_connection(ch)
+      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -144,12 +144,12 @@ module HTTPX
       @connected_connections -= 1
     end
 
-    def coalesce_connections(ch1, ch2)
-      if ch1.coalescable?(ch2)
-        ch1.merge(ch2)
-        @connections.delete(ch2)
+    def coalesce_connections(conn1, conn2)
+      if conn1.coalescable?(conn2)
+        conn1.merge(conn2)
+        @connections.delete(conn2)
       else
-        register_connection(ch2)
+        register_connection(conn2)
       end
     end
 
@@ -168,12 +168,6 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      rescue ArgumentError
-        # this block is here because of an error which happens on CI from time to time
-        warn "tried resolver: #{resolver_type}"
-        warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
-        warn "new: #{resolver_type.method(:new).source_location}"
-        raise
       end
     end
   end

data/lib/httpx/request.rb

@@ -106,7 +106,7 @@ module HTTPX
 
       query = []
       if (q = @options.params)
-        query << URI.encode_www_form(q)
+        query << Transcoder.registry("form").encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")

data/lib/httpx/resolver.rb

@@ -101,5 +101,3 @@ module HTTPX
     end
   end
 end
-
-require "httpx/resolver/options"

data/lib/httpx/resolver/https.rb

@@ -21,6 +21,7 @@ module HTTPX
     DEFAULTS = {
       uri: NAMESERVER,
       use_get: false,
+      record_types: RECORD_TYPES.keys,
     }.freeze
 
     def_delegator :@connections, :empty?
@@ -29,12 +30,12 @@ module HTTPX
 
     def initialize(options)
       @options = Options.new(options)
-      @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
-      @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
+      @resolver_options = DEFAULTS.merge(@options.resolver_options)
+      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @queries = {}
       @requests = {}
       @connections = []
-      @uri = URI(@resolver_options.uri)
+      @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
     end
 
@@ -44,12 +45,12 @@ module HTTPX
       @uri_addresses ||= Resolv.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
-        ex = ResolveError.new("Can't resolve #{connection.origin.host}")
+        ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        emit(:error, connection, ex)
-      else
-        early_resolve(connection) || resolve(connection)
+        throw(:resolve_error, ex)
       end
+
+      early_resolve(connection) || resolve(connection)
     end
 
     def timeout
@@ -70,12 +71,6 @@ module HTTPX
 
     private
 
-    def connect
-      return if @queries.empty?
-
-      resolver_connection.__send__(__method__)
-    end
-
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
@@ -100,10 +95,12 @@ module HTTPX
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
       end
-      type = @_record_types[hostname].first
+      type = @_record_types[hostname].first || "A"
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
+        request.on(:response, &method(:on_response).curry[request])
+        request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)
         @queries[hostname] = connection
@@ -118,9 +115,7 @@ module HTTPX
     rescue StandardError => e
       connection = @requests[request]
       hostname = @queries.key(connection)
-      error = ResolveError.new("Can't resolve #{hostname}: #{e.message}")
-      error.set_backtrace(e.backtrace)
-      emit(:error, connection, error)
+      emit_resolve_error(connection, hostname, e)
     else
       parse(response)
     ensure
@@ -143,7 +138,7 @@ module HTTPX
           return
         end
       end
-      if answers.empty?
+      if answers.nil? || answers.empty?
         host, connection = @queries.first
         @_record_types[host].shift
         if @_record_types[host].empty?
@@ -177,7 +172,7 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
-          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options.cache
+          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
           emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
@@ -191,7 +186,7 @@ module HTTPX
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
 
-      if @resolver_options.use_get
+      if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", type]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
@@ -202,8 +197,6 @@ module HTTPX
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
-      request.on(:response, &method(:on_response).curry[request])
-      request.on(:promise, &method(:on_promise))
       request
     end