httpx 0.10.2 → 0.12.0

data/lib/httpx/plugins/push_promise.rb

@@ -43,7 +43,7 @@ module HTTPX
         end
 
         def __on_promise_request(parser, stream, h)
-          log(level: 1) do
+          log(level: 1, color: :yellow) do
             # :nocov:
             h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
             # :nocov:
@@ -57,6 +57,8 @@ module HTTPX
             request.merge_headers(headers)
             promise_headers[stream] = request
             parser.pending.delete(request)
+            parser.streams[request] = stream
+            request.transition(:done)
           else
             stream.refuse
           end
@@ -67,11 +69,10 @@ module HTTPX
           return unless request
 
           parser.__send__(:on_stream_headers, stream, request, h)
-          request.transition(:done)
           response = request.response
           response.mark_as_pushed!
-          stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
-          stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
+          stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
+          stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
         end
       end
     end

data/lib/httpx/plugins/retries.rb

@@ -17,7 +17,7 @@ module HTTPX
                           Errno::ECONNRESET,
                           Errno::ECONNABORTED,
                           Errno::EPIPE,
-                          (OpenSSL::SSL::SSLError if defined?(OpenSSL)),
+                          (TLSError if defined?(TLSError)),
                           TimeoutError,
                           Parser::Error,
                           Errno::EINVAL,

data/lib/httpx/plugins/stream.rb

@@ -119,11 +119,9 @@ module HTTPX
         end
 
         def method_missing(meth, *args, &block)
-          if @options.response_class.public_method_defined?(meth)
-            response.__send__(meth, *args, &block)
-          else
-            super
-          end
+          return super unless @options.response_class.public_method_defined?(meth)
+
+          response.__send__(meth, *args, &block)
         end
       end
     end

data/lib/httpx/pool.rb

@@ -135,7 +135,6 @@ module HTTPX
       connection.on(:close) do
         unregister_connection(connection)
       end
-      return if connection.state == :open
     end
 
     def unregister_connection(connection)

data/lib/httpx/registry.rb

@@ -62,13 +62,7 @@ module HTTPX
         handler = @registry.fetch(tag)
         raise(Error, "#{tag} is not registered in #{self}") unless handler
 
-        case handler
-        when Symbol, String
-          obj = const_get(handler)
-          @registry[tag] = obj
-        else
-          handler
-        end
+        handler
       end
 
       # @param [Object] tag the identifier for the handler in the registry

data/lib/httpx/request.rb

@@ -81,6 +81,10 @@ module HTTPX
     def response=(response)
       return unless response
 
+      if response.status == 100
+        @informational_status = response.status
+        return
+      end
       @response = response
     end
 
@@ -170,6 +174,12 @@ module HTTPX
         end
       end
 
+      def rewind
+        return if empty?
+
+        @body.rewind if @body.respond_to?(:rewind)
+      end
+
       def empty?
         return true if @body.nil?
         return false if chunked?
@@ -207,11 +217,22 @@ module HTTPX
         "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
       end
       # :nocov:
+
+      def respond_to_missing?(meth, *args)
+        @body.respond_to?(meth, *args) || super
+      end
+
+      def method_missing(meth, *args, &block)
+        return super unless @body.respond_to?(meth)
+
+        @body.__send__(meth, *args, &block)
+      end
     end
 
     def transition(nextstate)
       case nextstate
       when :idle
+        @body.rewind
         @response = nil
         @drainer = nil
       when :headers
@@ -221,28 +242,27 @@ module HTTPX
                       @state == :expect
 
         if @headers.key?("expect")
-          unless @response
-            @state = :expect
-            return
-          end
-
-          case @response.status
-          when 100
-            # deallocate
-            @response = nil
+          if @informational_status && @informational_status == 100
+            # check for 100 Continue response, and deallocate the var
+            # if @informational_status == 100
+            #   @response = nil
+            # end
+          else
+            return if @state == :expect # do not re-set it
+
+            nextstate = :expect
           end
         end
       when :done
         return if @state == :expect
       end
       @state = nextstate
-      emit(@state)
+      emit(@state, self)
       nil
     end
 
     def expects?
-      @headers["expect"] == "100-continue" &&
-        @response && @response.status == 100
+      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
 
     class ProcIO

data/lib/httpx/resolver.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "httpx/resolver/resolver_mixin"
-require "httpx/resolver/system"
-require "httpx/resolver/native"
-require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
     extend Registry
 
+    RESOLVE_TIMEOUT = 5
+
+    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/system"
+    require "httpx/resolver/native"
+    require "httpx/resolver/https"
+
     register :system, System
     register :native, Native
     register :https,  HTTPS

data/lib/httpx/resolver/https.rb

@@ -37,12 +37,14 @@ module HTTPX
       @connections = []
       @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
+      @resolver = Resolv::DNS.new
+      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
     end
 
     def <<(connection)
       return if @uri.origin == connection.origin.to_s
 
-      @uri_addresses ||= Resolv.getaddresses(@uri.host)
+      @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -53,20 +55,12 @@ module HTTPX
       early_resolve(connection) || resolve(connection)
     end
 
-    def timeout
-      @connections.map(&:timeout).min
-    end
-
     def closed?
-      return true unless @resolver_connection
-
-      resolver_connection.closed?
+      true
     end
 
-    def interests
-      return if @queries.empty?
-
-      resolver_connection.__send__(__method__)
+    def empty?
+      true
     end
 
     private
@@ -99,7 +93,7 @@ module HTTPX
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
-        request.on(:response, &method(:on_response).curry[request])
+        request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)

data/lib/httpx/resolver/native.rb

@@ -9,7 +9,6 @@ module HTTPX
     include Resolver::ResolverMixin
     using URIExtensions
 
-    RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
       "A" => Resolv::DNS::Resource::IN::A,
       "AAAA" => Resolv::DNS::Resource::IN::AAAA,
@@ -19,7 +18,7 @@ module HTTPX
       {
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     else
@@ -27,7 +26,7 @@ module HTTPX
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     end
@@ -148,14 +147,19 @@ module HTTPX
           queries[h] = connection
           next
         end
+
         @timeouts[host].shift
         if @timeouts[host].empty?
           @timeouts.delete(host)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, host)
+          # This loop_time passed to the exception is bogus. Ideally we would pass the total
+          # resolve timeout, including from the previous retries.
+          raise ResolveTimeoutError.new(loop_time, "Timed out")
+          # raise NativeResolveError.new(connection, host)
         else
+          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
           connections << connection
-          log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
+          queries[h] = connection
         end
       end
       @queries = queries
@@ -279,7 +283,7 @@ module HTTPX
         @io.connect
         return unless @io.connected?
 
-        resolve if @queries.empty?
+        resolve if @queries.empty? && !@connections.empty?
       when :closed
         return unless @state == :open
 

data/lib/httpx/resolver/system.rb

@@ -20,7 +20,7 @@ module HTTPX
       timeouts = resolv_options.delete(:timeouts)
       resolv_options.delete(:cache)
       @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
-      @resolver.timeouts = timeouts if timeouts
+      @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
     end
 
     def closed?

data/lib/httpx/response.rb

@@ -268,8 +268,15 @@ module HTTPX
       @error.message
     end
 
-    def to_s
-      @error.backtrace.join("\n")
+    if Exception.method_defined?(:full_message)
+      def to_s
+        @error.full_message
+      end
+    else
+      def to_s
+        "#{@error.message} (#{@error.class})\n" \
+        "#{@error.backtrace.join("\n") if @error.backtrace}"
+      end
     end
 
     def raise_for_status

data/lib/httpx/selector.rb

@@ -69,6 +69,11 @@ class HTTPX::Selector
 
         if @selectables.empty?
           @selectables = selectables
+
+          # do not run event loop if there's nothing to wait on.
+          # this might happen if connect failed and connection was unregistered.
+          return if (!r || r.empty?) && (!w || w.empty?)
+
           break
         else
           @selectables = [*selectables, @selectables]
@@ -117,6 +122,7 @@ class HTTPX::Selector
     yield io
   rescue IOError, SystemCallError
     @selectables.reject!(&:closed?)
+    raise unless @selectables.empty?
   end
 
   def select(interval, &block)

data/lib/httpx/session.rb

@@ -41,7 +41,7 @@ module HTTPX
     def build_request(verb, uri, options = EMPTY_HASH)
       rklass = @options.request_class
       request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
-      request.on(:response, &method(:on_response).curry[request])
+      request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
       request
     end
@@ -136,23 +136,20 @@ module HTTPX
     def build_requests(*args, options)
       request_options = @options.merge(options)
 
-      requests = case args.size
-                 when 1
-                   reqs = args.first
-                   reqs.map do |verb, uri, opts = EMPTY_HASH|
-                     build_request(verb, uri, request_options.merge(opts))
-                   end
-                 when 2
-                   verb, uris = args
-                   if uris.respond_to?(:each)
-                     uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
-                       build_request(verb, uri, request_options.merge(opts))
-                     end
-                   else
-                     [build_request(verb, uris, request_options)]
-                   end
-                 else
-                   raise ArgumentError, "unsupported number of arguments"
+      requests = if args.size == 1
+        reqs = args.first
+        reqs.map do |verb, uri, opts = EMPTY_HASH|
+          build_request(verb, uri, request_options.merge(opts))
+        end
+      else
+        verb, uris = args
+        if uris.respond_to?(:each)
+          uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
+            build_request(verb, uri, request_options.merge(opts))
+          end
+        else
+          [build_request(verb, uris, request_options)]
+        end
       end
       raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
 
@@ -202,7 +199,18 @@ module HTTPX
           responses << response
           requests.shift
 
-          break if requests.empty? || pool.empty?
+          break if requests.empty?
+
+          next unless pool.empty?
+
+          # in some cases, the pool of connections might have been drained because there was some
+          # handshake error, and the error responses have already been emitted, but there was no
+          # opportunity to traverse the requests, hence we're returning only a fraction of the errors
+          # we were supposed to. This effectively fetches the existing responses and return them.
+          while (request = requests.shift)
+            responses << fetch_response(request, connections, request_options)
+          end
+          break
         end
         responses
       ensure
@@ -272,7 +280,19 @@ module HTTPX
       end
       # :nocov:
     end
+  end
+
+  unless ENV.grep(/https?_proxy$/i).empty?
+    proxy_session = plugin(:proxy)
+    ::HTTPX.send(:remove_const, :Session)
+    ::HTTPX.send(:const_set, :Session, proxy_session.class)
+  end
 
-    plugin(:proxy) unless ENV.grep(/https?_proxy$/i).empty?
+  # :nocov:
+  if Session.default_options.debug_level > 2
+    proxy_session = plugin(:internal_telemetry)
+    ::HTTPX.send(:remove_const, :Session)
+    ::HTTPX.send(:const_set, :Session, proxy_session.class)
   end
+  # :nocov:
 end

data/lib/httpx/transcoder.rb

@@ -4,18 +4,20 @@ module HTTPX
   module Transcoder
     extend Registry
 
-    def self.normalize_keys(key, value, &block)
-      if value.respond_to?(:to_ary)
+    def self.normalize_keys(key, value, cond = nil, &block)
+      if (cond && cond.call(value))
+        block.call(key.to_s, value)
+      elsif value.respond_to?(:to_ary)
         if value.empty?
           block.call("#{key}[]")
         else
           value.to_ary.each do |element|
-            normalize_keys("#{key}[]", element, &block)
+            normalize_keys("#{key}[]", element, cond, &block)
           end
         end
       elsif value.respond_to?(:to_hash)
         value.to_hash.each do |child_key, child_value|
-          normalize_keys("#{key}[#{child_key}]", child_value, &block)
+          normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
         end
       else
         block.call(key.to_s, value)