httpx 0.15.4 → 0.18.0

data/lib/httpx/plugins/multipart.rb

@@ -29,13 +29,14 @@ module HTTPX
               # in order not to break legacy code, we'll keep loading http/form_data for them.
               require "http/form_data"
               warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
-                "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
-                "If you'd like to stop seeing this message, require 'http/form_data' yourself."
+                   "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
+                   "If you'd like to stop seeing this message, require 'http/form_data' yourself."
             end
           rescue LoadError
           end
           # :nocov:
           require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/decoder"
           require "httpx/plugins/multipart/part"
           require "httpx/plugins/multipart/mime_type_detector"
         end
@@ -56,6 +57,19 @@ module HTTPX
           end
         end
 
+        def decode(response)
+          content_type = response.content_type.mime_type
+
+          case content_type
+          when "application/x-www-form-urlencoded"
+            Transcoder::Form.decode(response)
+          when "multipart/form-data"
+            Decoder.new(response)
+          else
+            raise Error, "invalid form mime type (#{content_type})"
+          end
+        end
+
         def multipart?(data)
           data.any? do |_, v|
             MULTIPART_VALUE_COND.call(v) ||

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -15,13 +15,15 @@ module HTTPX
         end
 
         def extra_options(options)
-          Class.new(options.class) do
-            def_option(:ntlm, <<-OUT)
-              raise Error, ":ntlm must be a #{NTLMParams}" unless value.is_a?(#{NTLMParams})
+          options.merge(max_concurrent_requests: 1)
+        end
+      end
+
+      module OptionsMethods
+        def option_ntlm(value)
+          raise TypeError, ":ntlm must be a #{NTLMParams}" unless value.is_a?(NTLMParams)
 
-              value
-            OUT
-          end.new(options).merge(max_concurrent_requests: 1)
+          value
         end
       end
 
@@ -32,13 +34,13 @@ module HTTPX
 
         alias_method :ntlm_auth, :ntlm_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             ntlm = request.options.ntlm
 
             if ntlm
               request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
                  probe_response.headers.key?("www-authenticate") &&
@@ -50,12 +52,12 @@ module HTTPX
                 request.transition(:idle)
 
                 request.headers["authorization"] = "NTLM #{ntlm_challenge}"
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,8 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  Socks4Error = Class.new(Error)
+  class Socks4Error < Error; end
+
   module Plugins
     module Proxy
       module Socks4

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module HTTPX
-  Socks5Error = Class.new(Error)
+  class Socks5Error < Error; end
+
   module Plugins
     module Proxy
       module Socks5

data/lib/httpx/plugins/proxy/ssh.rb

@@ -6,23 +6,27 @@ module HTTPX
   module Plugins
     module Proxy
       module SSH
-        def self.load_dependencies(*)
-          require "net/ssh/gateway"
+        class << self
+          def load_dependencies(*)
+            require "net/ssh/gateway"
+          end
         end
 
-        def self.extra_options(options)
-          Class.new(options.class) do
-            def_option(:proxy, <<-OUT)
-              Hash[value]
-            OUT
-          end.new(options)
+        module OptionsMethods
+          def option_proxy(value)
+            Hash[value]
+          end
         end
 
         module InstanceMethods
-          private
+          def request(*args, **options)
+            raise ArgumentError, "must perform at least one request" if args.empty?
+
+            requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
 
-          def send_requests(*requests, options)
-            request_options = @options.merge(options)
+            request = requests.first or return super
+
+            request_options = request.options
 
             return super unless request_options.proxy
 
@@ -37,18 +41,21 @@ module HTTPX
             if request_options.debug
               ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
             end
+
             request_uri = URI(requests.first.uri)
             @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
             begin
               @_gateway.open(request_uri.host, request_uri.port) do |local_port|
                 io = build_gateway_socket(local_port, request_uri, request_options)
-                super(*requests, options.merge(io: io))
+                super(*args, **options.merge(io: io))
               end
             ensure
               @_gateway.shutdown!
             end
           end
 
+          private
+
           def build_gateway_socket(port, request_uri, options)
             case request_uri.scheme
             when "https"
@@ -65,7 +72,7 @@ module HTTPX
             when "http"
               TCPSocket.open("localhost", port)
             else
-              raise Error, "unexpected scheme: #{request_uri.scheme}"
+              raise TypeError, "unexpected scheme: #{request_uri.scheme}"
             end
           end
         end

data/lib/httpx/plugins/proxy.rb

@@ -5,7 +5,8 @@ require "ipaddr"
 require "forwardable"
 
 module HTTPX
-  HTTPProxyError = Class.new(Error)
+  class HTTPProxyError < Error; end
+
   module Plugins
     #
     # This plugin adds support for proxies. It ships with support for:
@@ -64,13 +65,11 @@ module HTTPX
           klass.plugin(:"proxy/socks4")
           klass.plugin(:"proxy/socks5")
         end
+      end
 
-        def extra_options(options)
-          Class.new(options.class) do
-            def_option(:proxy, <<-OUT)
-              value.is_a?(#{Parameters}) ? value : Hash[value]
-            OUT
-          end.new(options)
+      module OptionsMethods
+        def option_proxy(value)
+          value.is_a?(Parameters) ? value : Hash[value]
         end
       end
 
@@ -138,10 +137,11 @@ module HTTPX
         def __proxy_error?(response)
           error = response.error
           case error
-          when ResolveError
+          when NativeResolveError
             # failed resolving proxy domain
-            proxy_uri = error.connection.options.proxy.uri
-            proxy_uri.to_s == @_proxy_uris.first
+            error.connection.origin.to_s == @_proxy_uris.first
+          when ResolveError
+            error.message.end_with?(@_proxy_uris.first)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true

data/lib/httpx/plugins/response_cache/store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module HTTPX::Plugins
+  module ResponseCache
+    class Store
+      extend Forwardable
+
+      def_delegator :@store, :clear
+
+      def initialize
+        @store = {}
+      end
+
+      def lookup(uri)
+        @store[uri]
+      end
+
+      def cached?(uri)
+        @store.key?(uri)
+      end
+
+      def cache(uri, response)
+        @store[uri] = response
+      end
+
+      def prepare(request)
+        cached_response = @store[request.uri]
+
+        return unless cached_response
+
+        original_request = cached_response.instance_variable_get(:@request)
+
+        if (vary = cached_response.headers["vary"])
+          if vary == "*"
+            return unless request.headers.same_headers?(original_request.headers)
+          else
+            return unless vary.split(/ *, */).all? do |cache_field|
+              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+            end
+          end
+        end
+
+        if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
+          request.headers.add("if-modified-since", last_modified)
+        end
+
+        if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          request.headers.add("if-none-match", etag)
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when certain errors happen.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    #
+    module ResponseCache
+      CACHEABLE_VERBS = %i[get head].freeze
+      private_constant :CACHEABLE_VERBS
+
+      class << self
+        def load_dependencies(*)
+          require_relative "response_cache/store"
+        end
+
+        def cacheable_request?(request)
+          CACHEABLE_VERBS.include?(request.verb)
+        end
+
+        def cacheable_response?(response)
+          response.is_a?(Response) &&
+            # partial responses shall not be cached, only full ones.
+            response.status != 206 && (
+            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+          )
+        end
+
+        def cached_response?(response)
+          response.is_a?(Response) && response.status == 304
+        end
+
+        def extra_options(options)
+          options.merge(response_cache_store: Store.new)
+        end
+      end
+
+      module OptionsMethods
+        def option_response_cache_store(value)
+          raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+
+          @options.response_cache_store.prepare(request)
+
+          request
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          if response && ResponseCache.cached_response?(response)
+            log { "returning cached response for #{request.uri}" }
+            cached_response = @options.response_cache_store.lookup(request.uri)
+
+            response.copy_from_cached(cached_response)
+          end
+
+          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+
+          response
+        end
+      end
+
+      module ResponseMethods
+        def copy_from_cached(other)
+          @body = other.body
+
+          @body.__send__(:rewind)
+        end
+      end
+    end
+    register_plugin :response_cache, ResponseCache
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -17,39 +17,57 @@ module HTTPX
                           Errno::ECONNRESET,
                           Errno::ECONNABORTED,
                           Errno::EPIPE,
-                          (TLSError if defined?(TLSError)),
+                          TLSError,
                           TimeoutError,
                           Parser::Error,
                           Errno::EINVAL,
                           Errno::ETIMEDOUT].freeze
+      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
 
-      def self.extra_options(options)
-        Class.new(options.class) do
-          def_option(:retry_after, <<-OUT)
-            # return early if callable
-            unless value.respond_to?(:call)
-              value = Integer(value)
-              raise Error, ":retry_after must be positive" unless value.positive?
-            end
+      if ENV.key?("HTTPX_NO_JITTER")
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES)
+        end
+      else
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+        end
+      end
+
+      module OptionsMethods
+        def option_retry_after(value)
+          # return early if callable
+          unless value.respond_to?(:call)
+            value = Integer(value)
+            raise TypeError, ":retry_after must be positive" unless value.positive?
+          end
+
+          value
+        end
 
-            value
-          OUT
+        def option_retry_jitter(value)
+          # return early if callable
+          raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
 
-          def_option(:max_retries, <<-OUT)
-            num = Integer(value)
-            raise Error, ":max_retries must be positive" unless num.positive?
+          value
+        end
 
-            num
-          OUT
+        def option_max_retries(value)
+          num = Integer(value)
+          raise TypeError, ":max_retries must be positive" unless num.positive?
 
-          def_option(:retry_change_requests)
+          num
+        end
 
-          def_option(:retry_on, <<-OUT)
-            raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+        def option_retry_change_requests(v)
+          v
+        end
 
-            value
-          OUT
-        end.new(options).merge(max_retries: MAX_RETRIES)
+        def option_retry_on(value)
+          raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
       end
 
       module InstanceMethods
@@ -83,10 +101,15 @@ module HTTPX
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
 
             if retry_after
+              # apply jitter
+              if (jitter = request.options.retry_jitter)
+                retry_after = jitter.call(retry_after)
+              end
 
+              retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do
-                log { "retrying!!" }
+                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 connection = find_connection(request, connections, options)
                 connection.send(request)
               end

data/lib/httpx/plugins/stream.rb

@@ -6,11 +6,10 @@ module HTTPX
       @request = request
       @session = session
       @connections = connections
-      @options = @request.options
     end
 
     def each(&block)
-      return enum_for(__method__) unless block_given?
+      return enum_for(__method__) unless block
 
       raise Error, "response already streamed" if @response
 
@@ -72,7 +71,7 @@ module HTTPX
     private
 
     def response
-      @session.__send__(:receive_requests, [@request], @connections, @options) until @request.response
+      @session.__send__(:receive_requests, [@request], @connections) until @request.response
 
       @request.response
     end
@@ -106,7 +105,7 @@ module HTTPX
 
           request = requests.first
 
-          connections = _send_requests(requests, request.options)
+          connections = _send_requests(requests)
 
           StreamResponse.new(request, self, connections)
         end

data/lib/httpx/plugins/upgrade.rb

@@ -18,14 +18,15 @@ module HTTPX
           upgrade_handlers = Module.new do
             extend Registry
           end
+          options.merge(upgrade_handlers: upgrade_handlers)
+        end
+      end
 
-          Class.new(options.class) do
-            def_option(:upgrade_handlers, <<-OUT)
-              raise Error, ":upgrade_handlers must be a registry" unless value.respond_to?(:registry)
+      module OptionsMethods
+        def option_upgrade_handlers(value)
+          raise TypeError, ":upgrade_handlers must be a registry" unless value.respond_to?(:registry)
 
-              value
-            OUT
-          end.new(options).merge(upgrade_handlers: upgrade_handlers)
+          value
         end
       end
 

data/lib/httpx/pool.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "timers"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
 
 module HTTPX
   class Pool
+    using ArrayExtensions
     extend Forwardable
 
     def_delegator :@timers, :after
@@ -15,7 +15,7 @@ module HTTPX
     def initialize
       @resolvers = {}
       @_resolver_ios = {}
-      @timers = Timers::Group.new
+      @timers = Timers.new
       @selector = Selector.new
       @connections = []
       @connected_connections = 0
@@ -27,15 +27,18 @@ module HTTPX
 
     def next_tick
       catch(:jump_tick) do
-        timeout = [next_timeout, @timers.wait_interval].compact.min
+        timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
           throw(:jump_tick)
         end
 
-        @selector.select(timeout, &:call)
-
-        @timers.fire
+        begin
+          @selector.select(timeout, &:call)
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
       end
     rescue StandardError => e
       @connections.each do |connection|
@@ -64,6 +67,16 @@ module HTTPX
       connection.on(:open) do
         @connected_connections += 1
       end
+      connection.on(:activate) do
+        select_connection(connection)
+      end
+    end
+
+    def deactivate(connections)
+      connections.each do |connection|
+        connection.deactivate
+        deselect_connection(connection) if connection.state == :inactive
+      end
     end
 
     # opens a connection to the IP reachable through +uri+.
@@ -81,7 +94,7 @@ module HTTPX
     def resolve_connection(connection)
       @connections << connection unless @connections.include?(connection)
 
-      if connection.addresses || connection.state == :open
+      if connection.addresses || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:
@@ -98,7 +111,7 @@ module HTTPX
       resolver << connection
       return if resolver.empty?
 
-      @_resolver_ios[resolver] ||= @selector.register(resolver)
+      @_resolver_ios[resolver] ||= select_connection(resolver)
     end
 
     def on_resolver_connection(connection)
@@ -107,7 +120,7 @@ module HTTPX
       end
       return register_connection(connection) unless found_connection
 
-      if found_connection.state == :open
+      if found_connection.open?
         coalesce_connections(found_connection, connection)
         throw(:coalesced, found_connection)
       else
@@ -129,7 +142,7 @@ module HTTPX
 
       @resolvers.delete(resolver_type)
 
-      @selector.deregister(resolver)
+      deselect_connection(resolver)
       @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
@@ -140,7 +153,7 @@ module HTTPX
         # consider it connected already.
         @connected_connections += 1
       end
-      @selector.register(connection)
+      select_connection(connection)
       connection.on(:close) do
         unregister_connection(connection)
       end
@@ -148,10 +161,18 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      @selector.deregister(connection)
+      deselect_connection(connection)
       @connected_connections -= 1
     end
 
+    def select_connection(connection)
+      @selector.register(connection)
+    end
+
+    def deselect_connection(connection)
+      @selector.deregister(connection)
+    end
+
     def coalesce_connections(conn1, conn2)
       if conn1.coalescable?(conn2)
         conn1.merge(conn2)
@@ -162,7 +183,11 @@ module HTTPX
     end
 
     def next_timeout
-      @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
+      [
+        @timers.wait_interval,
+        *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
+        *@connections.filter_map(&:timeout),
+      ].compact.min
     end
 
     def find_resolver_for(connection)
@@ -172,6 +197,7 @@ module HTTPX
 
       @resolvers[resolver_type] ||= begin
         resolver = resolver_type.new(connection_options)
+        resolver.pool = self if resolver.respond_to?(:pool=)
         resolver.on(:resolve, &method(:on_resolver_connection))
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }

data/lib/httpx/registry.rb

@@ -31,7 +31,7 @@ module HTTPX
   #
   module Registry
     # Base Registry Error
-    Error = Class.new(Error)
+    class Error < Error; end
 
     def self.extended(klass)
       super
@@ -59,7 +59,7 @@ module HTTPX
         @registry ||= {}
         return @registry if tag.nil?
 
-        handler = @registry.fetch(tag)
+        handler = @registry[tag]
         raise(Error, "#{tag} is not registered in #{self}") unless handler
 
         handler