httpx 0.24.5 → 1.0.0

data/lib/httpx/options.rb

@@ -7,11 +7,10 @@ module HTTPX
     BUFFER_SIZE = 1 << 14
     WINDOW_SIZE = 1 << 14 # 16K
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
-    CONNECT_TIMEOUT = 60
-    OPERATION_TIMEOUT = 60
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
-    READ_TIMEOUT = WRITE_TIMEOUT = REQUEST_TIMEOUT = Float::INFINITY
+    CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
+    REQUEST_TIMEOUT = OPERATION_TIMEOUT = Float::INFINITY
 
     # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
     ip_address_families = begin
@@ -31,6 +30,9 @@ module HTTPX
       :ssl => {},
       :http2_settings => { settings_enable_push: 0 },
       :fallback_protocol => "http/1.1",
+      :supported_compression_formats => %w[gzip deflate],
+      :decompress_response_body => true,
+      :compress_request_body => true,
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
@@ -52,7 +54,6 @@ module HTTPX
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
-      :transport_options => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
@@ -60,28 +61,6 @@ module HTTPX
       :ip_families => ip_address_families,
     }.freeze
 
-    begin
-      module HashExtensions
-        refine Hash do
-          def >=(other)
-            Hash[other] <= self
-          end
-
-          def <=(other)
-            other = Hash[other]
-            return false unless size <= other.size
-
-            each do |k, v|
-              v2 = other.fetch(k) { return false }
-              return false unless v2 == v
-            end
-            true
-          end
-        end
-      end
-      using HashExtensions
-    end unless Hash.method_defined?(:>=)
-
     class << self
       def new(options = {})
         # let enhanced options go through
@@ -100,38 +79,10 @@ module HTTPX
 
         attr_reader(optname)
       end
-
-      def def_option(optname, *args, &block)
-        if args.empty? && !block
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(v); v; end # def option_smth(v); v; end
-          OUT
-          return
-        end
-
-        deprecated_def_option(optname, *args, &block)
-      end
-
-      def deprecated_def_option(optname, layout = nil, &interpreter)
-        warn "DEPRECATION WARNING: using `def_option(#{optname})` for setting options is deprecated. " \
-             "Define module OptionsMethods and `def option_#{optname}(val)` instead."
-
-        if layout
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(value)  # def option_origin(v)
-              #{layout}                   #   URI(v)
-            end                           # end
-          OUT
-        elsif interpreter
-          define_method(:"option_#{optname}") do |value|
-            instance_exec(value, &interpreter)
-          end
-        end
-      end
     end
 
     def initialize(options = {})
-      __initialize__(options)
+      do_initialize(options)
       freeze
     end
 
@@ -142,6 +93,7 @@ module HTTPX
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
+      @supported_compression_formats.freeze
     end
 
     def option_origin(value)
@@ -157,14 +109,11 @@ module HTTPX
     end
 
     def option_timeout(value)
-      timeouts = Hash[value]
-
-      if timeouts.key?(:loop_timeout)
-        warn ":loop_timeout is deprecated, use :operation_timeout instead"
-        timeouts[:operation_timeout] = timeouts.delete(:loop_timeout)
-      end
+      Hash[value]
+    end
 
-      timeouts
+    def option_supported_compression_formats(value)
+      Array(value).map(&:to_s)
     end
 
     def option_max_concurrent_requests(value)
@@ -196,7 +145,10 @@ module HTTPX
     end
 
     def option_body_threshold_size(value)
-      Integer(value)
+      bytes = Integer(value)
+      raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
+
+      bytes
     end
 
     def option_transport(value)
@@ -218,10 +170,13 @@ module HTTPX
       params form json xml body ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
-      io fallback_protocol debug debug_level transport_options resolver_class resolver_options
+      io fallback_protocol debug debug_level resolver_class resolver_options
+      compress_request_body decompress_response_body
       persistent
     ].each do |method_name|
-      def_option(method_name)
+      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        def option_#{method_name}(v); v; end # def option_smth(v); v; end
+      OUT
     end
 
     REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
@@ -270,30 +225,15 @@ module HTTPX
       end
     end
 
-    if RUBY_VERSION > "2.4.0"
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
-        end
-      end
-    else
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          value = other.instance_variable_get(ivar)
-          value = case value
-                  when Symbol, Numeric, TrueClass, FalseClass
-                    value
-                  else
-                    value.dup
-          end
-          instance_variable_set(ivar, value)
-        end
+    def initialize_dup(other)
+      instance_variables.each do |ivar|
+        instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
       end
     end
 
     private
 
-    def __initialize__(options = {})
+    def do_initialize(options = {})
       defaults = DEFAULT_OPTIONS.merge(options)
       defaults.each do |k, v|
         next if v.nil?

data/lib/httpx/plugins/{authentication → auth}/basic.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 
 module HTTPX
   module Plugins
@@ -11,10 +11,6 @@ module HTTPX
           @password = password
         end
 
-        def can_authenticate?(authenticate)
-          authenticate && /Basic .*/.match?(authenticate)
-        end
-
         def authenticate(*)
           "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
         end

data/lib/httpx/plugins/{authentication → auth}/digest.rb

@@ -8,8 +8,6 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
         def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
@@ -31,9 +29,8 @@ module HTTPX
           # discard first token, it's Digest
           auth_info = authenticate[/^(\w+) (.*)/, 2]
 
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
+          params = auth_info.split(/ *, */)
+                            .to_h { |val| val.split("=") }.transform_values { |v| v.delete("\"") }
           nonce = params["nonce"]
           nc = next_nonce
 

data/lib/httpx/plugins/{authentication → auth}/ntlm.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 require "ntlm"
 
 module HTTPX
   module Plugins
     module Authentication
       class Ntlm
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
         def initialize(user, password, domain: nil)
           @user = user
           @password = password

data/lib/httpx/plugins/{authentication → auth}/socks5.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "base64"
-
 module HTTPX
   module Plugins
     module Authentication

data/lib/httpx/plugins/auth.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds a shim +authorization+ method to the session, which will fill
+    # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
+    # in its value.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#authorization
+    #
+    module Auth
+      module InstanceMethods
+        def authorization(token)
+          with(headers: { "authorization" => token })
+        end
+
+        def bearer_auth(token)
+          authorization("Bearer #{token}")
+        end
+      end
+    end
+    register_plugin :auth, Auth
+  end
+end

data/lib/httpx/plugins/aws_sigv4.rb

@@ -146,7 +146,6 @@ module HTTPX
 
         def configure(klass)
           klass.plugin(:expect)
-          klass.plugin(:compression)
         end
       end
 

data/lib/httpx/plugins/{basic_authentication.rb → basic_auth.rb}

@@ -5,26 +5,25 @@ module HTTPX
     #
     # This plugin adds helper methods to implement HTTP Basic Auth (https://tools.ietf.org/html/rfc7617)
     #
-    # https://gitlab.com/os85/httpx/wikis/Authentication#basic-authentication
+    # https://gitlab.com/os85/httpx/wikis/Authorization#basic-auth
     #
     module BasicAuth
       class << self
         def load_dependencies(_klass)
-          require_relative "authentication/basic"
+          require_relative "auth/basic"
         end
 
         def configure(klass)
-          klass.plugin(:authentication)
+          klass.plugin(:auth)
         end
       end
 
       module InstanceMethods
         def basic_auth(user, password)
-          authentication(Authentication::Basic.new(user, password).authenticate)
+          authorization(Authentication::Basic.new(user, password).authenticate)
         end
-        alias_method :basic_authentication, :basic_auth
       end
     end
-    register_plugin :basic_authentication, BasicAuth
+    register_plugin :basic_auth, BasicAuth
   end
 end

data/lib/httpx/plugins/brotli.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    module Brotli
+      class Deflater < Transcoder::Deflater
+        def deflate(chunk)
+          return unless chunk
+
+          ::Brotli.deflate(chunk)
+        end
+      end
+
+      module RequestBodyClassMethods
+        def initialize_deflater_body(body, encoding)
+          return Brotli.encode(body) if encoding == "br"
+
+          super
+        end
+      end
+
+      module ResponseBodyClassMethods
+        def initialize_inflater_by_encoding(encoding, response, **kwargs)
+          return Brotli.decode(response, **kwargs) if encoding == "br"
+
+          super
+        end
+      end
+
+      module_function
+
+      def load_dependencies(*)
+        require "brotli"
+      end
+
+      def self.extra_options(options)
+        options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
+      end
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(_response, **)
+        ::Brotli.method(:inflate)
+      end
+    end
+    register_plugin :brotli, Brotli
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -15,8 +15,11 @@ module HTTPX
         @max_attempts = max_attempts
         @reset_attempts_in = reset_attempts_in
         @break_in = break_in
-        @circuit_breaker_half_open_drip_rate = 1 - circuit_breaker_half_open_drip_rate
+        @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
         @attempts = 0
+
+        total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
+        @drip_factor = (@max_attempts / total_real_attempts).round
         @state = :closed
       end
 
@@ -27,8 +30,13 @@ module HTTPX
         when :closed
           nil
         when :half_open
-          # return nothing or smth based on ratio
-          return if Random.rand >= @circuit_breaker_half_open_drip_rate
+          @attempts += 1
+
+          # do real requests while drip rate valid
+          if (@real_attempts % @drip_factor).zero?
+            @real_attempts += 1
+            return
+          end
 
           @response
         when :open
@@ -38,23 +46,31 @@ module HTTPX
       end
 
       def try_open(response)
-        return unless @state == :closed
+        case @state
+        when :closed
+          now = Utils.now
 
-        now = Utils.now
+          if @attempts.positive?
+            # reset if error happened long ago
+            @attempts = 0 if now - @attempted_at > @reset_attempts_in
+          else
+            @attempted_at = now
+          end
 
-        if @attempts.positive?
-          @attempts = 0 if now - @attempted_at > @reset_attempts_in
-        else
-          @attempted_at = now
-        end
+          @attempts += 1
 
-        @attempts += 1
+          return unless @attempts >= @max_attempts
 
-        return unless @attempts >= @max_attempts
+          @state = :open
+          @opened_at = now
+          @response = response
+        when :half_open
+          # open immediately
 
-        @state = :open
-        @opened_at = now
-        @response = response
+          @state = :open
+          @attempted_at = @opened_at = Utils.now
+          @response = response
+        end
       end
 
       def try_close
@@ -62,13 +78,21 @@ module HTTPX
         when :closed
           nil
         when :half_open
+
+          # do not close circuit unless attempts exhausted
+          return unless @attempts >= @max_attempts
+
           # reset!
           @attempts = 0
           @opened_at = @attempted_at = @response = nil
           @state = :closed
 
         when :open
-          @state = :half_open if Utils.elapsed_time(@opened_at) > @break_in
+          if Utils.elapsed_time(@opened_at) > @break_in
+            @state = :half_open
+            @attempts = 0
+            @real_attempts = 0
+          end
         end
       end
     end

data/lib/httpx/plugins/circuit_breaker/circuit_store.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "mutex_m"
+
 module HTTPX::Plugins::CircuitBreaker
   using HTTPX::URIExtensions
 
@@ -13,18 +15,29 @@ module HTTPX::Plugins::CircuitBreaker
           options.circuit_breaker_half_open_drip_rate
         )
       end
+      @circuits.extend(Mutex_m)
     end
 
     def try_open(uri, response)
-      circuit = get_circuit_for_uri(uri)
+      circuit = @circuits.synchronize { get_circuit_for_uri(uri) }
 
       circuit.try_open(response)
     end
 
+    def try_close(uri)
+      circuit = @circuits.synchronize do
+        return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
+
+        get_circuit_for_uri(uri)
+      end
+
+      circuit.try_close
+    end
+
     # if circuit is open, it'll respond with the stored response.
     # if not, nil.
     def try_respond(request)
-      circuit = get_circuit_for_uri(request.uri)
+      circuit = @circuits.synchronize { get_circuit_for_uri(request.uri) }
 
       circuit.respond
     end
@@ -32,9 +45,7 @@ module HTTPX::Plugins::CircuitBreaker
     private
 
     def get_circuit_for_uri(uri)
-      uri = URI(uri)
-
-      if @circuits.key?(uri.origin)
+      if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
         @circuits[uri.origin]
       else
         @circuits[uri.to_s]

data/lib/httpx/plugins/circuit_breaker.rb

@@ -16,8 +16,12 @@ module HTTPX
       end
 
       def self.extra_options(options)
-        options.merge(circuit_breaker_max_attempts: 3, circuit_breaker_reset_attempts_in: 60, circuit_breaker_break_in: 60,
-                      circuit_breaker_half_open_drip_rate: 1)
+        options.merge(
+          circuit_breaker_max_attempts: 3,
+          circuit_breaker_reset_attempts_in: 60,
+          circuit_breaker_break_in: 60,
+          circuit_breaker_half_open_drip_rate: 1
+        )
       end
 
       module InstanceMethods
@@ -47,13 +51,13 @@ module HTTPX
 
           # run all requests through the circuit breaker, see if the circuit is
           # open for any of them.
-          real_requests = requests.each_with_object([]) do |req, real_reqs|
+          real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
             short_circuit_response = @circuit_store.try_respond(req)
             if short_circuit_response.nil?
               real_reqs << req
               next
             end
-            short_circuit_responses[requests.index(req)] = short_circuit_response
+            short_circuit_responses[idx] = short_circuit_response
           end
 
           # run requests for the remainder
@@ -84,6 +88,9 @@ module HTTPX
             end
           elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
             @circuit_store.try_open(request.uri, response)
+          else
+            @circuit_store.try_close(request.uri)
+            nil
           end
         end
       end

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -6,8 +6,6 @@ require "time"
 module HTTPX
   module Plugins::Cookies
     module SetCookieParser
-      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
-
       # Whitespace.
       RE_WSP = /[ \t]+/.freeze
 

data/lib/httpx/plugins/cookies.rb

@@ -71,7 +71,7 @@ module HTTPX
       end
 
       module OptionsMethods
-        def __initialize__(*)
+        def do_initialize(*)
           super
 
           return unless @headers.key?("cookie")

data/lib/httpx/plugins/{digest_authentication.rb → digest_auth.rb}

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin adds helper methods to implement HTTP Digest Auth (https://tools.ietf.org/html/rfc7616)
     #
-    # https://gitlab.com/os85/httpx/wikis/Authentication#authentication
+    # https://gitlab.com/os85/httpx/wikis/Authorization#digest-auth
     #
     module DigestAuth
       DigestError = Class.new(Error)
@@ -16,7 +16,7 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require_relative "authentication/digest"
+          require_relative "auth/digest"
         end
       end
 
@@ -29,11 +29,11 @@ module HTTPX
       end
 
       module InstanceMethods
-        def digest_authentication(user, password, hashed: false)
+        def digest_auth(user, password, hashed: false)
           with(digest: Authentication::Digest.new(user, password, hashed: hashed))
         end
 
-        alias_method :digest_auth, :digest_authentication
+        private
 
         def send_requests(*requests)
           requests.flat_map do |request|
@@ -57,6 +57,6 @@ module HTTPX
       end
     end
 
-    register_plugin :digest_authentication, DigestAuth
+    register_plugin :digest_auth, DigestAuth
   end
 end

data/lib/httpx/plugins/follow_redirects.rb

@@ -48,7 +48,27 @@ module HTTPX
           return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
-          retry_request = build_redirect_request(redirect_request, response, options)
+          # build redirect request
+          redirect_uri = __get_location_from_response(response)
+
+          if response.status == 305 && options.respond_to?(:proxy)
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            retry_options = options.merge(headers: redirect_request.headers,
+                                          proxy: { uri: redirect_uri },
+                                          body: redirect_request.body,
+                                          max_redirects: max_redirects - 1)
+            redirect_uri = redirect_request.uri
+            options = retry_options
+          else
+
+            # redirects are **ALWAYS** GET
+            retry_options = options.merge(headers: redirect_request.headers,
+                                          body: redirect_request.body,
+                                          max_redirects: max_redirects - 1)
+          end
+
+          retry_request = build_request("GET", redirect_uri, retry_options)
 
           request.redirect_request = retry_request
 
@@ -83,29 +103,6 @@ module HTTPX
           nil
         end
 
-        def build_redirect_request(request, response, options)
-          redirect_uri = __get_location_from_response(response)
-          max_redirects = request.max_redirects
-
-          if response.status == 305 && options.respond_to?(:proxy)
-            # The requested resource MUST be accessed through the proxy given by
-            # the Location field. The Location field gives the URI of the proxy.
-            retry_options = options.merge(headers: request.headers,
-                                          proxy: { uri: redirect_uri },
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
-            redirect_uri = request.url
-          else
-
-            # redirects are **ALWAYS** GET
-            retry_options = options.merge(headers: request.headers,
-                                          body: request.body,
-                                          max_redirects: max_redirects - 1)
-          end
-
-          build_request("GET", redirect_uri, retry_options)
-        end
-
         def __get_location_from_response(response)
           location_uri = URI(response.headers["location"])
           location_uri = response.uri.merge(location_uri) if location_uri.relative?