httpx 0.21.0 → 1.2.1

data/lib/httpx/plugins/{authentication → auth}/digest.rb

@@ -8,12 +8,11 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
-        def initialize(user, password, **)
+        def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
           @nonce = 0
+          @hashed = hashed
         end
 
         def can_authenticate?(authenticate)
@@ -21,7 +20,7 @@ module HTTPX
         end
 
         def authenticate(request, authenticate)
-          "Digest #{generate_header(request.verb.to_s.upcase, request.path, authenticate)}"
+          "Digest #{generate_header(request.verb, request.path, authenticate)}"
         end
 
         private
@@ -30,9 +29,8 @@ module HTTPX
           # discard first token, it's Digest
           auth_info = authenticate[/^(\w+) (.*)/, 2]
 
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
+          params = auth_info.split(/ *, */)
+                            .to_h { |val| val.split("=") }.transform_values { |v| v.delete("\"") }
           nonce = params["nonce"]
           nc = next_nonce
 
@@ -45,7 +43,6 @@ module HTTPX
             raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
 
             sess = Regexp.last_match(2)
-            params.delete("algorithm")
           else
             algorithm = ::Digest::MD5
           end
@@ -56,11 +53,13 @@ module HTTPX
           end
 
           a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
+            [
+              (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
+              nonce,
+              cnonce,
+            ].join ":"
           else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
           end
 
           ha1 = algorithm.hexdigest(a1)
@@ -77,11 +76,11 @@ module HTTPX
             %(response="#{algorithm.hexdigest(request_digest)}"),
           ]
           header << %(realm="#{params["realm"]}") if params.key?("realm")
-          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
-          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
+          header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
           header << %(cnonce="#{cnonce}") if cnonce
           header << %(nc=#{nc})
           header << %(qop=#{qop}) if qop
+          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
           header.join ", "
         end
 

data/lib/httpx/plugins/{authentication → auth}/ntlm.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 require "ntlm"
 
 module HTTPX
   module Plugins
     module Authentication
       class Ntlm
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
         def initialize(user, password, domain: nil)
           @user = user
           @password = password

data/lib/httpx/plugins/{authentication → auth}/socks5.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "base64"
-
 module HTTPX
   module Plugins
     module Authentication

data/lib/httpx/plugins/auth.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds a shim +authorization+ method to the session, which will fill
+    # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
+    # in its value.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#auth
+    #
+    module Auth
+      module InstanceMethods
+        def authorization(token)
+          with(headers: { "authorization" => token })
+        end
+
+        def bearer_auth(token)
+          authorization("Bearer #{token}")
+        end
+      end
+    end
+    register_plugin :auth, Auth
+  end
+end

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -20,9 +20,7 @@ module HTTPX
           true
         end
 
-        def method_missing(*)
-          nil
-        end
+        def method_missing(*); end
       end
 
       #

data/lib/httpx/plugins/aws_sigv4.rb

@@ -5,9 +5,9 @@ module HTTPX
     #
     # This plugin adds AWS Sigv4 authentication.
     #
-    # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+    # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
+    # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
     #
     module AWSSigV4
       Credentials = Struct.new(:username, :password, :security_token)
@@ -71,7 +71,7 @@ module HTTPX
           end.join
 
           # canonical request
-          creq = "#{request.verb.to_s.upcase}" \
+          creq = "#{request.verb}" \
                  "\n#{request.canonical_path}" \
                  "\n#{request.canonical_query}" \
                  "\n#{canonical_headers}" \
@@ -115,7 +115,7 @@ module HTTPX
           elsif value.respond_to?(:each)
             digest = OpenSSL::Digest.new(@algorithm)
 
-            mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
+            mb_buffer = value.each.with_object("".b) do |chunk, buffer|
               buffer << chunk
               break if buffer.bytesize >= 1024 * 1024
             end
@@ -146,7 +146,6 @@ module HTTPX
 
         def configure(klass)
           klass.plugin(:expect)
-          klass.plugin(:compression)
         end
       end
 
@@ -186,7 +185,7 @@ module HTTPX
         def canonical_query
           params = query.split("&")
           # params = params.map { |p| p.match(/=/) ? p : p + '=' }
-          # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+          # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
           # Sort the parameter names by character code point in ascending order.
           # Parameters with duplicate names should be sorted by value.
           #

data/lib/httpx/plugins/basic_auth.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
+    #
+    module BasicAuth
+      class << self
+        def load_dependencies(_klass)
+          require_relative "auth/basic"
+        end
+
+        def configure(klass)
+          klass.plugin(:auth)
+        end
+      end
+
+      module InstanceMethods
+        def basic_auth(user, password)
+          authorization(Authentication::Basic.new(user, password).authenticate)
+        end
+      end
+    end
+    register_plugin :basic_auth, BasicAuth
+  end
+end

data/lib/httpx/plugins/brotli.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    module Brotli
+      class Deflater < Transcoder::Deflater
+        def deflate(chunk)
+          return unless chunk
+
+          ::Brotli.deflate(chunk)
+        end
+      end
+
+      module RequestBodyClassMethods
+        def initialize_deflater_body(body, encoding)
+          return Brotli.encode(body) if encoding == "br"
+
+          super
+        end
+      end
+
+      module ResponseBodyClassMethods
+        def initialize_inflater_by_encoding(encoding, response, **kwargs)
+          return Brotli.decode(response, **kwargs) if encoding == "br"
+
+          super
+        end
+      end
+
+      module_function
+
+      def load_dependencies(*)
+        require "brotli"
+      end
+
+      def self.extra_options(options)
+        options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
+      end
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(_response, **)
+        ::Brotli.method(:inflate)
+      end
+    end
+    register_plugin :brotli, Brotli
+  end
+end

data/lib/httpx/plugins/callbacks.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds suppoort for callbacks around the request/response lifecycle.
+    #
+    # https://gitlab.com/os85/httpx/-/wikis/Events
+    #
+    module Callbacks
+      # connection closed user-space errors happen after errors can be surfaced to requests,
+      # so they need to pierce through the scheduler, which is only possible by simulating an
+      # interrupt.
+      class CallbackError < Exception; end # rubocop:disable Lint/InheritException
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        %i[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def on_#{meth}(&blk)   # def on_connection_opened(&blk)
+              on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+            end                    # end
+          MOD
+        end
+
+        private
+
+        def init_connection(uri, options)
+          connection = super
+          connection.on(:open) do
+            emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
+          end
+          connection.on(:close) do
+            emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
+          end
+
+          connection
+        end
+
+        def set_request_callbacks(request)
+          super
+
+          request.on(:headers) do
+            emit_or_callback_error(:request_started, request)
+          end
+          request.on(:body_chunk) do |chunk|
+            emit_or_callback_error(:request_body_chunk, request, chunk)
+          end
+          request.on(:done) do
+            emit_or_callback_error(:request_completed, request)
+          end
+
+          request.on(:response_started) do |res|
+            if res.is_a?(Response)
+              emit_or_callback_error(:response_started, request, res)
+              res.on(:chunk_received) do |chunk|
+                emit_or_callback_error(:response_body_chunk, request, res, chunk)
+              end
+            else
+              emit_or_callback_error(:request_error, request, res.error)
+            end
+          end
+          request.on(:response) do |res|
+            emit_or_callback_error(:response_completed, request, res)
+          end
+        end
+
+        def emit_or_callback_error(*args)
+          emit(*args)
+        rescue StandardError => e
+          ex = CallbackError.new(e.message)
+          ex.set_backtrace(e.backtrace)
+          raise ex
+        end
+
+        def receive_requests(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+      end
+    end
+    register_plugin :callbacks, Callbacks
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -15,8 +15,11 @@ module HTTPX
         @max_attempts = max_attempts
         @reset_attempts_in = reset_attempts_in
         @break_in = break_in
-        @circuit_breaker_half_open_drip_rate = 1 - circuit_breaker_half_open_drip_rate
+        @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
         @attempts = 0
+
+        total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
+        @drip_factor = (@max_attempts / total_real_attempts).round
         @state = :closed
       end
 
@@ -27,8 +30,13 @@ module HTTPX
         when :closed
           nil
         when :half_open
-          # return nothing or smth based on ratio
-          return if Random::DEFAULT.rand >= @circuit_breaker_half_open_drip_rate
+          @attempts += 1
+
+          # do real requests while drip rate valid
+          if (@real_attempts % @drip_factor).zero?
+            @real_attempts += 1
+            return
+          end
 
           @response
         when :open
@@ -38,23 +46,31 @@ module HTTPX
       end
 
       def try_open(response)
-        return unless @state == :closed
+        case @state
+        when :closed
+          now = Utils.now
 
-        now = Utils.now
+          if @attempts.positive?
+            # reset if error happened long ago
+            @attempts = 0 if now - @attempted_at > @reset_attempts_in
+          else
+            @attempted_at = now
+          end
 
-        if @attempts.positive?
-          @attempts = 0 if now - @attempted_at > @reset_attempts_in
-        else
-          @attempted_at = now
-        end
+          @attempts += 1
 
-        @attempts += 1
+          return unless @attempts >= @max_attempts
 
-        return unless @attempts >= @max_attempts
+          @state = :open
+          @opened_at = now
+          @response = response
+        when :half_open
+          # open immediately
 
-        @state = :open
-        @opened_at = now
-        @response = response
+          @state = :open
+          @attempted_at = @opened_at = Utils.now
+          @response = response
+        end
       end
 
       def try_close
@@ -62,13 +78,21 @@ module HTTPX
         when :closed
           nil
         when :half_open
+
+          # do not close circuit unless attempts exhausted
+          return unless @attempts >= @max_attempts
+
           # reset!
           @attempts = 0
           @opened_at = @attempted_at = @response = nil
           @state = :closed
 
         when :open
-          @state = :half_open if Utils.elapsed_time(@opened_at) > @break_in
+          if Utils.elapsed_time(@opened_at) > @break_in
+            @state = :half_open
+            @attempts = 0
+            @real_attempts = 0
+          end
         end
       end
     end

data/lib/httpx/plugins/circuit_breaker/circuit_store.rb

@@ -13,18 +13,29 @@ module HTTPX::Plugins::CircuitBreaker
           options.circuit_breaker_half_open_drip_rate
         )
       end
+      @circuits_mutex = Thread::Mutex.new
     end
 
     def try_open(uri, response)
-      circuit = get_circuit_for_uri(uri)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
 
       circuit.try_open(response)
     end
 
+    def try_close(uri)
+      circuit = @circuits_mutex.synchronize do
+        return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
+
+        get_circuit_for_uri(uri)
+      end
+
+      circuit.try_close
+    end
+
     # if circuit is open, it'll respond with the stored response.
     # if not, nil.
     def try_respond(request)
-      circuit = get_circuit_for_uri(request.uri)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
 
       circuit.respond
     end
@@ -32,9 +43,7 @@ module HTTPX::Plugins::CircuitBreaker
     private
 
     def get_circuit_for_uri(uri)
-      uri = URI(uri)
-
-      if @circuits.key?(uri.origin)
+      if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
         @circuits[uri.origin]
       else
         @circuits[uri.to_s]

data/lib/httpx/plugins/circuit_breaker.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin implements a circuit breaker around connection errors.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Circuit-Breaker
+    # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
     #
     module CircuitBreaker
       using URIExtensions
@@ -16,11 +16,17 @@ module HTTPX
       end
 
       def self.extra_options(options)
-        options.merge(circuit_breaker_max_attempts: 3, circuit_breaker_reset_attempts_in: 60, circuit_breaker_break_in: 60,
-                      circuit_breaker_half_open_drip_rate: 1)
+        options.merge(
+          circuit_breaker_max_attempts: 3,
+          circuit_breaker_reset_attempts_in: 60,
+          circuit_breaker_break_in: 60,
+          circuit_breaker_half_open_drip_rate: 1
+        )
       end
 
       module InstanceMethods
+        include HTTPX::Callbacks
+
         def initialize(*)
           super
           @circuit_store = CircuitStore.new(@options)
@@ -31,19 +37,29 @@ module HTTPX
           @circuit_store = orig.instance_variable_get(:@circuit_store).dup
         end
 
+        %i[circuit_open].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        def on_#{meth}(&blk)   # def on_circuit_open(&blk)
+          on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+        end                    # end
+          MOD
+        end
+
+        private
+
         def send_requests(*requests)
           # @type var short_circuit_responses: Array[response]
           short_circuit_responses = []
 
           # run all requests through the circuit breaker, see if the circuit is
           # open for any of them.
-          real_requests = requests.each_with_object([]) do |req, real_reqs|
+          real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
             short_circuit_response = @circuit_store.try_respond(req)
             if short_circuit_response.nil?
               real_reqs << req
               next
             end
-            short_circuit_responses[requests.index(req)] = short_circuit_response
+            short_circuit_responses[idx] = short_circuit_response
           end
 
           # run requests for the remainder
@@ -59,6 +75,12 @@ module HTTPX
         end
 
         def on_response(request, response)
+          emit(:circuit_open, request) if try_circuit_open(request, response)
+
+          super
+        end
+
+        def try_circuit_open(request, response)
           if response.is_a?(ErrorResponse)
             case response.error
             when RequestTimeoutError
@@ -68,9 +90,10 @@ module HTTPX
             end
           elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
             @circuit_store.try_open(request.uri, response)
+          else
+            @circuit_store.try_close(request.uri)
+            nil
           end
-
-          super
         end
       end
 

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -6,8 +6,6 @@ require "time"
 module HTTPX
   module Plugins::Cookies
     module SetCookieParser
-      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
-
       # Whitespace.
       RE_WSP = /[ \t]+/.freeze
 

data/lib/httpx/plugins/cookies.rb

@@ -9,7 +9,7 @@ module HTTPX
     #
     # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Cookies
+    # https://gitlab.com/os85/httpx/wikis/Cookies
     #
     module Cookies
       def self.load_dependencies(*)
@@ -71,22 +71,32 @@ module HTTPX
       end
 
       module OptionsMethods
-        def __initialize__(*)
-          super
+        def option_headers(*)
+          value = super
+
+          merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
 
-          return unless @headers.key?("cookie")
+          value
+        end
 
-          @headers.delete("cookie").each do |ck|
+        def option_cookies(value)
+          jar = value.is_a?(Jar) ? value : Jar.new(value)
+
+          merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
+
+          jar
+        end
+
+        private
+
+        def merge_cookie_in_jar(cookies, jar)
+          cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              @cookies.add(Cookie.new(name, value))
+              jar.add(Cookie.new(name, value))
             end
           end
         end
-
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
       end
     end
     register_plugin :cookies, Cookies