httpx 0.20.0 → 1.3.1

data/lib/httpx/plugins/grpc.rb

@@ -16,7 +16,7 @@ module HTTPX
     #
     # This plugin adds DSL to build GRPC interfaces.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/GRPC
+    # https://gitlab.com/os85/httpx/wikis/GRPC
     #
     module GRPC
       unless String.method_defined?(:underscore)
@@ -49,20 +49,19 @@ module HTTPX
       class << self
         def load_dependencies(*)
           require "stringio"
+          require "httpx/plugins/grpc/grpc_encoding"
           require "httpx/plugins/grpc/message"
           require "httpx/plugins/grpc/call"
         end
 
         def configure(klass)
           klass.plugin(:persistent)
-          klass.plugin(:compression)
           klass.plugin(:stream)
         end
 
         def extra_options(options)
           options.merge(
             fallback_protocol: "h2",
-            http2_settings: { wait_for_handshake: false },
             grpc_rpcs: {}.freeze,
             grpc_compression: false,
             grpc_deadline: DEADLINE
@@ -108,14 +107,24 @@ module HTTPX
           @trailing_metadata = Hash[trailers]
           super
         end
+      end
+
+      module RequestBodyMethods
+        def initialize(*, **)
+          super
 
-        def encoders
-          @options.encodings
+          if (compression = @headers["grpc-encoding"])
+            deflater_body = self.class.initialize_deflater_body(@body, compression)
+            @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
+          else
+            @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
+          end
         end
       end
 
       module InstanceMethods
         def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
+          # @type var ssl_params: ::Hash[::Symbol, untyped]
           ssl_params = {
             **ssl_opts,
             ca_file: ca_path,
@@ -141,17 +150,29 @@ module HTTPX
             deadline: @options.grpc_deadline,
           }.merge(opts)
 
+          local_rpc_name = rpc_name.underscore
+
           session_class = Class.new(self.class) do
+            # define rpc method with ruby style name
             class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-              def #{rpc_name}(input, **opts)              # def grpc_action(input, **opts)
-                rpc_execute("#{rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
-              end                                         # end
+              def #{local_rpc_name}(input, **opts)              # def grpc_action(input, **opts)
+                rpc_execute("#{local_rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+              end                                               # end
             OUT
+
+            # define rpc method with original name
+            unless local_rpc_name == rpc_name
+              class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+                def #{rpc_name}(input, **opts)                    # def grpcAction(input, **opts)
+                  rpc_execute("#{local_rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+                end                                               # end
+              OUT
+            end
           end
 
           session_class.new(@options.merge(
                               grpc_rpcs: @options.grpc_rpcs.merge(
-                                rpc_name.underscore => [rpc_name, input, output, rpc_opts]
+                                local_rpc_name => [rpc_name, input, output, rpc_opts]
                               ).freeze
                             ))
         end
@@ -195,7 +216,7 @@ module HTTPX
                     **opts)
           grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
           response = request(grpc_request, **opts)
-          response.raise_for_status
+          response.raise_for_status unless opts[:stream]
           GRPC::Call.new(response)
         end
 
@@ -233,7 +254,7 @@ module HTTPX
           uri.path = rpc_method
 
           headers = HEADERS.merge(
-            "grpc-accept-encoding" => ["identity", *@options.encodings.registry.keys]
+            "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
           )
           unless deadline == Float::INFINITY
             # convert to milliseconds
@@ -241,30 +262,16 @@ module HTTPX
             headers["grpc-timeout"] = "#{deadline}m"
           end
 
-          headers = headers.merge(metadata) if metadata
+          headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
 
           # prepare compressor
-          deflater = nil
           compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
 
-          if compression
-            headers["grpc-encoding"] = compression
-            deflater = @options.encodings.registry(compression).deflater
-          end
-
-          headers.merge!(@options.call_credentials.call) if @options.call_credentials
+          headers["grpc-encoding"] = compression if compression
 
-          body = if input.respond_to?(:each)
-            Enumerator.new do |y|
-              input.each do |message|
-                y << Message.encode(message, deflater: deflater)
-              end
-            end
-          else
-            Message.encode(input, deflater: deflater)
-          end
+          headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
-          build_request(:post, uri, headers: headers, body: body)
+          build_request("POST", uri, headers: headers, body: input)
         end
       end
     end

data/lib/httpx/plugins/h2c.rb

@@ -4,21 +4,16 @@ module HTTPX
   module Plugins
     #
     # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-    # (https://tools.ietf.org/html/rfc7540#section-3.2)
+    # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2c
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
     #
     module H2C
-      VALID_H2C_VERBS = %i[get options head].freeze
+      VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
 
       class << self
-        def load_dependencies(*)
-          require "base64"
-        end
-
-        def configure(klass)
+        def load_dependencies(klass)
           klass.plugin(:upgrade)
-          klass.default_options.upgrade_handlers.register "h2c", self
         end
 
         def call(connection, request, response)
@@ -26,7 +21,7 @@ module HTTPX
         end
 
         def extra_options(options)
-          options.merge(max_concurrent_requests: 1)
+          options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
         end
       end
 
@@ -38,13 +33,13 @@ module HTTPX
 
           connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
-          return super if connection && connection.upgrade_protocol == :h2c
+          return super if connection && connection.upgrade_protocol == "h2c"
 
           # build upgrade request
           upgrade_request.headers.add("connection", "upgrade")
           upgrade_request.headers.add("connection", "http2-settings")
           upgrade_request.headers["upgrade"] = "h2c"
-          upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
+          upgrade_request.headers["http2-settings"] = ::HTTP2::Client.settings_header(upgrade_request.options.http2_settings)
 
           super(upgrade_request, *remainder)
         end
@@ -78,22 +73,34 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
-          @parser = H2CParser.new(@write_buffer, parser_options)
+          @parser = H2CParser.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @inflight += 1
           @parser.upgrade(request, response)
-          @upgrade_protocol = :h2c
-
-          if request.options.max_concurrent_requests != @options.max_concurrent_requests
-            @options = @options.merge(max_concurrent_requests: nil)
-          end
+          @upgrade_protocol = "h2c"
 
           prev_parser.requests.each do |req|
             req.transition(:idle)
             send(req)
           end
         end
+
+        private
+
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
+
+          max_concurrent_requests = parser.max_concurrent_requests
+
+          return if max_concurrent_requests == 1
+
+          parser.max_concurrent_requests = 1
+          request.once(:response) do
+            parser.max_concurrent_requests = max_concurrent_requests
+          end
+        end
       end
     end
     register_plugin(:h2c, H2C)

data/lib/httpx/plugins/internal_telemetry.rb

@@ -32,6 +32,15 @@ module HTTPX
         end
       end
 
+      module NativeResolverMethods
+        def transition(nextstate)
+          state = @state
+          val = super
+          meter_elapsed_time("Resolver::Native: #{state} -> #{nextstate}")
+          val
+        end
+      end
+
       module InstanceMethods
         def self.included(klass)
           klass.prepend TrackTimeMethods
@@ -42,6 +51,13 @@ module HTTPX
           meter_elapsed_time("Session: initializing...")
           super
           meter_elapsed_time("Session: initialized!!!")
+          resolver_type = @options.resolver_class
+          resolver_type = Resolver.resolver_for(resolver_type)
+          return unless resolver_type <= Resolver::Native
+
+          resolver_type.prepend TrackTimeMethods
+          resolver_type.prepend NativeResolverMethods
+          @options = @options.merge(resolver_class: resolver_type)
         end
 
         def close(*)

data/lib/httpx/plugins/{ntlm_authentication.rb → ntlm_auth.rb}

@@ -3,12 +3,12 @@
 module HTTPX
   module Plugins
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#ntlm-authentication
+    # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
     #
     module NTLMAuth
       class << self
         def load_dependencies(_klass)
-          require_relative "authentication/ntlm"
+          require_relative "auth/ntlm"
         end
 
         def extra_options(options)
@@ -25,11 +25,11 @@ module HTTPX
       end
 
       module InstanceMethods
-        def ntlm_authentication(user, password, domain = nil)
+        def ntlm_auth(user, password, domain = nil)
           with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
         end
 
-        alias_method :ntlm_auth, :ntlm_authentication
+        private
 
         def send_requests(*requests)
           requests.flat_map do |request|
@@ -39,6 +39,8 @@ module HTTPX
               request.headers["authorization"] = ntlm.negotiate
               probe_response = wrap { super(request).first }
 
+              return probe_response unless probe_response.is_a?(Response)
+
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
                 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
@@ -53,6 +55,6 @@ module HTTPX
         end
       end
     end
-    register_plugin :ntlm_authentication, NTLMAuth
+    register_plugin :ntlm_auth, NTLMAuth
   end
 end

data/lib/httpx/plugins/oauth.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # https://gitlab.com/os85/httpx/wikis/OAuth
+    #
+    module OAuth
+      class << self
+        def load_dependencies(_klass)
+          require_relative "auth/basic"
+        end
+      end
+
+      SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
+      SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
+
+      class OAuthSession
+        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
+
+        def initialize(
+          issuer:,
+          client_id:,
+          client_secret:,
+          access_token: nil,
+          refresh_token: nil,
+          scope: nil,
+          token_endpoint: nil,
+          response_type: nil,
+          grant_type: nil,
+          token_endpoint_auth_method: nil
+        )
+          @issuer = URI(issuer)
+          @client_id = client_id
+          @client_secret = client_secret
+          @token_endpoint = URI(token_endpoint) if token_endpoint
+          @response_type = response_type
+          @scope = case scope
+                   when String
+                     scope.split
+                   when Array
+                     scope
+          end
+          @access_token = access_token
+          @refresh_token = refresh_token
+          @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
+          @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+
+          unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
+            raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
+          end
+
+          return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
+
+          raise Error, "#{@grant_type} is not a supported grant type"
+        end
+
+        def token_endpoint
+          @token_endpoint || "#{@issuer}/token"
+        end
+
+        def token_endpoint_auth_method
+          @token_endpoint_auth_method || "client_secret_basic"
+        end
+
+        def load(http)
+          return if @grant_type && @scope
+
+          metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+          nil
+        end
+
+        def merge(other)
+          obj = dup
+
+          case other
+          when OAuthSession
+            other.instance_variables.each do |ivar|
+              val = other.instance_variable_get(ivar)
+              next unless val
+
+              obj.instance_variable_set(ivar, val)
+            end
+          when Hash
+            other.each do |k, v|
+              obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
+            end
+          end
+          obj
+        end
+      end
+
+      module OptionsMethods
+        def option_oauth_session(value)
+          case value
+          when Hash
+            OAuthSession.new(**value)
+          when OAuthSession
+            value
+          else
+            raise TypeError, ":oauth_session must be a #{OAuthSession}"
+          end
+        end
+      end
+
+      module InstanceMethods
+        def oauth_auth(**args)
+          with(oauth_session: OAuthSession.new(**args))
+        end
+
+        def with_access_token
+          oauth_session = @options.oauth_session
+
+          oauth_session.load(self)
+
+          grant_type = oauth_session.grant_type
+
+          headers = {}
+          form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
+
+          # auth
+          case oauth_session.token_endpoint_auth_method
+          when "client_secret_post"
+            form_post["client_id"] = oauth_session.client_id
+            form_post["client_secret"] = oauth_session.client_secret
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
+          end
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            form_post["refresh_token"] = oauth_session.refresh_token
+          end
+
+          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
+          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = request(token_request)
+          token_response.raise_for_status
+
+          payload = token_response.json
+
+          access_token = payload["access_token"]
+          refresh_token = payload["refresh_token"]
+
+          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+        end
+
+        def build_request(*)
+          request = super
+
+          return request if request.headers.key?("authorization")
+
+          oauth_session = @options.oauth_session
+
+          return request unless oauth_session && oauth_session.access_token
+
+          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+
+          request
+        end
+      end
+    end
+    register_plugin :oauth, OAuth
+  end
+end

data/lib/httpx/plugins/persistent.rb

@@ -15,7 +15,7 @@ module HTTPX
     # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
     # So when you use this, make sure that you don't fall into this trap.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Persistent
+    # https://gitlab.com/os85/httpx/wikis/Persistent
     #
     module Persistent
       def self.load_dependencies(klass)

data/lib/httpx/plugins/proxy/http.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
-require "base64"
-
 module HTTPX
   module Plugins
     module Proxy
       module HTTP
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
+          end
+        end
+
         module InstanceMethods
           def with_proxy_basic_auth(opts)
             with(proxy: opts.merge(scheme: "basic"))
@@ -23,17 +27,23 @@ module HTTPX
             response = super
 
             if response &&
+               response.is_a?(Response) &&
                response.status == 407 &&
                !request.headers.key?("proxy-authorization") &&
                response.headers.key?("proxy-authenticate")
 
-              connection = find_connection(request, connections, options)
+              uri = request.uri
+
+              proxy_options = proxy_options(uri, options)
+              connection = connections.find do |conn|
+                conn.match?(uri, proxy_options)
+              end
 
-              if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+              if connection && connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
                 request.transition(:idle)
                 request.headers["proxy-authorization"] =
                   connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-                connection.send(request)
+                send_request(request, connections)
                 return
               end
             end
@@ -60,7 +70,7 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
@@ -71,7 +81,6 @@ module HTTPX
                   else
                     transition(:closing)
                     transition(:closed)
-                    emit(:reset)
 
                     parser.reset if @parser
                     transition(:idle)
@@ -113,13 +122,14 @@ module HTTPX
 
           def __http_on_connect(request, response)
             @inflight -= 1
-            if response.status == 200
+            if response.is_a?(Response) && response.status == 200
               req = @pending.first
               request_uri = req.uri
               @io = ProxySSL.new(@io, request_uri, @options)
               transition(:connected)
               throw(:called)
-            elsif response.status == 407 &&
+            elsif response.is_a?(Response) &&
+                  response.status == 407 &&
                   !request.headers.key?("proxy-authorization") &&
                   @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
 
@@ -139,9 +149,9 @@ module HTTPX
 
         module ProxyParser
           def join_headline(request)
-            return super if request.verb == :connect
+            return super if request.verb == "CONNECT"
 
-            "#{request.verb.to_s.upcase} #{request.uri} HTTP/#{@version.join(".")}"
+            "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
           end
 
           def set_protocol_headers(request)
@@ -158,8 +168,8 @@ module HTTPX
         end
 
         class ConnectRequest < Request
-          def initialize(uri, _options)
-            super(:connect, uri, {})
+          def initialize(uri, options)
+            super("CONNECT", uri, options)
             @headers.delete("accept")
           end
 

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,7 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  class Socks4Error < Error; end
+  class Socks4Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -16,6 +16,12 @@ module HTTPX
 
         Error = Socks4Error
 
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
+          end
+        end
+
         module ConnectionMethods
           def interests
             if @state == :connecting
@@ -79,15 +85,11 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
-          end
-
-          def timeout
-            @options.timeout[:operation_timeout]
+            @options = options
           end
 
           def close; end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class Socks5Error < Error; end
+  class Socks5Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -18,8 +18,14 @@ module HTTPX
 
         Error = Socks5Error
 
-        def self.load_dependencies(*)
-          require_relative "../authentication/socks5"
+        class << self
+          def load_dependencies(*)
+            require_relative "../auth/socks5"
+          end
+
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
+          end
         end
 
         module ConnectionMethods
@@ -131,15 +137,11 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer
-            @options = Options.new(options)
-          end
-
-          def timeout
-            @options.timeout[:operation_timeout]
+            @options = options
           end
 
           def close; end