httpx 0.3.1 → 0.4.0

data/lib/httpx/plugins/h2c.rb

@@ -2,53 +2,51 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2.
+    #
+    # https://tools.ietf.org/html/rfc7540#section-3.2
+    #
     module H2C
       def self.load_dependencies(*)
         require "base64"
       end
 
       module InstanceMethods
-        def request(*args, keep_open: @keep_open, **options)
-          return super if @_h2c_probed
-          begin
-            requests = __build_reqs(*args, **options)
-
-            upgrade_request = requests.first
-            return super unless valid_h2c_upgrade_request?(upgrade_request)
-            upgrade_request.headers["upgrade"] = "h2c"
-            upgrade_request.headers.add("connection", "upgrade")
-            upgrade_request.headers.add("connection", "http2-settings")
-            upgrade_request.headers["http2-settings"] = HTTP2::Client.settings_header(@options.http2_settings)
-            # TODO: validate!
-            upgrade_response = __send_reqs(*upgrade_request, **options).first
-
-            if upgrade_response.status == 101
-              channel = find_channel(upgrade_request)
-              parser = channel.upgrade_parser("h2")
-              parser.extend(UpgradeExtensions)
-              parser.upgrade(upgrade_request, upgrade_response, **options)
-              data = upgrade_response.to_s
-              parser << data
-              response = upgrade_request.response
-              if response.status == 200
-                requests.delete(upgrade_request)
-                return response if requests.empty?
-              end
-              responses = __send_reqs(*requests)
-            else
-              # proceed as usual
-              responses = [upgrade_response] + __send_reqs(*requests[1..-1])
-            end
-            return responses.first if responses.size == 1
-            responses
-          ensure
-            @_h2c_probed = true
-            close unless keep_open
-          end
+        def request(*args, **options)
+          h2c_options = options.merge(fallback_protocol: "h2c")
+
+          requests = build_requests(*args, h2c_options)
+
+          upgrade_request = requests.first
+          return super unless valid_h2c_upgrade_request?(upgrade_request)
+
+          upgrade_request.headers.add("connection", "upgrade")
+          upgrade_request.headers.add("connection", "http2-settings")
+          upgrade_request.headers["upgrade"] = "h2c"
+          upgrade_request.headers["http2-settings"] = HTTP2::Client.settings_header(upgrade_request.options.http2_settings)
+          wrap { send_requests(*upgrade_request, h2c_options).first }
+
+          responses = send_requests(*requests, h2c_options)
+
+          return responses.first if responses.size == 1
+
+          responses
         end
 
         private
 
+        def fetch_response(request, connections, options)
+          response = super
+          if response && valid_h2c_upgrade?(request, response, options)
+            log { "upgrading to h2c..." }
+            connection = find_connection(request, connections, options)
+            connections << connection unless connections.include?(connection)
+            connection.upgrade(request, response)
+          end
+          response
+        end
+
         VALID_H2C_METHODS = %i[get options head].freeze
         private_constant :VALID_H2C_METHODS
 
@@ -56,16 +54,57 @@ module HTTPX
           VALID_H2C_METHODS.include?(request.verb) &&
             request.scheme == "http"
         end
+
+        def valid_h2c_upgrade?(request, response, options)
+          options.fallback_protocol == "h2c" &&
+            request.headers.get("connection").include?("upgrade") &&
+            request.headers.get("upgrade").include?("h2c") &&
+            response.status == 101
+        end
       end
 
-      module UpgradeExtensions
-        def upgrade(request, _response, **)
+      class H2CParser < Connection::HTTP2
+        def upgrade(request, response)
           @connection.send_connection_preface
           # skip checks, it is assumed that this is the first
           # request in the connection
           stream = @connection.upgrade
           handle_stream(stream, request)
           @streams[request] = stream
+
+          # clean up data left behind in the buffer, if the server started
+          # sending frames
+          data = response.to_s
+          @connection << data
+        end
+      end
+
+      module ConnectionMethods
+        using URIExtensions
+
+        def match?(uri, options)
+          return super unless uri.scheme == "http" && @options.fallback_protocol == "h2c"
+
+          super && options.fallback_protocol == "h2c"
+        end
+
+        def coalescable?(connection)
+          return super unless @options.fallback_protocol == "h2c" && @origin.scheme == "http"
+
+          @origin == connection.origin && connection.options.fallback_protocol == "h2c"
+        end
+
+        def upgrade(request, response)
+          @parser.reset if @parser
+          @parser = H2CParser.new(@write_buffer, @options)
+          set_parser_callbacks(@parser)
+          @parser.upgrade(request, response)
+        end
+
+        def build_parser(*)
+          return super unless @origin.scheme == "http"
+
+          super("http/1.1")
         end
       end
 

data/lib/httpx/plugins/multipart.rb

@@ -2,6 +2,11 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for passing `http-form_data` objects (like file objects) as "multipart/form-data";
+    #
+    #   HTTPX.post(URL, form: form: { image: HTTP::FormData::File.new("path/to/file")})
+    #
     module Multipart
       module FormTranscoder
         module_function

data/lib/httpx/plugins/persistent.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    # This plugin implements a session that persists connections over the duration of the process.
+    #
+    # This will improve connection reuse in a long-running process.
+    #
+    # One important caveat to note is, although this session might not close connections,
+    # other sessions from the same process that don't have this plugin turned on might.
+    #
+    # This session will still be able to work with it, as if, when expecting a connection
+    # terminated by a different session, it will just retry on a new one and keep it open.
+    #
+    # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
+    # So when you use this, make sure that you don't fall into this trap.
+    #
+    module Persistent
+      def self.load_dependencies(klass, *)
+        klass.plugin(:retries) # TODO: pass default max_retries -> 1 as soon as this is a parameter
+      end
+
+      def self.extra_options(options)
+        options.merge(persistent: true)
+      end
+    end
+    register_plugin :persistent, Persistent
+  end
+end

data/lib/httpx/plugins/proxy.rb

@@ -6,11 +6,19 @@ require "forwardable"
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for proxies. It ships with support for:
+    #
+    # * HTTP proxies
+    # * HTTPS proxies
+    # * Socks4/4a proxies
+    # * Socks5 proxies
+    #
     module Proxy
       Error = Class.new(Error)
-      class Parameters
-        extend Registry
+      PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
+      class Parameters
         attr_reader :uri, :username, :password
 
         def initialize(uri:, username: nil, password: nil)
@@ -26,6 +34,32 @@ module HTTPX
         def token_authentication
           Base64.strict_encode64("#{user}:#{password}")
         end
+
+        def ==(other)
+          if other.is_a?(Parameters)
+            @uri == other.uri &&
+              @username == other.username &&
+              @password == other.password
+          else
+            super
+          end
+        end
+      end
+
+      class << self
+        def configure(klass, *)
+          klass.plugin(:"proxy/http")
+          klass.plugin(:"proxy/socks4")
+          klass.plugin(:"proxy/socks5")
+        end
+
+        def extra_options(options)
+          Class.new(options.class) do
+            def_option(:proxy) do |pr|
+              Hash[pr]
+            end
+          end.new(options)
+        end
       end
 
       module InstanceMethods
@@ -35,118 +69,131 @@ module HTTPX
 
         private
 
-        def proxy_params(uri)
+        def proxy_uris(uri, options)
           @_proxy_uris ||= begin
-            uris = @options.proxy ? Array(@options.proxy[:uri]) : []
+            uris = options.proxy ? Array(options.proxy[:uri]) : []
             if uris.empty?
               uri = URI(uri).find_proxy
               uris << uri if uri
             end
             uris
           end
-          @options.proxy.merge(uri: @_proxy_uris.shift) unless @_proxy_uris.empty?
+          options.proxy.merge(uri: @_proxy_uris.first) unless @_proxy_uris.empty?
         end
 
-        def find_channel(request, **options)
+        def find_connection(request, connections, options)
+          return super unless options.respond_to?(:proxy)
+
           uri = URI(request.uri)
-          proxy = proxy_params(uri)
-          raise Error, "Failed to connect to proxy" unless proxy
-          @connection.find_channel(proxy) || build_channel(proxy, options)
+          next_proxy = proxy_uris(uri, options)
+          raise Error, "Failed to connect to proxy" unless next_proxy
+
+          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
+          connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
+          unless connections.nil? || connections.include?(connection)
+            connections << connection
+            set_connection_callbacks(connection, options)
+          end
+          connection
         end
 
-        def build_channel(proxy, options)
-          return super if proxy.is_a?(URI::Generic)
-          channel = build_proxy_channel(proxy, **options)
-          set_channel_callbacks(channel, options)
-          channel
-        end
+        def build_connection(uri, options)
+          proxy = options.proxy
+          return super unless proxy
 
-        def build_proxy_channel(proxy, **options)
-          parameters = Parameters.new(**proxy)
-          uri = parameters.uri
-          log { "proxy: #{uri}" }
-          proxy_type = Parameters.registry(parameters.uri.scheme)
-          channel = proxy_type.new("tcp", uri, parameters, @options.merge(options), &method(:on_response))
-          @connection.__send__(:resolve_channel, channel)
-          channel
+          connection = options.connection_class.new("tcp", uri, options)
+          pool.init_connection(connection, options)
+          connection
         end
 
-        def fetch_response(request)
+        def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
              # either it was a timeout error connecting, or it was a proxy error
-             (((response.error.is_a?(TimeoutError) || response.error.is_a?(IOError)) && request.state == :idle) ||
-              response.error.is_a?(Error)) &&
-             !@_proxy_uris.empty?
+             PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
+            @_proxy_uris.shift
             log { "failed connecting to proxy, trying next..." }
-            channel = find_channel(request)
-            channel.send(request)
+            request.transition(:idle)
+            connection = find_connection(request, connections, options)
+            connections << connection unless connections.include?(connection)
+            connection.send(request)
             return
           end
           response
         end
       end
 
-      module OptionsMethods
-        def self.included(klass)
+      module ConnectionMethods
+        using URIExtensions
+
+        def initialize(*)
           super
-          klass.def_option(:proxy) do |pr|
-            Hash[pr]
-          end
+          return unless @options.proxy
+
+          # redefining the connection origin as the proxy's URI,
+          # as this will be used as the tcp peer ip.
+          @origin = URI(@options.proxy.uri.origin)
         end
-      end
 
-      def self.configure(klass, *)
-        klass.plugin(:"proxy/http")
-        klass.plugin(:"proxy/socks4")
-        klass.plugin(:"proxy/socks5")
-      end
-    end
-    register_plugin :proxy, Proxy
-  end
+        def match?(uri, options)
+          return super unless @options.proxy
 
-  class ProxyChannel < Channel
-    def initialize(type, uri, parameters, options, &blk)
-      super(type, uri, options, &blk)
-      @parameters = parameters
-    end
+          super && @options.proxy == options.proxy
+        end
 
-    def match?(*)
-      true
-    end
+        # should not coalesce connections here, as the IP is the IP of the proxy
+        def coalescable?(*)
+          return super unless @options.proxy
 
-    def send(request, **args)
-      @pending << [request, args]
-    end
+          false
+        end
 
-    def connecting?
-      super || @state == :connecting || @state == :connected
-    end
+        def send(request)
+          return super unless @options.proxy
+          return super unless connecting?
 
-    def to_io
-      case @state
-      when :idle
-        transition(:connecting)
-      when :connected
-        transition(:open)
-      end
-      @io.to_io
-    end
+          @pending << request
+        end
 
-    def call
-      super
-      case @state
-      when :connecting
-        consume
-      end
-    end
+        def connecting?
+          return super unless @options.proxy
+
+          super || @state == :connecting || @state == :connected
+        end
+
+        def to_io
+          return super unless @options.proxy
 
-    def reset
-      @state = :open
-      transition(:closing)
-      transition(:closed)
-      emit(:close)
+          case @state
+          when :idle
+            transition(:connecting)
+          when :connected
+            transition(:open)
+          end
+          @io.to_io
+        end
+
+        def call
+          super
+          return unless @options.proxy
+
+          case @state
+          when :connecting
+            consume
+          end
+        end
+
+        def reset
+          return super unless @options.proxy
+
+          @state = :open
+          transition(:closing)
+          transition(:closed)
+          emit(:close)
+        end
+      end
     end
+    register_plugin :proxy, Proxy
   end
 
   class ProxySSL < SSL