httpx 0.20.0 → 1.3.1

data/lib/httpx/session.rb

@@ -1,12 +1,20 @@
 # frozen_string_literal: true
 
 module HTTPX
+  EMPTY_HASH = {}.freeze
+
+  # Class implementing the APIs being used publicly.
+  #
+  #   HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
+  #   HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
   class Session
     include Loggable
     include Chainable
 
-    EMPTY_HASH = {}.freeze
-
+    # initializes the session with a set of +options+, which will be shared by all
+    # requests sent from it.
+    #
+    # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
     def initialize(options = EMPTY_HASH, &blk)
       @options = self.class.default_options.merge(options)
       @responses = {}
@@ -14,63 +22,102 @@ module HTTPX
       wrap(&blk) if blk
     end
 
+    # Yields itself the block, then closes it after the block is evaluated.
+    #
+    #   session.wrap do |http|
+    #     http.get("https://wikipedia.com")
+    #   end # wikipedia connection closes here
     def wrap
-      begin
-        prev_persistent = @persistent
-        @persistent = true
-        yield self
-      ensure
-        @persistent = prev_persistent
-        close unless @persistent
+      prev_persistent = @persistent
+      @persistent = true
+      pool.wrap do
+        begin
+          yield self
+        ensure
+          @persistent = prev_persistent
+          close unless @persistent
+        end
       end
     end
 
+    # closes all the active connections from the session
     def close(*args)
       pool.close(*args)
     end
 
-    def request(*args, **options)
+    # performs one, or multple requests; it accepts:
+    #
+    # 1. one or multiple HTTPX::Request objects;
+    # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
+    # 3. one or multiple HTTP verb / uri / (optional) options tuples;
+    #
+    # when present, the set of +options+ kwargs is applied to all of the
+    # sent requests.
+    #
+    # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
+    #
+    #  resp1 = session.request(req1)
+    #  resp1, resp2 = session.request(req1, req2)
+    #  resp1 = session.request("GET", "https://server.org/a")
+    #  resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
+    #  resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
+    #  resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
+    #  resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
+    #  resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
+    #
+    def request(*args, **params)
       raise ArgumentError, "must perform at least one request" if args.empty?
 
-      requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+      requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
       responses = send_requests(*requests)
       return responses.first if responses.size == 1
 
       responses
     end
 
-    def build_request(verb, uri, options = EMPTY_HASH)
-      rklass = @options.request_class
-      options = @options.merge(options) unless options.is_a?(Options)
-      request = rklass.new(verb, uri, options.merge(persistent: @persistent))
-      request.on(:response, &method(:on_response).curry(2)[request])
-      request.on(:promise, &method(:on_promise))
+    # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
+    # the optional set of request-specific +options+. This request **must** be sent through
+    # the same session it was built from.
+    #
+    #   req = session.build_request("GET", "https://server.com")
+    #   resp = session.request(req)
+    def build_request(verb, uri, params = EMPTY_HASH, options = @options)
+      rklass = options.request_class
+      request = rklass.new(verb, uri, options, params)
+      request.persistent = @persistent
+      set_request_callbacks(request)
       request
     end
 
     private
 
+    # returns the HTTPX::Pool object which manages the networking required to
+    # perform requests.
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
 
+    # callback executed when a response for a given request has been received.
     def on_response(request, response)
       @responses[request] = response
     end
 
+    # callback executed when an HTTP/2 promise frame has been received.
     def on_promise(_, stream)
       log(level: 2) { "#{stream.id}: refusing stream!" }
       stream.refuse
     end
 
+    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
     def fetch_response(request, _, _)
       @responses.delete(request)
     end
 
+    # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request, connections, options)
       uri = request.uri
 
-      connection = pool.find_connection(uri, options) || build_connection(uri, options)
+      connection = pool.find_connection(uri, options) || init_connection(uri, options)
       unless connections.nil? || connections.include?(connection)
         connections << connection
         set_connection_callbacks(connection, connections, options)
@@ -78,7 +125,20 @@ module HTTPX
       connection
     end
 
-    def set_connection_callbacks(connection, connections, options)
+    # sends the +request+ to the corresponding HTTPX::Connection
+    def send_request(request, connections, options = request.options)
+      error = catch(:resolve_error) do
+        connection = find_connection(request, connections, options)
+        connection.send(request)
+      end
+      return unless error.is_a?(Error)
+
+      request.emit(:response, ErrorResponse.new(request, error))
+    end
+
+    # sets the callbacks on the +connection+ required to process certain specific
+    # connection lifecycle events which deal with request rerouting.
+    def set_connection_callbacks(connection, connections, options, cloned: false)
       connection.only(:misdirected) do |misdirected_request|
         other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
         other_connection.merge(connection)
@@ -94,17 +154,13 @@ module HTTPX
         other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
         connections << other_connection if other_connection
       end
-      connection.only(:exhausted) do
-        other_connection = connection.create_idle
-        other_connection.merge(connection)
-        catch(:coalesced) do
-          pool.init_connection(other_connection, options)
-        end
-        set_connection_callbacks(other_connection, connections, options)
-        connections << other_connection
-      end
+      connection.only(:cloned) do |cloned_conn|
+        set_connection_callbacks(cloned_conn, connections, options, cloned: true)
+        connections << cloned_conn
+      end unless cloned
     end
 
+    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
     def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
       # do not allow security downgrades on altsvc negotiation
       return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
@@ -116,46 +172,46 @@ module HTTPX
 
       alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
 
-      connection = pool.find_connection(alt_origin, alt_options) || build_connection(alt_origin, alt_options)
+      connection = pool.find_connection(alt_origin, alt_options) || init_connection(alt_origin, alt_options)
+
       # advertised altsvc is the same origin being used, ignore
       return if connection == existing_connection
 
+      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+
       set_connection_callbacks(connection, connections, alt_options)
 
       log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
 
-      # get uninitialized requests
-      # incidentally, all requests will be re-routed to the first
-      # advertised alt-svc, which incidentally follows the spec.
-      existing_connection.purge_pending do |request|
-        request.origin == origin &&
-          request.state == :idle &&
-          !request.headers.key?("alt-used")
-      end
-
       connection.merge(existing_connection)
+      existing_connection.terminate
       connection
     rescue UnsupportedSchemeError
       altsvc["noop"] = true
       nil
     end
 
-    def build_requests(*args, options)
-      request_options = @options.merge(options)
-
+    # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
+    def build_requests(*args, params)
       requests = if args.size == 1
         reqs = args.first
-        reqs.map do |verb, uri, opts = EMPTY_HASH|
-          build_request(verb, uri, request_options.merge(opts))
+        # TODO: find a way to make requests share same options object
+        reqs.map do |verb, uri, ps = EMPTY_HASH|
+          request_params = params
+          request_params = request_params.merge(ps) unless ps.empty?
+          build_request(verb, uri, request_params)
         end
       else
         verb, uris = args
         if uris.respond_to?(:each)
-          uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
-            build_request(verb, uri, request_options.merge(opts))
+          # TODO: find a way to make requests share same options object
+          uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
+            request_params = params
+            request_params = request_params.merge(ps) unless ps.empty?
+            build_request(verb, uri, request_params)
           end
         else
-          [build_request(verb, uris, request_options)]
+          [build_request(verb, uris, params)]
         end
       end
       raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
@@ -163,46 +219,47 @@ module HTTPX
       requests
     end
 
-    def build_connection(uri, options)
-      type = options.transport || begin
-        case uri.scheme
-        when "http"
-          "tcp"
-        when "https"
-          "ssl"
-        else
-          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
-        end
-      end
-      connection = options.connection_class.new(type, uri, options)
+    def set_request_callbacks(request)
+      request.on(:response, &method(:on_response).curry(2)[request])
+      request.on(:promise, &method(:on_promise))
+    end
+
+    def init_connection(uri, options)
+      connection = options.connection_class.new(uri, options)
       catch(:coalesced) do
         pool.init_connection(connection, options)
         connection
       end
     end
 
+    def deactivate_connection(request, connections, options)
+      conn = connections.find do |c|
+        c.match?(request.uri, options)
+      end
+
+      pool.deactivate(conn) if conn
+    end
+
+    # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
     def send_requests(*requests)
       connections = _send_requests(requests)
       receive_requests(requests, connections)
     end
 
+    # sends an array of HTTPX::Request objects
     def _send_requests(requests)
       connections = []
 
       requests.each do |request|
-        error = catch(:resolve_error) do
-          connection = find_connection(request, connections, request.options)
-          connection.send(request)
-        end
-        next unless error.is_a?(ResolveError)
-
-        request.emit(:response, ErrorResponse.new(request, error, request.options))
+        send_request(request, connections)
       end
 
       connections
     end
 
+    # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, connections)
+      # @type var responses: Array[response]
       responses = []
 
       begin
@@ -213,6 +270,7 @@ module HTTPX
           return responses unless request
 
           catch(:coalesced) { pool.next_tick } until (response = fetch_response(request, connections, request.options))
+          request.emit(:complete, response)
 
           responses << response
           requests.shift
@@ -226,14 +284,16 @@ module HTTPX
           # opportunity to traverse the requests, hence we're returning only a fraction of the errors
           # we were supposed to. This effectively fetches the existing responses and return them.
           while (request = requests.shift)
-            responses << fetch_response(request, connections, request.options)
+            response = fetch_response(request, connections, request.options)
+            request.emit(:complete, response) if response
+            responses << response
           end
           break
         end
         responses
       ensure
         if @persistent
-          pool.deactivate(connections)
+          pool.deactivate(*connections)
         else
           close(connections)
         end
@@ -251,8 +311,14 @@ module HTTPX
         super
         klass.instance_variable_set(:@default_options, @default_options)
         klass.instance_variable_set(:@plugins, @plugins.dup)
+        klass.instance_variable_set(:@callbacks, @callbacks.dup)
       end
 
+      # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
+      #
+      #   session_with_retries = session.plugin(:retries)
+      #   session_with_custom = session.plugin(CustomPlugin)
+      #
       def plugin(pl, options = nil, &block)
         # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
@@ -266,19 +332,8 @@ module HTTPX
           extend(pl::ClassMethods) if defined?(pl::ClassMethods)
 
           opts = @default_options
-          opts.request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
-          opts.request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
-          opts.response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
-          opts.response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
-          opts.headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
-          opts.headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
-          opts.request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
-          opts.request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
-          opts.response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
-          opts.response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
-          opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
+          opts.extend_with_plugin_classes(pl)
           if defined?(pl::OptionsMethods)
-            opts.options_class.__send__(:include, pl::OptionsMethods)
 
             (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
               opts.options_class.method_added(meth)
@@ -302,16 +357,9 @@ module HTTPX
         end
         self
       end
-
-      # :nocov:
-      def plugins(pls)
-        warn ":#{__method__} is deprecated, use :plugin instead"
-        pls.each do |pl|
-          plugin(pl)
-        end
-        self
-      end
-      # :nocov:
     end
   end
+
+  # session may be overridden by certain adapters.
+  S = Session
 end

data/lib/httpx/session_extensions.rb

@@ -9,10 +9,13 @@ module HTTPX
     # redefine the default options static var, which needs to
     # refresh options_class
     options = proxy_session.class.default_options.to_hash
-    options.freeze
     original_verbosity = $VERBOSE
     $VERBOSE = nil
+    const_set(:Options, proxy_session.class.default_options.options_class)
+    options[:options_class] = Class.new(options[:options_class])
+    options.freeze
     Options.send(:const_set, :DEFAULT_OPTIONS, options)
+    Session.instance_variable_set(:@default_options, Options.new(options))
     $VERBOSE = original_verbosity
   end
 

data/lib/httpx/timers.rb

@@ -6,20 +6,27 @@ module HTTPX
       @intervals = []
     end
 
-    def after(interval_in_secs, &blk)
+    def after(interval_in_secs, cb = nil, &blk)
       return unless interval_in_secs
 
+      callback = cb || blk
+
       # I'm assuming here that most requests will have the same
       # request timeout, as in most cases they share common set of
       # options. A user setting different request timeouts for 100s of
       # requests will already have a hard time dealing with that.
-      unless (interval = @intervals.find { |t| t == interval_in_secs })
+      unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
         interval = Interval.new(interval_in_secs)
+        interval.on_empty { @intervals.delete(interval) }
         @intervals << interval
         @intervals.sort!
       end
 
-      interval << blk
+      interval << callback
+
+      @next_interval_at = nil
+
+      interval
     end
 
     def wait_interval
@@ -36,11 +43,9 @@ module HTTPX
 
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
-      @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
-    end
+      @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
 
-    def cancel
-      @intervals.clear
+      @next_interval_at = nil if @intervals.empty?
     end
 
     class Interval
@@ -51,6 +56,11 @@ module HTTPX
       def initialize(interval)
         @interval = interval
         @callbacks = []
+        @on_empty = nil
+      end
+
+      def on_empty(&blk)
+        @on_empty = blk
       end
 
       def <=>(other)
@@ -71,10 +81,26 @@ module HTTPX
         @callbacks << callback
       end
 
+      def delete(callback)
+        @callbacks.delete(callback)
+        @on_empty.call if @callbacks.empty?
+      end
+
+      def no_callbacks?
+        @callbacks.empty?
+      end
+
+      def elapsed?
+        @interval <= 0
+      end
+
       def elapse(elapsed)
         @interval -= elapsed
 
-        @callbacks.each(&:call) if @interval <= 0
+        if @interval <= 0
+          cb = @callbacks.dup
+          cb.each(&:call)
+        end
 
         @interval
       end

data/lib/httpx/transcoder/body.rb

@@ -9,7 +9,6 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
-      using HTTPX::ArrayExtensions
       extend Forwardable
 
       def_delegator :@raw, :to_s
@@ -55,5 +54,4 @@ module HTTPX::Transcoder
       Encoder.new(body)
     end
   end
-  register "body", Body
 end

data/lib/httpx/transcoder/chunker.rb

@@ -112,5 +112,4 @@ module HTTPX::Transcoder
       Encoder.new(chunks)
     end
   end
-  register "chunker", Chunker
 end

data/lib/httpx/transcoder/deflate.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "zlib"
+require_relative "utils/deflater"
+
+module HTTPX
+  module Transcoder
+    module Deflate
+      class Deflater < Transcoder::Deflater
+        def deflate(chunk)
+          @deflater ||= Zlib::Deflate.new
+
+          if chunk.nil?
+            unless @deflater.closed?
+              last = @deflater.finish
+              @deflater.close
+              last.empty? ? nil : last
+            end
+          else
+            @deflater.deflate(chunk)
+          end
+        end
+      end
+
+      module_function
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(response, bytesize: nil)
+        bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
+        GZIP::Inflater.new(bytesize)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/form.rb

@@ -2,58 +2,77 @@
 
 require "forwardable"
 require "uri"
+require_relative "multipart"
 
-module HTTPX::Transcoder
-  module Form
-    module_function
+module HTTPX
+  module Transcoder
+    module Form
+      module_function
 
-    PARAM_DEPTH_LIMIT = 32
+      PARAM_DEPTH_LIMIT = 32
 
-    class Encoder
-      extend Forwardable
+      class Encoder
+        extend Forwardable
 
-      def_delegator :@raw, :to_s
+        def_delegator :@raw, :to_s
 
-      def_delegator :@raw, :to_str
+        def_delegator :@raw, :to_str
 
-      def_delegator :@raw, :bytesize
+        def_delegator :@raw, :bytesize
 
-      def initialize(form)
-        @raw = form.each_with_object("".b) do |(key, val), buf|
-          HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
-            buf << "&" unless buf.empty?
-            buf << URI.encode_www_form_component(k)
-            buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
+        def initialize(form)
+          @raw = form.each_with_object("".b) do |(key, val), buf|
+            HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
+              buf << "&" unless buf.empty?
+              buf << URI.encode_www_form_component(k)
+              buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
+            end
           end
         end
-      end
 
-      def content_type
-        "application/x-www-form-urlencoded"
+        def content_type
+          "application/x-www-form-urlencoded"
+        end
       end
-    end
 
-    module Decoder
-      module_function
+      module Decoder
+        module_function
 
-      def call(response, _)
-        URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
-          HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
+        def call(response, *)
+          URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
+            HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
+          end
         end
       end
-    end
 
-    def encode(form)
-      Encoder.new(form)
-    end
+      def encode(form)
+        if multipart?(form)
+          Multipart::Encoder.new(form)
+        else
+          Encoder.new(form)
+        end
+      end
 
-    def decode(response)
-      content_type = response.content_type.mime_type
+      def decode(response)
+        content_type = response.content_type.mime_type
 
-      raise HTTPX::Error, "invalid form mime type (#{content_type})" unless content_type == "application/x-www-form-urlencoded"
+        case content_type
+        when "application/x-www-form-urlencoded"
+          Decoder
+        when "multipart/form-data"
+          Multipart::Decoder.new(response)
+        else
+          raise Error, "invalid form mime type (#{content_type})"
+        end
+      end
 
-      Decoder
+      def multipart?(data)
+        data.any? do |_, v|
+          Multipart::MULTIPART_VALUE_COND.call(v) ||
+            (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
+            (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+        end
+      end
     end
   end
-  register "form", Form
 end