httpx 0.21.0 → 1.2.1

data/lib/httpx/session.rb

@@ -1,12 +1,20 @@
 # frozen_string_literal: true
 
 module HTTPX
+  # Class implementing the APIs being used publicly.
+  #
+  #   HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
+  #   HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
   class Session
     include Loggable
     include Chainable
 
     EMPTY_HASH = {}.freeze
 
+    # initializes the session with a set of +options+, which will be shared by all
+    # requests sent from it.
+    #
+    # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
     def initialize(options = EMPTY_HASH, &blk)
       @options = self.class.default_options.merge(options)
       @responses = {}
@@ -14,6 +22,11 @@ module HTTPX
       wrap(&blk) if blk
     end
 
+    # Yields itself the block, then closes it after the block is evaluated.
+    #
+    #   session.wrap do |http|
+    #     http.get("https://wikipedia.com")
+    #   end # wikipedia connection closes here
     def wrap
       begin
         prev_persistent = @persistent
@@ -25,10 +38,31 @@ module HTTPX
       end
     end
 
+    # closes all the active connections from the session
     def close(*args)
       pool.close(*args)
     end
 
+    # performs one, or multple requests; it accepts:
+    #
+    # 1. one or multiple HTTPX::Request objects;
+    # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
+    # 3. one or multiple HTTP verb / uri / (optional) options tuples;
+    #
+    # when present, the set of +options+ kwargs is applied to all of the
+    # sent requests.
+    #
+    # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
+    #
+    #  resp1 = session.request(req1)
+    #  resp1, resp2 = session.request(req1, req2)
+    #  resp1 = session.request("GET", "https://server.org/a")
+    #  resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
+    #  resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
+    #  resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
+    #  resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
+    #  resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
+    #
     def request(*args, **options)
       raise ArgumentError, "must perform at least one request" if args.empty?
 
@@ -39,38 +73,50 @@ module HTTPX
       responses
     end
 
+    # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
+    # the optional set of request-specific +options+. This request **must** be sent through
+    # the same session it was built from.
+    #
+    #   req = session.build_request("GET", "https://server.com")
+    #   resp = session.request(req)
     def build_request(verb, uri, options = EMPTY_HASH)
       rklass = @options.request_class
       options = @options.merge(options) unless options.is_a?(Options)
-      request = rklass.new(verb, uri, options.merge(persistent: @persistent))
-      request.on(:response, &method(:on_response).curry(2)[request])
-      request.on(:promise, &method(:on_promise))
+      request = rklass.new(verb, uri, options)
+      request.persistent = @persistent
+      set_request_callbacks(request)
       request
     end
 
     private
 
+    # returns the HTTPX::Pool object which manages the networking required to
+    # perform requests.
     def pool
       Thread.current[:httpx_connection_pool] ||= Pool.new
     end
 
+    # callback executed when a response for a given request has been received.
     def on_response(request, response)
       @responses[request] = response
     end
 
+    # callback executed when an HTTP/2 promise frame has been received.
     def on_promise(_, stream)
       log(level: 2) { "#{stream.id}: refusing stream!" }
       stream.refuse
     end
 
+    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
     def fetch_response(request, _, _)
       @responses.delete(request)
     end
 
+    # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request, connections, options)
       uri = request.uri
 
-      connection = pool.find_connection(uri, options) || build_connection(uri, options)
+      connection = pool.find_connection(uri, options) || init_connection(uri, options)
       unless connections.nil? || connections.include?(connection)
         connections << connection
         set_connection_callbacks(connection, connections, options)
@@ -78,6 +124,18 @@ module HTTPX
       connection
     end
 
+    def send_request(request, connections, options = request.options)
+      error = catch(:resolve_error) do
+        connection = find_connection(request, connections, options)
+        connection.send(request)
+      end
+      return unless error.is_a?(Error)
+
+      request.emit(:response, ErrorResponse.new(request, error, options))
+    end
+
+    # sets the callbacks on the +connection+ required to process certain specific
+    # connection lifecycle events which deal with request rerouting.
     def set_connection_callbacks(connection, connections, options)
       connection.only(:misdirected) do |misdirected_request|
         other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
@@ -94,17 +152,9 @@ module HTTPX
         other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
         connections << other_connection if other_connection
       end
-      connection.only(:exhausted) do
-        other_connection = connection.create_idle
-        other_connection.merge(connection)
-        catch(:coalesced) do
-          pool.init_connection(other_connection, options)
-        end
-        set_connection_callbacks(other_connection, connections, options)
-        connections << other_connection
-      end
     end
 
+    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
     def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
       # do not allow security downgrades on altsvc negotiation
       return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
@@ -116,30 +166,26 @@ module HTTPX
 
       alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
 
-      connection = pool.find_connection(alt_origin, alt_options) || build_connection(alt_origin, alt_options)
+      connection = pool.find_connection(alt_origin, alt_options) || init_connection(alt_origin, alt_options)
+
       # advertised altsvc is the same origin being used, ignore
       return if connection == existing_connection
 
+      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+
       set_connection_callbacks(connection, connections, alt_options)
 
       log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
 
-      # get uninitialized requests
-      # incidentally, all requests will be re-routed to the first
-      # advertised alt-svc, which incidentally follows the spec.
-      existing_connection.purge_pending do |request|
-        request.origin == origin &&
-          request.state == :idle &&
-          !request.headers.key?("alt-used")
-      end
-
       connection.merge(existing_connection)
+      existing_connection.terminate
       connection
     rescue UnsupportedSchemeError
       altsvc["noop"] = true
       nil
     end
 
+    # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
     def build_requests(*args, options)
       request_options = @options.merge(options)
 
@@ -163,46 +209,39 @@ module HTTPX
       requests
     end
 
-    def build_connection(uri, options)
-      type = options.transport || begin
-        case uri.scheme
-        when "http"
-          "tcp"
-        when "https"
-          "ssl"
-        else
-          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
-        end
-      end
-      connection = options.connection_class.new(type, uri, options)
+    def set_request_callbacks(request)
+      request.on(:response, &method(:on_response).curry(2)[request])
+      request.on(:promise, &method(:on_promise))
+    end
+
+    def init_connection(uri, options)
+      connection = options.connection_class.new(uri, options)
       catch(:coalesced) do
         pool.init_connection(connection, options)
         connection
       end
     end
 
+    # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
     def send_requests(*requests)
       connections = _send_requests(requests)
       receive_requests(requests, connections)
     end
 
+    # sends an array of HTTPX::Request objects
     def _send_requests(requests)
       connections = []
 
       requests.each do |request|
-        error = catch(:resolve_error) do
-          connection = find_connection(request, connections, request.options)
-          connection.send(request)
-        end
-        next unless error.is_a?(ResolveError)
-
-        request.emit(:response, ErrorResponse.new(request, error, request.options))
+        send_request(request, connections)
       end
 
       connections
     end
 
+    # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, connections)
+      # @type var responses: Array[response]
       responses = []
 
       begin
@@ -251,8 +290,14 @@ module HTTPX
         super
         klass.instance_variable_set(:@default_options, @default_options)
         klass.instance_variable_set(:@plugins, @plugins.dup)
+        klass.instance_variable_set(:@callbacks, @callbacks.dup)
       end
 
+      # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
+      #
+      #   session_with_retries = session.plugin(:retries)
+      #   session_with_custom = session.plugin(CustomPlugin)
+      #
       def plugin(pl, options = nil, &block)
         # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
@@ -266,19 +311,8 @@ module HTTPX
           extend(pl::ClassMethods) if defined?(pl::ClassMethods)
 
           opts = @default_options
-          opts.request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
-          opts.request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
-          opts.response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
-          opts.response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
-          opts.headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
-          opts.headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
-          opts.request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
-          opts.request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
-          opts.response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
-          opts.response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
-          opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
+          opts.extend_with_plugin_classes(pl)
           if defined?(pl::OptionsMethods)
-            opts.options_class.__send__(:include, pl::OptionsMethods)
 
             (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
               opts.options_class.method_added(meth)
@@ -302,16 +336,9 @@ module HTTPX
         end
         self
       end
-
-      # :nocov:
-      def plugins(pls)
-        warn ":#{__method__} is deprecated, use :plugin instead"
-        pls.each do |pl|
-          plugin(pl)
-        end
-        self
-      end
-      # :nocov:
     end
   end
+
+  # session may be overridden by certain adapters.
+  S = Session
 end

data/lib/httpx/session_extensions.rb

@@ -9,10 +9,13 @@ module HTTPX
     # redefine the default options static var, which needs to
     # refresh options_class
     options = proxy_session.class.default_options.to_hash
-    options.freeze
     original_verbosity = $VERBOSE
     $VERBOSE = nil
+    const_set(:Options, proxy_session.class.default_options.options_class)
+    options[:options_class] = Class.new(options[:options_class])
+    options.freeze
     Options.send(:const_set, :DEFAULT_OPTIONS, options)
+    Session.instance_variable_set(:@default_options, Options.new(options))
     $VERBOSE = original_verbosity
   end
 

data/lib/httpx/timers.rb

@@ -6,20 +6,27 @@ module HTTPX
       @intervals = []
     end
 
-    def after(interval_in_secs, &blk)
+    def after(interval_in_secs, cb = nil, &blk)
       return unless interval_in_secs
 
+      callback = cb || blk
+
       # I'm assuming here that most requests will have the same
       # request timeout, as in most cases they share common set of
       # options. A user setting different request timeouts for 100s of
       # requests will already have a hard time dealing with that.
-      unless (interval = @intervals.find { |t| t == interval_in_secs })
+      unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
         interval = Interval.new(interval_in_secs)
+        interval.on_empty { @intervals.delete(interval) }
         @intervals << interval
         @intervals.sort!
       end
 
-      interval << blk
+      interval << callback
+
+      @next_interval_at = nil
+
+      interval
     end
 
     def wait_interval
@@ -41,11 +48,6 @@ module HTTPX
       @next_interval_at = nil if @intervals.empty?
     end
 
-    def cancel
-      @next_interval_at = nil
-      @intervals.clear
-    end
-
     class Interval
       include Comparable
 
@@ -54,6 +56,11 @@ module HTTPX
       def initialize(interval)
         @interval = interval
         @callbacks = []
+        @on_empty = nil
+      end
+
+      def on_empty(&blk)
+        @on_empty = blk
       end
 
       def <=>(other)
@@ -74,6 +81,19 @@ module HTTPX
         @callbacks << callback
       end
 
+      def delete(callback)
+        @callbacks.delete(callback)
+        @on_empty.call if @callbacks.empty?
+      end
+
+      def no_callbacks?
+        @callbacks.empty?
+      end
+
+      def elapsed?
+        @interval <= 0
+      end
+
       def elapse(elapsed)
         @interval -= elapsed
 

data/lib/httpx/transcoder/body.rb

@@ -9,7 +9,6 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
-      using HTTPX::ArrayExtensions::Sum
       extend Forwardable
 
       def_delegator :@raw, :to_s
@@ -55,5 +54,4 @@ module HTTPX::Transcoder
       Encoder.new(body)
     end
   end
-  register "body", Body
 end

data/lib/httpx/transcoder/chunker.rb

@@ -112,5 +112,4 @@ module HTTPX::Transcoder
       Encoder.new(chunks)
     end
   end
-  register "chunker", Chunker
 end

data/lib/httpx/transcoder/deflate.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "zlib"
+require_relative "utils/deflater"
+
+module HTTPX
+  module Transcoder
+    module Deflate
+      class Deflater < Transcoder::Deflater
+        def deflate(chunk)
+          @deflater ||= Zlib::Deflate.new
+
+          if chunk.nil?
+            unless @deflater.closed?
+              last = @deflater.finish
+              @deflater.close
+              last.empty? ? nil : last
+            end
+          else
+            @deflater.deflate(chunk)
+          end
+        end
+      end
+
+      module_function
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(response, bytesize: nil)
+        bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
+        GZIP::Inflater.new(bytesize)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/form.rb

@@ -2,58 +2,77 @@
 
 require "forwardable"
 require "uri"
+require_relative "multipart"
 
-module HTTPX::Transcoder
-  module Form
-    module_function
+module HTTPX
+  module Transcoder
+    module Form
+      module_function
 
-    PARAM_DEPTH_LIMIT = 32
+      PARAM_DEPTH_LIMIT = 32
 
-    class Encoder
-      extend Forwardable
+      class Encoder
+        extend Forwardable
 
-      def_delegator :@raw, :to_s
+        def_delegator :@raw, :to_s
 
-      def_delegator :@raw, :to_str
+        def_delegator :@raw, :to_str
 
-      def_delegator :@raw, :bytesize
+        def_delegator :@raw, :bytesize
 
-      def initialize(form)
-        @raw = form.each_with_object("".b) do |(key, val), buf|
-          HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
-            buf << "&" unless buf.empty?
-            buf << URI.encode_www_form_component(k)
-            buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
+        def initialize(form)
+          @raw = form.each_with_object("".b) do |(key, val), buf|
+            HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
+              buf << "&" unless buf.empty?
+              buf << URI.encode_www_form_component(k)
+              buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
+            end
           end
         end
-      end
 
-      def content_type
-        "application/x-www-form-urlencoded"
+        def content_type
+          "application/x-www-form-urlencoded"
+        end
       end
-    end
 
-    module Decoder
-      module_function
+      module Decoder
+        module_function
 
-      def call(response, *)
-        URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
-          HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
+        def call(response, *)
+          URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
+            HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
+          end
         end
       end
-    end
 
-    def encode(form)
-      Encoder.new(form)
-    end
+      def encode(form)
+        if multipart?(form)
+          Multipart::Encoder.new(form)
+        else
+          Encoder.new(form)
+        end
+      end
 
-    def decode(response)
-      content_type = response.content_type.mime_type
+      def decode(response)
+        content_type = response.content_type.mime_type
 
-      raise HTTPX::Error, "invalid form mime type (#{content_type})" unless content_type == "application/x-www-form-urlencoded"
+        case content_type
+        when "application/x-www-form-urlencoded"
+          Decoder
+        when "multipart/form-data"
+          Multipart::Decoder.new(response)
+        else
+          raise Error, "invalid form mime type (#{content_type})"
+        end
+      end
 
-      Decoder
+      def multipart?(data)
+        data.any? do |_, v|
+          Multipart::MULTIPART_VALUE_COND.call(v) ||
+            (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
+            (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+        end
+      end
     end
   end
-  register "form", Form
 end

data/lib/httpx/transcoder/gzip.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "uri"
+require "stringio"
+require "zlib"
+
+module HTTPX
+  module Transcoder
+    module GZIP
+      class Deflater < Transcoder::Deflater
+        def initialize(body)
+          @compressed_chunk = "".b
+          super
+        end
+
+        def deflate(chunk)
+          @deflater ||= Zlib::GzipWriter.new(self)
+
+          if chunk.nil?
+            unless @deflater.closed?
+              @deflater.flush
+              @deflater.close
+              compressed_chunk
+            end
+          else
+            @deflater.write(chunk)
+            compressed_chunk
+          end
+        end
+
+        private
+
+        def write(chunk)
+          @compressed_chunk << chunk
+        end
+
+        def compressed_chunk
+          @compressed_chunk.dup
+        ensure
+          @compressed_chunk.clear
+        end
+      end
+
+      class Inflater
+        def initialize(bytesize)
+          @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+          @bytesize = bytesize
+        end
+
+        def call(chunk)
+          buffer = @inflater.inflate(chunk)
+          @bytesize -= chunk.bytesize
+          if @bytesize <= 0
+            buffer << @inflater.finish
+            @inflater.close
+          end
+          buffer
+        end
+      end
+
+      module_function
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(response, bytesize: nil)
+        bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
+        Inflater.new(bytesize)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/json.rb

@@ -4,12 +4,10 @@ require "forwardable"
 
 module HTTPX::Transcoder
   module JSON
-    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+)?json\b}i.freeze
-
-    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
-
     module_function
 
+    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+|hal\+)?json\b}i.freeze
+
     class Encoder
       extend Forwardable
 
@@ -56,5 +54,4 @@ module HTTPX::Transcoder
     end
     # rubocop:enable Style/SingleLineMethods
   end
-  register "json", JSON
 end