httpx 0.15.4 → 0.18.0

data/lib/httpx/session.rb

@@ -11,18 +11,17 @@ module HTTPX
       @options = self.class.default_options.merge(options)
       @responses = {}
       @persistent = @options.persistent
-      wrap(&blk) if block_given?
+      wrap(&blk) if blk
     end
 
     def wrap
-      return unless block_given?
-
       begin
         prev_persistent = @persistent
         @persistent = true
         yield self
       ensure
         @persistent = prev_persistent
+        close unless @persistent
       end
     end
 
@@ -31,8 +30,10 @@ module HTTPX
     end
 
     def request(*args, **options)
+      raise ArgumentError, "must perform at least one request" if args.empty?
+
       requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
-      responses = send_requests(*requests, options)
+      responses = send_requests(*requests)
       return responses.first if responses.size == 1
 
       responses
@@ -40,7 +41,8 @@ module HTTPX
 
     def build_request(verb, uri, options = EMPTY_HASH)
       rklass = @options.request_class
-      request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
+      options = @options.merge(options) unless options.is_a?(Options)
+      request = rklass.new(verb, uri, options.merge(persistent: @persistent))
       request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
       request
@@ -174,37 +176,38 @@ module HTTPX
       end
     end
 
-    def send_requests(*requests, options)
-      request_options = @options.merge(options)
-
-      connections = _send_requests(requests, request_options)
-      receive_requests(requests, connections, request_options)
+    def send_requests(*requests)
+      connections = _send_requests(requests)
+      receive_requests(requests, connections)
     end
 
-    def _send_requests(requests, options)
+    def _send_requests(requests)
       connections = []
 
       requests.each do |request|
         error = catch(:resolve_error) do
-          connection = find_connection(request, connections, options)
+          connection = find_connection(request, connections, request.options)
           connection.send(request)
         end
         next unless error.is_a?(ResolveError)
 
-        request.emit(:response, ErrorResponse.new(request, error, options))
+        request.emit(:response, ErrorResponse.new(request, error, request.options))
       end
 
       connections
     end
 
-    def receive_requests(requests, connections, options)
+    def receive_requests(requests, connections)
       responses = []
 
       begin
         # guarantee ordered responses
         loop do
           request = requests.first
-          pool.next_tick until (response = fetch_response(request, connections, options))
+
+          return responses unless request
+
+          pool.next_tick until (response = fetch_response(request, connections, request.options))
 
           responses << response
           requests.shift
@@ -218,13 +221,17 @@ module HTTPX
           # opportunity to traverse the requests, hence we're returning only a fraction of the errors
           # we were supposed to. This effectively fetches the existing responses and return them.
           while (request = requests.shift)
-            responses << fetch_response(request, connections, options)
+            responses << fetch_response(request, connections, request.options)
           end
           break
         end
         responses
       ensure
-        close(connections) unless @persistent
+        if @persistent
+          pool.deactivate(connections)
+        else
+          close(connections)
+        end
       end
     end
 
@@ -247,9 +254,8 @@ module HTTPX
         if !@plugins.include?(pl)
           @plugins << pl
           pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
+
           @default_options = @default_options.dup
-          @default_options = pl.extra_options(@default_options, &block) if pl.respond_to?(:extra_options)
-          @default_options = @default_options.merge(options) if options
 
           include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
           extend(pl::ClassMethods) if defined?(pl::ClassMethods)
@@ -266,14 +272,26 @@ module HTTPX
           opts.response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
           opts.response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
           opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
+          if defined?(pl::OptionsMethods)
+            opts.options_class.__send__(:include, pl::OptionsMethods)
+
+            (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
+              opts.options_class.method_added(meth)
+            end
+            @default_options = opts.options_class.new(opts)
+          end
+
+          @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
+          @default_options = @default_options.merge(options) if options
+
           pl.configure(self, &block) if pl.respond_to?(:configure)
 
           @default_options.freeze
         elsif options
           # this can happen when two plugins are loaded, an one of them calls the other under the hood,
           # albeit changing some default.
-          @default_options = @default_options.dup
-          @default_options = @default_options.merge(options)
+          @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
+          @default_options = @default_options.merge(options) if options
 
           @default_options.freeze
         end
@@ -283,8 +301,8 @@ module HTTPX
       # :nocov:
       def plugins(pls)
         warn ":#{__method__} is deprecated, use :plugin instead"
-        pls.each do |pl, *args|
-          plugin(pl, *args)
+        pls.each do |pl|
+          plugin(pl)
         end
         self
       end

data/lib/httpx/session2.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "session"
+module HTTPX
+  class Session
+    def initialize(options = EMPTY_HASH, &blk)
+      @options = self.class.default_options.merge(options)
+      @responses = {}
+      @persistent = @options.persistent
+      wrap(&blk) if blk
+    end
+
+    def wrap
+      begin
+        prev_persistent = @persistent
+        @persistent = true
+        yield self
+      ensure
+        @persistent = prev_persistent
+      end
+    end
+  end
+end

data/lib/httpx/timers.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class Timers
+    def initialize
+      @intervals = []
+    end
+
+    def after(interval_in_secs, &blk)
+      return unless interval_in_secs
+
+      # I'm assuming here that most requests will have the same
+      # request timeout, as in most cases they share common set of
+      # options. A user setting different request timeouts for 100s of
+      # requests will already have a hard time dealing with that.
+      unless (interval = @intervals.find { |t| t == interval_in_secs })
+        interval = Interval.new(interval_in_secs)
+        @intervals << interval
+        @intervals.sort!
+      end
+
+      interval << blk
+    end
+
+    def wait_interval
+      return if @intervals.empty?
+
+      @next_interval_at = Utils.now
+
+      @intervals.first.interval
+    end
+
+    def fire(error = nil)
+      raise error if error && error.timeout != @intervals.first
+      return if @intervals.empty? || !@next_interval_at
+
+      elapsed_time = Utils.elapsed_time(@next_interval_at)
+
+      @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
+    end
+
+    def cancel
+      @intervals.clear
+    end
+
+    class Interval
+      include Comparable
+
+      attr_reader :interval
+
+      def initialize(interval)
+        @interval = interval
+        @callbacks = []
+      end
+
+      def <=>(other)
+        @interval <=> other.interval
+      end
+
+      def ==(other)
+        return @interval == other if other.is_a?(Numeric)
+
+        @interval == other.to_f # rubocop:disable Lint/FloatComparison
+      end
+
+      def to_f
+        @interval
+      end
+
+      def <<(callback)
+        @callbacks << callback
+      end
+
+      def elapse(elapsed)
+        @interval -= elapsed
+
+        @callbacks.each(&:call) if @interval <= 0
+
+        @interval
+      end
+    end
+    private_constant :Interval
+  end
+end

data/lib/httpx/transcoder/body.rb

@@ -4,11 +4,12 @@ require "forwardable"
 
 module HTTPX::Transcoder
   module Body
-    Error = Class.new(HTTPX::Error)
+    class Error < HTTPX::Error; end
 
     module_function
 
     class Encoder
+      using HTTPX::ArrayExtensions
       extend Forwardable
 
       def_delegator :@raw, :to_s
@@ -21,7 +22,7 @@ module HTTPX::Transcoder
         if @raw.respond_to?(:bytesize)
           @raw.bytesize
         elsif @raw.respond_to?(:to_ary)
-          @raw.map(&:bytesize).reduce(0, :+)
+          @raw.sum(&:bytesize)
         elsif @raw.respond_to?(:size)
           @raw.size || Float::INFINITY
         elsif @raw.respond_to?(:length)

data/lib/httpx/transcoder/chunker.rb

@@ -4,7 +4,8 @@ require "forwardable"
 
 module HTTPX::Transcoder
   module Chunker
-    Error = Class.new(HTTPX::Error)
+    class Error < HTTPX::Error; end
+
     CRLF = "\r\n".b
 
     class Encoder

data/lib/httpx/transcoder/form.rb

@@ -7,6 +7,8 @@ module HTTPX::Transcoder
   module Form
     module_function
 
+    PARAM_DEPTH_LIMIT = 32
+
     class Encoder
       extend Forwardable
 
@@ -31,9 +33,27 @@ module HTTPX::Transcoder
       end
     end
 
+    module Decoder
+      module_function
+
+      def call(response, _)
+        URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
+          HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
+        end
+      end
+    end
+
     def encode(form)
       Encoder.new(form)
     end
+
+    def decode(response)
+      content_type = response.content_type.mime_type
+
+      raise HTTPX::Error, "invalid form mime type (#{content_type})" unless content_type == "application/x-www-form-urlencoded"
+
+      Decoder
+    end
   end
   register "form", Form
 end

data/lib/httpx/transcoder/json.rb

@@ -5,6 +5,10 @@ require "json"
 
 module HTTPX::Transcoder
   module JSON
+    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+)?json\b}i.freeze
+
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
     module_function
 
     class Encoder
@@ -27,6 +31,14 @@ module HTTPX::Transcoder
     def encode(json)
       Encoder.new(json)
     end
+
+    def decode(response)
+      content_type = response.content_type.mime_type
+
+      raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
+
+      ::JSON.method(:parse)
+    end
   end
   register "json", JSON
 end

data/lib/httpx/transcoder.rb

@@ -4,7 +4,11 @@ module HTTPX
   module Transcoder
     extend Registry
 
-    def self.normalize_keys(key, value, cond = nil, &block)
+    using RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    module_function
+
+    def normalize_keys(key, value, cond = nil, &block)
       if (cond && cond.call(value))
         block.call(key.to_s, value)
       elsif value.respond_to?(:to_ary)
@@ -23,6 +27,63 @@ module HTTPX
         block.call(key.to_s, value)
       end
     end
+
+    # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
+    def normalize_query(params, name, v, depth)
+      raise Error, "params depth surpasses what's supported" if depth <= 0
+
+      name =~ /\A[\[\]]*([^\[\]]+)\]*/
+      k = Regexp.last_match(1) || ""
+      after = Regexp.last_match ? Regexp.last_match.post_match : ""
+
+      if k.empty?
+        return Array(v) if !v.empty? && name == "[]"
+
+        return
+      end
+
+      case after
+      when ""
+        params[k] = v
+      when "["
+        params[name] = v
+      when "[]"
+        params[k] ||= []
+        raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
+
+        params[k] << v
+      when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
+        child_key = Regexp.last_match(1)
+        params[k] ||= []
+        raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
+
+        if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
+          normalize_query(params[k].last, child_key, v, depth - 1)
+        else
+          params[k] << normalize_query({}, child_key, v, depth - 1)
+        end
+      else
+        params[k] ||= {}
+        raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
+
+        params[k] = normalize_query(params[k], after, v, depth - 1)
+      end
+
+      params
+    end
+
+    def params_hash_has_key?(hash, key)
+      return false if /\[\]/.match?(key)
+
+      key.split(/[\[\]]+/).inject(hash) do |h, part|
+        next h if part == ""
+        return false unless h.is_a?(Hash) && h.key?(part)
+
+        h[part]
+      end
+
+      true
+    end
   end
 end
 

data/lib/httpx/utils.rb

@@ -6,6 +6,14 @@ module HTTPX
 
     module_function
 
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def elapsed_time(monotonic_timestamp)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
+    end
+
     # The value of this field can be either an HTTP-date or a number of
     # seconds to delay after the response is received.
     def parse_retry_after(retry_after)
@@ -28,9 +36,9 @@ module HTTPX
       URIParser = URI::RFC2396_Parser.new
 
       def to_uri(uri)
-        return Kernel.URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+        return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
 
-        uri = Kernel.URI(URIParser.escape(uri))
+        uri = URI(URIParser.escape(uri))
 
         non_ascii_hostname = URIParser.unescape(uri.host)
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.15.4"
+  VERSION = "0.18.0"
 end

data/lib/httpx.rb

@@ -13,6 +13,7 @@ require "httpx/callbacks"
 require "httpx/loggable"
 require "httpx/registry"
 require "httpx/transcoder"
+require "httpx/timers"
 require "httpx/pool"
 require "httpx/headers"
 require "httpx/request"
@@ -20,6 +21,7 @@ require "httpx/response"
 require "httpx/options"
 require "httpx/chainable"
 
+require "mutex_m"
 # Top-Level Namespace
 #
 module HTTPX
@@ -28,15 +30,16 @@ module HTTPX
   #
   module Plugins
     @plugins = {}
+    @plugins.extend(Mutex_m)
 
     # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
     # it from the load path under "httpx/plugins/" directory.
     #
     def self.load_plugin(name)
       h = @plugins
-      unless (plugin = h[name])
+      unless (plugin = h.synchronize { h[name] })
         require "httpx/plugins/#{name}"
-        raise "Plugin #{name} hasn't been registered" unless (plugin = h[name])
+        raise "Plugin #{name} hasn't been registered" unless (plugin = h.synchronize { h[name] })
       end
       plugin
     end
@@ -44,7 +47,8 @@ module HTTPX
     # Registers a plugin (+mod+) in the central store indexed by +name+.
     #
     def self.register_plugin(name, mod)
-      @plugins[name] = mod
+      h = @plugins
+      h.synchronize { h[name] = mod }
     end
   end
 

data/sig/buffer.rbs

@@ -1,5 +1,7 @@
 module HTTPX
   class Buffer
+    extend Forwardable
+
     include _ToS
     include _ToStr
 
@@ -11,7 +13,7 @@ module HTTPX
     def shift!: (Integer) -> void
 
     # delegated
-    def <<: (string data) -> void
+    def <<: (string data) -> String
     def empty?: () -> bool
     def bytesize: () -> Integer
     def clear: () -> void

data/sig/chainable.rbs

@@ -1,43 +1,45 @@
 module HTTPX
   module Chainable
-    def request: (*untyped, **untyped) -> (response | Array[response])
+    def request: (*Request, **untyped) -> Array[response]
+               | (Request, **untyped) -> response
+               | (verb | string, uri | [uri], **untyped) -> response
+               | (Array[[verb | string, uri] | [verb | string, uri, options]], **untyped) -> Array[response]
+               | (verb | string, _Each[uri | [uri, options]], **untyped) -> Array[response]
+
     def accept: (String) -> Session
     def wrap: () { (Session) -> void } -> void
-            | () -> void
 
     def with: (options) -> Session
-            | (options) { (Session) -> Session} -> Session
-
-
-
+            | (options) { (Session) -> void } -> void
 
-    def plugin: (:authentication) -> Plugins::sessionAuthentication
-              | (:basic_authentication) -> Plugins::sessionBasicAuthentication
-              | (:digest_authentication) -> Plugins::sessionDigestAuthentication
-              | (:ntlm_authentication) -> Plugins::sessionNTLMAuthentication
-              | (:aws_sdk_authentication) -> Plugins::sessionAwsSdkAuthentication
-              | (:compression) -> Session
-              | (:cookies) -> Plugins::sessionCookies
-              | (:expect) -> Session
-              | (:follow_redirects) -> Plugins::sessionFollowRedirects
-              | (:upgrade) -> Session
-              | (:h2c) -> Session
-              | (:multipart) -> Session
-              | (:persistent) -> Plugins::sessionPersistent
-              | (:proxy) -> Plugins::sessionProxy
-              | (:push_promise) -> Plugins::sessionPushPromise
-              | (:retries) -> Plugins::sessionRetries
-              | (:rate_limiter) -> Session
-              | (:stream) -> Plugins::sessionStream
-              | (:aws_sigv4) -> Plugins::awsSigV4Session
-              | (:grpc) -> Plugins::grpcSession
-              | (Symbol | Module, ?options?) { (Class) -> void } -> Session
-              | (Symbol | Module, ?options?) -> Session
+    def plugin: (:authentication, ?options) -> Plugins::sessionAuthentication
+              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuthentication
+              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuthentication
+              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuthentication
+              | (:aws_sdk_authentication, ?options) -> Plugins::sessionAwsSdkAuthentication
+              | (:compression, ?options) -> Session
+              | (:cookies, ?options) -> Plugins::sessionCookies
+              | (:expect, ?options) -> Session
+              | (:follow_redirects, ?options) -> Plugins::sessionFollowRedirects
+              | (:upgrade, ?options) -> Session
+              | (:h2c, ?options) -> Session
+              | (:multipart, ?options) -> Session
+              | (:persistent, ?options) -> Plugins::sessionPersistent
+              | (:proxy, ?options) -> Plugins::sessionProxy
+              | (:push_promise, ?options) -> Plugins::sessionPushPromise
+              | (:retries, ?options) -> Plugins::sessionRetries
+              | (:rate_limiter, ?options) -> Session
+              | (:stream, ?options) -> Plugins::sessionStream
+              | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session
+              | (:grpc, ?options) -> Plugins::grpcSession
+              | (:response_cache, ?options) -> Plugins::sessionResponseCache
+              | (Symbol | Module, ?options) { (Class) -> void } -> Session
+              | (Symbol | Module, ?options) -> Session
 
     private
 
     def default_options: () -> Options
     def branch: (options) -> Session
-              | (options) { (Session) -> Session } -> Session
+              | (options) { (Session) -> void } -> Session
   end
 end

data/sig/connection/http1.rbs

@@ -3,13 +3,17 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    UPCASED: Hash[String, String]
+    MAX_REQUESTS: Integer
+    CRLF: String
+
     attr_reader pending: Array[Request]
     attr_reader requests: Array[Request]
 
     @options: Options
     @max_concurrent_requests: Integer
     @max_requests: Integer
-    @parser: HTTP1
+    @parser: Parser::HTTP1
     @buffer: Buffer
 
     def interests: () -> io_interests?
@@ -30,11 +34,13 @@ module HTTPX
 
     def handle_error: (StandardError ex) -> void
 
+    def on_start: () -> void
+
     def on_headers: (Hash[String, Array[String]] headers) -> void
 
     def on_trailers: (Hash[String, Array[String]] headers) -> void
 
-    def on_data: (string chunk) -> void
+    def on_data: (String chunk) -> void
 
     def on_complete: () -> void
 
@@ -42,7 +48,7 @@ module HTTPX
 
     def ping: () -> void
 
-    def timeout: () -> Integer
+    def timeout: () -> Numeric
 
     private
 
@@ -54,7 +60,7 @@ module HTTPX
 
     def disable_pipelining: () -> void
 
-    def set_protocol_headers: (Request) -> _Each[headers_key, String]
+    def set_protocol_headers: (Request) -> _Each[[String, String]]
 
     def headline_uri: (Request) -> String
 
@@ -64,7 +70,7 @@ module HTTPX
 
     def join_trailers: (Request request) -> void
 
-    def join_headers2: (_Each[headers_key, String] headers) -> void
+    def join_headers2: (_Each[[String, String]] headers) -> void
 
     def join_body: (Request request) -> void