httpx 1.6.3 → 1.7.1

data/lib/httpx/plugins/stream.rb

@@ -55,9 +55,9 @@ module HTTPX
         line << chunk
 
         while (idx = line.index("\n"))
-          yield line.byteslice(0..idx - 1)
+          yield line.byteslice(0..(idx - 1))
 
-          line = line.byteslice(idx + 1..-1)
+          line = line.byteslice((idx + 1)..-1)
         end
       end
 
@@ -121,20 +121,71 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
+      STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
+
       def self.extra_options(options)
-        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+        options.merge(
+          stream: false,
+          timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
+          stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
+        )
+      end
+
+      # adds support for the following options:
+      #
+      # :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
+      # :stream_response_class :: Class used to build the stream response object.
+      module OptionsMethods
+        def option_stream(val)
+          val
+        end
+
+        def option_stream_response_class(value)
+          value
+        end
+
+        def extend_with_plugin_classes(pl)
+          return super unless defined?(pl::StreamResponseMethods)
+
+          @stream_response_class = @stream_response_class.dup
+          Options::SET_TEMPORARY_NAME[@stream_response_class, pl]
+          @stream_response_class.__send__(:include, pl::StreamResponseMethods) if defined?(pl::StreamResponseMethods)
+
+          super
+        end
       end
 
       module InstanceMethods
-        def request(*args, stream: false, **options)
-          return super(*args, **options) unless stream
+        def request(*args, **options)
+          if args.first.is_a?(Request)
+            requests = args
+
+            request = requests.first
+
+            unless request.options.stream && !request.stream
+              if options[:stream]
+                warn "passing `stream: true` with a request object is not supported anymore. " \
+                     "You can instead build the request object with `stream :true`"
+              end
+              return super
+            end
+          else
+            return super unless options[:stream]
+
+            requests = build_requests(*args, options)
+
+            request = requests.first
+          end
 
-          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
           raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
 
-          request = requests.first
+          @options.stream_response_class.new(request, self)
+        end
+
+        def build_request(verb, uri, params = EMPTY_HASH, options = @options)
+          return super unless params[:stream]
 
-          StreamResponse.new(request, self)
+          super(verb, uri, params, options.merge(STREAM_REQUEST_OPTIONS.merge(stream: true)))
         end
       end
 
@@ -166,7 +217,7 @@ module HTTPX
 
           @stream.on_chunk(chunk.dup)
 
-          chunk.size
+          chunk.bytesize
         end
 
         private

data/lib/httpx/plugins/stream_bidi.rb

@@ -26,6 +26,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             def #{lock_meth}(*)                # def close(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -43,6 +45,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             private def #{lock_meth}(*)                # private def join_headers(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -55,6 +59,8 @@ module HTTPX
         end
 
         def handle_stream(stream, request)
+          return super unless @options.stream
+
           request.on(:body) do
             next unless request.headers_sent
 
@@ -67,6 +73,8 @@ module HTTPX
 
         # when there ain't more chunks, it makes the buffer as full.
         def send_chunk(request, stream, chunk, next_chunk)
+          return super unless @options.stream
+
           super
 
           return if next_chunk
@@ -77,39 +85,60 @@ module HTTPX
 
         # sets end-stream flag when the request is closed.
         def end_stream?(request, next_chunk)
+          return super unless @options.stream
+
           request.closed? && next_chunk.nil?
         end
       end
 
-      # BidiBuffer is a Buffer which can be receive data from threads othr
-      # than the thread of the corresponding Connection/Session.
+      # BidiBuffer is a thread-safe Buffer which can receive data from any thread.
       #
-      # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
-      # is reconciled to the main internal +@buffer+.
+      # It uses a dual-buffer strategy with mutex protection:
+      # - +@buffer+ is the main buffer, protected by +@buffer_mutex+
+      # - +@oob_buffer+ receives data when +@buffer_mutex+ is contended
+      #
+      # This allows non-blocking writes from any thread while maintaining thread safety.
       class BidiBuffer < Buffer
         def initialize(*)
           super
-          @parent_thread = Thread.current
+          @buffer_mutex = Thread::Mutex.new
           @oob_mutex = Thread::Mutex.new
           @oob_buffer = "".b
         end
 
-        # buffers the +chunk+ to be sent
+        # buffers the +chunk+ to be sent (thread-safe, non-blocking)
         def <<(chunk)
-          return super if Thread.current == @parent_thread
-
-          @oob_mutex.synchronize { @oob_buffer << chunk }
+          if @buffer_mutex.try_lock
+            begin
+              super
+            ensure
+              @buffer_mutex.unlock
+            end
+          else
+            # another thread holds the lock, use OOB buffer to avoid blocking
+            @oob_mutex.synchronize { @oob_buffer << chunk }
+          end
         end
 
-        # reconciles the main and secondary buffer (which receives data from other threads).
+        # reconciles the main and secondary buffer (thread-safe, callable from any thread).
         def rebuffer
-          raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
+          @buffer_mutex.synchronize do
+            @oob_mutex.synchronize do
+              return if @oob_buffer.empty?
 
-          @oob_mutex.synchronize do
-            @buffer << @oob_buffer
-            @oob_buffer.clear
+              @buffer << @oob_buffer
+              @oob_buffer.clear
+            end
           end
         end
+
+        Buffer.instance_methods - Object.instance_methods - %i[<<].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def #{meth}(*) # def empty?
+              @buffer_mutex.synchronize { super }
+            end
+          MOD
+        end
       end
 
       # Proxy to wake up the session main loop when one
@@ -156,7 +185,13 @@ module HTTPX
 
         def timeout; end
 
+        def inflight?
+          !@closed
+        end
+
         def terminate
+          return if @closed
+
           @pipe_write.close
           @pipe_read.close
           @closed = true
@@ -190,16 +225,20 @@ module HTTPX
         def close(selector = Selector.new)
           @signal.terminate
           selector.deregister(@signal)
-          super(selector)
+          super
         end
 
         def select_connection(connection, selector)
+          return super unless connection.options.stream
+
           super
           selector.register(@signal)
           connection.signal = @signal
         end
 
         def deselect_connection(connection, *)
+          return super unless connection.options.stream
+
           super
 
           connection.signal = nil
@@ -219,10 +258,14 @@ module HTTPX
         end
 
         def closed?
+          return super unless @options.stream
+
           @closed
         end
 
         def can_buffer?
+          return super unless @options.stream
+
           super && @state != :waiting_for_chunk
         end
 
@@ -230,9 +273,13 @@ module HTTPX
         # +:waiting_for_chunk+ state, which the request transitions to once payload
         # is buffered.
         def transition(nextstate)
+          return super unless @options.stream
+
           headers_sent = @headers_sent
 
           case nextstate
+          when :idle
+            headers_sent = false
           when :waiting_for_chunk
             return unless @state == :body
           when :body
@@ -264,6 +311,8 @@ module HTTPX
         end
 
         def close
+          return super unless @options.stream
+
           @mutex.synchronize do
             return if @closed
 
@@ -278,10 +327,22 @@ module HTTPX
       module RequestBodyMethods
         def initialize(*, **)
           super
+
+          return unless @options.stream
+
           @headers.delete("content-length")
+
+          return unless @body
+
+          return if @body.is_a?(Transcoder::Body::Encoder)
+
+          raise Error, "bidirectional streams only allow the usage of the `:body` param to set request bodies." \
+                       "You must encode it yourself if you wish to do so."
         end
 
         def empty?
+          return super unless @options.stream
+
           false
         end
       end
@@ -293,18 +354,23 @@ module HTTPX
 
         def initialize(*)
           super
+
+          return unless @options.stream
+
           @write_buffer = BidiBuffer.new(@options.buffer_size)
         end
 
         # rebuffers the +@write_buffer+ before calculating interests.
         def interests
+          return super unless @options.stream
+
           @write_buffer.rebuffer
 
           super
         end
 
         def call
-          return super unless (error = @signal.error)
+          return super unless @options.stream && (error = @signal.error)
 
           on_error(error)
         end
@@ -312,6 +378,8 @@ module HTTPX
         private
 
         def set_parser_callbacks(parser)
+          return super unless @options.stream
+
           super
           parser.on(:flush_buffer) do
             @signal.wakeup if @signal

data/lib/httpx/pool.rb

@@ -122,6 +122,12 @@ module HTTPX
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
+
+        # Observed situations where a session handling multiple requests in a loop
+        # across multiple threads checks the same connection in and out, while another
+        # thread which is waiting on the same connection never gets the chance to pick
+        # it up, because ruby's thread scheduler never switched on to it in the process.
+        Thread.pass
       end
     end
 
@@ -142,7 +148,6 @@ module HTTPX
 
     def checkout_resolver(options)
       resolver_type = options.resolver_class
-      resolver_type = Resolver.resolver_for(resolver_type, options)
 
       @resolver_mtx.synchronize do
         resolvers = @resolvers[resolver_type]
@@ -193,15 +198,19 @@ module HTTPX
     end
 
     def checkout_new_connection(uri, options)
-      options.connection_class.new(uri, options)
+      connection = options.connection_class.new(uri, options)
+      connection.log(level: 2) { "created connection##{connection.object_id} in pool##{object_id}" }
+      connection
     end
 
     def checkout_new_resolver(resolver_type, options)
-      if resolver_type.multi?
+      resolver = if resolver_type.multi?
         Resolver::Multi.new(resolver_type, options)
       else
         resolver_type.new(options)
       end
+      resolver.log(level: 2) { "created resolver##{resolver.object_id} in pool##{object_id}" }
+      resolver
     end
 
     # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),

data/lib/httpx/request/body.rb

@@ -128,7 +128,7 @@ module HTTPX
           Transcoder::Body.encode(body)
         elsif (form = params.delete(:form))
           if Transcoder::Multipart.multipart?(form)
-            # @type var form: Transcoder::multipart_input
+            # @type var form: Transcoder::Multipart::multipart_input
             Transcoder::Multipart.encode(form)
           else
             # @type var form: Transcoder::urlencoded_input

data/lib/httpx/request.rb

@@ -10,6 +10,7 @@ module HTTPX
     extend Forwardable
     include Loggable
     include Callbacks
+
     using URIExtensions
 
     ALLOWED_URI_SCHEMES = %w[https http].freeze
@@ -90,12 +91,20 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @context = @informational_status = nil
+      @response = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
 
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      @uri = orig.instance_variable_get(:@uri).dup
+      @headers = orig.instance_variable_get(:@headers).dup
+      @body = orig.instance_variable_get(:@body).dup
+    end
+
     def complete!(response = @response)
       emit(:complete, response)
     end

data/lib/httpx/resolver/cache/base.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "resolv"
+
+module HTTPX
+  module Resolver::Cache
+    # Base class of the Resolver Cache adapter implementations.
+    #
+    # While resolver caches are not required to inherit from this class, it nevertheless provides
+    # common useful functions for desired functionality, such as singleton object ractor-safe access,
+    # or a default #resolve implementation which deals with IPs and the system hosts file.
+    #
+    class Base
+      MAX_CACHE_SIZE = 512
+      CACHE_MUTEX = Thread::Mutex.new
+      HOSTS = Resolv::Hosts.new
+      @cache = nil
+
+      class << self
+        attr_reader :hosts_resolver
+
+        # returns the singleton instance to be used within the current ractor.
+        def cache(label)
+          return Ractor.store_if_absent(:"httpx_resolver_cache_#{label}") { new } if Utils.in_ractor?
+
+          @cache ||= CACHE_MUTEX.synchronize do
+            @cache || new
+          end
+        end
+      end
+
+      # resolves +hostname+ into an instance of HTTPX::Resolver::Entry if +hostname+ is an IP,
+      # or can be found in the cache, or can be found in the system hosts file.
+      def resolve(hostname)
+        ip_resolve(hostname) || get(hostname) || hosts_resolve(hostname)
+      end
+
+      private
+
+      # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
+      def ip_resolve(hostname)
+        [Resolver::Entry.new(hostname)]
+      rescue ArgumentError
+      end
+
+      # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
+      # found, or there is no hosts file.
+      def hosts_resolve(hostname)
+        ips = if Utils.in_ractor?
+          Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
+        else
+          HOSTS
+        end.getaddresses(hostname)
+
+        return if ips.empty?
+
+        ips.map { |ip| Resolver::Entry.new(ip) }
+      rescue IOError
+      end
+
+      # not to be used directly!
+      def _get(hostname, lookups, hostnames, ttl)
+        return unless lookups.key?(hostname)
+
+        entries = lookups[hostname]
+
+        return unless entries
+
+        entries.delete_if do |address|
+          address["TTL"] < ttl
+        end
+
+        if entries.empty?
+          lookups.delete(hostname)
+          hostnames.delete(hostname)
+        end
+
+        ips = entries.flat_map do |address|
+          if (als = address["alias"])
+            _get(als, lookups, hostnames, ttl)
+          else
+            Resolver::Entry.new(address["data"], address["TTL"])
+          end
+        end.compact
+
+        ips unless ips.empty?
+      end
+
+      def _set(hostname, family, entries, lookups, hostnames)
+        # lru cleanup
+        while lookups.size >= MAX_CACHE_SIZE
+          hs = hostnames.shift
+          lookups.delete(hs)
+        end
+        hostnames << hostname
+
+        lookups[hostname] ||= [] # when there's no default proc
+
+        case family
+        when Socket::AF_INET6
+          lookups[hostname].concat(entries)
+        when Socket::AF_INET
+          lookups[hostname].unshift(*entries)
+        end
+        entries.each do |entry|
+          name = entry["name"]
+          next unless name != hostname
+
+          lookups[name] ||= []
+
+          case family
+          when Socket::AF_INET6
+            lookups[name] << entry
+          when Socket::AF_INET
+            lookups[name].unshift(entry)
+          end
+        end
+      end
+
+      def _evict(hostname, ip, lookups, hostnames)
+        return unless lookups.key?(hostname)
+
+        entries = lookups[hostname]
+
+        return unless entries
+
+        entries.delete_if { |entry| entry["data"] == ip }
+
+        return unless entries.empty?
+
+        lookups.delete(hostname)
+        hostnames.delete(hostname)
+      end
+    end
+  end
+end

data/lib/httpx/resolver/cache/memory.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Resolver::Cache
+    # Implementation of a thread-safe in-memory LRU resolver cache.
+    class Memory < Base
+      def initialize
+        super
+        @hostnames = []
+        @lookups = Hash.new { |h, k| h[k] = [] }
+        @lookup_mutex = Thread::Mutex.new
+      end
+
+      def get(hostname)
+        now = Utils.now
+        synchronize do |lookups, hostnames|
+          _get(hostname, lookups, hostnames, now)
+        end
+      end
+
+      def set(hostname, family, entries)
+        synchronize do |lookups, hostnames|
+          _set(hostname, family, entries, lookups, hostnames)
+        end
+      end
+
+      def evict(hostname, ip)
+        ip = ip.to_s
+
+        synchronize do |lookups, hostnames|
+          _evict(hostname, ip, lookups, hostnames)
+        end
+      end
+
+      private
+
+      def synchronize
+        @lookup_mutex.synchronize { yield(@lookups, @hostnames) }
+      end
+    end
+  end
+end

data/lib/httpx/resolver/cache.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "httpx/resolver/cache/base"
+require "httpx/resolver/cache/memory"
+
+module HTTPX::Resolver
+  # The internal resolvers cache adapters are defined under this namespace.
+  #
+  # Adapters must comply with the Resolver Cache Adapter API and implement the following methods:
+  #
+  # * #resolve: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache or system)
+  # * #get: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache)
+  # * #set: (String hostname, Integer ip_family, Array[dns_result]) -> void => stores the set of results in the cache indexes for
+  #         the hostname and the IP family
+  # * #evict: (String hostname, _ToS ip) -> void => evicts the ip for the hostname from the cache (usually done when no longer reachable)
+  module Cache
+  end
+end