rdkafka 0.29.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu

data/lib/rdkafka/config.rb

@@ -3,24 +3,24 @@
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on
+  # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
-    # @private
+    # @!visibility private
     @@logger = Logger.new($stdout)
-    # @private
+    # @!visibility private
     @@statistics_callback = nil
-    # @private
+    # @!visibility private
     @@error_callback = nil
-    # @private
+    # @!visibility private
     @@opaques = ObjectSpace::WeakMap.new
-    # @private
+    # @!visibility private
     @@log_queue = Queue.new
-    # We memoize thread on the first log flush
-    # This allows us also to restart logger thread on forks
+    # We memoize thread on first log flush. This allows us also to restart logger thread on forks
     @@log_thread = nil
-    # @private
+    # @!visibility private
     @@log_mutex = Mutex.new
-    # @private
+    # @!visibility private
     @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
@@ -69,7 +69,8 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here:
+    # https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call, nil] callable object or nil to clear
     # @return [nil]
@@ -86,7 +87,8 @@ module Rdkafka
     end
 
     # Set a callback that will be called every time the underlying client emits an error.
-    # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
+    # If this callback is not set, global errors such as brokers becoming unavailable will only be
+    # sent to the logger, as defined by librdkafka.
     # The callback is called with an instance of RdKafka::Error.
     #
     # @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
@@ -197,6 +199,11 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
 
+      # Set callback to receive background events, so admin-style operations issued on the
+      # consumer handle (e.g. Consumer#list_offsets) get their results dispatched back to their
+      # handles. librdkafka spawns its internally-managed background thread because of this.
+      Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+
       if @consumer_rebalance_listener
         opaque.consumer_rebalance_listener = @consumer_rebalance_listener
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
@@ -290,6 +297,9 @@ module Rdkafka
     # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
     # internal properties like `client.software.name`) as a flat Hash.
     #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
     # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
     #   and global properties at the configuration level. All properties are returned
     #   regardless of the intended client type.
@@ -297,10 +307,6 @@ module Rdkafka
     # @note The returned Hash may include sensitive values such as authentication
     #   credentials and key passwords. Do not log or serialize the returned data
     #   unless you have explicitly redacted secret entries.
-    #
-    # @return [Hash{Symbol => String}] property names mapped to their current values
-    #
-    # @raise [ConfigError] When the configuration contains invalid options
     def describe_properties
       config = nil
       dump_ptr = nil
@@ -325,7 +331,8 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
     end
 
-    # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
+    # Error that is returned by the underlying rdkafka error if an invalid configuration option is
+    # present.
     class ConfigError < RuntimeError; end
 
     # Error that is returned by the underlying rdkafka library if the client cannot be created.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -6,7 +6,8 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an
+      #   empty list
       #
       # @return [TopicPartitionList]
       def initialize(data = nil)
@@ -34,10 +35,12 @@ module Rdkafka
       end
 
       # Add a topic with optionally partitions to the list.
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
+      #   partition count
       #
       # @return [nil]
       #
@@ -61,11 +64,12 @@ module Rdkafka
       end
 
       # Add a topic with partitions and offsets set to the list
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
-      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
+      #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
         @data[topic.to_s] = partitions_with_offsets.map do |p, o|
@@ -73,7 +77,8 @@ module Rdkafka
         end
       end
 
-      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      # Return a `Hash` with the topics as keys and and an array of partition information as the
+      # value if present.
       #
       # @return [Hash{String => Array<Partition>,nil}]
       def to_h
@@ -97,7 +102,8 @@ module Rdkafka
       #
       # @private
       #
-      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will
+      #   be copied.
       #
       # @return [TopicPartitionList]
       def self.from_native_tpl(pointer)

data/lib/rdkafka/consumer.rb

@@ -15,6 +15,7 @@ module Rdkafka
     include Helpers::Time
     include Helpers::OAuth
     include Helpers::Metadata
+    include Helpers::ListOffsets
 
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
@@ -34,11 +35,13 @@ module Rdkafka
     # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
     # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
     # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
-    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the handle.
+    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
+    # handle.
     #
     # @private
     # @param native_kafka [NativeKafka] the wrapped native client
-    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer (or empty)
+    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
+    #   (or empty)
     # @return [Proc] finalizer proc that must not reference the consumer instance
     def self.finalizer(native_kafka, queue_holder)
       proc do
@@ -124,11 +127,6 @@ module Rdkafka
     # @return [nil]
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     #
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other consumer operations will wait until this method returns.
-    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
-    # @note Do NOT use this if `consumer_poll_set` was set to `true`
-    #
     # @example Drain all pending events
     #   consumer.events_poll_nb_each { |_count| }
     #
@@ -137,6 +135,10 @@ module Rdkafka
     #   consumer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
     def events_poll_nb_each
       closed_consumer_check(__method__)
 
@@ -162,14 +164,6 @@ module Rdkafka
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
     #
-    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
-    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
-    #   statistics, etc.). For consumers, use this method to receive messages and
-    #   `events_poll_nb_each` for processing background events.
-    # @note This method holds the inner lock for the duration. Other consumer operations
-    #   will wait until this method returns.
-    # @note Timeout/max_messages logic should be implemented by the caller
-    #
     # @example Process messages until queue is empty
     #   consumer.poll_nb_each do |message|
     #     process(message)
@@ -182,6 +176,13 @@ module Rdkafka
     #     count += 1
     #     :stop if count >= 10
     #   end
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
     def poll_nb_each
       closed_consumer_check(__method__)
 
@@ -446,9 +447,11 @@ module Rdkafka
     end
 
     # Return the current positions (offsets) for topics and partitions.
-    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    # The offset field of each requested partition will be set to the offset of the last consumed
+    # message + 1, or nil in case there was no previous message.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     #
     # @return [TopicPartitionList]
     #
@@ -515,28 +518,65 @@ module Rdkafka
     # possible to create one yourself, in this case you have to provide a list that
     # already contains all the partitions you need the lag for.
     #
+    # The end offsets of all requested partitions are fetched in a single batched
+    # {#list_offsets} query - librdkafka fans it out to the involved partition leaders
+    # internally and concurrently - instead of one blocking {#query_watermark_offsets}
+    # broker roundtrip per partition.
+    #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
-    # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
+    # @param watermark_timeout_ms [Integer] The timeout for the batched end-offsets query.
     # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
+    # @raise [ClosedConsumerError] when the consumer is closed
     # @raise [RdkafkaError] When querying the broker fails.
     def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
+      closed_consumer_check(__method__)
+
       out = {}
+      request = {}
+      partitions_by_topic = topic_partition_list.to_h
+
+      partitions_by_topic.each do |topic, partitions|
+        out[topic] = {}
 
-      topic_partition_list.to_h.each do |topic, partitions|
-        # Query high watermarks for this topic's partitions and compare to the offset in the list.
-        topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          _low, high = query_watermark_offsets(
-            topic,
-            p.partition,
-            watermark_timeout_ms
-          )
-          topic_out[p.partition] = high - p.offset
+
+          (request[topic] ||= []) << { partition: p.partition, offset: :latest }
+        end
+      end
+
+      return out if request.empty?
+
+      report = begin
+        # The isolation level is forwarded so the end offsets match what the old per-partition
+        # watermark query returned: librdkafka resolves that query with the consumer's configured
+        # `isolation.level` (LSO for the default read_committed), while the admin-style batched
+        # query would otherwise default to read_uncommitted (true high watermark).
+        list_offsets(request, isolation_level: isolation_level)
+          .wait(max_wait_timeout_ms: watermark_timeout_ms)
+      rescue AbstractHandle::WaitTimeoutError
+        # Keep the pre-batching contract: a slow broker surfaced as a timed-out RdkafkaError
+        # from the per-partition watermark query, not as a handle wait timeout.
+        raise RdkafkaError.new(
+          Rdkafka::Bindings::RD_KAFKA_RESP_ERR__TIMED_OUT,
+          "Error querying watermark offsets of '#{request.keys.join(", ")}'"
+        )
+      end
+
+      end_offsets = {}
+      report.offsets.each do |result|
+        (end_offsets[result[:topic]] ||= {})[result[:partition]] = result[:offset]
+      end
+
+      partitions_by_topic.each do |topic, partitions|
+        partitions.each do |p|
+          next if p.offset.nil?
+
+          out[topic][p.partition] = end_offsets.fetch(topic).fetch(p.partition) - p.offset
         end
-        out[topic] = topic_out
       end
+
       out
     end
 
@@ -846,7 +886,7 @@ module Rdkafka
     # returns without further waiting.
     #
     # Error events (e.g. `:partition_eof`) are returned inline as {RdkafkaError} objects
-    # rather than raised, so callers receive the complete batch — both messages and errors —
+    # rather than raised, so callers receive the complete batch - both messages and errors -
     # and can decide how to handle each. This is particularly useful when multiple partitions
     # signal EOF simultaneously: all signals appear in the returned array rather than only
     # the first one being raised and the rest silently discarded.
@@ -921,15 +961,15 @@ module Rdkafka
     # particularly useful in fiber scheduler contexts where GVL release/reacquire
     # overhead is wasteful since we don't expect to wait.
     #
+    # @param timeout_ms [Integer] Timeout waiting for the first message
+    #   (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
+    # @raise [ClosedConsumerError] When called on a closed consumer
     # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
     #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
     #
     # Error events are returned inline as {RdkafkaError} objects; see {#poll_batch} for details.
-    #
-    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
-    # @param max_items [Integer] Maximum number of messages to return per call
-    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
-    # @raise [ClosedConsumerError] When called on a closed consumer
     def poll_batch_nb(timeout_ms = 0, max_items: 100)
       closed_consumer_check(__method__)
 
@@ -1081,6 +1121,35 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_mem_free(inner, ptr) unless ptr.null?
     end
 
+    # Reads this consumer's effective `isolation.level` from the live librdkafka configuration
+    # and maps it to the numeric isolation level constant. Memoized: the value cannot change
+    # after client creation.
+    #
+    # @return [Integer] `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` or
+    #   `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED`
+    # @raise [Rdkafka::Config::ConfigError] when the property cannot be read
+    def isolation_level
+      @isolation_level ||= @native_kafka.with_inner do |inner|
+        conf = Rdkafka::Bindings.rd_kafka_conf(inner)
+
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        size_ptr[:value] = 64
+        value_ptr = FFI::MemoryPointer.new(:char, 64)
+
+        result = Rdkafka::Bindings.rd_kafka_conf_get(conf, "isolation.level", value_ptr, size_ptr)
+
+        if result != :config_ok
+          raise Rdkafka::Config::ConfigError.new("Could not read isolation.level: #{result}")
+        end
+
+        if value_ptr.read_string == "read_committed"
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED
+        else
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED
+        end
+      end
+    end
+
     # Returns the consumer queue pointer, lazily initialized
     # @return [FFI::Pointer] consumer queue handle
     def consumer_queue

data/lib/rdkafka/defaults.rb

@@ -3,19 +3,17 @@
 module Rdkafka
   # Default timeout and timing values used throughout rdkafka-ruby.
   #
-  # All timeout values can be overridden per-call via method parameters.
-  # These constants provide a central place to understand and reference
-  # the default values used across the library.
-  #
-  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
-  #   For librdkafka options, see:
-  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+  # All timeout values can be overridden per-call via method parameters. These constants provide
+  # a central place to understand and reference the default values used across the library.
   #
   # @example Overriding a timeout per-call
   #   consumer.committed(timeout_ms: 5_000)  # Use 5 seconds instead of default 2 seconds
   #
   # @example Checking the default value
   #   Rdkafka::Defaults::CONSUMER_COMMITTED_TIMEOUT_MS  # => 2000
+  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
+  #   For librdkafka options, see:
+  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
   module Defaults
     # Consumer timeouts (in milliseconds)
 

data/lib/rdkafka/helpers/list_offsets.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Helpers
+    # Shared `#list_offsets` implementation for Admin and Consumer.
+    #
+    # The librdkafka Admin API is handle-agnostic: `rd_kafka_ListOffsets()` can be issued on any
+    # `rd_kafka_t` instance regardless of its type, so this Ruby-level implementation is identical
+    # for admin and consumer clients. Includers must provide a private `#closed_check(method)`
+    # that raises their own `Closed*Error`, and must have the background event callback
+    # registered on their native config (see `Config#admin` and `Config#consumer`) so the result
+    # event is dispatched back to the returned handle.
+    module ListOffsets
+      # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
+      # timestamp) without requiring a consumer group.
+      #
+      # The query is batched: all requested partitions are carried in one `ListOffsets` request
+      # that librdkafka fans out to the involved partition leaders internally and concurrently.
+      #
+      # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
+      #   arrays of partition offset specifications. Each specification is a hash with:
+      #   - `:partition` [Integer] partition number
+      #   - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
+      #     `:max_timestamp`, or an integer timestamp in milliseconds
+      # @param isolation_level [Integer, nil] optional isolation level:
+      #   - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
+      #   - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
+      #
+      # @return [Admin::ListOffsetsHandle] handle that can be used to wait for the result
+      #
+      # @raise [ClosedAdminError, ClosedConsumerError] when the client is closed
+      # @raise [ConfigError] when the background queue is unavailable
+      #
+      # @example Query earliest and latest offsets
+      #   handle = client.list_offsets(
+      #     { "my_topic" => [
+      #       { partition: 0, offset: :earliest },
+      #       { partition: 1, offset: :latest }
+      #     ] }
+      #   )
+      #   report = handle.wait(max_wait_timeout_ms: 15_000)
+      #   report.offsets
+      #   # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
+      def list_offsets(topic_partition_offsets, isolation_level: nil)
+        closed_check(__method__)
+
+        # Parse and validate every offset spec before allocating the native list, so a missing key
+        # or an unknown offset specification raises with nothing to clean up. Previously the
+        # ArgumentError (or KeyError) was raised after `rd_kafka_topic_partition_list_new`, leaking
+        # the native list.
+        parsed = topic_partition_offsets.flat_map do |topic, partitions|
+          partitions.map do |spec|
+            offset = spec.fetch(:offset)
+
+            native_offset = case offset
+            when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
+            when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
+            when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
+            when Integer then offset
+            else
+              raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
+            end
+
+            [topic, spec.fetch(:partition), native_offset]
+          end
+        end
+
+        # Build native topic partition list
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
+
+        parsed.each do |topic, partition, native_offset|
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
+        end
+
+        # Get a pointer to the queue that our request will be enqueued on
+        queue_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        end
+
+        if queue_ptr.null?
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
+
+        # Create and register the handle we will return to the caller
+        handle = Admin::ListOffsetsHandle.new
+        handle[:pending] = true
+        handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+
+        admin_options_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+            inner,
+            Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
+          )
+        end
+
+        if isolation_level
+          Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
+        end
+
+        Admin::ListOffsetsHandle.register(handle)
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+        begin
+          @native_kafka.with_inner do |inner|
+            Rdkafka::Bindings.rd_kafka_ListOffsets(
+              inner,
+              tpl,
+              admin_options_ptr,
+              queue_ptr
+            )
+          end
+        rescue Exception
+          Admin::ListOffsetsHandle.remove(handle.to_ptr.address)
+          raise
+        ensure
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        end
+
+        handle
+      end
+    end
+  end
+end

data/lib/rdkafka/helpers/oauth.rb

@@ -4,10 +4,14 @@ module Rdkafka
     module OAuth
       # Set the OAuthBearer token
       #
-      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
-      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS
+      #   compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds
+      #   since the epoch. See https://currentmillis.com/.
       # @param principal_name [String] the mandatory Kafka principal name associated with the token.
-      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the
+      #   broker as additional key-value pairs during the initial client response as per
+      #   https://tools.ietf.org/html/rfc7628#section-3.1.
       # @return [Integer] 0 on success
       def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
         error_buffer = FFI::MemoryPointer.from_string(" " * 256)
@@ -49,7 +53,8 @@ module Rdkafka
       # Convert extensions hash to FFI::MemoryPointer (`const char **`).
       #
       # @param extensions [Hash, nil] extension key-value pairs
-      # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string pointers
+      # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string
+      #   pointers
       # @note The returned pointers must be freed manually (autorelease = false).
       def map_extensions(extensions)
         return [nil, nil] if extensions.nil? || extensions.empty?
@@ -80,7 +85,7 @@ module Rdkafka
       #
       # @param extensions [Hash, nil] extension key-value pairs
       # @return [Integer] non-negative even number representing keys + values count
-      # @see https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      # @see github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
       def extension_size(extensions)
         return 0 unless extensions
         extensions.size * 2

data/lib/rdkafka/native_kafka.rb

@@ -13,6 +13,14 @@ module Rdkafka
     def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
       @inner = inner
       @opaque = opaque
+      # Process that owns `@inner`. librdkafka is not fork-safe: `fork` copies only the calling
+      # thread, so the background/broker threads backing this handle do not exist in a child
+      # process. An inherited handle therefore must not be polled or destroyed in the child -
+      # `rd_kafka_destroy` would walk thread state that no longer exists (segfault) and the
+      # inherited mutexes may have been copied in a locked state (deadlock). We record the creator
+      # pid so a forked child can recognise an inherited handle and leave its teardown to the
+      # parent, which still owns the running threads.
+      @creator_pid = Process.pid
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -115,17 +123,21 @@ module Rdkafka
     end
 
     # Returns whether this native Kafka handle is closed or closing
+    #
+    # A handle inherited across `fork` is reported as closed in the child: it belongs to another
+    # process whose threads back the native client, so it is not usable here and must not be
+    # destroyed here (see the `@creator_pid` note in `#initialize`). This makes every `#close`
+    # path - including the GC finalizers that run during a child's exit - skip the native teardown
+    # for inherited handles, which is what would otherwise segfault the child.
+    #
     # @return [Boolean] true if closed or closing
     def closed?
-      @closing || @inner.nil?
+      @closing || @inner.nil? || @creator_pid != Process.pid
     end
 
     # Enable IO event notifications on the main queue
     # Librdkafka will write to your FD when the queue transitions from empty to non-empty
     #
-    # @note This method is incompatible with background polling threads.
-    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
-    #
     # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
     # @param payload [String] data to write to fd when queue has data (default: "\x01")
     # @return [nil]
@@ -142,6 +154,8 @@ module Rdkafka
     #   if readable
     #     consumer.poll(0)  # Get messages
     #   end
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
     def enable_main_queue_io_events(fd, payload = "\x01")
       if @run_polling_thread
         raise "Cannot enable IO events while background polling thread is active. " \
@@ -159,14 +173,13 @@ module Rdkafka
     # Enable IO event notifications on the background queue
     # Librdkafka will write to your FD when the background queue transitions from empty to non-empty
     #
-    # @note This method is incompatible with background polling threads.
-    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
-    #
     # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
     # @param payload [String] data to write to fd when queue has data (default: "\x01")
     # @return [nil]
     # @raise [ClosedInnerError] when the handle is closed
     # @raise [RuntimeError] when background polling thread is active
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
     def enable_background_queue_io_events(fd, payload = "\x01")
       if @run_polling_thread
         raise "Cannot enable IO events while background polling thread is active. " \

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,8 +2,7 @@
 
 module Rdkafka
   class Producer
-    # Handle to wait for a delivery report which is returned when
-    # producing a message.
+    # Handle to wait for a delivery report which is returned when producing a message.
     class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
         :response, :int,