rdkafka 0.29.1 → 0.29.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9df14d82195673f4747cc9a5d20eeb705a547295b1fa25f6554a5f9fac415ff0
-  data.tar.gz: d98858ef1d0c6694acd2de1726e0727a17aae1694adf2d75ee9890bdf8608d57
+  metadata.gz: f2acf73116281ea22d96af51b56fecc615cb32616d25834a5a990fae1213a1ba
+  data.tar.gz: 04c06f93561b22719261fab54fc0191f1a6edba01926596124f430adcc11d7d5
 SHA512:
-  metadata.gz: 82715ad307088ce3559789b9445c968fb1dcbfa2e87ba70b8e90c786ff92066c37497b8592153e8512ed297706a8499d38e2c04754129390665478a3174bcf0e
-  data.tar.gz: 00fd0221b7afde5cf110437d3b83f7672b820b3ebc596d2ad7126c248f31bb89b35d29ced75c24da0a17af2828a2fc4f7feb4397cdcfe6d04bf78882f2ee02b6
+  metadata.gz: 7ea74276532d8cd48ee913a46166497ace6426bd8859eb41d72f5407bc5a1eb4273bbe2d82a793618e3420a956b95a86919e416c46e0ac657d8b8056167dcdc0
+  data.tar.gz: 20e3c8ac1bb1113673f5af90e977dc3e59e6e6b9f9b0b550e1f6b1a9796bd19b1915d8ba8e345e5cfb0e574d61e8fe354788e0f3445ee0333a8634c7f81e4d97

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Rdkafka Changelog
 
+## 0.29.2 (2026-09-11)
+- [Fix] Close live clients from an `at_exit` hook before Ruby's shutdown finalization, so librdkafka is no longer `dlclose`d while its native threads are still running (which could segfault on exit).
+- [Fix] Make client construction exception-safe and destroy the native handle when setup fails after `rd_kafka_new`, so a later error no longer orphans the native client.
+- [Fix] Destroy each polled message exactly once in `Consumer#poll_batch`/`#poll_batch_nb` when building a message raises a non-`RdkafkaError`, fixing a double-free that could abort the process and closing the matching leak window.
+- [Fix] Free the background queue, `AdminOptions` and any already-built `ConfigResource`s (and remove the handle) when `Admin#describe_configs`/`#incremental_alter_configs` raise while building native resources (e.g. a non-String resource name).
+- [Fix] Raise a clear `ConfigError` instead of segfaulting when `Admin#describe_configs`/`#incremental_alter_configs` are given an empty resource name or a negative resource type (`rd_kafka_ConfigResource_new` returns NULL), or when `rd_kafka_AdminOptions_new` returns NULL.
+
 ## 0.29.1 (2026-09-04)
 - [Enhancement] Add `Admin#delete_records` to delete all messages in a partition up to a given offset.
 - [Enhancement] Add `Admin#list_consumer_groups` to list the cluster's consumer groups.
@@ -8,6 +15,7 @@
 - [Fix] Make `NativeKafka#close` fork-aware so a forked child no longer segfaults on exit.
 - [Fix] Stabilize the `to_native_tpl` leak integration spec.
 - [Fix] Stabilize the flaky partitions count cache statistics spec.
+- [Fix] Use global allocator in topic_partition_list for pointer freed by librdkafka.
 
 ## 0.29.0 (2026-07-10)
 - [Enhancement] Bump librdkafka to `2.14.2`

data/lib/rdkafka/admin.rb

@@ -898,35 +898,64 @@ module Rdkafka
       handle[:pending] = true
       handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
-      queue_ptr = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
-      end
+      # All native allocation happens inside the begin so a raise at any point (a null background
+      # queue, a non-String resource name making `from_string` raise mid-build, etc.) is cleaned up
+      # by the ensure rather than leaking the queue, the AdminOptions, the handle and the
+      # ConfigResources already built.
+      queue_ptr = nil
+      admin_options_ptr = nil
+      pointer_array = []
+      registered = false
 
-      if queue_ptr.null?
-        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
-      end
+      begin
+        queue_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        end
 
-      admin_options_ptr = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
-          inner,
-          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
-        )
-      end
+        if queue_ptr.null?
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
 
-      DescribeConfigsHandle.register(handle)
-      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+        admin_options_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+            inner,
+            Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
+          )
+        end
 
-      pointer_array = parsed_resources.map do |resource_type, resource_name|
-        Rdkafka::Bindings.rd_kafka_ConfigResource_new(
-          resource_type,
-          FFI::MemoryPointer.from_string(resource_name)
-        )
-      end
+        # A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
+        # before registering the handle (the ensure frees the queue already acquired).
+        if admin_options_ptr.null?
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
+        end
 
-      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
-      configs_array_ptr.write_array_of_pointer(pointer_array)
+        DescribeConfigsHandle.register(handle)
+        registered = true
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+        # Build resources one at a time so a raise on a later element still leaves the earlier ones
+        # in pointer_array for the ensure to destroy.
+        parsed_resources.each do |resource_type, resource_name|
+          resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+            resource_type,
+            FFI::MemoryPointer.from_string(resource_name)
+          )
+
+          # librdkafka returns NULL for an empty resource name or a negative resource type. Passing
+          # NULL on to DescribeConfigs and then destroying it both dereference NULL and segfault, so
+          # reject it here (the ensure frees everything already built).
+          if resource_ptr.null?
+            raise Rdkafka::Config::ConfigError.new(
+              "rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
+            )
+          end
+
+          pointer_array << resource_ptr
+        end
+
+        configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+        configs_array_ptr.write_array_of_pointer(pointer_array)
 
-      begin
         @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_DescribeConfigs(
             inner,
@@ -937,18 +966,20 @@ module Rdkafka
           )
         end
       rescue Exception
-        DescribeConfigsHandle.remove(handle.to_ptr.address)
+        DescribeConfigsHandle.remove(handle.to_ptr.address) if registered
 
         raise
       ensure
-        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
-        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        if admin_options_ptr && !admin_options_ptr.null?
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        end
 
-        if configs_array_ptr
-          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
-            configs_array_ptr,
-            pointer_array.size
-          )
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
+
+        pointer_array.each do |config_resource_ptr|
+          next if config_resource_ptr.null?
+
+          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
         end
       end
 
@@ -987,66 +1018,91 @@ module Rdkafka
       handle[:pending] = true
       handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
-      queue_ptr = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
-      end
+      # All native allocation happens inside the begin so a raise at any point (a null background
+      # queue, a non-String resource name, or FFI marshaling of a config name/value failing while
+      # adding configs) is cleaned up by the ensure rather than leaking the queue, the AdminOptions,
+      # the handle and the ConfigResources already built.
+      queue_ptr = nil
+      admin_options_ptr = nil
+      pointer_array = []
+      registered = false
+      add_error = nil
 
-      if queue_ptr.null?
-        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
-      end
+      begin
+        queue_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        end
 
-      admin_options_ptr = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
-          inner,
-          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
-        )
-      end
+        if queue_ptr.null?
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
 
-      IncrementalAlterConfigsHandle.register(handle)
-      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+        admin_options_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+            inner,
+            Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
+          )
+        end
 
-      add_error = nil
+        # A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
+        # before registering the handle (the ensure frees the queue already acquired).
+        if admin_options_ptr.null?
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
+        end
 
-      pointer_array = parsed_resources.map do |resource_type, resource_name, configs|
-        # First build the appropriate resource representation
-        resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
-          resource_type,
-          FFI::MemoryPointer.from_string(resource_name)
-        )
+        IncrementalAlterConfigsHandle.register(handle)
+        registered = true
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
 
-        configs.each do |name, op_type, value|
-          # rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
-          # an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
-          # used to be ignored: the entry was silently dropped, the alter request still reported
-          # success, and the error object leaked. Capture the first error (always destroying the
-          # native error object) and raise it below, before the request is sent.
-          error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
-            resource_ptr,
-            name,
-            op_type,
-            value
+        parsed_resources.each do |resource_type, resource_name, configs|
+          # First build the appropriate resource representation
+          resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+            resource_type,
+            FFI::MemoryPointer.from_string(resource_name)
           )
 
-          unless error_ptr.null?
-            code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
-            Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
+          # librdkafka returns NULL for an empty resource name or a negative resource type. Passing
+          # NULL on to add_incremental_config/IncrementalAlterConfigs and then destroying it both
+          # dereference NULL and segfault, so reject it here (the ensure frees everything built).
+          if resource_ptr.null?
+            raise Rdkafka::Config::ConfigError.new(
+              "rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
+            )
+          end
 
-            unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-              add_error ||= Rdkafka::RdkafkaError.new(
-                code,
-                "rd_kafka_ConfigResource_add_incremental_config"
-              )
+          # Track it immediately so a raise while adding its configs still frees it via the ensure.
+          pointer_array << resource_ptr
+
+          configs.each do |name, op_type, value|
+            # rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
+            # an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
+            # used to be ignored: the entry was silently dropped, the alter request still reported
+            # success, and the error object leaked. Capture the first error (always destroying the
+            # native error object) and raise it below, before the request is sent.
+            error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
+              resource_ptr,
+              name,
+              op_type,
+              value
+            )
+
+            unless error_ptr.null?
+              code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
+              Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
+
+              unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                add_error ||= Rdkafka::RdkafkaError.new(
+                  code,
+                  "rd_kafka_ConfigResource_add_incremental_config"
+                )
+              end
             end
           end
         end
 
-        resource_ptr
-      end
-
-      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
-      configs_array_ptr.write_array_of_pointer(pointer_array)
+        configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+        configs_array_ptr.write_array_of_pointer(pointer_array)
 
-      begin
         # Raise only after the full array is built so the ensure below frees every ConfigResource
         # we created, and before the request is sent so a rejected entry can't report success.
         raise add_error if add_error
@@ -1061,18 +1117,20 @@ module Rdkafka
           )
         end
       rescue Exception
-        IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
+        IncrementalAlterConfigsHandle.remove(handle.to_ptr.address) if registered
 
         raise
       ensure
-        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
-        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        if admin_options_ptr && !admin_options_ptr.null?
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        end
 
-        if configs_array_ptr
-          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
-            configs_array_ptr,
-            pointer_array.size
-          )
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
+
+        pointer_array.each do |config_resource_ptr|
+          next if config_resource_ptr.null?
+
+          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
         end
       end
 

data/lib/rdkafka/bindings.rb

@@ -161,6 +161,7 @@ module Rdkafka
 
     attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
     attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_destroy, [:pointer], :void
     attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
     attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
     attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer

data/lib/rdkafka/clients.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Tracks live public clients so they can be closed before Ruby begins forced finalization.
+  # @private
+  module Clients
+    # Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
+    # foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
+    # the moment the value is touched. Track weakly only where that is safe and fall back to a
+    # strong registry - pruned as clients close - on older Rubies. The strong registry holds an
+    # open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
+    WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
+
+    @pid = Process.pid
+    @mutex = Mutex.new
+
+    if WEAK_TRACKING
+      @clients = ObjectSpace::WeakMap.new
+      @tokens = [].freeze
+    else
+      @clients = {}.freeze
+    end
+
+    # Registers a successfully constructed client so it can be closed during process shutdown.
+    # @param client [Consumer, Producer, Admin] client to close during process shutdown
+    # @return [Consumer, Producer, Admin] the registered client
+    def self.register(client)
+      reset_after_fork
+
+      @mutex.synchronize do
+        if WEAK_TRACKING
+          live_clients
+
+          # The strongly-held token is the key and the client is the weak value, so registering a
+          # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
+          # look the value up by token instead.
+          token = Object.new
+          @clients[token] = client
+          @tokens = (@tokens + [token]).freeze
+        else
+          # Strong references, so drop clients that have already been closed to bound growth.
+          live = @clients.reject { |_id, existing| existing.closed? }
+          live[client.object_id] = client
+          @clients = live.freeze
+        end
+      end
+
+      client
+    end
+
+    # Closes each live client registered in this process.
+    # @return [nil]
+    # @raise [StandardError] the first close error, after attempting every client
+    def self.close_all
+      return unless @pid == Process.pid
+
+      clients = @mutex.synchronize { current_clients }
+      first_error = nil
+
+      clients.each do |client|
+        client.close unless client.closed?
+      rescue => error
+        first_error ||= error
+      end
+
+      raise first_error if first_error
+    end
+
+    # Returns the currently registered live clients.
+    # @return [Array<Consumer, Producer, Admin>] live clients
+    # @private
+    def self.current_clients
+      WEAK_TRACKING ? live_clients : @clients.values
+    end
+    private_class_method :current_clients
+
+    # Returns the live clients and releases tokens whose clients have been collected.
+    # Weak-tracking Rubies only.
+    # @return [Array<Consumer, Producer, Admin>] live clients
+    # @private
+    def self.live_clients
+      clients = []
+      tokens = []
+
+      @tokens.each do |token|
+        client = @clients[token]
+        clients << client if client
+        tokens << token if client
+      end
+
+      @tokens = tokens.freeze
+      clients
+    end
+    private_class_method :live_clients
+
+    # Replaces inherited state before registering a client in a forked child.
+    # @return [nil]
+    # @private
+    def self.reset_after_fork
+      return if @pid == Process.pid
+
+      if WEAK_TRACKING
+        @clients = ObjectSpace::WeakMap.new
+        @tokens = [].freeze
+      else
+        @clients = {}.freeze
+      end
+
+      @pid = Process.pid
+      @mutex = Mutex.new
+    end
+    private_class_method :reset_after_fork
+  end
+end

data/lib/rdkafka/config.rb

@@ -209,21 +209,16 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
-      # Create native client
-      kafka = native_kafka(config, :rd_kafka_consumer)
-
-      # Redirect the main queue to the consumer queue
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
-
-      # Return consumer with Kafka client
-      Rdkafka::Consumer.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: false,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start
-        )
-      )
+      build_native_client(
+        config,
+        :rd_kafka_consumer,
+        opaque: opaque,
+        run_polling_thread: false,
+        auto_start: native_kafka_auto_start
+      ) do |kafka, native_kafka|
+        Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
+        Rdkafka::Consumer.new(native_kafka)
+      end
     end
 
     # Create a producer with this configuration.
@@ -247,19 +242,17 @@ module Rdkafka
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
 
-      kafka = native_kafka(config, :rd_kafka_producer)
-
-      Rdkafka::Producer.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: run_polling_thread,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start,
-          timeout_ms: native_kafka_poll_timeout_ms
-        ),
-        partitioner_name
-      ).tap do |producer|
-        opaque.producer = producer
+      build_native_client(
+        config,
+        :rd_kafka_producer,
+        opaque: opaque,
+        run_polling_thread: run_polling_thread,
+        auto_start: native_kafka_auto_start,
+        timeout_ms: native_kafka_poll_timeout_ms
+      ) do |_kafka, native_kafka|
+        Rdkafka::Producer.new(native_kafka, partitioner_name).tap do |producer|
+          opaque.producer = producer
+        end
       end
     end
 
@@ -279,17 +272,16 @@ module Rdkafka
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
 
-      kafka = native_kafka(config, :rd_kafka_producer)
-
-      Rdkafka::Admin.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: run_polling_thread,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start,
-          timeout_ms: native_kafka_poll_timeout_ms
-        )
-      )
+      build_native_client(
+        config,
+        :rd_kafka_producer,
+        opaque: opaque,
+        run_polling_thread: run_polling_thread,
+        auto_start: native_kafka_auto_start,
+        timeout_ms: native_kafka_poll_timeout_ms
+      ) do |_kafka, native_kafka|
+        Rdkafka::Admin.new(native_kafka)
+      end
     end
 
     # Returns all configuration properties and their current values for this config.
@@ -343,6 +335,56 @@ module Rdkafka
 
     private
 
+    # Builds a public client around a native Kafka handle, destroying the current owner if setup
+    # fails before ownership is transferred to the caller.
+    # @param config [FFI::Pointer] native rdkafka configuration
+    # @param type [Symbol] native client type
+    # @param opaque [Object] callback context retained for the native client
+    # @param run_polling_thread [Boolean] whether to run a Ruby polling thread
+    # @param auto_start [Boolean] whether to start the polling thread after wrapping the client
+    # @param timeout_ms [Integer] polling timeout in milliseconds
+    # @yieldparam kafka [FFI::Pointer] native Kafka handle
+    # @yieldparam native_kafka [NativeKafka] Ruby owner of the native handle
+    # @return [Consumer, Producer, Admin] the constructed public client
+    # @private
+    def build_native_client(
+      config,
+      type,
+      opaque:,
+      run_polling_thread:,
+      auto_start:,
+      timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS
+    )
+      kafka = native_kafka(config, type)
+      native_kafka = nil
+      client = nil
+
+      begin
+        native_kafka = Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: run_polling_thread,
+          opaque: opaque,
+          auto_start: false,
+          timeout_ms: timeout_ms
+        )
+        client = yield(kafka, native_kafka)
+        native_kafka.start if auto_start
+        Rdkafka::Clients.register(client)
+      rescue Exception
+        owner = client || native_kafka
+
+        begin
+          owner ? owner.close : Rdkafka::Bindings.rd_kafka_destroy(kafka)
+        rescue Exception => err
+          # Never let a teardown failure replace the error that caused it: that error is what the
+          # caller needs to see, and it is re-raised below regardless.
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+
+        raise
+      end
+    end
+
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
     #
@@ -429,6 +471,9 @@ module Rdkafka
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle
+    rescue Exception
+      Rdkafka::Bindings.rd_kafka_destroy(handle) if handle && !handle.null?
+      raise
     end
   end
 

data/lib/rdkafka/consumer/allocator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Consumer
+    # Creates FFI pointers from the process-global allocator so that memory handed to librdkafka
+    # can be released by it with the matching `free`.
+    module Allocator
+      extend FFI::Library
+
+      ffi_lib FFI::CURRENT_PROCESS # use process-global malloc not FFI::Library::LIBC
+      attach_function :malloc, [:size_t], :pointer
+
+      class << self
+        # Return an FFI pointer to a string allocated from the global allocator
+        # so that it can be released by another library.
+        #
+        # @private
+        #
+        # @param string [String] Ruby string that will be copied into the new
+        #   string pointer.
+        #
+        # @return [FFI::Pointer]
+        def string_pointer(string)
+          bytes = string.to_s
+          pointer = malloc(bytes.bytesize + 1)
+          Kernel.raise(NoMemoryError, "malloc failed") if pointer.null?
+
+          pointer.put_bytes(0, bytes)
+          pointer.put_char(bytes.bytesize, 0) # NUL terminator
+
+          pointer.autorelease = false
+
+          pointer
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -160,10 +160,9 @@ module Rdkafka
 
                 if p.metadata
                   part = Rdkafka::Bindings::TopicPartition.new(ref)
-                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
                   # The metadata string is owned by librdkafka once handed over and released here:
                   # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
-                  str_ptr.autorelease = false
+                  str_ptr = Allocator.string_pointer(p.metadata)
                   part[:metadata] = str_ptr
                   part[:metadata_size] = p.metadata.bytesize
                 end

data/lib/rdkafka/consumer.rb

@@ -916,33 +916,35 @@ module Rdkafka
       begin
         while i < count
           ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          # Take ownership of this index before processing the pointer: advancing `i` up front means
+          # the cleanup loop in the `ensure` below starts past it, so a raise mid-iteration cannot
+          # double-free the same message.
+          i += 1
 
-          if ptr.null?
-            i += 1
-            next
-          end
+          next if ptr.null?
 
-          native_message = Rdkafka::Bindings::Message.new(ptr)
+          # One `ensure` around the whole body guarantees `ptr` is destroyed exactly once on every
+          # path - the error branch, a successful build, a rescued `RdkafkaError`, or any other
+          # exception unwinding through here - so nothing leaks in the window before the build.
+          begin
+            native_message = Rdkafka::Bindings::Message.new(ptr)
 
-          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            results << Rdkafka::RdkafkaError.new(native_message[:err])
-            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
-            i += 1
-            next
-          end
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              results << Rdkafka::RdkafkaError.new(native_message[:err])
+              next
+            end
 
-          begin
-            results << Rdkafka::Consumer::Message.new(native_message)
-          rescue Rdkafka::RdkafkaError => e
-            # A message that fails to build (e.g. a header read error) is surfaced inline as an
-            # error event rather than discarding the whole batch - including the messages already
-            # built - and raising, which silently lost them once their offsets had been stored.
-            results << e
+            begin
+              results << Rdkafka::Consumer::Message.new(native_message)
+            rescue Rdkafka::RdkafkaError => e
+              # A message that fails to build (e.g. a header read error) is surfaced inline as an
+              # error event rather than discarding the whole batch - including the messages already
+              # built - and raising, which silently lost them once their offsets had been stored.
+              results << e
+            end
           ensure
             Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           end
-
-          i += 1
         end
       ensure
         while i < count
@@ -991,33 +993,35 @@ module Rdkafka
       begin
         while i < count
           ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          # Take ownership of this index before processing the pointer: advancing `i` up front means
+          # the cleanup loop in the `ensure` below starts past it, so a raise mid-iteration cannot
+          # double-free the same message.
+          i += 1
 
-          if ptr.null?
-            i += 1
-            next
-          end
+          next if ptr.null?
 
-          native_message = Rdkafka::Bindings::Message.new(ptr)
+          # One `ensure` around the whole body guarantees `ptr` is destroyed exactly once on every
+          # path - the error branch, a successful build, a rescued `RdkafkaError`, or any other
+          # exception unwinding through here - so nothing leaks in the window before the build.
+          begin
+            native_message = Rdkafka::Bindings::Message.new(ptr)
 
-          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            results << Rdkafka::RdkafkaError.new(native_message[:err])
-            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
-            i += 1
-            next
-          end
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              results << Rdkafka::RdkafkaError.new(native_message[:err])
+              next
+            end
 
-          begin
-            results << Rdkafka::Consumer::Message.new(native_message)
-          rescue Rdkafka::RdkafkaError => e
-            # A message that fails to build (e.g. a header read error) is surfaced inline as an
-            # error event rather than discarding the whole batch - including the messages already
-            # built - and raising, which silently lost them once their offsets had been stored.
-            results << e
+            begin
+              results << Rdkafka::Consumer::Message.new(native_message)
+            rescue Rdkafka::RdkafkaError => e
+              # A message that fails to build (e.g. a header read error) is surfaced inline as an
+              # error event rather than discarding the whole batch - including the messages already
+              # built - and raising, which silently lost them once their offsets had been stored.
+              results << e
+            end
           ensure
             Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           end
-
-          i += 1
         end
       ensure
         while i < count

data/lib/rdkafka/version.rb

@@ -2,7 +2,7 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.29.1"
+  VERSION = "0.29.2"
   # Target librdkafka version to be used
   LIBRDKAFKA_VERSION = "2.14.2"
   # SHA256 hash of the librdkafka source tarball for verification

data/lib/rdkafka.rb

@@ -60,6 +60,7 @@ require "rdkafka/consumer"
 require "rdkafka/consumer/headers"
 require "rdkafka/consumer/message"
 require "rdkafka/consumer/partition"
+require "rdkafka/consumer/allocator"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
@@ -68,9 +69,12 @@ require "rdkafka/producer/partitions_count_cache"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"
+require "rdkafka/clients"
 
 # Main Rdkafka namespace of this gem
 module Rdkafka
 end
 
 Rdkafka::Bindings.rd_kafka_global_init
+
+at_exit { Rdkafka::Clients.close_all }

data/package-lock.json

@@ -103,9 +103,9 @@
       }
     },
     "node_modules/fastq": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
-      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
+      "version": "1.20.2",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
+      "integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
       "dev": true,
       "license": "ISC",
       "dependencies": {
@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.7.1",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
-      "integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
+      "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {

data/renovate.json

@@ -27,7 +27,7 @@
       ],
       "depNameTemplate": "openssl/openssl",
       "datasourceTemplate": "github-releases",
-      "extractVersionTemplate": "^OpenSSL_(?<version>.*)$"
+      "extractVersionTemplate": "^openssl-(?<version>.*)$"
     },
     {
       "customType": "regex",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.29.1
+  version: 0.29.2
 platform: ruby
 authors:
 - Thijs Cadier
@@ -143,8 +143,10 @@ files:
 - lib/rdkafka/callbacks/incremental_alter_configs_handler.rb
 - lib/rdkafka/callbacks/list_consumer_groups_handler.rb
 - lib/rdkafka/callbacks/list_offsets_handler.rb
+- lib/rdkafka/clients.rb
 - lib/rdkafka/config.rb
 - lib/rdkafka/consumer.rb
+- lib/rdkafka/consumer/allocator.rb
 - lib/rdkafka/consumer/headers.rb
 - lib/rdkafka/consumer/message.rb
 - lib/rdkafka/consumer/partition.rb