rdkafka 0.12.0 → 0.13.0.beta.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 821523c304fc7a1fbb2c7be2b58d98d56600b645b89fdb4093f976418650035d
-  data.tar.gz: '039b8e345fd8be5f295a293d64466071dbefd77d81b01460abb0fcf343a6bed3'
+  metadata.gz: eac016dc712d56e178c77a62f8af4e422319a783e80acbd7528453bfca6ca260
+  data.tar.gz: 8a4ed8df1339e0e74c96b259431012465c499ae9e9c8c6db8ff751497e1e5c43
 SHA512:
-  metadata.gz: 2c7ac2199a63aacd3b1420890981ed5d953ae5cdadb874886cc4e396fa1fd8f69333633319beef35a05a002d75d22335a526a126e518cc3fbbb877a1c11ef2f7
-  data.tar.gz: 5d23c6beec3759877013b040018111453e05c41238014b07a27c1a9d8b96e8af3bc037aacd1ebe89f856435cf0afb8e34a9f89443f87cf1a3682736efb79b4bd
+  metadata.gz: e3167de231aaabdfa512cabef13a4ffaabcc94757a2b7d0f360ed27a9005a40da5bbb262dffc49bda03d11557eef63dcf6b9c010fb944cf41fcb06bb7531c3ee
+  data.tar.gz: 8db04d46686505a9b1fef79c8544f3c27056ae5ff9e9eede9343d4f5cef4c63c844b9798011a5339ebbdf2cab883d1c3fd5df61842ea90c4862fa0d03dd88610

data/.semaphore/semaphore.yml

@@ -9,15 +9,19 @@ agent:
 blocks:
   - name: Run specs
     task:
+      prologue:
+        commands:
+          - sudo apt install -y valgrind
       jobs:
         - name: bundle exec rspec
           matrix:
             - env_var: RUBY_VERSION
-              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
+              values: [ "2.6.10", "2.7.6", "3.0.4", "3.1.2"]
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout
             - bundle install --path vendor/bundle
             - cd ext && bundle exec rake && cd ..
             - docker-compose up -d --no-recreate
-            - bundle exec rspec
+            - ulimit -c unlimited
+            - valgrind -v bundle exec rspec

data/CHANGELOG.md

@@ -1,5 +1,17 @@
+# 0.13.0
+* Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
+* Support both string and symbol header keys (ColinDKelley)
+* Handle tombstone messages properly (kgalieva)
+* Add topic name to delivery report (maeve)
+* Allow string partitioner config (mollyegibson)
+* Fix documented type for DeliveryReport#error (jimmydo)
+* Bump librdkafka to 1.9.2 (thijsc)
+* Use finalizers to cleanly exit producer and admin (thijsc)
+
 # 0.12.0
 * Bumps librdkafka to 1.9.0
+* Fix crash on empty partition key (mensfeld)
+* Pass the delivery handle to the callback (gvisokinskas)
 
 # 0.11.0
 * Upgrade librdkafka to 1.8.2

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gemspec

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Rakefile
 
 require 'bundler/gem_tasks'

data/ext/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
 require "mini_portile2"
 require "fileutils"

data/lib/rdkafka/abstract_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "ffi"
 
 module Rdkafka

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class CreateTopicHandle < AbstractHandle

data/lib/rdkafka/admin/create_topic_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class CreateTopicReport

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class DeleteTopicHandle < AbstractHandle

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class DeleteTopicReport

data/lib/rdkafka/admin.rb

@@ -1,33 +1,26 @@
+# frozen_string_literal: true
+
+require "objspace"
+
 module Rdkafka
   class Admin
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
-
-      # Start thread to poll client for callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+    end
+
+    def finalizer
+      ->(_) { close }
     end
 
     # Close this admin instance
     def close
-      return unless @native_kafka
-
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      ObjectSpace.undefine_finalizer(self)
+
+      @native_kafka.close
     end
 
     # Create a topic with the given partition count and replication factor
@@ -38,6 +31,7 @@ module Rdkafka
     #
     # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_topic(topic_name, partition_count, replication_factor, topic_config={})
+      closed_admin_check(__method__)
 
       # Create a rd_kafka_NewTopic_t representing the new topic
       error_buffer = FFI::MemoryPointer.from_string(" " * 256)
@@ -68,7 +62,7 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -79,16 +73,16 @@ module Rdkafka
       create_topic_handle[:pending] = true
       create_topic_handle[:response] = -1
       CreateTopicHandle.register(create_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
 
       begin
         Rdkafka::Bindings.rd_kafka_CreateTopics(
-            @native_kafka,
-            topics_array_ptr,
-            1,
-            admin_options_ptr,
-            queue_ptr
+          @native_kafka.inner,
+          topics_array_ptr,
+          1,
+          admin_options_ptr,
+          queue_ptr
         )
       rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
@@ -108,6 +102,7 @@ module Rdkafka
     #
     # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
     def delete_topic(topic_name)
+      closed_admin_check(__method__)
 
       # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
       delete_topic_ptr = Rdkafka::Bindings.rd_kafka_DeleteTopic_new(FFI::MemoryPointer.from_string(topic_name))
@@ -118,7 +113,7 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -129,16 +124,16 @@ module Rdkafka
       delete_topic_handle[:pending] = true
       delete_topic_handle[:response] = -1
       DeleteTopicHandle.register(delete_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
 
       begin
         Rdkafka::Bindings.rd_kafka_DeleteTopics(
-            @native_kafka,
-            topics_array_ptr,
-            1,
-            admin_options_ptr,
-            queue_ptr
+          @native_kafka.inner,
+          topics_array_ptr,
+          1,
+          admin_options_ptr,
+          queue_ptr
         )
       rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
@@ -151,5 +146,10 @@ module Rdkafka
 
       delete_topic_handle
     end
+
+    private
+    def closed_admin_check(method)
+      raise Rdkafka::ClosedAdminError.new(method) if @native_kafka.closed?
+    end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "ffi"
 require "json"
 require "logger"
@@ -15,7 +17,7 @@ module Rdkafka
       end
     end
 
-    ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
+    ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -33,6 +35,7 @@ module Rdkafka
 
     # Polling
 
+    attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
@@ -110,6 +113,7 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
+    attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
 
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -173,6 +177,8 @@ module Rdkafka
     attach_function :rd_kafka_unsubscribe, [:pointer], :int
     attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
     attach_function :rd_kafka_assign, [:pointer, :pointer], :int
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int
     attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
     attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
@@ -198,9 +204,17 @@ module Rdkafka
     ) do |client_ptr, code, partitions_ptr, opaque_ptr|
       case code
       when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        end
       else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        end
       end
 
       opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
@@ -256,11 +270,11 @@ module Rdkafka
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
-      str_ptr = FFI::MemoryPointer.from_string(str)
+      str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
       method_name = PARTITIONERS.fetch(partitioner_name) do
         raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
       end
-      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
+      public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
     end
 
     # Create Topics

data/lib/rdkafka/callbacks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   module Callbacks
 
@@ -90,14 +92,18 @@ module Rdkafka
         message = Rdkafka::Bindings::Message.new(message_ptr)
         delivery_handle_ptr_address = message[:_private].address
         if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
+          topic_name = Rdkafka::Bindings.rd_kafka_topic_name(message[:rkt])
+
           # Update delivery handle
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
+          delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
           delivery_handle[:pending] = false
+
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]), delivery_handle)
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
           end
         end
       end

data/lib/rdkafka/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 module Rdkafka
@@ -30,7 +32,6 @@ module Rdkafka
       @@logger
     end
 
-
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -47,7 +48,7 @@ module Rdkafka
     # @return [nil]
     def self.logger=(logger)
       raise NoLoggerError if logger.nil?
-      @@logger=logger
+      @@logger = logger
     end
 
     # Set a callback that will be called every time the underlying client emits statistics.
@@ -179,7 +180,8 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
+      partitioner_name = self[:partitioner] || self["partitioner"]
+      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)), partitioner_name).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -194,7 +196,7 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)))
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
@@ -210,7 +212,7 @@ module Rdkafka
 
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
-    def native_config(opaque=nil)
+    def native_config(opaque = nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
         # Create config
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|

data/lib/rdkafka/consumer/headers.rb

@@ -1,10 +1,26 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
-    # A message headers
-    class Headers
-      # Reads a native kafka's message header into ruby's hash
+    # Interface to return headers for a consumer message
+    module Headers
+      class HashWithSymbolKeysTreatedLikeStrings < Hash
+        def [](key)
+          if key.is_a?(Symbol)
+            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
+                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+            super(key.to_s)
+          else
+            super
+          end
+        end
+      end
+
+      # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      #
+      # @param [librdkakfa message] native_message
       #
-      # @return [Hash<String, String>] a message headers
+      # @return [Hash<String, String>] headers Hash for the native_message
       #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       #
@@ -24,7 +40,8 @@ module Rdkafka
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
-        headers = {}
+
+        headers = HashWithSymbolKeysTreatedLikeStrings.new
 
         idx = 0
         loop do
@@ -51,12 +68,12 @@ module Rdkafka
 
           value = value_ptr.read_string(size)
 
-          headers[name.to_sym] = value
+          headers[name] = value
 
           idx += 1
         end
 
-        headers
+        headers.freeze
       end
     end
   end

data/lib/rdkafka/consumer/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A message that was consumed from a topic.
@@ -18,7 +20,7 @@ module Rdkafka
       # @return [String, nil]
       attr_reader :key
 
-      # This message's offset in it's partition
+      # This message's offset in its partition
       # @return [Integer]
       attr_reader :offset
 

data/lib/rdkafka/consumer/partition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # Information about a partition, used in {TopicPartitionList}.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A list of topics with their partition information

data/lib/rdkafka/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
   # brokers automatically assign partitions and load balance partitions over consumers that
@@ -14,20 +16,27 @@ module Rdkafka
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
+    end
+
+    def finalizer
+      ->(_) { close }
     end
 
     # Close this consumer
     # @return [nil]
     def close
-      return unless @native_kafka
+      return if closed?
 
-      @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
       Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
       @native_kafka = nil
     end
 
+    # Whether this consumer has closed
+    def closed?
+      @native_kafka.nil?
+    end
+
     # Subscribe to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
@@ -459,7 +468,7 @@ module Rdkafka
         if message
           yield(message)
         else
-          if @closing
+          if closed?
             break
           else
             next
@@ -468,10 +477,6 @@ module Rdkafka
       end
     end
 
-    def closed_consumer_check(method)
-      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
-    end
-
     # Poll for new messages and yield them in batches that may contain
     # messages from more than one partition.
     #
@@ -527,7 +532,7 @@ module Rdkafka
       bytes = 0
       end_time = monotonic_now + timeout_ms / 1000.0
       loop do
-        break if @closing
+        break if closed?
         max_wait = end_time - monotonic_now
         max_wait_ms = if max_wait <= 0
                         0  # should not block, but may retrieve a message
@@ -545,7 +550,7 @@ module Rdkafka
         end
         if message
           slice << message
-          bytes += message.payload.bytesize
+          bytes += message.payload.bytesize if message.payload
         end
         if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
           yield slice.dup, nil
@@ -561,5 +566,9 @@ module Rdkafka
       # needed because Time.now can go backwards
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if closed?
+    end
   end
 end

data/lib/rdkafka/error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # Base error class.
   class BaseError < RuntimeError; end
@@ -83,4 +85,11 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed producer")
     end
   end
+
+  # Error class for public consumer method calls on a closed admin.
+  class ClosedAdminError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed admin")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Metadata
     attr_reader :brokers, :topics

data/lib/rdkafka/native_kafka.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # @private
+  # A wrapper around a native kafka that polls and cleanly exits
+  class NativeKafka
+    def initialize(inner)
+      @inner = inner
+
+      # Start thread to poll client for delivery callbacks
+      @polling_thread = Thread.new do
+        loop do
+          Rdkafka::Bindings.rd_kafka_poll(inner, 250)
+          # Exit thread if closing and the poll queue is empty
+          if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+            break
+          end
+        end
+      end
+      @polling_thread.abort_on_exception = true
+      @polling_thread[:closing] = false
+    end
+
+    def inner
+      @inner
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    def closed?
+      @inner.nil?
+    end
+
+    def close(object_id=nil)
+      return if closed?
+
+      # Flush outstanding activity
+      Rdkafka::Bindings.rd_kafka_flush(@inner, 30 * 1000)
+
+      # Indicate to polling thread that we're closing
+      @polling_thread[:closing] = true
+      # Wait for the polling thread to finish up
+      @polling_thread.join
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+
+      @inner = nil
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
@@ -6,7 +8,8 @@ module Rdkafka
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
-             :offset, :int64
+             :offset, :int64,
+             :topic_name, :pointer
 
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
@@ -15,7 +18,7 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset])
+        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
       end
     end
   end