rdkafka 0.29.2 → 0.31.0

data/lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class AlterConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the alter consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_AlterConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_AlterConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::AlterConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation, so a missing group result
+              # means the broker returned nothing to act on rather than a partial success.
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # Kafka reports a rejected offset change per partition, leaving the group level
+                # error unset - a group with live members comes back with the group error NULL
+                # and `unknown_member_id` on each partition. Resolving on the group error alone
+                # would hand back a report that looks like success, so any failed partition has
+                # to fail the handle too.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/delete_consumer_group_offsets_handler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class DeleteConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the delete consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_DeleteConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_DeleteConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::DeleteConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation, so a missing group result
+              # means the broker returned nothing to act on rather than a partial success.
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # Kafka reports a rejected offset change per partition, leaving the group level
+                # error unset - a group with live members comes back with the group error NULL
+                # and `unknown_member_id` on each partition. Resolving on the group error alone
+                # would hand back a report that looks like success, so any failed partition has
+                # to fail the handle too.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_cluster_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT` events
+    # @private
+    class DescribeClusterHandler < BaseHandler
+      class << self
+        # Resolves the describe cluster handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeCluster_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeClusterHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeClusterReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_consumer_groups_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT` events
+    # @private
+    class DescribeConsumerGroupsHandler < BaseHandler
+      class << self
+        # Resolves the describe-consumer-groups handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeConsumerGroups_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeConsumerGroupsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeConsumerGroupsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_topics_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT` events
+    # @private
+    class DescribeTopicsHandler < BaseHandler
+      class << self
+        # Resolves the describe topics handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeTopics_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeTopicsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeTopicsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -21,10 +21,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<TopicResult>] array of topic results
       def self.create_topic_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -53,10 +52,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<GroupResult>] array of group results
       def self.create_group_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -77,10 +75,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<CreateAclResult>] array of ACL results
       def self.create_acl_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -108,10 +105,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<DeleteAclResult>] array of delete ACL results
       def self.delete_acl_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -213,6 +209,8 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
         when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT then AlterConsumerGroupOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT then DeleteConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
@@ -221,6 +219,9 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT then DescribeConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT then DescribeClusterHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT then DescribeTopicsHandler
         end
 
         handler&.call(event_ptr)
@@ -249,6 +250,9 @@ module Rdkafka
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
+          delivery_handle[:status] = Rdkafka::Bindings.rd_kafka_message_status(message_ptr)
+          delivery_handle[:latency] = Rdkafka::Bindings.rd_kafka_message_latency(message_ptr)
+          delivery_handle[:broker_id] = Rdkafka::Bindings.rd_kafka_message_broker_id(message_ptr)
           # The topic is not stored on the handle here: it is already set as the handle's
           # `topic` Ruby attribute during produce, which spares a native string copy that
           # would otherwise be allocated and retained for every message.
@@ -262,7 +266,10 @@ module Rdkafka
                   message[:offset],
                   topic_name,
                   message[:err],
-                  delivery_handle.label
+                  delivery_handle.label,
+                  status: delivery_handle[:status],
+                  latency: delivery_handle[:latency],
+                  broker_id: delivery_handle[:broker_id]
                 ),
                 delivery_handle
               )

data/lib/rdkafka/clients.rb

@@ -4,22 +4,10 @@ module Rdkafka
   # Tracks live public clients so they can be closed before Ruby begins forced finalization.
   # @private
   module Clients
-    # Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
-    # foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
-    # the moment the value is touched. Track weakly only where that is safe and fall back to a
-    # strong registry - pruned as clients close - on older Rubies. The strong registry holds an
-    # open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
-    WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
-
     @pid = Process.pid
     @mutex = Mutex.new
-
-    if WEAK_TRACKING
-      @clients = ObjectSpace::WeakMap.new
-      @tokens = [].freeze
-    else
-      @clients = {}.freeze
-    end
+    @clients = ObjectSpace::WeakMap.new
+    @tokens = [].freeze
 
     # Registers a successfully constructed client so it can be closed during process shutdown.
     # @param client [Consumer, Producer, Admin] client to close during process shutdown
@@ -28,21 +16,14 @@ module Rdkafka
       reset_after_fork
 
       @mutex.synchronize do
-        if WEAK_TRACKING
-          live_clients
-
-          # The strongly-held token is the key and the client is the weak value, so registering a
-          # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
-          # look the value up by token instead.
-          token = Object.new
-          @clients[token] = client
-          @tokens = (@tokens + [token]).freeze
-        else
-          # Strong references, so drop clients that have already been closed to bound growth.
-          live = @clients.reject { |_id, existing| existing.closed? }
-          live[client.object_id] = client
-          @clients = live.freeze
-        end
+        live_clients
+
+        # The strongly-held token is the key and the client is the weak value, so registering a
+        # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
+        # look the value up by token instead.
+        token = Object.new
+        @clients[token] = client
+        @tokens = (@tokens + [token]).freeze
       end
 
       client
@@ -54,7 +35,7 @@ module Rdkafka
     def self.close_all
       return unless @pid == Process.pid
 
-      clients = @mutex.synchronize { current_clients }
+      clients = @mutex.synchronize { live_clients }
       first_error = nil
 
       clients.each do |client|
@@ -66,16 +47,7 @@ module Rdkafka
       raise first_error if first_error
     end
 
-    # Returns the currently registered live clients.
-    # @return [Array<Consumer, Producer, Admin>] live clients
-    # @private
-    def self.current_clients
-      WEAK_TRACKING ? live_clients : @clients.values
-    end
-    private_class_method :current_clients
-
     # Returns the live clients and releases tokens whose clients have been collected.
-    # Weak-tracking Rubies only.
     # @return [Array<Consumer, Producer, Admin>] live clients
     # @private
     def self.live_clients
@@ -99,13 +71,8 @@ module Rdkafka
     def self.reset_after_fork
       return if @pid == Process.pid
 
-      if WEAK_TRACKING
-        @clients = ObjectSpace::WeakMap.new
-        @tokens = [].freeze
-      else
-        @clients = {}.freeze
-      end
-
+      @clients = ObjectSpace::WeakMap.new
+      @tokens = [].freeze
       @pid = Process.pid
       @mutex = Mutex.new
     end

data/lib/rdkafka/config.rb

@@ -22,6 +22,12 @@ module Rdkafka
     @@log_mutex = Mutex.new
     # @!visibility private
     @@oauthbearer_token_refresh_callback = nil
+    # @!visibility private
+    # Whether the partition key length passed to the partitioner is the byte length (`bytesize`)
+    # rather than the character length (`size`). `false` (character length) is the legacy default
+    # to avoid changing partition assignment for existing multibyte/binary keys; set to `true` to
+    # opt into the correct, other-client-consistent byte-length hashing.
+    @@partitioner_key_uses_bytesize = false
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -122,6 +128,34 @@ module Rdkafka
       @@oauthbearer_token_refresh_callback
     end
 
+    # Whether the partitioner hashes the partition key by its byte length instead of its character
+    # length. Defaults to `false` (character length) to preserve existing partition assignment for
+    # multibyte/binary keys. Passing the byte length is the correct, other-client-consistent
+    # behavior; enable it (and be aware it can move such keys to a different partition) with
+    # `Rdkafka::Config.partitioner_key_uses_bytesize = true`. This default is expected to change in
+    # a future major version.
+    #
+    # @return [Boolean]
+    def self.partitioner_key_uses_bytesize
+      @@partitioner_key_uses_bytesize
+    end
+
+    # Sets whether the partitioner hashes the partition key by its byte length rather than its
+    # character length. The key length method is resolved here once rather than on every produced
+    # message, and each call re-resolves it, so changing the setting at runtime takes effect
+    # immediately.
+    #
+    # @param value [Boolean] true to hash by byte length, false (default) for character length
+    # @return [void]
+    def self.partitioner_key_uses_bytesize=(value)
+      @@partitioner_key_uses_bytesize = value
+
+      Rdkafka::Bindings.singleton_class.alias_method(
+        :partition_key_length,
+        value ? :partition_key_bytesize : :partition_key_size
+      )
+    end
+
     # @private
     def self.opaques
       @@opaques

data/lib/rdkafka/consumer/message.rb

@@ -35,19 +35,16 @@ module Rdkafka
       # @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
       def initialize(native_message)
         # Set topic
-        unless native_message[:rkt].null?
-          @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
-        end
+        rkt = native_message[:rkt]
+        @topic = Rdkafka::Bindings.rd_kafka_topic_name(rkt) unless rkt.null?
         # Set partition
         @partition = native_message[:partition]
         # Set payload
-        unless native_message[:payload].null?
-          @payload = native_message[:payload].read_string(native_message[:len])
-        end
+        payload = native_message[:payload]
+        @payload = payload.read_string(native_message[:len]) unless payload.null?
         # Set key
-        unless native_message[:key].null?
-          @key = native_message[:key].read_string(native_message[:key_len])
-        end
+        key = native_message[:key]
+        @key = key.read_string(native_message[:key_len]) unless key.null?
         # Set offset
         @offset = native_message[:offset]
         # Set timestamp

data/lib/rdkafka/producer/delivery_handle.rb

@@ -7,7 +7,10 @@ module Rdkafka
       layout :pending, :bool,
         :response, :int,
         :partition, :int,
-        :offset, :int64
+        :offset, :int64,
+        :status, :int,
+        :latency, :int64,
+        :broker_id, :int32
 
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
@@ -31,7 +34,10 @@ module Rdkafka
           # we should not return it
           topic,
           (self[:response] != 0) ? RdkafkaError.new(self[:response]) : nil,
-          label
+          label,
+          status: self[:status],
+          latency: self[:latency],
+          broker_id: self[:broker_id]
         )
       end
     end

data/lib/rdkafka/producer/delivery_report.rb

@@ -25,6 +25,26 @@ module Rdkafka
       # @return [Object, nil] label set during message production or nil by default
       attr_reader :label
 
+      # Persistence status of the message, one of `Bindings::RD_KAFKA_MSG_STATUS_*`:
+      #   - `RD_KAFKA_MSG_STATUS_NOT_PERSISTED` - the message was not written to the broker. A retry
+      #     risks reordering, but not a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED` - the message may have been written (e.g. the
+      #     request timed out before an ack). A retry risks reordering and a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_PERSISTED` - the broker acknowledged the message. Trust this only
+      #     with `acks=all`.
+      # `nil` when the report was not built from a delivery callback.
+      # @return [Integer, nil]
+      attr_reader :status
+
+      # Time from the `produce` call until the delivery report, in microseconds, or `nil` when
+      #   not available
+      # @return [Integer, nil]
+      attr_reader :latency
+
+      # Id of the broker the message was produced to, or `nil` when not known
+      # @return [Integer, nil]
+      attr_reader :broker_id
+
       # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
       # where the topic name is under `#topic` method. That way we have a consistent name that
       # is present in both places
@@ -38,12 +58,42 @@ module Rdkafka
       # @param topic_name [String, nil] topic name
       # @param error [Integer, nil] error code if any
       # @param label [Object, nil] user-defined label
-      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
+      # @param status [Integer, nil] `Bindings::RD_KAFKA_MSG_STATUS_*` persistence status
+      # @param latency [Integer, nil] produce latency in microseconds, negative when not available
+      # @param broker_id [Integer, nil] broker id, negative when not known
+      def initialize(
+        partition,
+        offset,
+        topic_name = nil,
+        error = nil,
+        label = nil,
+        status: nil,
+        latency: nil,
+        broker_id: nil
+      )
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
         @label = label
+        @status = status
+        @latency = (latency.nil? || latency.negative?) ? nil : latency
+        @broker_id = (broker_id.nil? || broker_id.negative?) ? nil : broker_id
+      end
+
+      # @return [Boolean] true when the broker acknowledged the message
+      def persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_PERSISTED
+      end
+
+      # @return [Boolean] true when the message may have been written, so a retry may duplicate it
+      def possibly_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED
+      end
+
+      # @return [Boolean] true when the message was not written, so a retry does not duplicate it
+      def not_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_NOT_PERSISTED
       end
     end
   end

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.29.2"
+  VERSION = "0.31.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.14.2"
+  LIBRDKAFKA_VERSION = "2.15.1"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "d7eec9c31c817fa44402f679c252dfbf97e4c338a849a25c3579a31fd127beb8"
+  LIBRDKAFKA_SOURCE_SHA256 = "23c8575c7d1ced07246cb9cf200c11325b72201fd4134a02414ca869fbdd8ed3"
 end

data/lib/rdkafka.rb

@@ -13,6 +13,10 @@ require "rdkafka/helpers/list_offsets"
 require "rdkafka/defaults"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
+require "rdkafka/admin/alter_consumer_group_offsets_handle"
+require "rdkafka/admin/alter_consumer_group_offsets_report"
+require "rdkafka/admin/delete_consumer_group_offsets_handle"
+require "rdkafka/admin/delete_consumer_group_offsets_report"
 require "rdkafka/admin/create_topic_handle"
 require "rdkafka/admin/create_topic_report"
 require "rdkafka/admin/delete_groups_handle"
@@ -37,6 +41,13 @@ require "rdkafka/admin/list_offsets_handle"
 require "rdkafka/admin/list_offsets_report"
 require "rdkafka/admin/list_consumer_groups_handle"
 require "rdkafka/admin/list_consumer_groups_report"
+require "rdkafka/admin/describe_result_parsing"
+require "rdkafka/admin/describe_cluster_handle"
+require "rdkafka/admin/describe_cluster_report"
+require "rdkafka/admin/describe_topics_handle"
+require "rdkafka/admin/describe_topics_report"
+require "rdkafka/admin/describe_consumer_groups_handle"
+require "rdkafka/admin/describe_consumer_groups_report"
 require "rdkafka/admin/acl_binding_result"
 require "rdkafka/admin/config_binding_result"
 require "rdkafka/admin/config_resource_binding_result"
@@ -46,6 +57,8 @@ require "rdkafka/callbacks/base_handler"
 require "rdkafka/callbacks/create_topic_handler"
 require "rdkafka/callbacks/delete_topic_handler"
 require "rdkafka/callbacks/create_partitions_handler"
+require "rdkafka/callbacks/alter_consumer_group_offsets_handler"
+require "rdkafka/callbacks/delete_consumer_group_offsets_handler"
 require "rdkafka/callbacks/delete_groups_handler"
 require "rdkafka/callbacks/delete_records_handler"
 require "rdkafka/callbacks/create_acl_handler"
@@ -55,6 +68,9 @@ require "rdkafka/callbacks/describe_configs_handler"
 require "rdkafka/callbacks/incremental_alter_configs_handler"
 require "rdkafka/callbacks/list_offsets_handler"
 require "rdkafka/callbacks/list_consumer_groups_handler"
+require "rdkafka/callbacks/describe_cluster_handler"
+require "rdkafka/callbacks/describe_topics_handler"
+require "rdkafka/callbacks/describe_consumer_groups_handler"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"