rdkafka 0.25.1-aarch64-linux-gnu → 0.27.0-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb4d5531f159f367ce708bf7f673ac532f3abc4b252cb27cc8c8048ee40f5469
-  data.tar.gz: d7470b6f2eb99a44750dcd9703e39d7f978764e4ab281c1ed4090c9d7834a6c0
+  metadata.gz: 33a3c2a2bd564e933625d3d72b29e29ea0332ec31d601ac0c7eb28c4078a9830
+  data.tar.gz: e8534bdd1aa7ccb88db946d691f4127cf708ef9fb2e672591502dd191998921f
 SHA512:
-  metadata.gz: 8c767c43a9bb81d866423289f79695d46f78e4e4afc85f5d5e8ae8444a2835ddb7381455463fccfcb60ec3759b0bdc9c8abcc974bfd26d440ba1ddee35cbec7a
-  data.tar.gz: 2721303ffa4c4bdcdd1a9c11ec13153003202e91968fd22fc882c24cd4efbc6c75ba093d1c43ba0047d0d3bb9aaca6f858a45177895516186db4bc35087d4685
+  metadata.gz: 54874a764ab1fb6e0fa2c04f8276f47b5c708b6ed69a9b20445ad006300d2d80b42d25c3f61a959583e35879a4842694a64469cbfc63ce7e1d67b922c74d2732
+  data.tar.gz: cfe7ea3d073a2e73ffbfd9531bf2a866502a7b14f287c21fed1b27259aac96bcd1bcb11dfc1df7165a60fe7f047eee45fffac7a7e9046713e71f21d07941fa79

data/CHANGELOG.md

@@ -1,6 +1,23 @@
 # Rdkafka Changelog
 
+## 0.27.0 (2026-05-07)
+- [Feature] Add `Consumer#poll_batch(timeout_ms, max_items:)` and `Consumer#poll_batch_nb(timeout_ms, max_items:)` for batch message polling via `rd_kafka_consume_batch_queue`.
+- [Feature] Add `Config#describe_properties` to dump all librdkafka configuration properties (including defaults and hidden properties) as a Hash via `rd_kafka_conf_dump`.
+- [Enhancement] Bump librdkafka to `2.14.0`
+- [Fix] Fix resource leak in `Admin#describe_configs` and `Admin#incremental_alter_configs` where `admin_options_ptr` and `queue_ptr` were not destroyed in the ensure block.
+- [Fix] Fix leaked queue reference in `Config#native_kafka` where `rd_kafka_queue_get_main` return value was not destroyed after passing to `rd_kafka_set_log_queue`.
+- [Fix] Fix native topic partition list leak in `Consumer#position` where `tpl` was never destroyed.
+
+## 0.26.0 (2026-04-02)
+- [Enhancement] Bump librdkafka to `2.13.2`
+- [Enhancement] Embed a per-file SPEC_HASH in test topic and consumer group names for tracing Kafka warnings back to specific spec files.
+- [Fix] Register `ObjectSpace.define_finalizer` in `Rdkafka::Consumer` to prevent segfaults when a consumer is GC'd without being explicitly closed (jturkel).
+- [Fix] Remove dead `#finalizer` instance methods from `Consumer` and `Admin` that could never work as GC finalizers.
+- [Fix] Prevent cascading test failures in admin specs when a single handle leaks into the registry.
+- [Feature] Extend `Rdkafka::RdkafkaError` with `instance_name` attribute containing the `rd_kafka_name` for tying errors back to specific native Kafka instances (#181).
+
 ## 0.25.1 (2026-02-26)
+- [Feature] Support `rd_kafka_ListOffsets` admin API for querying partition offsets by specification (earliest, latest, max_timestamp, or by timestamp) without requiring a consumer group (#613).
 - [Enhancement] Use native ARM64 runners instead of QEMU emulation for Alpine musl aarch64 builds, improving build performance and reliability.
 - [Enhancement] Enable parallel compilation (`make -j$(nproc)`) for ARM64 Alpine musl builds.
 - [Enhancement] Add file descriptor API for fiber scheduler integration. Expose `enable_queue_io_events` and `enable_background_queue_io_events` on `Consumer`, `Producer`, and `Admin` to enable non-blocking monitoring with select/poll/epoll for integration with Ruby fiber schedulers (Falcon, Async) and custom event loops.

data/README.md

@@ -163,6 +163,8 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.27.x (2026-05-07) | 2.14.0 (2026-04-01) | yes |
+| 0.26.x (2026-03-30) | 2.13.2 (2026-03-02) | yes |
 | 0.25.x (2026-01-21) | 2.12.1 (2025-10-21) | yes |
 | 0.24.x (2025-10-10) | 2.11.1 (2025-08-18) | yes |
 | 0.23.x (2025-09-04) | 2.11.0 (2025-07-03) | yes |

data/bin/verify_kafka_warnings

@@ -12,6 +12,8 @@ allowed_patterns=(
   "Creating new"
   "Unloaded transaction metadata"
   "closing connection"
+  "sent a heartbeat request but received error REQUEST_TIMED_OUT"
+  "Topic '__consumer_offsets' already exists"
 )
 
 # Get all warnings

data/docker-compose-ssl.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.1.1
+    image: confluentinc/cp-kafka:8.2.0
     ports:
       - 9092:9092 # Support PLAINTEXT so we can run one docker setup for SSL and PLAINTEXT
       - 9093:9093
@@ -19,7 +19,7 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9094
       ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.1.1
+    image: confluentinc/cp-kafka:8.2.0
 
     ports:
       - 9092:9092
@@ -18,7 +18,7 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
       ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"

data/lib/rdkafka/admin/list_offsets_handle.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Handle for list offsets operation
+    class ListOffsetsHandle < AbstractHandle
+      layout :pending, :bool,
+        :response, :int,
+        :response_string, :pointer,
+        :result_infos, :pointer,
+        :result_count, :int
+
+      # @return [String] the name of the operation.
+      def operation_name
+        "list offsets"
+      end
+
+      # @return [ListOffsetsReport] instance with partition offset information.
+      def create_result
+        ListOffsetsReport.new(
+          result_infos: self[:result_infos],
+          result_count: self[:result_count]
+        )
+      end
+
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/list_offsets_report.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Report for list offsets operation result
+    class ListOffsetsReport
+      attr_reader :offsets
+
+      # @param result_infos [FFI::Pointer] pointer to result info array
+      # @param result_count [Integer] number of result info entries
+      def initialize(result_infos:, result_count:)
+        @offsets = []
+
+        return if result_infos.null?
+
+        result_infos
+          .read_array_of_pointer(result_count)
+          .each { |result_info_ptr| validate!(result_info_ptr) }
+          .each do |result_info_ptr|
+            tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
+            tp = Bindings::TopicPartition.new(tp_ptr)
+            timestamp = Bindings.rd_kafka_ListOffsetsResultInfo_timestamp(result_info_ptr)
+            leader_epoch = Bindings.rd_kafka_topic_partition_get_leader_epoch(tp_ptr)
+
+            @offsets << {
+              topic: tp[:topic],
+              partition: tp[:partition],
+              offset: tp[:offset],
+              timestamp: timestamp,
+              leader_epoch: (leader_epoch == -1) ? nil : leader_epoch
+            }
+          end
+      end
+
+      private
+
+      # Validates the partition result and raises an error if invalid
+      # @param result_info_ptr [FFI::Pointer] pointer to the result info
+      # @raise [RdkafkaError] when the partition has an error
+      def validate!(result_info_ptr)
+        tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
+        tp = Bindings::TopicPartition.new(tp_ptr)
+        code = tp[:err]
+
+        return if code.zero?
+
+        raise RdkafkaError.new(code)
+      end
+    end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -130,12 +130,6 @@ module Rdkafka
       end
     end
 
-    # @return [Proc] finalizer proc for closing the admin
-    # @private
-    def finalizer
-      ->(_) { close }
-    end
-
     # Performs the metadata request using admin
     #
     # @param topic_name [String, nil] metadat about particular topic or all if nil
@@ -783,6 +777,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,
@@ -871,6 +868,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,
@@ -882,6 +882,111 @@ module Rdkafka
       handle
     end
 
+    # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
+    # timestamp) without requiring a consumer group.
+    #
+    # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
+    #   arrays of partition offset specifications. Each specification is a hash with:
+    #   - `:partition` [Integer] partition number
+    #   - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
+    #     `:max_timestamp`, or an integer timestamp in milliseconds
+    # @param isolation_level [Integer, nil] optional isolation level:
+    #   - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
+    #   - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
+    #
+    # @return [ListOffsetsHandle] handle that can be used to wait for the result
+    #
+    # @raise [ClosedAdminError] when the admin is closed
+    # @raise [ConfigError] when the background queue is unavailable
+    #
+    # @example Query earliest and latest offsets
+    #   handle = admin.list_offsets(
+    #     { "my_topic" => [
+    #       { partition: 0, offset: :earliest },
+    #       { partition: 1, offset: :latest }
+    #     ] }
+    #   )
+    #   report = handle.wait(max_wait_timeout_ms: 15_000)
+    #   report.offsets
+    #   # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
+    def list_offsets(topic_partition_offsets, isolation_level: nil)
+      closed_admin_check(__method__)
+
+      # Count total partitions for pre-allocation
+      total = topic_partition_offsets.sum { |_, partitions| partitions.size }
+
+      # Build native topic partition list
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
+
+      topic_partition_offsets.each do |topic, partitions|
+        partitions.each do |spec|
+          partition = spec.fetch(:partition)
+          offset = spec.fetch(:offset)
+
+          native_offset = case offset
+          when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
+          when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
+          when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
+          when Integer then offset
+          else
+            raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
+          end
+
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
+        end
+      end
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      handle = ListOffsetsHandle.new
+      handle[:pending] = true
+      handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
+        )
+      end
+
+      if isolation_level
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
+      end
+
+      ListOffsetsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_ListOffsets(
+            inner,
+            tpl,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        ListOffsetsHandle.remove(handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+      end
+
+      handle
+    end
+
     private
 
     # Checks if the admin is closed and raises an error if so

data/lib/rdkafka/bindings.rb

@@ -188,6 +188,27 @@ module Rdkafka
     RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
     RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
 
+    # List Offsets
+    RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
+    RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
+
+    # rd_kafka_IsolationLevel_t
+    RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
+    RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
+
+    # rd_kafka_OffsetSpec_t
+    RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
+    RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
+    RD_KAFKA_OFFSET_SPEC_LATEST = -1
+
+    attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
+    attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
+    attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
+    attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
+
     # FFI struct for error description (rd_kafka_err_desc)
     class NativeErrorDesc < FFI::Struct
       layout :code, :int,
@@ -213,6 +234,9 @@ module Rdkafka
 
     attach_function :rd_kafka_conf_new, [], :pointer
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
+    attach_function :rd_kafka_conf_destroy, [:pointer], :void
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
     attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
@@ -300,9 +324,10 @@ module Rdkafka
 
     ErrorCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :pointer]
-    ) do |_client_prr, err_code, reason, _opaque|
+    ) do |client_ptr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
-        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
+        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason, instance_name: instance_name)
         error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
@@ -361,6 +386,9 @@ module Rdkafka
     # More efficient for poll(0) calls in fiber schedulers.
     attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/callbacks.rb

@@ -180,6 +180,27 @@ module Rdkafka
       end
     end
 
+    # Extracts attributes of rd_kafka_ListOffsets_result_t
+    #
+    # @private
+    class ListOffsetsResult
+      attr_reader :result_error, :error_string, :result_infos, :result_count
+
+      # @param event_ptr [FFI::Pointer] pointer to the event
+      def initialize(event_ptr)
+        @result_infos = FFI::Pointer::NULL
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          list_offsets_result = Rdkafka::Bindings.rd_kafka_event_ListOffsets_result(event_ptr)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          @result_infos = Rdkafka::Bindings.rd_kafka_ListOffsets_result_infos(list_offsets_result, pointer_to_size_t)
+          @result_count = pointer_to_size_t.read(:size_t)
+        end
+      end
+    end
+
     # @private
     class BackgroundEventCallback
       # Handles background events from librdkafka
@@ -206,6 +227,8 @@ module Rdkafka
           process_describe_acl(event_ptr)
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
           process_delete_groups(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT
+          process_list_offsets(event_ptr)
         end
       end
 
@@ -392,6 +415,26 @@ module Rdkafka
           describe_acl_handle.unlock
         end
       end
+
+      # Processes list offsets result event
+      # @param event_ptr [FFI::Pointer] pointer to the event
+      def self.process_list_offsets(event_ptr)
+        list_offsets = ListOffsetsResult.new(event_ptr)
+        list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
+          list_offsets_handle[:response] = list_offsets.result_error
+          list_offsets_handle[:response_string] = list_offsets.error_string
+          list_offsets_handle[:pending] = false
+
+          if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            list_offsets_handle[:result_infos] = list_offsets.result_infos
+            list_offsets_handle[:result_count] = list_offsets.result_count
+          end
+
+          list_offsets_handle.unlock
+        end
+      end
     end
 
     # @private

data/lib/rdkafka/config.rb

@@ -89,10 +89,10 @@ module Rdkafka
     # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
     # The callback is called with an instance of RdKafka::Error.
     #
-    # @param callback [Proc, #call] callable object to handle errors
+    # @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
     # @return [nil]
     def self.error_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
       @@error_callback = callback
     end
 
@@ -285,6 +285,46 @@ module Rdkafka
       )
     end
 
+    # Returns all configuration properties and their current values for this config.
+    #
+    # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
+    # internal properties like `client.software.name`) as a flat Hash.
+    #
+    # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
+    #   and global properties at the configuration level. All properties are returned
+    #   regardless of the intended client type.
+    #
+    # @note The returned Hash may include sensitive values such as authentication
+    #   credentials and key passwords. Do not log or serialize the returned data
+    #   unless you have explicitly redacted secret entries.
+    #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    def describe_properties
+      config = nil
+      dump_ptr = nil
+      count = 0
+
+      config = native_config
+      count_ptr = Rdkafka::Bindings::SizePtr.new
+      dump_ptr = Rdkafka::Bindings.rd_kafka_conf_dump(config, count_ptr)
+
+      count = count_ptr[:value]
+      result = {}
+
+      (0...count).step(2) do |i|
+        key = dump_ptr.get_pointer(i * FFI::Pointer.size).read_string
+        value = dump_ptr.get_pointer((i + 1) * FFI::Pointer.size).read_string
+        result[key.to_sym] = value
+      end
+
+      result
+    ensure
+      Rdkafka::Bindings.rd_kafka_conf_dump_free(dump_ptr, count) if dump_ptr
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 
@@ -364,10 +404,9 @@ module Rdkafka
       end
 
       # Redirect log to handle's queue
-      Rdkafka::Bindings.rd_kafka_set_log_queue(
-        handle,
-        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
-      )
+      main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
+      Rdkafka::Bindings.rd_kafka_queue_destroy(main_queue)
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle

data/lib/rdkafka/consumer.rb

@@ -19,6 +19,9 @@ module Rdkafka
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -172,12 +175,6 @@ module Rdkafka
       end
     end
 
-    # @return [Proc] finalizer proc for closing the consumer
-    # @private
-    def finalizer
-      ->(_) { close }
-    end
-
     # Close this consumer
     # @return [nil]
     def close
@@ -186,6 +183,11 @@ module Rdkafka
 
       @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+        if @consumer_queue
+          Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
+          @consumer_queue = nil
+        end
       end
 
       @native_kafka.close
@@ -437,6 +439,8 @@ module Rdkafka
       end
 
       TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -782,6 +786,130 @@ module Rdkafka
       end
     end
 
+    # Poll for a batch of messages from the consumer queue in a single FFI call.
+    #
+    # This is more efficient than calling {#poll} in a loop because it crosses the FFI
+    # boundary only once to fetch up to `max_items` messages.
+    #
+    # The timeout controls how long to wait for the **first** message. Once any message
+    # is available, librdkafka fills the buffer with whatever is immediately ready and
+    # returns without further waiting.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch(timeout_ms, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
+    # Poll for a batch of messages without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#poll_batch} for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
+    #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch_nb(timeout_ms = 0, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
@@ -854,5 +982,24 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+
+    # Returns the consumer queue pointer, lazily initialized
+    # @return [FFI::Pointer] consumer queue handle
+    def consumer_queue
+      @consumer_queue ||= @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+      end
+    end
+
+    # Returns a reusable FFI buffer for batch polling, growing if needed
+    # @param max_items [Integer] minimum buffer capacity
+    # @return [FFI::MemoryPointer] pointer buffer
+    def batch_buffer(max_items)
+      if @batch_buffer.nil? || @batch_buffer_size < max_items
+        @batch_buffer = FFI::MemoryPointer.new(:pointer, max_items)
+        @batch_buffer_size = max_items
+      end
+      @batch_buffer
+    end
   end
 end

data/lib/rdkafka/error.rb

@@ -18,15 +18,21 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    # The name of the rdkafka instance that generated this error
+    # @return [String, nil]
+    attr_reader :instance_name
+
     # @private
     # @param response [Integer] the raw error response code from librdkafka
     # @param message_prefix [String, nil] optional prefix for error messages
     # @param broker_message [String, nil] optional error message from the broker
-    def initialize(response, message_prefix = nil, broker_message: nil)
+    # @param instance_name [String, nil] optional name of the rdkafka instance
+    def initialize(response, message_prefix = nil, broker_message: nil, instance_name: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
       @broker_message = broker_message
+      @instance_name = instance_name
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -48,7 +54,12 @@ module Rdkafka
       else
         ""
       end
-      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+      instance_name_part = if instance_name
+        " [#{instance_name}]"
+      else
+        ""
+      end
+      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})#{instance_name_part}"
     end
 
     # Whether this error indicates the partition is EOF.

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.25.1"
+  VERSION = "0.27.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.12.1"
+  LIBRDKAFKA_VERSION = "2.14.0"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "ec103fa05cb0f251e375f6ea0b6112cfc9d0acd977dc5b69fdc54242ba38a16f"
+  LIBRDKAFKA_SOURCE_SHA256 = "c05c03ef00a13a8463fac3e8918c04843c416f11ced58c889d806a88ca92cf99"
 end

data/lib/rdkafka.rb

@@ -29,6 +29,8 @@ require "rdkafka/admin/describe_configs_handle"
 require "rdkafka/admin/describe_configs_report"
 require "rdkafka/admin/incremental_alter_configs_handle"
 require "rdkafka/admin/incremental_alter_configs_report"
+require "rdkafka/admin/list_offsets_handle"
+require "rdkafka/admin/list_offsets_report"
 require "rdkafka/admin/acl_binding_result"
 require "rdkafka/admin/config_binding_result"
 require "rdkafka/admin/config_resource_binding_result"

data/package-lock.json

@@ -217,9 +217,9 @@
       }
     },
     "node_modules/picomatch": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
-      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
+      "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
       "dev": true,
       "license": "MIT",
       "engines": {

data/renovate.json

@@ -90,5 +90,8 @@
   ],
   "labels": [
     "dependencies"
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true
+  }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.25.1
+  version: 0.27.0
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier
@@ -122,6 +122,8 @@ files:
 - lib/rdkafka/admin/describe_configs_report.rb
 - lib/rdkafka/admin/incremental_alter_configs_handle.rb
 - lib/rdkafka/admin/incremental_alter_configs_report.rb
+- lib/rdkafka/admin/list_offsets_handle.rb
+- lib/rdkafka/admin/list_offsets_report.rb
 - lib/rdkafka/bindings.rb
 - lib/rdkafka/callbacks.rb
 - lib/rdkafka/config.rb
@@ -169,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+