rdkafka 0.26.0 → 0.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfa1b93687c48e464d6ed74591aaa63a719f3f8278d24729d47ed4517942a4a2
-  data.tar.gz: c9ae55f386c986a76bce3f1bb87b53647d8c657bb6a49a90eac556da32a272e8
+  metadata.gz: d818b1dce30cffe7b01f9961f05016b70bd99c823f487b3a2c075d335ab31dab
+  data.tar.gz: 5bfd08ae7392e5d13f7d06b4dc366515aacbd995d139b19a82dcc74b4324eaab
 SHA512:
-  metadata.gz: 0d32f50f1313ff623dfde28d1453f7dadb0e8fd23049bfbfd8fdcd89ce4ce87611a496ad3a82ac57763827ba80cebf1b3801a7eb690a2b21abcde706e950c9a4
-  data.tar.gz: 61f908671be7b0e72cd4670cb8c21ea7a644dbb69b36821f5896255da5c26f2dd6f18a75cfc30acfa05255c143e7826ea56f4d4ed27dcc8779cb99981917ff42
+  metadata.gz: b022eff9faf7d80689e17f832fc1017c395c3463100884fcd3c2fb3610a546cf5bf62362b6f2766ef3de13cf2c4ba1b0d6322dcd1908a38ab2db25a8530a3fda
+  data.tar.gz: f489a965325b6c7ab4faa6430b4fdc4912d9223e0cdf623df865e54f4c2baeae8e3d09f167f478d94b77b6f73ddad506b5b7d4f3098dc79c1a3dc58f85ebd941

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Rdkafka Changelog
 
+## 0.27.0 (2026-05-07)
+- [Feature] Add `Consumer#poll_batch(timeout_ms, max_items:)` and `Consumer#poll_batch_nb(timeout_ms, max_items:)` for batch message polling via `rd_kafka_consume_batch_queue`.
+- [Feature] Add `Config#describe_properties` to dump all librdkafka configuration properties (including defaults and hidden properties) as a Hash via `rd_kafka_conf_dump`.
+- [Enhancement] Bump librdkafka to `2.14.0`
+- [Fix] Fix resource leak in `Admin#describe_configs` and `Admin#incremental_alter_configs` where `admin_options_ptr` and `queue_ptr` were not destroyed in the ensure block.
+- [Fix] Fix leaked queue reference in `Config#native_kafka` where `rd_kafka_queue_get_main` return value was not destroyed after passing to `rd_kafka_set_log_queue`.
+- [Fix] Fix native topic partition list leak in `Consumer#position` where `tpl` was never destroyed.
+
 ## 0.26.0 (2026-04-02)
 - [Enhancement] Bump librdkafka to `2.13.2`
 - [Enhancement] Embed a per-file SPEC_HASH in test topic and consumer group names for tracing Kafka warnings back to specific spec files.

data/README.md

@@ -163,7 +163,8 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
-| 0.26.x (2026-03-30) | 2.13.2 (2026-03-30) | yes |
+| 0.27.x (2026-05-07) | 2.14.0 (2026-04-01) | yes |
+| 0.26.x (2026-03-30) | 2.13.2 (2026-03-02) | yes |
 | 0.25.x (2026-01-21) | 2.12.1 (2025-10-21) | yes |
 | 0.24.x (2025-10-10) | 2.11.1 (2025-08-18) | yes |
 | 0.23.x (2025-09-04) | 2.11.0 (2025-07-03) | yes |

data/lib/rdkafka/admin.rb

@@ -777,6 +777,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,
@@ -865,6 +868,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,

data/lib/rdkafka/bindings.rb

@@ -234,6 +234,9 @@ module Rdkafka
 
     attach_function :rd_kafka_conf_new, [], :pointer
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
+    attach_function :rd_kafka_conf_destroy, [:pointer], :void
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
     attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
@@ -383,6 +386,9 @@ module Rdkafka
     # More efficient for poll(0) calls in fiber schedulers.
     attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/config.rb

@@ -285,6 +285,46 @@ module Rdkafka
       )
     end
 
+    # Returns all configuration properties and their current values for this config.
+    #
+    # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
+    # internal properties like `client.software.name`) as a flat Hash.
+    #
+    # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
+    #   and global properties at the configuration level. All properties are returned
+    #   regardless of the intended client type.
+    #
+    # @note The returned Hash may include sensitive values such as authentication
+    #   credentials and key passwords. Do not log or serialize the returned data
+    #   unless you have explicitly redacted secret entries.
+    #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    def describe_properties
+      config = nil
+      dump_ptr = nil
+      count = 0
+
+      config = native_config
+      count_ptr = Rdkafka::Bindings::SizePtr.new
+      dump_ptr = Rdkafka::Bindings.rd_kafka_conf_dump(config, count_ptr)
+
+      count = count_ptr[:value]
+      result = {}
+
+      (0...count).step(2) do |i|
+        key = dump_ptr.get_pointer(i * FFI::Pointer.size).read_string
+        value = dump_ptr.get_pointer((i + 1) * FFI::Pointer.size).read_string
+        result[key.to_sym] = value
+      end
+
+      result
+    ensure
+      Rdkafka::Bindings.rd_kafka_conf_dump_free(dump_ptr, count) if dump_ptr
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 
@@ -364,10 +404,9 @@ module Rdkafka
       end
 
       # Redirect log to handle's queue
-      Rdkafka::Bindings.rd_kafka_set_log_queue(
-        handle,
-        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
-      )
+      main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
+      Rdkafka::Bindings.rd_kafka_queue_destroy(main_queue)
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle

data/lib/rdkafka/consumer.rb

@@ -183,6 +183,11 @@ module Rdkafka
 
       @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+        if @consumer_queue
+          Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
+          @consumer_queue = nil
+        end
       end
 
       @native_kafka.close
@@ -434,6 +439,8 @@ module Rdkafka
       end
 
       TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -779,6 +786,130 @@ module Rdkafka
       end
     end
 
+    # Poll for a batch of messages from the consumer queue in a single FFI call.
+    #
+    # This is more efficient than calling {#poll} in a loop because it crosses the FFI
+    # boundary only once to fetch up to `max_items` messages.
+    #
+    # The timeout controls how long to wait for the **first** message. Once any message
+    # is available, librdkafka fills the buffer with whatever is immediately ready and
+    # returns without further waiting.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch(timeout_ms, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
+    # Poll for a batch of messages without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#poll_batch} for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
+    #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch_nb(timeout_ms = 0, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
@@ -851,5 +982,24 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+
+    # Returns the consumer queue pointer, lazily initialized
+    # @return [FFI::Pointer] consumer queue handle
+    def consumer_queue
+      @consumer_queue ||= @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+      end
+    end
+
+    # Returns a reusable FFI buffer for batch polling, growing if needed
+    # @param max_items [Integer] minimum buffer capacity
+    # @return [FFI::MemoryPointer] pointer buffer
+    def batch_buffer(max_items)
+      if @batch_buffer.nil? || @batch_buffer_size < max_items
+        @batch_buffer = FFI::MemoryPointer.new(:pointer, max_items)
+        @batch_buffer_size = max_items
+      end
+      @batch_buffer
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.26.0"
+  VERSION = "0.27.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.13.2"
+  LIBRDKAFKA_VERSION = "2.14.0"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f"
+  LIBRDKAFKA_SOURCE_SHA256 = "c05c03ef00a13a8463fac3e8918c04843c416f11ced58c889d806a88ca92cf99"
 end

data/renovate.json

@@ -90,5 +90,8 @@
   ],
   "labels": [
     "dependencies"
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true
+  }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.0
 platform: ruby
 authors:
 - Thijs Cadier
@@ -96,7 +96,7 @@ files:
 - README.md
 - Rakefile
 - bin/verify_kafka_warnings
-- dist/librdkafka-2.13.2.tar.gz
+- dist/librdkafka-2.14.0.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose-ssl.yml
 - docker-compose.yml