karafka-rdkafka 0.26.1-aarch64-linux-gnu → 0.27.0-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a435733d9fee98c645fb787d72bf367708827b890d00858c247fb1f56e3b9a7e
-  data.tar.gz: a297c932f096bb9b86f756eed38cd57872857b3964d9fb35ea531c80cb076a9f
+  metadata.gz: f997aa68021eb19e490afc55841e4009da71e85a03a98decf53009b0284471e5
+  data.tar.gz: ddab9a03905ae72c7d984718f66736e1e1e9dad9ee48a40f04d55e9ab697a672
 SHA512:
-  metadata.gz: 6119ccf54d991f536f73d95aecb0c96849f724500696ba30657f7338c6a39705cc1c5bedfa6a7b2fd692fd2dec0aa0c3910d13917d82152c43bf75895bd91424
-  data.tar.gz: c73a1eb6a70d059927ae6fdbf1a35f7f3c2a3be719d8301b7f0e15ea39e9185cead70cb966b5cea742953103c7861c0c66ce3949aa2c9b91d03fa22c519fa5e2
+  metadata.gz: b0af437d10ef0c21da45c0d33f79adcd190ce2db7e6243cd2058aa6988067f319f4ca6fb942893a79e24e19d8298b324de00577168e3d792693161c722c5a221
+  data.tar.gz: 3bf2f40df79b4754751bb250f1adf2775d0c5c18fb887366cb75812e7e63b093fa660c229b8f2d1f58059cbbdcad7ab4a3a4bf507be8ba214eb00ad8e0f5dff4

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Rdkafka Changelog
 
+## 0.27.0 (2026-05-08)
+- [Feature] Add `Consumer#poll_batch(timeout_ms, max_items:)` and `Consumer#poll_batch_nb(timeout_ms, max_items:)` for batch message polling via `rd_kafka_consume_batch_queue` (from upstream).
+- [Enhancement] Bump librdkafka to `2.14.1`.
+- [Fix] Fix resource leak in `Admin#describe_configs` and `Admin#incremental_alter_configs` where `admin_options_ptr` and `queue_ptr` were not destroyed in the ensure block (from upstream).
+- [Fix] Fix leaked queue reference in `Config#native_kafka` where `rd_kafka_queue_get_main` return value was not destroyed after passing to `rd_kafka_set_log_queue` (from upstream).
+- [Fix] Fix native topic partition list leak in `Consumer#position` where `tpl` was never destroyed (from upstream).
+
 ## 0.26.1 (2026-04-13)
 - [Feature] Add `Config#describe_properties` to dump all librdkafka configuration properties (including defaults and hidden properties) as a Hash via `rd_kafka_conf_dump` (from upstream).
 

data/README.md

@@ -63,8 +63,9 @@ Contributions should generally be made to the upstream [rdkafka-ruby repository]
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.27.x (2026-05-08) | 2.14.1 (2026-04-15)  | yes |
 | 0.26.x (2026-04-11) | 2.14.0 (2026-04-01)  | yes |
-| 0.25.x (2026-04-02) | 2.13.2 (2026-03-30)  | yes |
+| 0.25.x (2026-04-02) | 2.13.2 (2026-03-02)  | yes |
 | 0.24.x (2026-02-25) | 2.13.0 (2026-01-05)  | yes |
 | 0.23.x (2025-11-01) | 2.12.1 (2025-10-16)  | yes |
 | 0.22.x (2025-09-26) | 2.11.1 (2025-08-18)  | yes |

data/lib/rdkafka/admin.rb

@@ -777,6 +777,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,
@@ -865,6 +868,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,

data/lib/rdkafka/bindings.rb

@@ -435,6 +435,9 @@ module Rdkafka
     # More efficient for poll(0) calls in fiber schedulers.
     attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/config.rb

@@ -404,10 +404,9 @@ module Rdkafka
       end
 
       # Redirect log to handle's queue
-      Rdkafka::Bindings.rd_kafka_set_log_queue(
-        handle,
-        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
-      )
+      main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
+      Rdkafka::Bindings.rd_kafka_queue_destroy(main_queue)
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle

data/lib/rdkafka/consumer.rb

@@ -183,6 +183,11 @@ module Rdkafka
 
       @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+        if @consumer_queue
+          Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
+          @consumer_queue = nil
+        end
       end
 
       @native_kafka.close
@@ -421,6 +426,8 @@ module Rdkafka
       end
 
       TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -783,6 +790,130 @@ module Rdkafka
       end
     end
 
+    # Poll for a batch of messages from the consumer queue in a single FFI call.
+    #
+    # This is more efficient than calling {#poll} in a loop because it crosses the FFI
+    # boundary only once to fetch up to `max_items` messages.
+    #
+    # The timeout controls how long to wait for the **first** message. Once any message
+    # is available, librdkafka fills the buffer with whatever is immediately ready and
+    # returns without further waiting.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch(timeout_ms, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
+    # Poll for a batch of messages without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#poll_batch} for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
+    #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch_nb(timeout_ms = 0, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
@@ -855,5 +986,24 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+
+    # Returns the consumer queue pointer, lazily initialized
+    # @return [FFI::Pointer] consumer queue handle
+    def consumer_queue
+      @consumer_queue ||= @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+      end
+    end
+
+    # Returns a reusable FFI buffer for batch polling, growing if needed
+    # @param max_items [Integer] minimum buffer capacity
+    # @return [FFI::MemoryPointer] pointer buffer
+    def batch_buffer(max_items)
+      if @batch_buffer.nil? || @batch_buffer_size < max_items
+        @batch_buffer = FFI::MemoryPointer.new(:pointer, max_items)
+        @batch_buffer_size = max_items
+      end
+      @batch_buffer
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.26.1"
+  VERSION = "0.27.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.14.0"
+  LIBRDKAFKA_VERSION = "2.14.1"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "c05c03ef00a13a8463fac3e8918c04843c416f11ced58c889d806a88ca92cf99"
+  LIBRDKAFKA_SOURCE_SHA256 = "bb246e754dee3560e9b42bf4e844dc05de4b146a3cae937e36301ffacdc456e7"
 end

data/renovate.json

@@ -90,5 +90,8 @@
   ],
   "labels": [
     "dependencies"
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true
+  }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.26.1
+  version: 0.27.0
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier