karafka-rdkafka 0.26.0-aarch64-linux-gnu → 0.27.0-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad9d2d2175395ecc174e0ae513db335c01f866bb0d241b11c3b2e6cf0fd9e0aa
-  data.tar.gz: 5a3a373ac001b4f73c758d30f69c8b7f82272e65db16bccda0240c3529606539
+  metadata.gz: f997aa68021eb19e490afc55841e4009da71e85a03a98decf53009b0284471e5
+  data.tar.gz: ddab9a03905ae72c7d984718f66736e1e1e9dad9ee48a40f04d55e9ab697a672
 SHA512:
-  metadata.gz: 55010dbdc16f9d0e415d6dc0bcf25763f2b641aaa2bbc1eef3079da195fc3916860ac01bbe735d1c9d1451972ef986331d6a43b2ab82a0e7fd2db7fc709a8cd0
-  data.tar.gz: 13b30111dd2f6bfb5cc876c8f1ee82f245deb39a6cc9021bf5f4f71787760bb998067184fe46611ad90bde1f1155d3b8ab83e91b9e734324124f24735d1bd2cf
+  metadata.gz: b0af437d10ef0c21da45c0d33f79adcd190ce2db7e6243cd2058aa6988067f319f4ca6fb942893a79e24e19d8298b324de00577168e3d792693161c722c5a221
+  data.tar.gz: 3bf2f40df79b4754751bb250f1adf2775d0c5c18fb887366cb75812e7e63b093fa660c229b8f2d1f58059cbbdcad7ab4a3a4bf507be8ba214eb00ad8e0f5dff4

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Rdkafka Changelog
 
+## 0.27.0 (2026-05-08)
+- [Feature] Add `Consumer#poll_batch(timeout_ms, max_items:)` and `Consumer#poll_batch_nb(timeout_ms, max_items:)` for batch message polling via `rd_kafka_consume_batch_queue` (from upstream).
+- [Enhancement] Bump librdkafka to `2.14.1`.
+- [Fix] Fix resource leak in `Admin#describe_configs` and `Admin#incremental_alter_configs` where `admin_options_ptr` and `queue_ptr` were not destroyed in the ensure block (from upstream).
+- [Fix] Fix leaked queue reference in `Config#native_kafka` where `rd_kafka_queue_get_main` return value was not destroyed after passing to `rd_kafka_set_log_queue` (from upstream).
+- [Fix] Fix native topic partition list leak in `Consumer#position` where `tpl` was never destroyed (from upstream).
+
+## 0.26.1 (2026-04-13)
+- [Feature] Add `Config#describe_properties` to dump all librdkafka configuration properties (including defaults and hidden properties) as a Hash via `rd_kafka_conf_dump` (from upstream).
+
 ## 0.26.0 (2026-04-11)
 - [Enhancement] Bump librdkafka to `2.14.0`.
 - [Enhancement] Add `advertised.listeners` to macOS ARM64 CI KRaft broker config to fix flaky tests (from upstream).

data/README.md

@@ -31,7 +31,7 @@ This fork is actively maintained and kept in sync with the upstream rdkafka-ruby
 
 ## Long-term Plan
 
-Our long-term goal is to work with the rdkafka-ruby maintainers to eventually merge the beneficial changes back upstream. This would allow us to:
+The goal is to eventually merge the beneficial changes back upstream. This would allow to:
 
 - Reduce maintenance overhead
 - Benefit the broader Ruby Kafka community
@@ -63,8 +63,9 @@ Contributions should generally be made to the upstream [rdkafka-ruby repository]
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.27.x (2026-05-08) | 2.14.1 (2026-04-15)  | yes |
 | 0.26.x (2026-04-11) | 2.14.0 (2026-04-01)  | yes |
-| 0.25.x (2026-04-02) | 2.13.2 (2026-03-30)  | yes |
+| 0.25.x (2026-04-02) | 2.13.2 (2026-03-02)  | yes |
 | 0.24.x (2026-02-25) | 2.13.0 (2026-01-05)  | yes |
 | 0.23.x (2025-11-01) | 2.12.1 (2025-10-16)  | yes |
 | 0.22.x (2025-09-26) | 2.11.1 (2025-08-18)  | yes |

data/lib/rdkafka/admin.rb

@@ -777,6 +777,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,
@@ -865,6 +868,9 @@ module Rdkafka
 
         raise
       ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+
         if configs_array_ptr
           Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
             configs_array_ptr,

data/lib/rdkafka/bindings.rb

@@ -242,6 +242,9 @@ module Rdkafka
 
     attach_function :rd_kafka_conf_new, [], :pointer
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
+    attach_function :rd_kafka_conf_destroy, [:pointer], :void
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
     attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
@@ -432,6 +435,9 @@ module Rdkafka
     # More efficient for poll(0) calls in fiber schedulers.
     attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
+    attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
+    attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/config.rb

@@ -285,6 +285,46 @@ module Rdkafka
       )
     end
 
+    # Returns all configuration properties and their current values for this config.
+    #
+    # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
+    # internal properties like `client.software.name`) as a flat Hash.
+    #
+    # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
+    #   and global properties at the configuration level. All properties are returned
+    #   regardless of the intended client type.
+    #
+    # @note The returned Hash may include sensitive values such as authentication
+    #   credentials and key passwords. Do not log or serialize the returned data
+    #   unless you have explicitly redacted secret entries.
+    #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    def describe_properties
+      config = nil
+      dump_ptr = nil
+      count = 0
+
+      config = native_config
+      count_ptr = Rdkafka::Bindings::SizePtr.new
+      dump_ptr = Rdkafka::Bindings.rd_kafka_conf_dump(config, count_ptr)
+
+      count = count_ptr[:value]
+      result = {}
+
+      (0...count).step(2) do |i|
+        key = dump_ptr.get_pointer(i * FFI::Pointer.size).read_string
+        value = dump_ptr.get_pointer((i + 1) * FFI::Pointer.size).read_string
+        result[key.to_sym] = value
+      end
+
+      result
+    ensure
+      Rdkafka::Bindings.rd_kafka_conf_dump_free(dump_ptr, count) if dump_ptr
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 
@@ -364,10 +404,9 @@ module Rdkafka
       end
 
       # Redirect log to handle's queue
-      Rdkafka::Bindings.rd_kafka_set_log_queue(
-        handle,
-        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
-      )
+      main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
+      Rdkafka::Bindings.rd_kafka_queue_destroy(main_queue)
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle

data/lib/rdkafka/consumer.rb

@@ -183,6 +183,11 @@ module Rdkafka
 
       @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+        if @consumer_queue
+          Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
+          @consumer_queue = nil
+        end
       end
 
       @native_kafka.close
@@ -421,6 +426,8 @@ module Rdkafka
       end
 
       TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -783,6 +790,130 @@ module Rdkafka
       end
     end
 
+    # Poll for a batch of messages from the consumer queue in a single FFI call.
+    #
+    # This is more efficient than calling {#poll} in a loop because it crosses the FFI
+    # boundary only once to fetch up to `max_items` messages.
+    #
+    # The timeout controls how long to wait for the **first** message. Once any message
+    # is available, librdkafka fills the buffer with whatever is immediately ready and
+    # returns without further waiting.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch(timeout_ms, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
+    # Poll for a batch of messages without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#poll_batch} for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
+    #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch_nb(timeout_ms = 0, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
@@ -855,5 +986,24 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+
+    # Returns the consumer queue pointer, lazily initialized
+    # @return [FFI::Pointer] consumer queue handle
+    def consumer_queue
+      @consumer_queue ||= @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+      end
+    end
+
+    # Returns a reusable FFI buffer for batch polling, growing if needed
+    # @param max_items [Integer] minimum buffer capacity
+    # @return [FFI::MemoryPointer] pointer buffer
+    def batch_buffer(max_items)
+      if @batch_buffer.nil? || @batch_buffer_size < max_items
+        @batch_buffer = FFI::MemoryPointer.new(:pointer, max_items)
+        @batch_buffer_size = max_items
+      end
+      @batch_buffer
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.26.0"
+  VERSION = "0.27.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.14.0"
+  LIBRDKAFKA_VERSION = "2.14.1"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "c05c03ef00a13a8463fac3e8918c04843c416f11ced58c889d806a88ca92cf99"
+  LIBRDKAFKA_SOURCE_SHA256 = "bb246e754dee3560e9b42bf4e844dc05de4b146a3cae937e36301ffacdc456e7"
 end

data/renovate.json

@@ -90,5 +90,8 @@
   ],
   "labels": [
     "dependencies"
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true
+  }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.0
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier