karafka-rdkafka 0.30.0 → 0.30.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 290a1e79997166534339567e0f46bf560191141a22c754bdd67c656ea68594f4
-  data.tar.gz: 972ca0af039c92a2b68d6767214ede8c06d7da0eaffde72f98c79bfcef0b8943
+  metadata.gz: 7b68458fe1759b2645cce196bd5a602c236fe36853e428940641301272d5ecaf
+  data.tar.gz: 6248362a62a402e6004286abc9a7c1f7f29baa9ac375983fa1fc0a2d64159fc3
 SHA512:
-  metadata.gz: 33ba78845a20d938ad79d61c859b08303dfe28ffdbf6a0fa5e99184805fa1d842153c0d676ba6fa25640241b71b2188a65d8ac58208e78526a5e2e3eedc433ad
-  data.tar.gz: 92f7639fca0c51e06844f705285df809a8680315d44531953d6d9b0049aff2964a7633933c31522b8fd75f4bb1606cb13f6a1a5c1d097e8a3f7011b5f3e04083
+  metadata.gz: c69f725864465b708ac3159db0df5a8597e8cf6894f6108ed5ca34654bb4e905fc6dc1e79bd6689bbd2ff4378f4e7bb2f1edfb1bda4416e944912154fda608de
+  data.tar.gz: e0193cf81b907d704b97a27c1d0998f14b83ffb12361b8f0557e87b4345ae58c29caa2d2544192e8524a9feb719b3d707bb42b24cf2d4079ace9d481075277db

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Rdkafka Changelog
 
+## 0.30.2 (2026-10-01)
+- [Feature] Add `ShareConsumer#events_poll` (and `#events_poll_nb`) to service the statistics, error, log and OAuthBearer callbacks without acquiring records.
+
+## 0.30.1 (2026-09-30)
+- [Fix] Derive `ShareConsumer#name` from the native handle at creation (mirroring `Consumer#name`) instead of leaving it `nil` until an OAuthBearer callback happened to set it. A `nil` name meant downstreams that route the global statistics and error callbacks by client name (e.g. Karafka) silently dropped every share-consumer statistic and background error.
+
 ## 0.30.0 (2026-09-25)
 - [Fix] Register share consumers in `Rdkafka::Clients` and destroy the native handle when `Config#share_consumer` fails part-way. Share consumers own the native handle directly rather than a `NativeKafka` wrapper, so they bypass `build_native_client` and were missed by the `at_exit` shutdown hook, leaving librdkafka to be unloaded with a live share handle.
 - [Feature] Add preview support for KIP-932 share groups ("Queues for Kafka") via `Rdkafka::ShareConsumer`, created with `Config#share_consumer`. Members consume the same partitions cooperatively with per-record acknowledgements (`:accept`, `:release`, `:reject`) instead of committed offsets. Like the regular `Consumer`, `ShareConsumer` is a thin binding that exposes the librdkafka share primitives (`#subscribe`/`#unsubscribe`/`#subscription`, batch `#poll`, `#acknowledge`, `#commit_sync`, `#commit_async`, `#acknowledgement_commit_callback=`, `#close`) but drives none of them itself - there is no built-in poll loop, acknowledgement strategy or offset/lifecycle orchestration, leaving that to a higher layer such as Karafka. The handle is single-threaded by design, servicing log, statistics and error callbacks from within `#poll`. Requires a broker with share groups enabled (Apache Kafka 4.2.0+); librdkafka marks the feature as preview and not production-ready.

data/lib/rdkafka/bindings.rb

@@ -517,6 +517,18 @@ module Rdkafka
     # rd_kafka_share_consumer_new instead of rd_kafka_new. It is single-threaded by design and
     # librdkafka itself rejects concurrent access with RD_KAFKA_RESP_ERR__CONFLICT.
 
+    # Minimal mirror of the head of librdkafka's `struct rd_kafka_share_s`, whose first member is
+    # the wrapped `rd_kafka_t` (`rkshare_rk`). `rd_kafka_share_t` has no public accessors of its
+    # own - not even for the client name - but the handle it wraps is a normal `rd_kafka_t` and is
+    # the very handle librdkafka passes to the statistics, error and OAuthBearer callbacks. Reading
+    # it lets a share consumer report a name that matches its statistics `name` field, which
+    # `rd_kafka_name` cannot do when handed the `rd_kafka_share_t` directly (it would read an
+    # unrelated field and return garbage). This mirrors how the gem already maps other librdkafka
+    # structs by layout for the pinned librdkafka version.
+    class NativeShareConsumer < FFI::Struct
+      layout :rkshare_rk, :pointer
+    end
+
     # Acknowledge types for share consumer records
     RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT = 1
     RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE = 2

data/lib/rdkafka/defaults.rb

@@ -51,6 +51,10 @@ module Rdkafka
     # @see ShareConsumer#poll
     SHARE_CONSUMER_POLL_TIMEOUT_MS = 250
 
+    # Share consumer timeout for events_poll (0 = non-blocking async)
+    # @see ShareConsumer#events_poll
+    SHARE_CONSUMER_EVENTS_POLL_TIMEOUT_MS = 0
+
     # Share consumer timeout for synchronous acknowledgement commits
     # @see ShareConsumer#commit_sync
     SHARE_CONSUMER_COMMIT_SYNC_TIMEOUT_MS = 5_000

data/lib/rdkafka/share_consumer.rb

@@ -22,9 +22,10 @@ module Rdkafka
   #   share consumer per thread. librdkafka enforces this and calls from a second thread raise
   #   an `RdkafkaError` with code `conflict`. This class is deliberately not built on
   #   {NativeKafka}: there is no background polling thread (statistics, error and log callbacks
-  #   are all serviced from within {#poll}) and the native handle must never be used from a
-  #   polling thread, so only the {#close} interaction is synchronized here. {#close} may be
-  #   called from any thread and waits for in-flight calls to finish before tearing down.
+  #   are all serviced from within {#poll}, or from {#events_poll} when the caller is not polling
+  #   for records) and the native handle must never be used from a polling thread, so only the
+  #   {#close} interaction is synchronized here. {#close} may be called from any thread and waits
+  #   for in-flight calls to finish before tearing down.
   #
   # Preview limitations inherited from librdkafka worth knowing at this layer:
   # - `max.poll.records` (default 500) is a soft bound
@@ -57,14 +58,17 @@ module Rdkafka
     private_constant :State
 
     # The client name librdkafka reports for this consumer (e.g. "rdkafka#consumer-1"), used to
-    # correlate global callbacks (such as the OAuthBearer token refresh callback) with this
-    # instance. librdkafka exposes no name accessor on the share handle, so this is captured
-    # from the first callback that carries it and is nil until then.
-    # @return [String, nil]
+    # correlate the global callbacks (statistics, error and the OAuthBearer token refresh
+    # callback, all of which carry the client name) with this instance. Captured from the native
+    # handle when the consumer is created, so it is available immediately - before the first
+    # {#poll} and without OAuth configured - and stays readable after the consumer is closed or
+    # inherited by a forked child.
+    # @return [String] the librdkafka client name
     attr_reader :name
 
     # @private
-    # Captures the librdkafka client name once known (from callback context)
+    # Lets the OAuthBearer token refresh callback set the name from its callback context. It
+    # carries the same `rd_kafka_name` value already captured at creation, so both paths agree.
     attr_writer :name
 
     # @private
@@ -80,6 +84,18 @@ module Rdkafka
       # every teardown path can skip the native destroy in a forked child (librdkafka is not
       # fork-safe).
       @state = State.new(native, nil, Process.pid)
+      # Capture the librdkafka client name up front. It is needed to correlate this consumer with
+      # the global statistics and error callbacks (which carry the client name) - the common
+      # downstream pattern (e.g. Karafka) filters those broadcast callbacks by matching `name`.
+      #
+      # `rd_kafka_name` has no `rd_kafka_share_t` overload: handing it the share handle reads an
+      # unrelated field and returns garbage. The share handle's first member is the `rd_kafka_t`
+      # it wraps (`rkshare_rk`), which is the same handle librdkafka passes to those callbacks, so
+      # the name read from it matches the statistics `name` field. Capturing it here (rather than
+      # lazily like the regular consumer) keeps it available before the first poll, without OAuth,
+      # and after close or in a forked child, where the native handle must not be touched.
+      # librdkafka assigns the name at creation, so no broker connection is required.
+      @name = Rdkafka::Bindings.rd_kafka_name(Rdkafka::Bindings::NativeShareConsumer.new(native)[:rkshare_rk])
       # Guards handle teardown against in-flight calls: increments happen under this mutex and
       # close holds it while draining and destroying (mirrors NativeKafka's approach)
       @access_mutex = Mutex.new
@@ -249,6 +265,50 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue (not the share one), serving queued global callbacks without
+    # acquiring any records.
+    #
+    # Events will cause application-provided callbacks to be called:
+    #   - error callbacks
+    #   - stats callbacks
+    #   - log and OAuthBearer token refresh callbacks
+    #
+    # Lets a caller keep those callbacks flowing while it is not polling for records (waiting on
+    # in-flight work, quieting, or draining on shutdown). Does **NOT** replace {#poll}, which
+    # drains the same queue while acquiring records; the "one share consumer per thread" rule
+    # means the two never run concurrently.
+    #
+    # @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
+    #   block until any events available.
+    # @return [Integer] number of events served
+    # @raise [ClosedConsumerError] when the consumer is closed or closing
+    def events_poll(timeout_ms = Defaults::SHARE_CONSUMER_EVENTS_POLL_TIMEOUT_MS)
+      with_native(__method__) do |native|
+        # The main queue lives on the rd_kafka_t the share handle wraps (rkshare_rk), which is the
+        # handle librdkafka hands the callbacks; rd_kafka_poll has to run against it (see #name).
+        inner = Rdkafka::Bindings::NativeShareConsumer.new(native)[:rkshare_rk]
+        Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
+      end
+    end
+
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#events_poll} for non-blocking `poll(0)` calls, particularly
+    # useful in fiber scheduler contexts where GVL release/reacquire overhead is wasteful since we
+    # don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] number of events served
+    # @raise [ClosedConsumerError] when the consumer is closed or closing
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      with_native(__method__) do |native|
+        inner = Rdkafka::Bindings::NativeShareConsumer.new(native)[:rkshare_rk]
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
     # Acknowledges a message delivered by {#poll} in explicit acknowledgement mode
     # (`share.acknowledgement.mode` set to `explicit`).
     #

data/lib/rdkafka/version.rb

@@ -2,7 +2,7 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.30.0"
+  VERSION = "0.30.2"
   # Target librdkafka version to be used
   LIBRDKAFKA_VERSION = "2.15.1"
   # SHA256 hash of the librdkafka source tarball for verification

data/renovate.json

@@ -1,4 +1,7 @@
 {
+  "bundler": {
+    "managerFilePatterns": ["/(^|/)Gemfile\\.lint$/"]
+  },
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
     "config:recommended"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.30.0
+  version: 0.30.2
 platform: ruby
 authors:
 - Thijs Cadier