karafka-rdkafka 0.29.0-aarch64-linux-gnu → 0.30.1-aarch64-linux-gnu

data/lib/rdkafka/bindings.rb

@@ -201,7 +201,7 @@ module Rdkafka
     RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
 
     # List Offsets
-    RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
+    RD_KAFKA_ADMIN_OP_LISTOFFSETS = 21
     RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
 
     # rd_kafka_IsolationLevel_t
@@ -412,9 +412,17 @@ module Rdkafka
     # ```
     OAuthbearerTokenRefreshCallback = FFI::Function.new(
       :void, [:pointer, :string, :pointer]
-    ) do |client_ptr, config, _opaque|
+    ) do |client_ptr, config, opaque_ptr|
       if Rdkafka::Config.oauthbearer_token_refresh_callback && !client_ptr.null?
-        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
+        client_name = Rdkafka::Bindings.rd_kafka_name(client_ptr)
+
+        # Share consumers have no way to query their librdkafka client name (the share handle
+        # exposes no name accessor), so hand it over here - where librdkafka provides it -
+        # letting applications correlate this callback's client name with a ShareConsumer via
+        # ShareConsumer#name.
+        Rdkafka::Config.opaques[opaque_ptr.to_i]&.capture_client_name(client_name)
+
+        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, client_name)
       end
     end
 
@@ -503,6 +511,68 @@ module Rdkafka
       end
     end
 
+    # Share consumer (KIP-932, preview)
+    #
+    # The share consumer is a distinct librdkafka handle type (rd_kafka_share_t) created with
+    # rd_kafka_share_consumer_new instead of rd_kafka_new. It is single-threaded by design and
+    # librdkafka itself rejects concurrent access with RD_KAFKA_RESP_ERR__CONFLICT.
+
+    # Minimal mirror of the head of librdkafka's `struct rd_kafka_share_s`, whose first member is
+    # the wrapped `rd_kafka_t` (`rkshare_rk`). `rd_kafka_share_t` has no public accessors of its
+    # own - not even for the client name - but the handle it wraps is a normal `rd_kafka_t` and is
+    # the very handle librdkafka passes to the statistics, error and OAuthBearer callbacks. Reading
+    # it lets a share consumer report a name that matches its statistics `name` field, which
+    # `rd_kafka_name` cannot do when handed the `rd_kafka_share_t` directly (it would read an
+    # unrelated field and return garbage). This mirrors how the gem already maps other librdkafka
+    # structs by layout for the pinned librdkafka version.
+    class NativeShareConsumer < FFI::Struct
+      layout :rkshare_rk, :pointer
+    end
+
+    # Acknowledge types for share consumer records
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT = 1
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE = 2
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT = 3
+
+    attach_function :rd_kafka_share_consumer_new, [:pointer, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_share_consumer_close, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_consumer_closed, [:pointer], :int
+    attach_function :rd_kafka_share_destroy, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_set_log_queue, [:pointer, :pointer], :pointer
+
+    attach_function :rd_kafka_share_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_share_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_share_subscription, [:pointer, :pointer], :int, blocking: true
+
+    attach_function :rd_kafka_share_poll, [:pointer, :int, :pointer], :pointer, blocking: true
+    attach_function :rd_kafka_messages_count, [:pointer], :size_t
+    attach_function :rd_kafka_messages_get, [:pointer, :size_t], :pointer
+    attach_function :rd_kafka_messages_destroy, [:pointer], :void
+    attach_function :rd_kafka_message_delivery_count, [:pointer], :int16
+
+    # The by-coordinates acknowledge variant is the only one we bind: Ruby messages are copies
+    # that do not retain the native rd_kafka_message_t pointer, so the message-pointer variants
+    # (rd_kafka_share_acknowledge / rd_kafka_share_acknowledge_type) cannot be used safely here.
+    attach_function :rd_kafka_share_acknowledge_offset, [:pointer, :string, :int32, :int64, :int], :int
+
+    attach_function :rd_kafka_share_commit_sync, [:pointer, :int, :pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_commit_async, [:pointer], :pointer
+
+    callback :share_acknowledgement_commit_cb, [:pointer, :pointer, :int, :pointer], :void
+    attach_function :rd_kafka_share_set_acknowledgement_commit_cb, [:pointer, :share_acknowledgement_commit_cb, :pointer], :pointer
+
+    # Accessors for the rd_kafka_share_partition_offsets_list_t handed to the acknowledgement
+    # commit callback. The list is owned by librdkafka for the duration of the callback: read it
+    # inside the callback only, never destroy or retain it.
+    attach_function :rd_kafka_share_partition_offsets_list_count, [:pointer], :size_t
+    attach_function :rd_kafka_share_partition_offsets_list_get, [:pointer, :size_t], :pointer
+    attach_function :rd_kafka_share_partition_offsets_partition, [:pointer], :pointer
+    attach_function :rd_kafka_share_partition_offsets_offsets, [:pointer], :pointer
+    attach_function :rd_kafka_share_partition_offsets_offsets_cnt, [:pointer], :size_t
+
+    attach_function :rd_kafka_share_oauthbearer_set_token, [:pointer, :string, :int64, :string, :pointer, :size_t, :pointer, :size_t], :int
+    attach_function :rd_kafka_share_oauthbearer_set_token_failure, [:pointer, :string], :int
+
     # Stats
 
     attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int, blocking: true
@@ -559,9 +629,31 @@ module Rdkafka
         raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
       end
 
-      public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
+      public_send(method_name, topic_ptr, str_ptr, partition_key_length(str), partition_count, nil, nil)
+    end
+
+    # Partition key length as the character count (legacy default).
+    #
+    # @param str [String] the partition key string
+    # @return [Integer]
+    def self.partition_key_size(str)
+      str.size
     end
 
+    # Partition key length as the byte count, which is what librdkafka hashes (UTF-8 bytes copied
+    # into the key pointer) and what other Kafka clients use.
+    #
+    # @param str [String] the partition key string
+    # @return [Integer]
+    def self.partition_key_bytesize(str)
+      str.bytesize
+    end
+
+    # Length of the partition key passed to librdkafka. Aliased to one of the methods above by
+    # `Rdkafka::Config.partitioner_key_uses_bytesize=`, so the per-message path does not check the
+    # setting on every call.
+    singleton_class.alias_method :partition_key_length, :partition_key_size
+
     # Create Topics
     RD_KAFKA_ADMIN_OP_CREATETOPICS = 1   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
@@ -603,6 +695,26 @@ module Rdkafka
     attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
     attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
 
+    # Alter Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS = 15   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT = 0x10000 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_AlterConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_AlterConsumerGroupOffsets_t*
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_AlterConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_AlterConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_AlterConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_AlterConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+
+    # Delete Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_DELETECONSUMERGROUPOFFSETS = 8   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT = 107 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_DeleteConsumerGroupOffsets_t*
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_DeleteConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+
     # Delete Records
     RD_KAFKA_ADMIN_OP_DELETERECORDS = 6   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETERECORDS_RESULT = 105 # rd_kafka_event_type_t
@@ -752,5 +864,6 @@ module Rdkafka
 
     attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
     attach_function :rd_kafka_group_result_name, [:pointer], :pointer
+    attach_function :rd_kafka_group_result_partitions, [:pointer], :pointer # rd_kafka_group_result_t* => rd_kafka_topic_partition_list_t*
   end
 end

data/lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class AlterConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the alter consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_AlterConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_AlterConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::AlterConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation, so a missing group result
+              # means the broker returned nothing to act on rather than a partial success.
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # Kafka reports a rejected offset change per partition, leaving the group level
+                # error unset - a group with live members comes back with the group error NULL
+                # and `unknown_member_id` on each partition. Resolving on the group error alone
+                # would hand back a report that looks like success, so any failed partition has
+                # to fail the handle too.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/delete_consumer_group_offsets_handler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class DeleteConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the delete consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_DeleteConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_DeleteConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::DeleteConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation, so a missing group result
+              # means the broker returned nothing to act on rather than a partial success.
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # Kafka reports a rejected offset change per partition, leaving the group level
+                # error unset - a group with live members comes back with the group error NULL
+                # and `unknown_member_id` on each partition. Resolving on the group error alone
+                # would hand back a report that looks like success, so any failed partition has
+                # to fail the handle too.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -21,10 +21,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<TopicResult>] array of topic results
       def self.create_topic_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -53,10 +52,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<GroupResult>] array of group results
       def self.create_group_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -77,10 +75,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<CreateAclResult>] array of ACL results
       def self.create_acl_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -108,10 +105,9 @@ module Rdkafka
       # @param array_pointer [FFI::Pointer] pointer to the results array
       # @return [Array<DeleteAclResult>] array of delete ACL results
       def self.delete_acl_results_from_array(count, array_pointer)
-        (1..count).map do |index|
-          result_pointer = (array_pointer + (index - 1)).read_pointer
-          new(result_pointer)
-        end
+        return [] if count.zero?
+
+        array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
       end
     end
 
@@ -219,6 +215,8 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
         when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT then AlterConsumerGroupOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT then DeleteConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler

data/lib/rdkafka/config.rb

@@ -22,6 +22,12 @@ module Rdkafka
     @@log_mutex = Mutex.new
     # @!visibility private
     @@oauthbearer_token_refresh_callback = nil
+    # @!visibility private
+    # Whether the partition key length passed to the partitioner is the byte length (`bytesize`)
+    # rather than the character length (`size`). `false` (character length) is the legacy default
+    # to avoid changing partition assignment for existing multibyte/binary keys; set to `true` to
+    # opt into the correct, other-client-consistent byte-length hashing.
+    @@partitioner_key_uses_bytesize = false
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -122,6 +128,34 @@ module Rdkafka
       @@oauthbearer_token_refresh_callback
     end
 
+    # Whether the partitioner hashes the partition key by its byte length instead of its character
+    # length. Defaults to `false` (character length) to preserve existing partition assignment for
+    # multibyte/binary keys. Passing the byte length is the correct, other-client-consistent
+    # behavior; enable it (and be aware it can move such keys to a different partition) with
+    # `Rdkafka::Config.partitioner_key_uses_bytesize = true`. This default is expected to change in
+    # a future major version.
+    #
+    # @return [Boolean]
+    def self.partitioner_key_uses_bytesize
+      @@partitioner_key_uses_bytesize
+    end
+
+    # Sets whether the partitioner hashes the partition key by its byte length rather than its
+    # character length. The key length method is resolved here once rather than on every produced
+    # message, and each call re-resolves it, so changing the setting at runtime takes effect
+    # immediately.
+    #
+    # @param value [Boolean] true to hash by byte length, false (default) for character length
+    # @return [void]
+    def self.partitioner_key_uses_bytesize=(value)
+      @@partitioner_key_uses_bytesize = value
+
+      Rdkafka::Bindings.singleton_class.alias_method(
+        :partition_key_length,
+        value ? :partition_key_bytesize : :partition_key_size
+      )
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -221,6 +255,73 @@ module Rdkafka
       end
     end
 
+    # Creates a KIP-932 share group consumer with this configuration.
+    #
+    # `:"group.id"` is required. The acknowledgement mode is selected with the optional
+    # `:"share.acknowledgement.mode"` property (`implicit` by default, or `explicit`). librdkafka
+    # rejects a number of regular-consumer properties for share consumers (for example
+    # `enable.auto.commit`, `auto.offset.reset` or `partition.assignment.strategy`) with a
+    # {ClientCreationError}.
+    #
+    # @return [ShareConsumer] The created share consumer
+    #
+    # @raise [ConfigError] When the configuration contains invalid options or a rebalance
+    #   listener is set (share group assignment is broker-driven and has no rebalances)
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @note The share consumer is a preview librdkafka feature and requires a broker with share
+    #   groups enabled (Apache Kafka 4.2.0+).
+    def share_consumer
+      if @consumer_rebalance_listener
+        raise ConfigError.new("`consumer_rebalance_listener` is not supported for share consumers")
+      end
+
+      opaque = Opaque.new
+      config = native_config(opaque)
+
+      handle = create_native_handle(config) do |error_buffer|
+        Rdkafka::Bindings.rd_kafka_share_consumer_new(config, error_buffer, 256)
+      end
+
+      share_consumer = nil
+
+      # Share consumers cannot go through `build_native_client`: they own the native handle
+      # directly rather than a `NativeKafka` wrapper (single-threaded by design, no polling
+      # thread). They still need both of the things that factory provides - registration, so the
+      # `at_exit` hook closes them before Ruby's shutdown finalization rather than letting
+      # librdkafka be `dlclose`d with its threads still running, and a teardown that destroys the
+      # handle when construction fails part-way - so both are done here.
+      begin
+        # Forward the log queue (enabled via REQUIRED_CONFIG) to the main queue, which
+        # rd_kafka_share_poll drains. There is no background polling thread for share consumers:
+        # log, statistics and error callbacks are all serviced from within ShareConsumer#poll.
+        log_queue_error = Rdkafka::Bindings.rd_kafka_share_set_log_queue(handle, FFI::Pointer::NULL)
+        Rdkafka::Bindings.rd_kafka_error_destroy(log_queue_error) unless log_queue_error.null?
+
+        share_consumer = Rdkafka::ShareConsumer.new(handle, opaque: opaque)
+        opaque.share_consumer = share_consumer
+
+        Rdkafka::Clients.register(share_consumer)
+      rescue Exception
+        begin
+          if share_consumer
+            share_consumer.close
+          else
+            destroy_error = Rdkafka::Bindings.rd_kafka_share_destroy(handle)
+            Rdkafka::Bindings.rd_kafka_error_destroy(destroy_error) unless destroy_error.null?
+          end
+        rescue Exception => err
+          # Never let a teardown failure replace the error that caused it: that error is what the
+          # caller needs to see, and it is re-raised below regardless.
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+
+        raise
+      end
+
+      share_consumer
+    end
+
     # Create a producer with this configuration.
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
@@ -442,28 +543,44 @@ module Rdkafka
       raise
     end
 
-    # Creates a native Kafka handle
+    # Creates a native client handle via the given creation block, enforcing the conf ownership
+    # rule shared by all librdkafka constructors: on success librdkafka takes ownership of the
+    # conf and frees it itself; on failure the application keeps ownership (see rdkafka.c), so
+    # it must be destroyed here to avoid leaking the conf on every failed client creation.
+    #
     # @param config [FFI::Pointer] pointer to the native config
-    # @param type [Symbol] type of client (:rd_kafka_producer or :rd_kafka_consumer)
-    # @return [FFI::Pointer] pointer to the native Kafka handle
+    # @yield [error_buffer] performs the native creation call
+    # @yieldparam error_buffer [FFI::MemoryPointer] 256-byte buffer for the error string
+    # @return [FFI::Pointer] non-NULL native handle
+    # @raise [ClientCreationError] When the native client cannot be created
     # @private
-    def native_kafka(config, type)
-      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
-      handle = Rdkafka::Bindings.rd_kafka_new(
-        type,
-        config,
-        error_buffer,
-        256
-      )
+    def create_native_handle(config)
+      error_buffer = FFI::MemoryPointer.new(:char, 256)
+      handle = yield(error_buffer)
 
       if handle.null?
-        # On success rd_kafka_new takes ownership of the conf and frees it itself; on failure
-        # librdkafka keeps application ownership (see rdkafka.c), so we must destroy it here to
-        # avoid leaking the conf on every failed client creation.
         Rdkafka::Bindings.rd_kafka_conf_destroy(config)
         raise ClientCreationError.new(error_buffer.read_string)
       end
 
+      handle
+    end
+
+    # Creates a native Kafka handle
+    # @param config [FFI::Pointer] pointer to the native config
+    # @param type [Symbol] type of client (:rd_kafka_producer or :rd_kafka_consumer)
+    # @return [FFI::Pointer] pointer to the native Kafka handle
+    # @private
+    def native_kafka(config, type)
+      handle = create_native_handle(config) do |error_buffer|
+        Rdkafka::Bindings.rd_kafka_new(
+          type,
+          config,
+          error_buffer,
+          256
+        )
+      end
+
       # Redirect log to handle's queue
       main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
@@ -481,6 +598,18 @@ module Rdkafka
   class Opaque
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
+    # Backreference set for share consumers so conf-level callbacks (which receive the opaque)
+    # can hand instance-scoped data - currently the librdkafka client name - back to the
+    # ShareConsumer, which has no native accessor for it
+    attr_accessor :share_consumer
+
+    # Forwards the librdkafka client name to clients that cannot query it themselves. Regular
+    # clients resolve their name lazily via rd_kafka_name on their own handle, so only the
+    # share consumer (whose handle exposes no name accessor) consumes this.
+    # @param name [String] librdkafka client name (e.g. "rdkafka#consumer-1")
+    def capture_client_name(name)
+      share_consumer&.name = name
+    end
 
     # Invokes the delivery callback on the producer if one is set
     # @param delivery_report [Rdkafka::Producer::DeliveryReport] the delivery report

data/lib/rdkafka/defaults.rb

@@ -47,6 +47,14 @@ module Rdkafka
     # @see Consumer#events_poll
     CONSUMER_EVENTS_POLL_TIMEOUT_MS = 0
 
+    # Share consumer timeout for poll operations (used in each iteration)
+    # @see ShareConsumer#poll
+    SHARE_CONSUMER_POLL_TIMEOUT_MS = 250
+
+    # Share consumer timeout for synchronous acknowledgement commits
+    # @see ShareConsumer#commit_sync
+    SHARE_CONSUMER_COMMIT_SYNC_TIMEOUT_MS = 5_000
+
     # Producer timeout for flush operations
     # @see Producer#flush
     PRODUCER_FLUSH_TIMEOUT_MS = 5_000

data/lib/rdkafka/error.rb

@@ -40,9 +40,20 @@ module Rdkafka
       def build_from_c(response_ptr, message_prefix = nil, broker_message: nil, instance_name: nil)
         code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
 
-        return false if code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        if code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          # A no-op error object still has to be freed (librdkafka can construct code-0 errors)
+          Rdkafka::Bindings.rd_kafka_error_destroy(response_ptr) unless response_ptr.null?
 
-        message = broker_message || Rdkafka::Bindings.rd_kafka_err2str(code)
+          return false
+        end
+
+        # Prefer the human readable error string attached to this error object; librdkafka falls
+        # back to the generic per-code description internally when none was attached
+        message = broker_message
+        if message.nil?
+          message_ptr = Rdkafka::Bindings.rd_kafka_error_string(response_ptr)
+          message = message_ptr.null? ? Rdkafka::Bindings.rd_kafka_err2str(code) : message_ptr.read_string
+        end
         fatal = !Rdkafka::Bindings.rd_kafka_error_is_fatal(response_ptr).zero?
         retryable = !Rdkafka::Bindings.rd_kafka_error_is_retriable(response_ptr).zero?
         abortable = !Rdkafka::Bindings.rd_kafka_error_txn_requires_abort(response_ptr).zero?

data/lib/rdkafka/helpers/oauth.rb

@@ -18,12 +18,10 @@ module Rdkafka
         extensions_ptr, extensions_str_ptrs = map_extensions(extensions)
 
         begin
-          response = @native_kafka.with_inner do |inner|
-            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
-              inner, token, lifetime_ms, principal_name,
-              extensions_ptr, extension_size(extensions), error_buffer, 256
-            )
-          end
+          response = oauthbearer_native_set_token(
+            token, lifetime_ms, principal_name,
+            extensions_ptr, extension_size(extensions), error_buffer
+          )
         ensure
           extensions_str_ptrs&.each { |ptr| ptr.free }
           extensions_ptr&.free
@@ -50,6 +48,26 @@ module Rdkafka
 
       private
 
+      # Performs the native token set call. The default targets the regular client handle;
+      # clients with a different handle type (e.g. the share consumer) override just this hook
+      # while reusing the buffer/extension plumbing above.
+      #
+      # @param token [String] the token value
+      # @param lifetime_ms [Integer] token expiry, in milliseconds since the epoch
+      # @param principal_name [String] the Kafka principal name associated with the token
+      # @param extensions_ptr [FFI::Pointer, nil] `const char **` built by {#map_extensions}
+      # @param extensions_size [Integer] number of key-value pairs pointed to by `extensions_ptr`
+      # @param error_buffer [FFI::MemoryPointer] 256-byte buffer for the error string
+      # @return [Integer] 0 on success
+      def oauthbearer_native_set_token(token, lifetime_ms, principal_name, extensions_ptr, extensions_size, error_buffer)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            extensions_ptr, extensions_size, error_buffer, 256
+          )
+        end
+      end
+
       # Convert extensions hash to FFI::MemoryPointer (`const char **`).
       #
       # @param extensions [Hash, nil] extension key-value pairs