karafka-rdkafka 0.31.0-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/admin.rb

@@ -95,7 +95,7 @@ module Rdkafka
 
     # Polls for events in a non-blocking loop, yielding the count after each iteration.
     #
-    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # This method processes events (stats, errors, etc.) in a single GVL session,
     # which is more efficient than repeated individual polls. It uses non-blocking polls
     # internally (no GVL release between polls).
     #
@@ -116,8 +116,9 @@ module Rdkafka
     #   admin.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other admin operations will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
     def events_poll_nb_each
       closed_admin_check(__method__)
@@ -149,14 +150,35 @@ module Rdkafka
     # @param partition_count [Integer] number of partitions for the topic
     # @param replication_factor [Integer] replication factor for the topic
     # @param topic_config [Hash] optional topic configuration settings
+    # @param validate_only [Boolean] when true, the broker only validates the request and does not
+    #   create the topic
+    # @param operation_timeout_ms [Integer, nil] how long the broker waits for the topic creation
+    #   to complete before it responds (librdkafka default: 60 seconds)
+    # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
+    #   `socket.timeout.ms`)
+    # @param braceless_topic_config [Hash] topic configuration given as a hash without braces,
+    #   e.g. `create_topic("t", 1, 1, "retention.ms" => "1000")`, merged into `topic_config`
+    # @option braceless_topic_config [String] any topic configuration name, e.g. `"retention.ms"`
     # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
     #   creating the topic
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    def create_topic(topic_name, partition_count, replication_factor, topic_config = {})
+    # @raise [RdkafkaError] When librdkafka rejects a timeout value
+    def create_topic(
+      topic_name,
+      partition_count,
+      replication_factor,
+      topic_config = {},
+      validate_only: false,
+      operation_timeout_ms: nil,
+      request_timeout_ms: nil,
+      **braceless_topic_config
+    )
       closed_admin_check(__method__)
 
+      topic_config = (topic_config || {}).merge(braceless_topic_config) unless braceless_topic_config.empty?
+
       # Create a rd_kafka_NewTopic_t representing the new topic
       error_buffer = FFI::MemoryPointer.from_string(" " * 256)
       new_topic_ptr = Rdkafka::Bindings.rd_kafka_NewTopic_new(
@@ -203,6 +225,13 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
 
       begin
+        set_admin_options(
+          admin_options_ptr,
+          validate_only: validate_only,
+          operation_timeout_ms: operation_timeout_ms,
+          request_timeout_ms: request_timeout_ms
+        )
+
         @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_CreateTopics(
             inner,
@@ -224,6 +253,51 @@ module Rdkafka
       create_topic_handle
     end
 
+    # Reads the committed offsets of a consumer group from outside the group.
+    #
+    # Unlike {Consumer#committed}, this works for any group and does not join it, so it does not
+    # trigger a rebalance of the group it inspects.
+    #
+    # While the group coordinator is loading, not known yet or has moved (e.g. on a fresh cluster
+    # or after a broker restart), the handle raises `coordinator_load_in_progress`,
+    # `not_coordinator` or `coordinator_not_available`. Retry the call on these codes;
+    # `RdkafkaError#retryable?` is false for them.
+    #
+    # @param group_id [String] the group whose committed offsets we want to read
+    # @param topic_partition_list [TopicPartitionList, nil] partitions to read the offsets for, or
+    #   nil to read every partition the group has committed offsets for. List the partitions
+    #   explicitly: a topic without partitions is not expanded to its partitions
+    # @param require_stable_offsets [Boolean] pending offsets of an open transaction are never
+    #   returned. Without this option the last committed offset is returned for such a partition.
+    #   With it, the call fails with `unstable_offset_commit` until that transaction is committed
+    #   or aborted; retry on this code
+    # @return [ListConsumerGroupOffsetsHandle] handle that can be used to wait for the result
+    # @raise [RdkafkaError] when listing the offsets fails
+    def list_consumer_group_offsets(group_id, topic_partition_list = nil, require_stable_offsets: false)
+      closed_admin_check(__method__)
+
+      native_tpl = topic_partition_list&.to_native_tpl
+
+      begin
+        list_ptr = Rdkafka::Bindings.rd_kafka_ListConsumerGroupOffsets_new(
+          FFI::MemoryPointer.from_string(group_id),
+          native_tpl || FFI::Pointer::NULL
+        )
+      ensure
+        # librdkafka copies the list into the request object, so ours is no longer needed
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl) if native_tpl
+      end
+
+      enqueue_group_offsets_op(
+        request_ptr: list_ptr,
+        handle: ListConsumerGroupOffsetsHandle.new,
+        admin_op: Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS,
+        destroy: Rdkafka::Bindings.method(:rd_kafka_ListConsumerGroupOffsets_destroy),
+        enqueue: Rdkafka::Bindings.method(:rd_kafka_ListConsumerGroupOffsets),
+        configure: ->(admin_options_ptr) { set_require_stable_offsets(admin_options_ptr) if require_stable_offsets }
+      )
+    end
+
     # Sets the committed offsets of a consumer group from outside the group.
     #
     # Kafka requires the group to be empty (no active members) for this to be accepted; when it
@@ -748,10 +822,15 @@ module Rdkafka
     # Deletes the named topic
     #
     # @param topic_name [String] name of the topic to delete
+    # @param operation_timeout_ms [Integer, nil] how long the broker waits for the topic deletion
+    #   to complete before it responds (librdkafka default: 60 seconds)
+    # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
+    #   `socket.timeout.ms`)
     # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
     #   deleting the topic
     # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
-    def delete_topic(topic_name)
+    # @raise [RdkafkaError] When librdkafka rejects a timeout value
+    def delete_topic(topic_name, operation_timeout_ms: nil, request_timeout_ms: nil)
       closed_admin_check(__method__)
 
       # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
@@ -782,6 +861,12 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
 
       begin
+        set_admin_options(
+          admin_options_ptr,
+          operation_timeout_ms: operation_timeout_ms,
+          request_timeout_ms: request_timeout_ms
+        )
+
         @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_DeleteTopics(
             inner,
@@ -807,12 +892,25 @@ module Rdkafka
     #
     # @param topic_name [String] name of the topic
     # @param partition_count [Integer] how many partitions we want to end up with for given topic
+    # @param validate_only [Boolean] when true, the broker only validates the request and does not
+    #   add the partitions
+    # @param operation_timeout_ms [Integer, nil] how long the broker waits for the partitions to be
+    #   created before it responds (librdkafka default: 60 seconds)
+    # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
+    #   `socket.timeout.ms`)
     # @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
     #   result
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    def create_partitions(topic_name, partition_count)
+    # @raise [RdkafkaError] When librdkafka rejects a timeout value
+    def create_partitions(
+      topic_name,
+      partition_count,
+      validate_only: false,
+      operation_timeout_ms: nil,
+      request_timeout_ms: nil
+    )
       closed_admin_check(__method__)
 
       @native_kafka.with_inner do |inner|
@@ -847,6 +945,13 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
 
         begin
+          set_admin_options(
+            admin_options_ptr,
+            validate_only: validate_only,
+            operation_timeout_ms: operation_timeout_ms,
+            request_timeout_ms: request_timeout_ms
+          )
+
           Rdkafka::Bindings.rd_kafka_CreatePartitions(
             inner,
             topics_array_ptr,
@@ -1431,8 +1536,8 @@ module Rdkafka
 
     private
 
-    # Shared enqueue path for the two consumer group offset admin operations. They differ only
-    # in their C entry points, so keeping one body avoids two near identical copies of the
+    # Shared enqueue path for the consumer group offset admin operations. They differ only in
+    # their C entry points, so keeping one body avoids near identical copies of the
     # queue/handle/options dance and its cleanup.
     #
     # @param request_ptr [FFI::Pointer] the op specific request object
@@ -1440,8 +1545,13 @@ module Rdkafka
     # @param admin_op [Integer] the RD_KAFKA_ADMIN_OP_* constant for this operation
     # @param destroy [Method] bound function used to free the request object
     # @param enqueue [Method] bound function used to enqueue the request
+    # @param configure [Proc, nil] called with the `rd_kafka_AdminOptions_t` pointer before the
+    #   request is enqueued, to set an option only this operation supports. For example
+    #   `#list_consumer_group_offsets` always passes a lambda that takes `admin_options_ptr` and
+    #   calls `set_require_stable_offsets(admin_options_ptr)` only when
+    #   `require_stable_offsets: true`. nil (alter and delete) sets no extra option
     # @return [AbstractHandle] the registered handle
-    def enqueue_group_offsets_op(request_ptr:, handle:, admin_op:, destroy:, enqueue:)
+    def enqueue_group_offsets_op(request_ptr:, handle:, admin_op:, destroy:, enqueue:, configure: nil)
       array_ptr = FFI::MemoryPointer.new(:pointer)
       array_ptr.write_array_of_pointer([request_ptr])
 
@@ -1464,6 +1574,8 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
 
       begin
+        configure&.call(admin_options_ptr)
+
         @native_kafka.with_inner do |inner|
           enqueue.call(inner, array_ptr, 1, admin_options_ptr, queue_ptr)
         end
@@ -1483,8 +1595,25 @@ module Rdkafka
     # @param admin_options_ptr [FFI::Pointer] the admin options to update
     # @raise [RdkafkaError] when librdkafka rejects the option
     def set_include_authorized_operations(admin_options_ptr)
-      error_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_set_include_authorized_operations(admin_options_ptr, 1)
+      raise_admin_option_error(
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_include_authorized_operations(admin_options_ptr, 1)
+      )
+    end
+
+    # Makes the broker fail a partition with `unstable_offset_commit` while a transaction with
+    # pending offsets for it is open, instead of returning its last committed offset
+    # @param admin_options_ptr [FFI::Pointer] the admin options to update
+    # @raise [RdkafkaError] when librdkafka rejects the option
+    def set_require_stable_offsets(admin_options_ptr)
+      raise_admin_option_error(
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_require_stable_offsets(admin_options_ptr, 1)
+      )
+    end
 
+    # Raises and releases the error librdkafka returned from setting an admin option
+    # @param error_ptr [FFI::Pointer] the returned `rd_kafka_error_t*`, NULL on success
+    # @raise [RdkafkaError] when the error pointer is not NULL
+    def raise_admin_option_error(error_ptr)
       return if error_ptr.null?
 
       begin
@@ -1499,6 +1628,47 @@ module Rdkafka
       end
     end
 
+    # Applies the optional request settings to admin options
+    # @param admin_options_ptr [FFI::Pointer] the admin options to update
+    # @param validate_only [Boolean] whether the broker should only validate the request
+    # @param operation_timeout_ms [Integer, nil] broker operation timeout, librdkafka default if nil
+    # @param request_timeout_ms [Integer, nil] request timeout, librdkafka default if nil
+    # @raise [RdkafkaError] when librdkafka rejects a value
+    def set_admin_options(admin_options_ptr, validate_only: false, operation_timeout_ms: nil, request_timeout_ms: nil)
+      error_buffer = FFI::MemoryPointer.new(:char, 256)
+
+      if validate_only
+        check_admin_option(
+          Rdkafka::Bindings.rd_kafka_AdminOptions_set_validate_only(admin_options_ptr, 1, error_buffer, 256),
+          error_buffer
+        )
+      end
+
+      if operation_timeout_ms
+        check_admin_option(
+          Rdkafka::Bindings.rd_kafka_AdminOptions_set_operation_timeout(admin_options_ptr, operation_timeout_ms, error_buffer, 256),
+          error_buffer
+        )
+      end
+
+      if request_timeout_ms
+        check_admin_option(
+          Rdkafka::Bindings.rd_kafka_AdminOptions_set_request_timeout(admin_options_ptr, request_timeout_ms, error_buffer, 256),
+          error_buffer
+        )
+      end
+    end
+
+    # Raises when an admin options setter failed
+    # @param response [Integer] result code of an admin options setter
+    # @param error_buffer [FFI::MemoryPointer] buffer the setter wrote its error into
+    # @raise [RdkafkaError] when the setter failed
+    def check_admin_option(response, error_buffer)
+      return if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+      raise RdkafkaError.new(response, broker_message: error_buffer.read_string)
+    end
+
     # Checks if the admin is closed and raises an error if so
     # @param method [Symbol] name of the calling method for error context
     # @raise [ClosedAdminError] when the admin is closed

data/lib/rdkafka/bindings/admin/configs.rb

@@ -5,7 +5,7 @@ module Rdkafka
     attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
     attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
     attach_function :rd_kafka_ConfigResource_destroy, [:pointer], :void
-    attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
+    attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :size_t], :void
     attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
     attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
     attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer

data/lib/rdkafka/bindings/admin.rb

@@ -53,6 +53,17 @@ module Rdkafka
     attach_function :rd_kafka_event_AlterConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_AlterConsumerGroupOffsets_result_t*
     attach_function :rd_kafka_AlterConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_AlterConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
 
+    # List Consumer Group Offsets
+    RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS = 14   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT = 0x8000 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_ListConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_ListConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_ListConsumerGroupOffsets_t*
+    attach_function :rd_kafka_ListConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_ListConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_ListConsumerGroupOffsets_result_t*
+    attach_function :rd_kafka_ListConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_ListConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
+    attach_function :rd_kafka_AdminOptions_set_require_stable_offsets, [:pointer, :int], :pointer, blocking: true # => rd_kafka_error_t*
+
     # Delete Consumer Group Offsets
     RD_KAFKA_ADMIN_OP_DELETECONSUMERGROUPOFFSETS = 8   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT = 107 # rd_kafka_event_type_t
@@ -171,6 +182,9 @@ module Rdkafka
     attach_function :rd_kafka_AdminOptions_new, [:pointer, :int32], :pointer
     attach_function :rd_kafka_AdminOptions_set_opaque, [:pointer, :pointer], :void
     attach_function :rd_kafka_AdminOptions_destroy, [:pointer], :void
+    attach_function :rd_kafka_AdminOptions_set_request_timeout, [:pointer, :int, :pointer, :size_t], :int32
+    attach_function :rd_kafka_AdminOptions_set_operation_timeout, [:pointer, :int, :pointer, :size_t], :int32
+    attach_function :rd_kafka_AdminOptions_set_validate_only, [:pointer, :int, :pointer, :size_t], :int32
 
     # Extracting data from event types
 

data/lib/rdkafka/bindings/callbacks.rb

@@ -27,7 +27,10 @@ module Rdkafka
     StatsCallback = FFI::Function.new(
       :int, [:pointer, :string, :int, :pointer]
     ) do |_client_ptr, json, _json_len, _opaque|
-      if Rdkafka::Config.statistics_callback
+      # Read the callback once: another thread may clear it while this one runs
+      statistics_callback = Rdkafka::Config.statistics_callback
+
+      if statistics_callback
         stats = JSON.parse(json)
 
         # If user requested statistics callbacks, we can use the statistics data to get the
@@ -52,7 +55,7 @@ module Rdkafka
           Rdkafka::Producer.partitions_count_cache.set(topic_name, partitions_count)
         end
 
-        Rdkafka::Config.statistics_callback.call(stats)
+        statistics_callback.call(stats)
       end
 
       # Return 0 so librdkafka frees the json string
@@ -88,7 +91,9 @@ module Rdkafka
     ErrorCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :pointer]
     ) do |client_ptr, err_code, reason, _opaque|
-      if Rdkafka::Config.error_callback
+      error_callback = Rdkafka::Config.error_callback
+
+      if error_callback
         instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
 
         # Handle fatal errors according to librdkafka documentation:
@@ -106,7 +111,7 @@ module Rdkafka
         end
 
         error.set_backtrace(caller)
-        Rdkafka::Config.error_callback.call(error)
+        error_callback.call(error)
       end
     end
 
@@ -132,7 +137,9 @@ module Rdkafka
     OAuthbearerTokenRefreshCallback = FFI::Function.new(
       :void, [:pointer, :string, :pointer]
     ) do |client_ptr, config, opaque_ptr|
-      if Rdkafka::Config.oauthbearer_token_refresh_callback && !client_ptr.null?
+      refresh_callback = Rdkafka::Config.oauthbearer_token_refresh_callback
+
+      if refresh_callback && !client_ptr.null?
         client_name = Rdkafka::Bindings.rd_kafka_name(client_ptr)
 
         # Share consumers have no way to query their librdkafka client name (the share handle
@@ -141,7 +148,7 @@ module Rdkafka
         # ShareConsumer#name.
         Rdkafka::Config.opaques[opaque_ptr.to_i]&.capture_client_name(client_name)
 
-        Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, client_name)
+        refresh_callback.call(config, client_name)
       end
     end
   end

data/lib/rdkafka/bindings/configuration.rb

@@ -11,7 +11,7 @@ module Rdkafka
     ]
 
     attach_function :rd_kafka_conf_new, [], :pointer
-    attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :size_t], :kafka_config_response
     attach_function :rd_kafka_conf_get, [:pointer, :string, :pointer, :pointer], :kafka_config_response
     attach_function :rd_kafka_conf, [:pointer], :pointer
     attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
@@ -27,7 +27,7 @@ module Rdkafka
     attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
     callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
-    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :int, :pointer, :int], :int
+    attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :size_t, :pointer, :size_t], :int
     attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -45,6 +45,6 @@ module Rdkafka
     # Per topic configs
     attach_function :rd_kafka_topic_conf_new, [], :pointer
     attach_function :rd_kafka_topic_conf_destroy, [:pointer], :void
-    attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+    attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :size_t], :kafka_config_response
   end
 end

data/lib/rdkafka/bindings/consumer.rb

@@ -19,7 +19,7 @@ module Rdkafka
     # Non-blocking consumer poll variant (does not release GVL)
     # More efficient for poll(0) calls in fiber schedulers.
     attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
-    attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_consumer_close, [:pointer], :int, blocking: true
     attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
     attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
     attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
@@ -61,7 +61,7 @@ module Rdkafka
       end
 
       opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-      return unless opaque
+      next unless opaque
 
       tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
       begin
@@ -76,6 +76,48 @@ module Rdkafka
       end
     end
 
+    # Offset commit
+
+    callback :offset_commit_cb_function, [:pointer, :int, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_offset_commit_cb, [:pointer, :offset_commit_cb_function], :void, blocking: true
+
+    # Served from the consumer poll (or the events poll when `consumer_poll_set` is false) and
+    # from the consumer close, on the thread that calls them
+    OffsetCommitCallback = FFI::Function.new(
+      :void, [:pointer, :int, :pointer, :pointer]
+    ) do |_client_ptr, code, offsets_ptr, opaque_ptr|
+      opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+      next unless opaque
+
+      # Success and `no_offset` (a commit with nothing new to commit) are not failures
+      failed = code != RD_KAFKA_RESP_ERR_NO_ERROR && code != RD_KAFKA_RESP_ERR__NO_OFFSET
+      every = opaque.offsets_committed_listened?
+      on_failure = failed && opaque.offsets_commit_failed_listened?
+
+      # The list has an entry for every assigned partition, so build it only when it is used
+      if every || on_failure
+        error = code.zero? ? nil : Rdkafka::RdkafkaError.new(code)
+        tpl = if offsets_ptr.null?
+          Rdkafka::Consumer::TopicPartitionList.new
+        else
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(offsets_ptr)
+        end
+        tpl.freeze
+
+        begin
+          opaque.call_on_offsets_committed(error, tpl) if every
+        rescue Exception => err
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+
+        begin
+          opaque.call_on_offsets_commit_failed(error, tpl) if on_failure
+        rescue Exception => err
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+      end
+    end
+
     # Watermark offsets
 
     attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int, blocking: true

data/lib/rdkafka/bindings/errors.rb

@@ -11,7 +11,7 @@ module Rdkafka
     attach_function :rd_kafka_error_code, [:pointer], :int32
     attach_function :rd_kafka_error_string, [:pointer], :pointer
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
-    attach_function :rd_kafka_fatal_error, [:pointer, :pointer, :int], :int
+    attach_function :rd_kafka_fatal_error, [:pointer, :pointer, :size_t], :int
     attach_function :rd_kafka_test_fatal_error, [:pointer, :int, :string], :int
   end
 end

data/lib/rdkafka/bindings/handle.rb

@@ -9,7 +9,7 @@ module Rdkafka
       :rd_kafka_consumer
     ]
 
-    attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
+    attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :size_t], :pointer
 
     attach_function :rd_kafka_destroy, [:pointer], :void
   end

data/lib/rdkafka/bindings.rb

@@ -58,13 +58,16 @@ module Rdkafka
       end
     end
 
+    RD_KAFKA_RESP_ERR__DESTROY = -197
     RD_KAFKA_RESP_ERR__TIMED_OUT = -185
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
     RD_KAFKA_RESP_ERR__STATE = -172
+    RD_KAFKA_RESP_ERR__NO_OFFSET = -168
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR__FATAL = -150
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
+    RD_KAFKA_RESP_ERR_INVALID_PRODUCER_EPOCH = 47
 
     # Buffer size for fatal error strings, matches librdkafka expectations
     FATAL_ERROR_BUFFER_SIZE = 256

data/lib/rdkafka/callbacks/list_consumer_group_offsets_handler.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class ListConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the list consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_ListConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_ListConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::ListConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::ListConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::ListConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # The group level error can be clean while a partition failed (e.g.
+                # `unstable_offset_commit` with `require_stable_offsets`), which must not look
+                # like a committed offset of nil.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -216,6 +216,7 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT then AlterConsumerGroupOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT then ListConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT then DeleteConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
@@ -250,8 +251,6 @@ module Rdkafka
         message = Rdkafka::Bindings::Message.new(message_ptr)
         delivery_handle_ptr_address = message[:_private].address
         if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
-          topic_name = Rdkafka::Bindings.rd_kafka_topic_name(message[:rkt])
-
           # Update delivery handle
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
@@ -259,9 +258,9 @@ module Rdkafka
           delivery_handle[:status] = Rdkafka::Bindings.rd_kafka_message_status(message_ptr)
           delivery_handle[:latency] = Rdkafka::Bindings.rd_kafka_message_latency(message_ptr)
           delivery_handle[:broker_id] = Rdkafka::Bindings.rd_kafka_message_broker_id(message_ptr)
-          # The topic is not stored on the handle here: it is already set as the handle's
-          # `topic` Ruby attribute during produce, which spares a native string copy that
-          # would otherwise be allocated and retained for every message.
+          # The topic is taken from the handle's `topic` Ruby attribute set during produce, not
+          # read back from librdkafka, which spares an FFI call and a native string copy for
+          # every message.
 
           begin
             # Call delivery callback on opaque
@@ -270,7 +269,7 @@ module Rdkafka
                 Rdkafka::Producer::DeliveryReport.new(
                   message[:partition],
                   message[:offset],
-                  topic_name,
+                  delivery_handle.topic,
                   message[:err],
                   delivery_handle.label,
                   status: delivery_handle[:status],