karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/callbacks/describe_cluster_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT` events
+    # @private
+    class DescribeClusterHandler < BaseHandler
+      class << self
+        # Resolves the describe cluster handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeCluster_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeClusterHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeClusterReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_consumer_groups_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT` events
+    # @private
+    class DescribeConsumerGroupsHandler < BaseHandler
+      class << self
+        # Resolves the describe-consumer-groups handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeConsumerGroups_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeConsumerGroupsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeConsumerGroupsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_topics_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT` events
+    # @private
+    class DescribeTopicsHandler < BaseHandler
+      class << self
+        # Resolves the describe topics handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeTopics_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeTopicsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeTopicsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/list_consumer_group_offsets_handler.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT` events
+    # @private
+    class ListConsumerGroupOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the list consumer group offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result = Rdkafka::Bindings.rd_kafka_event_ListConsumerGroupOffsets_result(event_ptr)
+
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
+          group_result_array = Rdkafka::Bindings.rd_kafka_ListConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
+          group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if (handle = Rdkafka::Admin::ListConsumerGroupOffsetsHandle.remove(handle_ptr.address))
+            unless resolve_operation_error(event_ptr, handle)
+              group_result = group_results[0]
+
+              # librdkafka only supports one group per invocation
+              if group_result.nil?
+                handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                handle.result = Rdkafka::Admin::ListConsumerGroupOffsetsReport.new(
+                  FFI::Pointer::NULL,
+                  FFI::Pointer::NULL
+                )
+              else
+                report = Rdkafka::Admin::ListConsumerGroupOffsetsReport.new(
+                  group_result.error_string,
+                  group_result.result_name,
+                  Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
+                )
+                handle.result = report
+
+                # The group level error can be clean while a partition failed (e.g.
+                # `unstable_offset_commit` with `require_stable_offsets`), which must not look
+                # like a committed offset of nil.
+                failed = report.partitions.find { |partition| partition[:error] }
+
+                if !group_result.result_error.zero?
+                  handle[:response] = group_result.result_error
+                  handle.broker_message = report.error_string
+                elsif failed
+                  handle[:response] = failed[:error].rdkafka_response
+                  handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
+                else
+                  handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                end
+              end
+
+              handle.unlock
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -216,6 +216,7 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT then AlterConsumerGroupOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPOFFSETS_RESULT then ListConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT then DeleteConsumerGroupOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
@@ -225,6 +226,9 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT then DescribeConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT then DescribeClusterHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT then DescribeTopicsHandler
         end
 
         handler&.call(event_ptr)
@@ -247,15 +251,16 @@ module Rdkafka
         message = Rdkafka::Bindings::Message.new(message_ptr)
         delivery_handle_ptr_address = message[:_private].address
         if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
-          topic_name = Rdkafka::Bindings.rd_kafka_topic_name(message[:rkt])
-
           # Update delivery handle
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
-          # The topic is not stored on the handle here: it is already set as the handle's
-          # `topic` Ruby attribute during produce, which spares a native string copy that
-          # would otherwise be allocated and retained for every message.
+          delivery_handle[:status] = Rdkafka::Bindings.rd_kafka_message_status(message_ptr)
+          delivery_handle[:latency] = Rdkafka::Bindings.rd_kafka_message_latency(message_ptr)
+          delivery_handle[:broker_id] = Rdkafka::Bindings.rd_kafka_message_broker_id(message_ptr)
+          # The topic is taken from the handle's `topic` Ruby attribute set during produce, not
+          # read back from librdkafka, which spares an FFI call and a native string copy for
+          # every message.
 
           begin
             # Call delivery callback on opaque
@@ -264,9 +269,12 @@ module Rdkafka
                 Rdkafka::Producer::DeliveryReport.new(
                   message[:partition],
                   message[:offset],
-                  topic_name,
+                  delivery_handle.topic,
                   message[:err],
-                  delivery_handle.label
+                  delivery_handle.label,
+                  status: delivery_handle[:status],
+                  latency: delivery_handle[:latency],
+                  broker_id: delivery_handle[:broker_id]
                 ),
                 delivery_handle
               )

data/lib/rdkafka/clients.rb

@@ -4,22 +4,10 @@ module Rdkafka
   # Tracks live public clients so they can be closed before Ruby begins forced finalization.
   # @private
   module Clients
-    # Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
-    # foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
-    # the moment the value is touched. Track weakly only where that is safe and fall back to a
-    # strong registry - pruned as clients close - on older Rubies. The strong registry holds an
-    # open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
-    WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
-
     @pid = Process.pid
     @mutex = Mutex.new
-
-    if WEAK_TRACKING
-      @clients = ObjectSpace::WeakMap.new
-      @tokens = [].freeze
-    else
-      @clients = {}.freeze
-    end
+    @clients = ObjectSpace::WeakMap.new
+    @tokens = [].freeze
 
     # Registers a successfully constructed client so it can be closed during process shutdown.
     # @param client [Consumer, Producer, Admin] client to close during process shutdown
@@ -28,21 +16,14 @@ module Rdkafka
       reset_after_fork
 
       @mutex.synchronize do
-        if WEAK_TRACKING
-          live_clients
-
-          # The strongly-held token is the key and the client is the weak value, so registering a
-          # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
-          # look the value up by token instead.
-          token = Object.new
-          @clients[token] = client
-          @tokens = (@tokens + [token]).freeze
-        else
-          # Strong references, so drop clients that have already been closed to bound growth.
-          live = @clients.reject { |_id, existing| existing.closed? }
-          live[client.object_id] = client
-          @clients = live.freeze
-        end
+        live_clients
+
+        # The strongly-held token is the key and the client is the weak value, so registering a
+        # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
+        # look the value up by token instead.
+        token = Object.new
+        @clients[token] = client
+        @tokens = (@tokens + [token]).freeze
       end
 
       client
@@ -54,7 +35,7 @@ module Rdkafka
     def self.close_all
       return unless @pid == Process.pid
 
-      clients = @mutex.synchronize { current_clients }
+      clients = @mutex.synchronize { live_clients }
       first_error = nil
 
       clients.each do |client|
@@ -66,16 +47,7 @@ module Rdkafka
       raise first_error if first_error
     end
 
-    # Returns the currently registered live clients.
-    # @return [Array<Consumer, Producer, Admin>] live clients
-    # @private
-    def self.current_clients
-      WEAK_TRACKING ? live_clients : @clients.values
-    end
-    private_class_method :current_clients
-
     # Returns the live clients and releases tokens whose clients have been collected.
-    # Weak-tracking Rubies only.
     # @return [Array<Consumer, Producer, Admin>] live clients
     # @private
     def self.live_clients
@@ -99,13 +71,8 @@ module Rdkafka
     def self.reset_after_fork
       return if @pid == Process.pid
 
-      if WEAK_TRACKING
-        @clients = ObjectSpace::WeakMap.new
-        @tokens = [].freeze
-      else
-        @clients = {}.freeze
-      end
-
+      @clients = ObjectSpace::WeakMap.new
+      @tokens = [].freeze
       @pid = Process.pid
       @mutex = Mutex.new
     end

data/lib/rdkafka/config.rb

@@ -156,6 +156,25 @@ module Rdkafka
       )
     end
 
+    # Returns the time-to-live of the process-wide partitions count cache used by
+    # `produce(partition_key:)`.
+    #
+    # @return [Integer] time-to-live in milliseconds
+    def self.partition_count_cache_ttl_ms
+      Rdkafka::Producer.partitions_count_cache.ttl_ms
+    end
+
+    # Sets the time-to-live of the process-wide partitions count cache shared by all the producers.
+    # Cached counts are kept, so it can be changed at any time.
+    #
+    # @param value [Integer] time-to-live in milliseconds
+    #   (defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS})
+    # @return [void]
+    # @raise [ArgumentError] when the value is not a positive Integer
+    def self.partition_count_cache_ttl_ms=(value)
+      Rdkafka::Producer.partitions_count_cache.ttl_ms = value
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -180,6 +199,7 @@ module Rdkafka
 
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_offset_commit_listener = nil
       @consumer_poll_set = true
     end
 
@@ -207,6 +227,20 @@ module Rdkafka
     # @return [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
     attr_writer :consumer_rebalance_listener
 
+    # Get the result of every offset commit, including async commits that the broker rejects
+    #
+    # `#on_offsets_committed` gets an {RdkafkaError} (nil on success) and the
+    # {Consumer::TopicPartitionList} of the commit, whose partitions carry their own errors. It
+    # runs inside {Consumer#poll} (or {Consumer#events_poll} when `consumer_poll_set` is false)
+    # and inside {Consumer#close}, on the thread that calls them.
+    #
+    # `#on_offsets_commit_failed` gets the same arguments, but only for a failed commit: not on
+    # success and not on `no_offset` (nothing new to commit). A listener with only this method
+    # costs nothing for the other commits.
+    #
+    # @return [Object, #on_offsets_committed, #on_offsets_commit_failed] listener instance
+    attr_writer :consumer_offset_commit_listener
+
     # Should we use a single queue for the underlying consumer and events.
     #
     # This is an advanced API that allows for more granular control of the polling process.
@@ -243,6 +277,11 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      if @consumer_offset_commit_listener
+        opaque.consumer_offset_commit_listener = @consumer_offset_commit_listener
+        Rdkafka::Bindings.rd_kafka_conf_set_offset_commit_cb(config, Rdkafka::Bindings::OffsetCommitCallback)
+      end
+
       build_native_client(
         config,
         :rd_kafka_consumer,
@@ -598,6 +637,7 @@ module Rdkafka
   class Opaque
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
+    attr_accessor :consumer_offset_commit_listener
     # Backreference set for share consumers so conf-level callbacks (which receive the opaque)
     # can hand instance-scoped data - currently the librdkafka client name - back to the
     # ShareConsumer, which has no native accessor for it
@@ -635,5 +675,35 @@ module Rdkafka
 
       consumer_rebalance_listener.on_partitions_revoked(list)
     end
+
+    # Invokes the on_offsets_committed callback on the offset commit listener if set
+    # @param error [Rdkafka::RdkafkaError, nil] the commit error or nil on success
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the offsets of the commit
+    def call_on_offsets_committed(error, list)
+      return unless consumer_offset_commit_listener
+      return unless consumer_offset_commit_listener.respond_to?(:on_offsets_committed)
+
+      consumer_offset_commit_listener.on_offsets_committed(error, list)
+    end
+
+    # Invokes the on_offsets_commit_failed callback on the offset commit listener if set
+    # @param error [Rdkafka::RdkafkaError] the commit error
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the offsets of the commit
+    def call_on_offsets_commit_failed(error, list)
+      return unless consumer_offset_commit_listener
+      return unless consumer_offset_commit_listener.respond_to?(:on_offsets_commit_failed)
+
+      consumer_offset_commit_listener.on_offsets_commit_failed(error, list)
+    end
+
+    # @return [Boolean] true if the offset commit listener wants the result of every commit
+    def offsets_committed_listened?
+      consumer_offset_commit_listener.respond_to?(:on_offsets_committed)
+    end
+
+    # @return [Boolean] true if the offset commit listener wants the failed commits
+    def offsets_commit_failed_listened?
+      consumer_offset_commit_listener.respond_to?(:on_offsets_commit_failed)
+    end
   end
 end

data/lib/rdkafka/consumer/allocator.rb

@@ -28,8 +28,6 @@ module Rdkafka
           pointer.put_bytes(0, bytes)
           pointer.put_char(bytes.bytesize, 0) # NUL terminator
 
-          pointer.autorelease = false
-
           pointer
         end
       end

data/lib/rdkafka/consumer/message.rb

@@ -35,19 +35,16 @@ module Rdkafka
       # @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
       def initialize(native_message)
         # Set topic
-        unless native_message[:rkt].null?
-          @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
-        end
+        rkt = native_message[:rkt]
+        @topic = Rdkafka::Bindings.rd_kafka_topic_name(rkt) unless rkt.null?
         # Set partition
         @partition = native_message[:partition]
         # Set payload
-        unless native_message[:payload].null?
-          @payload = native_message[:payload].read_string(native_message[:len])
-        end
+        payload = native_message[:payload]
+        @payload = payload.read_string(native_message[:len]) unless payload.null?
         # Set key
-        unless native_message[:key].null?
-          @key = native_message[:key].read_string(native_message[:key_len])
-        end
+        key = native_message[:key]
+        @key = key.read_string(native_message[:key_len]) unless key.null?
         # Set offset
         @offset = native_message[:offset]
         # Set timestamp

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -38,7 +38,7 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous
       # configuraton.
       #
-      # @param topic [String] The topic's name
+      # @param topic_name [String] The topic's name
       # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
       #   partition count
       #
@@ -52,14 +52,14 @@ module Rdkafka
       #
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
-      def add_topic(topic, partitions = nil)
+      def add_topic(topic_name, partitions = nil)
         if partitions.nil?
-          @data[topic.to_s] = nil
+          @data[topic_name.to_s] = nil
         else
           if partitions.is_a? Integer
             partitions = (0..partitions - 1)
           end
-          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
+          @data[topic_name.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
         end
       end
 
@@ -67,12 +67,12 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous
       # configuraton.
       #
-      # @param topic [String] The topic's name
+      # @param topic_name [String] The topic's name
       # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
       #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
-      def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
-        @data[topic.to_s] = partitions_with_offsets.map do |p, o|
+      def add_topic_and_partitions_with_offsets(topic_name, partitions_with_offsets)
+        @data[topic_name.to_s] = partitions_with_offsets.map do |p, o|
           p.is_a?(Partition) ? p : Partition.new(p, o)
         end
       end