rdkafka 0.25.0-aarch64-linux-gnu → 0.26.0-aarch64-linux-gnu

data/lib/rdkafka/admin/list_offsets_handle.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Handle for list offsets operation
+    class ListOffsetsHandle < AbstractHandle
+      layout :pending, :bool,
+        :response, :int,
+        :response_string, :pointer,
+        :result_infos, :pointer,
+        :result_count, :int
+
+      # @return [String] the name of the operation.
+      def operation_name
+        "list offsets"
+      end
+
+      # @return [ListOffsetsReport] instance with partition offset information.
+      def create_result
+        ListOffsetsReport.new(
+          result_infos: self[:result_infos],
+          result_count: self[:result_count]
+        )
+      end
+
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/list_offsets_report.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Report for list offsets operation result
+    class ListOffsetsReport
+      attr_reader :offsets
+
+      # @param result_infos [FFI::Pointer] pointer to result info array
+      # @param result_count [Integer] number of result info entries
+      def initialize(result_infos:, result_count:)
+        @offsets = []
+
+        return if result_infos.null?
+
+        result_infos
+          .read_array_of_pointer(result_count)
+          .each { |result_info_ptr| validate!(result_info_ptr) }
+          .each do |result_info_ptr|
+            tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
+            tp = Bindings::TopicPartition.new(tp_ptr)
+            timestamp = Bindings.rd_kafka_ListOffsetsResultInfo_timestamp(result_info_ptr)
+            leader_epoch = Bindings.rd_kafka_topic_partition_get_leader_epoch(tp_ptr)
+
+            @offsets << {
+              topic: tp[:topic],
+              partition: tp[:partition],
+              offset: tp[:offset],
+              timestamp: timestamp,
+              leader_epoch: (leader_epoch == -1) ? nil : leader_epoch
+            }
+          end
+      end
+
+      private
+
+      # Validates the partition result and raises an error if invalid
+      # @param result_info_ptr [FFI::Pointer] pointer to the result info
+      # @raise [RdkafkaError] when the partition has an error
+      def validate!(result_info_ptr)
+        tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
+        tp = Bindings::TopicPartition.new(tp_ptr)
+        code = tp[:err]
+
+        return if code.zero?
+
+        raise RdkafkaError.new(code)
+      end
+    end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -36,8 +36,8 @@ module Rdkafka
           # Read values from the struct
           code = error_desc[:code]
 
-          name = ''
-          desc = ''
+          name = ""
+          desc = ""
 
           name = error_desc[:name].read_string unless error_desc[:name].null?
           desc = error_desc[:desc].read_string unless error_desc[:desc].null?
@@ -71,10 +71,63 @@ module Rdkafka
       end
     end
 
-    # @return [Proc] finalizer proc for closing the admin
-    # @private
-    def finalizer
-      ->(_) { close }
+    # Enable IO event notifications for fiber scheduler integration
+    # When admin operations complete, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the admin client is closed
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the admin client is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other admin operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending events
+    #   admin.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   admin.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
     end
 
     # Performs the metadata request using admin
@@ -113,7 +166,7 @@ module Rdkafka
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    def create_topic(topic_name, partition_count, replication_factor, topic_config={})
+    def create_topic(topic_name, partition_count, replication_factor, topic_config = {})
       closed_admin_check(__method__)
 
       # Create a rd_kafka_NewTopic_t representing the new topic
@@ -129,14 +182,12 @@ module Rdkafka
         raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
       end
 
-      unless topic_config.nil?
-        topic_config.each do |key, value|
-          Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
-            new_topic_ptr,
-            key.to_s,
-            value.to_s
-          )
-        end
+      topic_config&.each do |key, value|
+        Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
+          new_topic_ptr,
+          key.to_s,
+          value.to_s
+        )
       end
 
       # Note that rd_kafka_CreateTopics can create more than one topic at a time
@@ -372,6 +423,8 @@ module Rdkafka
     #   - RD_KAFKA_RESOURCE_BROKER  = 4
     # @param resource_name [String] name of the resource
     # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
+    #   - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN  = 0
+    #   - RD_KAFKA_RESOURCE_PATTERN_ANY      = 1
     #   - RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
     #   - RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
     #   - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
@@ -471,6 +524,8 @@ module Rdkafka
     #   - RD_KAFKA_RESOURCE_BROKER  = 4
     # @param resource_name [String, nil] name of the resource or nil for any
     # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
+    #   - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN  = 0
+    #   - RD_KAFKA_RESOURCE_PATTERN_ANY      = 1
     #   - RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
     #   - RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
     #   - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
@@ -572,6 +627,8 @@ module Rdkafka
     #   - RD_KAFKA_RESOURCE_BROKER  = 4
     # @param resource_name [String, nil] name of the resource or nil for any
     # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
+    #   - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN  = 0
+    #   - RD_KAFKA_RESOURCE_PATTERN_ANY      = 1
     #   - RD_KAFKA_RESOURCE_PATTERN_MATCH    = 2
     #   - RD_KAFKA_RESOURCE_PATTERN_LITERAL  = 3
     #   - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
@@ -720,10 +777,12 @@ module Rdkafka
 
         raise
       ensure
-        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
-          configs_array_ptr,
-          pointer_array.size
-        ) if configs_array_ptr
+        if configs_array_ptr
+          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+            configs_array_ptr,
+            pointer_array.size
+          )
+        end
       end
 
       handle
@@ -791,7 +850,6 @@ module Rdkafka
       configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
       configs_array_ptr.write_array_of_pointer(pointer_array)
 
-
       begin
         @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
@@ -807,10 +865,117 @@ module Rdkafka
 
         raise
       ensure
-        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
-          configs_array_ptr,
-          pointer_array.size
-        ) if configs_array_ptr
+        if configs_array_ptr
+          Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+            configs_array_ptr,
+            pointer_array.size
+          )
+        end
+      end
+
+      handle
+    end
+
+    # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
+    # timestamp) without requiring a consumer group.
+    #
+    # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
+    #   arrays of partition offset specifications. Each specification is a hash with:
+    #   - `:partition` [Integer] partition number
+    #   - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
+    #     `:max_timestamp`, or an integer timestamp in milliseconds
+    # @param isolation_level [Integer, nil] optional isolation level:
+    #   - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
+    #   - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
+    #
+    # @return [ListOffsetsHandle] handle that can be used to wait for the result
+    #
+    # @raise [ClosedAdminError] when the admin is closed
+    # @raise [ConfigError] when the background queue is unavailable
+    #
+    # @example Query earliest and latest offsets
+    #   handle = admin.list_offsets(
+    #     { "my_topic" => [
+    #       { partition: 0, offset: :earliest },
+    #       { partition: 1, offset: :latest }
+    #     ] }
+    #   )
+    #   report = handle.wait(max_wait_timeout_ms: 15_000)
+    #   report.offsets
+    #   # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
+    def list_offsets(topic_partition_offsets, isolation_level: nil)
+      closed_admin_check(__method__)
+
+      # Count total partitions for pre-allocation
+      total = topic_partition_offsets.sum { |_, partitions| partitions.size }
+
+      # Build native topic partition list
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
+
+      topic_partition_offsets.each do |topic, partitions|
+        partitions.each do |spec|
+          partition = spec.fetch(:partition)
+          offset = spec.fetch(:offset)
+
+          native_offset = case offset
+          when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
+          when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
+          when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
+          when Integer then offset
+          else
+            raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
+          end
+
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
+        end
+      end
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      handle = ListOffsetsHandle.new
+      handle[:pending] = true
+      handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
+        )
+      end
+
+      if isolation_level
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
+      end
+
+      ListOffsetsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_ListOffsets(
+            inner,
+            tpl,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        ListOffsetsHandle.remove(handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
 
       handle