rdkafka 0.28.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu

data/lib/rdkafka/helpers/list_offsets.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Helpers
+    # Shared `#list_offsets` implementation for Admin and Consumer.
+    #
+    # The librdkafka Admin API is handle-agnostic: `rd_kafka_ListOffsets()` can be issued on any
+    # `rd_kafka_t` instance regardless of its type, so this Ruby-level implementation is identical
+    # for admin and consumer clients. Includers must provide a private `#closed_check(method)`
+    # that raises their own `Closed*Error`, and must have the background event callback
+    # registered on their native config (see `Config#admin` and `Config#consumer`) so the result
+    # event is dispatched back to the returned handle.
+    module ListOffsets
+      # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
+      # timestamp) without requiring a consumer group.
+      #
+      # The query is batched: all requested partitions are carried in one `ListOffsets` request
+      # that librdkafka fans out to the involved partition leaders internally and concurrently.
+      #
+      # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
+      #   arrays of partition offset specifications. Each specification is a hash with:
+      #   - `:partition` [Integer] partition number
+      #   - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
+      #     `:max_timestamp`, or an integer timestamp in milliseconds
+      # @param isolation_level [Integer, nil] optional isolation level:
+      #   - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
+      #   - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
+      #
+      # @return [Admin::ListOffsetsHandle] handle that can be used to wait for the result
+      #
+      # @raise [ClosedAdminError, ClosedConsumerError] when the client is closed
+      # @raise [ConfigError] when the background queue is unavailable
+      #
+      # @example Query earliest and latest offsets
+      #   handle = client.list_offsets(
+      #     { "my_topic" => [
+      #       { partition: 0, offset: :earliest },
+      #       { partition: 1, offset: :latest }
+      #     ] }
+      #   )
+      #   report = handle.wait(max_wait_timeout_ms: 15_000)
+      #   report.offsets
+      #   # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
+      def list_offsets(topic_partition_offsets, isolation_level: nil)
+        closed_check(__method__)
+
+        # Parse and validate every offset spec before allocating the native list, so a missing key
+        # or an unknown offset specification raises with nothing to clean up. Previously the
+        # ArgumentError (or KeyError) was raised after `rd_kafka_topic_partition_list_new`, leaking
+        # the native list.
+        parsed = topic_partition_offsets.flat_map do |topic, partitions|
+          partitions.map do |spec|
+            offset = spec.fetch(:offset)
+
+            native_offset = case offset
+            when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
+            when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
+            when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
+            when Integer then offset
+            else
+              raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
+            end
+
+            [topic, spec.fetch(:partition), native_offset]
+          end
+        end
+
+        # Build native topic partition list
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
+
+        parsed.each do |topic, partition, native_offset|
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
+        end
+
+        # Get a pointer to the queue that our request will be enqueued on
+        queue_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        end
+
+        if queue_ptr.null?
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
+
+        # Create and register the handle we will return to the caller
+        handle = Admin::ListOffsetsHandle.new
+        handle[:pending] = true
+        handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+
+        admin_options_ptr = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+            inner,
+            Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
+          )
+        end
+
+        if isolation_level
+          Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
+        end
+
+        Admin::ListOffsetsHandle.register(handle)
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+        begin
+          @native_kafka.with_inner do |inner|
+            Rdkafka::Bindings.rd_kafka_ListOffsets(
+              inner,
+              tpl,
+              admin_options_ptr,
+              queue_ptr
+            )
+          end
+        rescue Exception
+          Admin::ListOffsetsHandle.remove(handle.to_ptr.address)
+          raise
+        ensure
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        end
+
+        handle
+      end
+    end
+  end
+end

data/lib/rdkafka/helpers/metadata.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Helpers
+    # Shared `#metadata` implementation for Admin, Consumer and Producer.
+    #
+    # `rd_kafka_metadata()` is handle-agnostic in librdkafka - it works on any `rd_kafka_t`
+    # (consumer, producer or admin) - so this Ruby-level implementation is identical across all
+    # three client types. Includers must provide a private `#closed_check(method)` that raises
+    # their own `Closed*Error`.
+    module Metadata
+      # Performs the metadata request using this client
+      #
+      # @param topic_name [String, nil] metadata about particular topic or all if nil
+      # @param timeout_ms [Integer] metadata request timeout
+      # @return [Rdkafka::Metadata] requested metadata
+      def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
+        closed_check(__method__)
+
+        @native_kafka.with_inner do |inner|
+          # Must stay fully qualified: this module is itself named `Metadata`, so a bare
+          # `Metadata.new` here would resolve to `Rdkafka::Helpers::Metadata` (no `.new`) instead
+          # of this class.
+          Rdkafka::Metadata.new(inner, topic_name, timeout_ms)
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/helpers/oauth.rb

@@ -4,10 +4,14 @@ module Rdkafka
     module OAuth
       # Set the OAuthBearer token
       #
-      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
-      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS
+      #   compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds
+      #   since the epoch. See https://currentmillis.com/.
       # @param principal_name [String] the mandatory Kafka principal name associated with the token.
-      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the
+      #   broker as additional key-value pairs during the initial client response as per
+      #   https://tools.ietf.org/html/rfc7628#section-3.1.
       # @return [Integer] 0 on success
       def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
         error_buffer = FFI::MemoryPointer.from_string(" " * 256)
@@ -49,7 +53,8 @@ module Rdkafka
       # Convert extensions hash to FFI::MemoryPointer (`const char **`).
       #
       # @param extensions [Hash, nil] extension key-value pairs
-      # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string pointers
+      # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string
+      #   pointers
       # @note The returned pointers must be freed manually (autorelease = false).
       def map_extensions(extensions)
         return [nil, nil] if extensions.nil? || extensions.empty?
@@ -80,7 +85,7 @@ module Rdkafka
       #
       # @param extensions [Hash, nil] extension key-value pairs
       # @return [Integer] non-negative even number representing keys + values count
-      # @see https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      # @see github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
       def extension_size(extensions)
         return 0 unless extensions
         extensions.size * 2

data/lib/rdkafka/metadata.rb

@@ -23,12 +23,57 @@ module Rdkafka
     # @param timeout_ms [Integer] timeout in milliseconds
     # @raise [RdkafkaError] when metadata fetch fails
     def initialize(native_client, topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
-      attempt ||= 0
-      attempt += 1
-
-      native_topic = if topic_name
-        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      attempt = 0
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) +
+        Defaults::METADATA_RETRY_BUDGET_MS / 1_000.0
+
+      begin
+        attempt += 1
+        fetch_metadata(native_client, topic_name, timeout_ms)
+      rescue ::Rdkafka::RdkafkaError => e
+        raise unless RETRIED_ERRORS.include?(e.code)
+        raise if attempt > Defaults::METADATA_MAX_RETRIES
+
+        # Stop once the wall-clock retry budget is spent, but only after at least
+        # METADATA_MIN_ATTEMPTS tries so a slow broker (whose requests each consume the full
+        # timeout) still gets a few tries rather than being cut off after one or two.
+        raise if attempt >= Defaults::METADATA_MIN_ATTEMPTS &&
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) >= deadline
+
+        # Exponential backoff between attempts, capped so a long retry sequence cannot block for
+        # minutes. The request timeout (`timeout_ms`) is intentionally left unchanged: it used to be
+        # overwritten with the backoff value, which shrank the first retries below the configured
+        # timeout (near-guaranteeing another timeout) and then inflated later ones to ~100s.
+        backoff_ms = [
+          (2**attempt) * Defaults::METADATA_RETRY_BACKOFF_BASE_MS,
+          Defaults::METADATA_RETRY_BACKOFF_MAX_MS
+        ].min
+
+        sleep(backoff_ms / 1_000.0)
+
+        retry
       end
+    end
+
+    private
+
+    # Performs a single metadata fetch attempt, freeing this attempt's native resources.
+    #
+    # Kept separate from {#initialize} so each retried attempt frees its own `native_topic` and
+    # metadata struct. `retry` re-enters `initialize`'s `begin` without running an `ensure` placed
+    # there, so doing the cleanup per attempt here is what prevents the per-retry native leak. The
+    # metadata struct is only allocated on success, so it is read and destroyed only after the
+    # result has been validated (avoids destroying a NULL/garbage pointer on a failed fetch).
+    #
+    # @param native_client [FFI::Pointer] pointer to the native Kafka client
+    # @param topic_name [String, nil] specific topic to fetch metadata for, or nil for all topics
+    # @param timeout_ms [Integer] timeout in milliseconds
+    # @raise [RdkafkaError] when the metadata fetch fails
+    def fetch_metadata(native_client, topic_name, timeout_ms)
+      native_topic = nil
+      metadata_ptr = nil
+
+      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil) if topic_name
 
       ptr = FFI::MemoryPointer.new(:pointer)
 
@@ -42,24 +87,16 @@ module Rdkafka
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
-      metadata_from_native(ptr.read_pointer)
-    rescue ::Rdkafka::RdkafkaError => e
-      raise unless RETRIED_ERRORS.include?(e.code)
-      raise if attempt > Defaults::METADATA_MAX_RETRIES
+      # rd_kafka_metadata only allocates the struct on success, so we read the pointer to destroy
+      # only after the result has been confirmed successful.
+      metadata_ptr = ptr.read_pointer
 
-      backoff_factor = 2**attempt
-      timeout_ms = backoff_factor * Defaults::METADATA_RETRY_BACKOFF_BASE_MS
-
-      sleep(timeout_ms / 1000.0)
-
-      retry
+      metadata_from_native(metadata_ptr)
     ensure
-      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
-      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if native_topic
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(metadata_ptr) if metadata_ptr && !metadata_ptr.null?
     end
 
-    private
-
     # Extracts metadata from native pointer
     # @param ptr [FFI::Pointer] pointer to native metadata
     def metadata_from_native(ptr)
@@ -134,6 +171,36 @@ module Rdkafka
         :replicas, :pointer,
         :in_sync_replica_brokers, :int,
         :isrs, :pointer
+
+      # The base `#to_h` skips FFI pointer members, which would drop the replica and in-sync
+      # replica assignments entirely. We dereference those pointers here so the partition hash
+      # exposes the broker ids backing the partition (needed e.g. to plan replication changes).
+      #
+      # @return [Hash{Symbol => Integer, Array<Integer>}] partition metadata:
+      #   * +:partition_id+ (Integer) - partition id
+      #   * +:leader+ (Integer) - broker id of the partition leader
+      #   * +:replica_count+ (Integer) - number of assigned replicas
+      #   * +:in_sync_replica_brokers+ (Integer) - number of in-sync replicas
+      #   * +:replicas+ (Array<Integer>) - broker ids of the assigned replicas
+      #   * +:isrs+ (Array<Integer>) - broker ids of the in-sync replicas
+      def to_h
+        super.merge(
+          replicas: read_broker_ids(self[:replicas], self[:replica_count]),
+          isrs: read_broker_ids(self[:isrs], self[:in_sync_replica_brokers])
+        )
+      end
+
+      private
+
+      # Reads `count` broker ids (int32) from a replicas/isrs pointer.
+      # @param pointer [FFI::Pointer] pointer to the broker ids array
+      # @param count [Integer] number of broker ids to read
+      # @return [Array<Integer>] broker ids (empty when there are none)
+      def read_broker_ids(pointer, count)
+        return [] if count.zero? || pointer.null?
+
+        pointer.read_array_of_int32(count)
+      end
     end
   end
 end

data/lib/rdkafka/native_kafka.rb

@@ -13,6 +13,14 @@ module Rdkafka
     def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
       @inner = inner
       @opaque = opaque
+      # Process that owns `@inner`. librdkafka is not fork-safe: `fork` copies only the calling
+      # thread, so the background/broker threads backing this handle do not exist in a child
+      # process. An inherited handle therefore must not be polled or destroyed in the child -
+      # `rd_kafka_destroy` would walk thread state that no longer exists (segfault) and the
+      # inherited mutexes may have been copied in a locked state (deadlock). We record the creator
+      # pid so a forked child can recognise an inherited handle and leave its teardown to the
+      # parent, which still owns the running threads.
+      @creator_pid = Process.pid
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -55,8 +63,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
 
         if @run_polling_thread
-          # Start thread to poll client for delivery callbacks,
-          # not used in consumer.
+          # Start thread to poll client for delivery callbacks, not used in consumer.
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
@@ -116,17 +123,21 @@ module Rdkafka
     end
 
     # Returns whether this native Kafka handle is closed or closing
+    #
+    # A handle inherited across `fork` is reported as closed in the child: it belongs to another
+    # process whose threads back the native client, so it is not usable here and must not be
+    # destroyed here (see the `@creator_pid` note in `#initialize`). This makes every `#close`
+    # path - including the GC finalizers that run during a child's exit - skip the native teardown
+    # for inherited handles, which is what would otherwise segfault the child.
+    #
     # @return [Boolean] true if closed or closing
     def closed?
-      @closing || @inner.nil?
+      @closing || @inner.nil? || @creator_pid != Process.pid
     end
 
     # Enable IO event notifications on the main queue
     # Librdkafka will write to your FD when the queue transitions from empty to non-empty
     #
-    # @note This method is incompatible with background polling threads.
-    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
-    #
     # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
     # @param payload [String] data to write to fd when queue has data (default: "\x01")
     # @return [nil]
@@ -143,6 +154,8 @@ module Rdkafka
     #   if readable
     #     consumer.poll(0)  # Get messages
     #   end
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
     def enable_main_queue_io_events(fd, payload = "\x01")
       if @run_polling_thread
         raise "Cannot enable IO events while background polling thread is active. " \
@@ -160,14 +173,13 @@ module Rdkafka
     # Enable IO event notifications on the background queue
     # Librdkafka will write to your FD when the background queue transitions from empty to non-empty
     #
-    # @note This method is incompatible with background polling threads.
-    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
-    #
     # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
     # @param payload [String] data to write to fd when queue has data (default: "\x01")
     # @return [nil]
     # @raise [ClosedInnerError] when the handle is closed
     # @raise [RuntimeError] when background polling thread is active
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
     def enable_background_queue_io_events(fd, payload = "\x01")
       if @run_polling_thread
         raise "Cannot enable IO events while background polling thread is active. " \
@@ -197,8 +209,7 @@ module Rdkafka
           # Indicate to polling thread that we're closing
           @polling_thread[:closing] = true
 
-          # Wait for the polling thread to finish up,
-          # this can be aborted in practice if this
+          # Wait for the polling thread to finish up, this can be aborted in practice if this
           # code runs from a finalizer.
           @polling_thread.join
         end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,20 +2,19 @@
 
 module Rdkafka
   class Producer
-    # Handle to wait for a delivery report which is returned when
-    # producing a message.
+    # Handle to wait for a delivery report which is returned when producing a message.
     class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
         :response, :int,
         :partition, :int,
-        :offset, :int64,
-        :topic_name, :pointer
+        :offset, :int64
 
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
 
       # @return [String] topic where we are trying to send the message
-      # We use this instead of reading from `topic_name` pointer to save on memory allocations
+      # Set in `#produce`, where the topic is known upfront. Keeping it as a Ruby attribute
+      # spares a per-message native string copy in the delivery callback.
       attr_accessor :topic
 
       # @return [String] the name of the operation (e.g. "delivery")

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -37,12 +37,11 @@ module Rdkafka
     #    contention in multi-threaded environments while ensuring data consistency.
     #
     # 6. Topic recreation handling
-    #    If a topic is deleted and recreated with fewer partitions, the cache will continue to
-    #    report the higher count until either the TTL expires or the process is restarted. This
-    #    design choice simplifies the implementation while relying on librdkafka's error handling
-    #    for edge cases. In production environments, topic recreation with different partition
-    #    counts is typically accompanied by application restarts to handle structural changes.
-    #    This also aligns with the previous cache implementation.
+    #    If a topic is deleted and recreated with fewer partitions, the cache keeps reporting the
+    #    higher count only until the entry's TTL expires. The first refresh after expiry performs
+    #    an authoritative metadata read and adopts the lower count. Within the TTL window a lower
+    #    value is still ignored, so a transient or racy lower read cannot clobber a correct higher
+    #    count.
     class PartitionsCountCache
       include Helpers::Time
 
@@ -91,28 +90,14 @@ module Rdkafka
         current_info = @counts[topic]
 
         if current_info.nil? || expired?(current_info[0])
+          # The cached entry is missing or expired, so the block performs an authoritative metadata
+          # read. We hand it to `set`, which adopts a higher count always and a lower count once the
+          # entry has expired (e.g. the topic was recreated with fewer partitions). We then return
+          # whatever `set` settled on so a concurrent refresh that wrote a higher value still wins.
           new_count = yield
+          set(topic, new_count)
 
-          if current_info.nil?
-            # No existing data, create a new entry with mutex
-            set(topic, new_count)
-
-            return new_count
-          else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Higher value needs mutex to update both timestamp and count
-              set(topic, new_count)
-
-              return new_count
-            else
-              # Same or lower value, just update timestamp without mutex
-              refresh_timestamp(topic)
-
-              return current_count
-            end
-          end
+          return @counts[topic][1]
         end
 
         current_info[1]
@@ -133,8 +118,11 @@ module Rdkafka
         # First check outside mutex to avoid unnecessary locking
         current_info = @counts[topic]
 
-        # For lower values, we don't update count but might need to refresh timestamp
-        if current_info && new_count < current_info[1]
+        # Within the TTL window a lower value is treated as a stale/racy read and ignored, since
+        # partition counts only grow during normal operation. Once the entry has expired a lower
+        # value is an authoritative refresh (e.g. the topic was recreated with fewer partitions),
+        # so we fall through and adopt it below.
+        if current_info && new_count < current_info[1] && !expired?(current_info[0])
           refresh_timestamp(topic)
 
           return
@@ -148,17 +136,15 @@ module Rdkafka
           if current_info.nil?
             # Create new entry
             @counts[topic] = [monotonic_now_ms, new_count]
+          elsif new_count > current_info[1] || expired?(current_info[0])
+            # A higher count always wins; a lower count is accepted only when the existing entry
+            # has expired, so a concurrent fresh higher value (which reset the timestamp) is never
+            # clobbered by a stale lower one.
+            current_info[0] = monotonic_now_ms
+            current_info[1] = new_count
           else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Update to higher count value
-              current_info[0] = monotonic_now_ms
-              current_info[1] = new_count
-            else
-              # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now_ms
-            end
+            # Same or lower count within the TTL window: refresh the timestamp only
+            current_info[0] = monotonic_now_ms
           end
         end
       end