rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/defaults.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Default timeout and timing values used throughout rdkafka-ruby.
+  #
+  # All timeout values can be overridden per-call via method parameters.
+  # These constants provide a central place to understand and reference
+  # the default values used across the library.
+  #
+  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
+  #   For librdkafka options, see:
+  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+  #
+  # @example Overriding a timeout per-call
+  #   consumer.committed(timeout_ms: 5_000)  # Use 5 seconds instead of default 2 seconds
+  #
+  # @example Checking the default value
+  #   Rdkafka::Defaults::CONSUMER_COMMITTED_TIMEOUT_MS  # => 2000
+  module Defaults
+    # Consumer timeouts (in milliseconds)
+
+    # Default timeout for fetching committed offsets
+    # @see Consumer#committed
+    CONSUMER_COMMITTED_TIMEOUT_MS = 2_000
+
+    # Default timeout for fetching the cluster id
+    # @see Consumer#cluster_id
+    CONSUMER_CLUSTER_ID_TIMEOUT_MS = 1_000
+
+    # Default timeout for querying watermark offsets
+    # @see Consumer#query_watermark_offsets
+    CONSUMER_QUERY_WATERMARK_TIMEOUT_MS = 1_000
+
+    # Default timeout for lag calculation watermark queries
+    # @see Consumer#lag
+    CONSUMER_LAG_TIMEOUT_MS = 1_000
+
+    # Default timeout for offsets_for_times operation
+    # @see Consumer#offsets_for_times
+    CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS = 1_000
+
+    # Default poll timeout for Consumer#each iterator
+    # @see Consumer#each
+    CONSUMER_POLL_TIMEOUT_MS = 250
+
+    # Seek operation timeout (0 = non-blocking)
+    # @see Consumer#seek_by
+    CONSUMER_SEEK_TIMEOUT_MS = 0
+
+    # Events poll timeout (0 = non-blocking/async)
+    # @see Consumer#events_poll
+    CONSUMER_EVENTS_POLL_TIMEOUT_MS = 0
+
+    # Producer timeouts (in milliseconds)
+
+    # Default timeout for producer flush operation
+    # @see Producer#flush
+    PRODUCER_FLUSH_TIMEOUT_MS = 5_000
+
+    # Default flush timeout during purge operation
+    # @see Producer#purge
+    PRODUCER_PURGE_FLUSH_TIMEOUT_MS = 100
+
+    # Metadata timeouts (in milliseconds)
+
+    # Default timeout for metadata requests
+    # @see Admin#metadata
+    # @see Metadata#initialize
+    METADATA_TIMEOUT_MS = 2_000
+
+    # Handle wait timeouts (in milliseconds)
+
+    # Default maximum wait timeout for async handles (delivery, admin operations)
+    # @see AbstractHandle#wait
+    HANDLE_WAIT_TIMEOUT_MS = 60_000
+
+    # Native Kafka polling (in milliseconds)
+
+    # Default poll timeout for producer/admin native polling thread
+    # @see Config#producer
+    # @see Config#admin
+    NATIVE_KAFKA_POLL_TIMEOUT_MS = 100
+
+    # Internal timing (in milliseconds)
+
+    # Sleep interval during purge wait loop
+    # @see Producer#purge
+    PRODUCER_PURGE_SLEEP_INTERVAL_MS = 1
+
+    # Sleep interval while waiting for operations to complete in NativeKafka#synchronize
+    # @see NativeKafka#synchronize
+    NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS = 10
+
+    # Base backoff factor for metadata retry in milliseconds (multiplied by 2^attempt)
+    # @see Metadata#initialize
+    METADATA_RETRY_BACKOFF_BASE_MS = 100
+
+    # Maximum backoff time between metadata retries. Caps the exponential backoff so a long retry
+    # sequence against an unhealthy cluster cannot block the calling thread for minutes.
+    # @see Metadata#initialize
+    METADATA_RETRY_BACKOFF_MAX_MS = 1_000
+
+    # Soft wall-clock budget for the whole metadata retry loop; past it (and past
+    # METADATA_MIN_ATTEMPTS) the loop stops so a synchronous fetch cannot block the caller for long
+    # @see Metadata#initialize
+    METADATA_RETRY_BUDGET_MS = 5_000
+
+    # Cache settings (in milliseconds)
+
+    # Default time-to-live for cached partition counts
+    # @see Producer::PartitionsCountCache
+    PARTITIONS_COUNT_CACHE_TTL_MS = 30_000
+
+    # Configuration values (not time-based)
+
+    # Maximum number of metadata fetch retry attempts
+    # @see Metadata#initialize
+    METADATA_MAX_RETRIES = 10
+
+    # Minimum metadata fetch attempts before the retry budget may end the loop, so a slow broker
+    # (whose requests each consume the full timeout) still gets a few tries
+    # @see Metadata#initialize
+    METADATA_MIN_ATTEMPTS = 3
+  end
+end

data/lib/rdkafka/error.rb

@@ -18,12 +18,21 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    # The name of the rdkafka instance that generated this error
+    # @return [String, nil]
+    attr_reader :instance_name
+
     # @private
-    def initialize(response, message_prefix=nil, broker_message: nil)
+    # @param response [Integer] the raw error response code from librdkafka
+    # @param message_prefix [String, nil] optional prefix for error messages
+    # @param broker_message [String, nil] optional error message from the broker
+    # @param instance_name [String, nil] optional name of the rdkafka instance
+    def initialize(response, message_prefix = nil, broker_message: nil, instance_name: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
       @broker_message = broker_message
+      @instance_name = instance_name
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -31,7 +40,7 @@ module Rdkafka
     def code
       code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
       if code[0] == "_"
-        code[1..-1].to_sym
+        code[1..].to_sym
       else
         code.to_sym
       end
@@ -41,11 +50,16 @@ module Rdkafka
     # @return [String]
     def to_s
       message_prefix_part = if message_prefix
-                       "#{message_prefix} - "
-                     else
-                       ''
-                     end
-      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+        "#{message_prefix} - "
+      else
+        ""
+      end
+      instance_name_part = if instance_name
+        " [#{instance_name}]"
+      else
+        ""
+      end
+      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})#{instance_name_part}"
     end
 
     # Whether this error indicates the partition is EOF.
@@ -55,8 +69,10 @@ module Rdkafka
     end
 
     # Error comparison
-    def ==(another_error)
-       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    # @param other [Object] object to compare with
+    # @return [Boolean]
+    def ==(other)
+      other.is_a?(self.class) && (to_s == other.to_s)
     end
   end
 
@@ -66,7 +82,10 @@ module Rdkafka
     attr_reader :topic_partition_list
 
     # @private
-    def initialize(response, topic_partition_list, message_prefix=nil)
+    # @param response [Integer] the raw error response code from librdkafka
+    # @param topic_partition_list [TopicPartitionList] the topic partition list with error info
+    # @param message_prefix [String, nil] optional prefix for error messages
+    def initialize(response, topic_partition_list, message_prefix = nil)
       super(response, message_prefix)
       @topic_partition_list = topic_partition_list
     end
@@ -74,28 +93,35 @@ module Rdkafka
 
   # Error class for public consumer method calls on a closed consumer.
   class ClosedConsumerError < BaseError
+    # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed consumer")
+      super("Illegal call to #{method} on a closed consumer")
     end
   end
 
   # Error class for public producer method calls on a closed producer.
   class ClosedProducerError < BaseError
+    # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed producer")
+      super("Illegal call to #{method} on a closed producer")
     end
   end
 
-  # Error class for public consumer method calls on a closed admin.
+  # Error class for public admin method calls on a closed admin.
   class ClosedAdminError < BaseError
+    # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed admin")
+      super("Illegal call to #{method} on a closed admin")
     end
   end
 
+  # Error class for calls on a closed inner librdkafka instance.
   class ClosedInnerError < BaseError
     def initialize
       super("Illegal call to a closed inner librdkafka instance")
     end
   end
+
+  # Error class for librdkafka library loading failures (e.g., glibc compatibility issues).
+  class LibraryLoadError < BaseError; end
 end

data/lib/rdkafka/helpers/metadata.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Helpers
+    # Shared `#metadata` implementation for Admin, Consumer and Producer.
+    #
+    # `rd_kafka_metadata()` is handle-agnostic in librdkafka - it works on any `rd_kafka_t`
+    # (consumer, producer or admin) - so this Ruby-level implementation is identical across all
+    # three client types. Includers must provide a private `#closed_check(method)` that raises
+    # their own `Closed*Error`.
+    module Metadata
+      # Performs the metadata request using this client
+      #
+      # @param topic_name [String, nil] metadata about particular topic or all if nil
+      # @param timeout_ms [Integer] metadata request timeout
+      # @return [Rdkafka::Metadata] requested metadata
+      def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
+        closed_check(__method__)
+
+        @native_kafka.with_inner do |inner|
+          # Must stay fully qualified: this module is itself named `Metadata`, so a bare
+          # `Metadata.new` here would resolve to `Rdkafka::Helpers::Metadata` (no `.new`) instead
+          # of this class.
+          Rdkafka::Metadata.new(inner, topic_name, timeout_ms)
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/helpers/oauth.rb

@@ -1,8 +1,7 @@
 module Rdkafka
   module Helpers
-
+    # OAuth helper methods for setting and refreshing SASL/OAUTHBEARER tokens
     module OAuth
-
       # Set the OAuthBearer token
       #
       # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
@@ -12,12 +11,18 @@ module Rdkafka
       # @return [Integer] 0 on success
       def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
         error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        extensions_ptr, extensions_str_ptrs = map_extensions(extensions)
 
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
-            inner, token, lifetime_ms, principal_name,
-            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
-          )
+        begin
+          response = @native_kafka.with_inner do |inner|
+            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+              inner, token, lifetime_ms, principal_name,
+              extensions_ptr, extension_size(extensions), error_buffer, 256
+            )
+          end
+        ensure
+          extensions_str_ptrs&.each { |ptr| ptr.free }
+          extensions_ptr&.free
         end
 
         return response if response.zero?
@@ -41,14 +46,41 @@ module Rdkafka
 
       private
 
-      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
-      def flatten_extensions(extensions)
-        return nil unless extensions
-        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      # Convert extensions hash to FFI::MemoryPointer (`const char **`).
+      #
+      # @param extensions [Hash, nil] extension key-value pairs
+      # @return [Array<FFI::MemoryPointer, Array<FFI::MemoryPointer>>] array pointer and string pointers
+      # @note The returned pointers must be freed manually (autorelease = false).
+      def map_extensions(extensions)
+        return [nil, nil] if extensions.nil? || extensions.empty?
+
+        # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+
+        # The method argument is const char **
+        array_ptr = FFI::MemoryPointer.new(:pointer, extension_size(extensions))
+        array_ptr.autorelease = false
+        str_ptrs = []
+
+        # Element i is the key, i + 1 is the value.
+        extensions.each_with_index do |(k, v), i|
+          k_ptr = FFI::MemoryPointer.from_string(k.to_s)
+          k_ptr.autorelease = false
+          str_ptrs << k_ptr
+          v_ptr = FFI::MemoryPointer.from_string(v.to_s)
+          v_ptr.autorelease = false
+          str_ptrs << v_ptr
+          array_ptr[i * 2].put_pointer(0, k_ptr)
+          array_ptr[i * 2 + 1].put_pointer(0, v_ptr)
+        end
+
+        [array_ptr, str_ptrs]
       end
 
-      # extension_size is the number of keys + values which should be a non-negative even number
-      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      # Returns the extension size (number of keys + values).
+      #
+      # @param extensions [Hash, nil] extension key-value pairs
+      # @return [Integer] non-negative even number representing keys + values count
+      # @see https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
       def extension_size(extensions)
         return 0 unless extensions
         extensions.size * 2

data/lib/rdkafka/helpers/time.rb

@@ -9,6 +9,11 @@ module Rdkafka
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
+
+      # @return [Integer] current monotonic time in milliseconds
+      def monotonic_now_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+      end
     end
   end
 end

data/lib/rdkafka/metadata.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
 module Rdkafka
+  # Provides cluster metadata information
   class Metadata
-    attr_reader :brokers, :topics
+    # @return [Array<Hash>] list of broker metadata
+    attr_reader :brokers
+    # @return [Array<Hash>] list of topic metadata
+    attr_reader :topics
 
     # Errors upon which we retry the metadata fetch
     RETRIED_ERRORS = %i[
@@ -12,13 +16,64 @@ module Rdkafka
 
     private_constant :RETRIED_ERRORS
 
-    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
-      attempt ||= 0
-      attempt += 1
-
-      native_topic = if topic_name
-        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+    # Fetches metadata from the Kafka cluster
+    #
+    # @param native_client [FFI::Pointer] pointer to the native Kafka client
+    # @param topic_name [String, nil] specific topic to fetch metadata for, or nil for all topics
+    # @param timeout_ms [Integer] timeout in milliseconds
+    # @raise [RdkafkaError] when metadata fetch fails
+    def initialize(native_client, topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
+      attempt = 0
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) +
+        Defaults::METADATA_RETRY_BUDGET_MS / 1_000.0
+
+      begin
+        attempt += 1
+        fetch_metadata(native_client, topic_name, timeout_ms)
+      rescue ::Rdkafka::RdkafkaError => e
+        raise unless RETRIED_ERRORS.include?(e.code)
+        raise if attempt > Defaults::METADATA_MAX_RETRIES
+
+        # Stop once the wall-clock retry budget is spent, but only after at least
+        # METADATA_MIN_ATTEMPTS tries so a slow broker (whose requests each consume the full
+        # timeout) still gets a few tries rather than being cut off after one or two.
+        raise if attempt >= Defaults::METADATA_MIN_ATTEMPTS &&
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) >= deadline
+
+        # Exponential backoff between attempts, capped so a long retry sequence cannot block for
+        # minutes. The request timeout (`timeout_ms`) is intentionally left unchanged: it used to be
+        # overwritten with the backoff value, which shrank the first retries below the configured
+        # timeout (near-guaranteeing another timeout) and then inflated later ones to ~100s.
+        backoff_ms = [
+          (2**attempt) * Defaults::METADATA_RETRY_BACKOFF_BASE_MS,
+          Defaults::METADATA_RETRY_BACKOFF_MAX_MS
+        ].min
+
+        sleep(backoff_ms / 1_000.0)
+
+        retry
       end
+    end
+
+    private
+
+    # Performs a single metadata fetch attempt, freeing this attempt's native resources.
+    #
+    # Kept separate from {#initialize} so each retried attempt frees its own `native_topic` and
+    # metadata struct. `retry` re-enters `initialize`'s `begin` without running an `ensure` placed
+    # there, so doing the cleanup per attempt here is what prevents the per-retry native leak. The
+    # metadata struct is only allocated on success, so it is read and destroyed only after the
+    # result has been validated (avoids destroying a NULL/garbage pointer on a failed fetch).
+    #
+    # @param native_client [FFI::Pointer] pointer to the native Kafka client
+    # @param topic_name [String, nil] specific topic to fetch metadata for, or nil for all topics
+    # @param timeout_ms [Integer] timeout in milliseconds
+    # @raise [RdkafkaError] when the metadata fetch fails
+    def fetch_metadata(native_client, topic_name, timeout_ms)
+      native_topic = nil
+      metadata_ptr = nil
+
+      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil) if topic_name
 
       ptr = FFI::MemoryPointer.new(:pointer)
 
@@ -32,24 +87,18 @@ module Rdkafka
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
-      metadata_from_native(ptr.read_pointer)
-    rescue ::Rdkafka::RdkafkaError => e
-      raise unless RETRIED_ERRORS.include?(e.code)
-      raise if attempt > 10
-
-      backoff_factor = 2**attempt
-      timeout = backoff_factor * 0.1
+      # rd_kafka_metadata only allocates the struct on success, so we read the pointer to destroy
+      # only after the result has been confirmed successful.
+      metadata_ptr = ptr.read_pointer
 
-      sleep(timeout)
-
-      retry
+      metadata_from_native(metadata_ptr)
     ensure
-      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
-      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if native_topic
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(metadata_ptr) if metadata_ptr && !metadata_ptr.null?
     end
 
-    private
-
+    # Extracts metadata from native pointer
+    # @param ptr [FFI::Pointer] pointer to native metadata
     def metadata_from_native(ptr)
       metadata = Metadata.new(ptr)
       @brokers = Array.new(metadata[:brokers_count]) do |i|
@@ -69,7 +118,11 @@ module Rdkafka
       end
     end
 
+    # Base class for metadata FFI structs with hash conversion
+    # @private
     class CustomFFIStruct < FFI::Struct
+      # Converts struct to a hash
+      # @return [Hash]
       def to_h
         members.each_with_object({}) do |mem, hsh|
           val = self.[](mem)
@@ -80,36 +133,74 @@ module Rdkafka
       end
     end
 
+    # @private
+    # FFI struct for rd_kafka_metadata_t
     class Metadata < CustomFFIStruct
       layout :brokers_count, :int,
-             :brokers_metadata, :pointer,
-             :topics_count, :int,
-             :topics_metadata, :pointer,
-             :broker_id, :int32,
-             :broker_name, :string
+        :brokers_metadata, :pointer,
+        :topics_count, :int,
+        :topics_metadata, :pointer,
+        :broker_id, :int32,
+        :broker_name, :string
     end
 
+    # @private
+    # FFI struct for rd_kafka_metadata_broker_t
     class BrokerMetadata < CustomFFIStruct
       layout :broker_id, :int32,
-             :broker_name, :string,
-             :broker_port, :int
+        :broker_name, :string,
+        :broker_port, :int
     end
 
+    # @private
+    # FFI struct for rd_kafka_metadata_topic_t
     class TopicMetadata < CustomFFIStruct
       layout :topic_name, :string,
-             :partition_count, :int,
-             :partitions_metadata, :pointer,
-             :rd_kafka_resp_err, :int
+        :partition_count, :int,
+        :partitions_metadata, :pointer,
+        :rd_kafka_resp_err, :int
     end
 
+    # @private
+    # FFI struct for rd_kafka_metadata_partition_t
     class PartitionMetadata < CustomFFIStruct
       layout :partition_id, :int32,
-             :rd_kafka_resp_err, :int,
-             :leader, :int32,
-             :replica_count, :int,
-             :replicas, :pointer,
-             :in_sync_replica_brokers, :int,
-             :isrs, :pointer
+        :rd_kafka_resp_err, :int,
+        :leader, :int32,
+        :replica_count, :int,
+        :replicas, :pointer,
+        :in_sync_replica_brokers, :int,
+        :isrs, :pointer
+
+      # The base `#to_h` skips FFI pointer members, which would drop the replica and in-sync
+      # replica assignments entirely. We dereference those pointers here so the partition hash
+      # exposes the broker ids backing the partition (needed e.g. to plan replication changes).
+      #
+      # @return [Hash{Symbol => Integer, Array<Integer>}] partition metadata:
+      #   * +:partition_id+ (Integer) - partition id
+      #   * +:leader+ (Integer) - broker id of the partition leader
+      #   * +:replica_count+ (Integer) - number of assigned replicas
+      #   * +:in_sync_replica_brokers+ (Integer) - number of in-sync replicas
+      #   * +:replicas+ (Array<Integer>) - broker ids of the assigned replicas
+      #   * +:isrs+ (Array<Integer>) - broker ids of the in-sync replicas
+      def to_h
+        super.merge(
+          replicas: read_broker_ids(self[:replicas], self[:replica_count]),
+          isrs: read_broker_ids(self[:isrs], self[:in_sync_replica_brokers])
+        )
+      end
+
+      private
+
+      # Reads `count` broker ids (int32) from a replicas/isrs pointer.
+      # @param pointer [FFI::Pointer] pointer to the broker ids array
+      # @param count [Integer] number of broker ids to read
+      # @return [Array<Integer>] broker ids (empty when there are none)
+      def read_broker_ids(pointer, count)
+        return [] if count.zero? || pointer.null?
+
+        pointer.read_array_of_int32(count)
+      end
     end
   end
 end