karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -42,9 +42,30 @@ module Rdkafka
     #    an authoritative metadata read and adopts the lower count. Within the TTL window a lower
     #    value is still ignored, so a transient or racy lower read cannot clobber a correct higher
     #    count.
+    #
+    # 7. Cluster outages
+    #    If the refresh of an expired entry fails on a transient error (for example all brokers
+    #    are unreachable), the last known count is served instead of raising, and the next refresh
+    #    is attempted after {Defaults::PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS}. This lets
+    #    `partition_key` produces keep buffering in librdkafka during an outage. Without a known
+    #    count the error is raised. Only one caller refreshes an expired entry at a time; the
+    #    others get the known count meanwhile.
     class PartitionsCountCache
       include Helpers::Time
 
+      # Refresh errors after which the last known count is still served
+      STALE_FALLBACK_ERRORS = %i[
+        transport
+        timed_out
+        all_brokers_down
+        resolve
+      ].freeze
+
+      private_constant :STALE_FALLBACK_ERRORS
+
+      # @return [Integer] time-to-live in milliseconds for cached values
+      attr_reader :ttl_ms
+
       # Creates a new partition count cache
       #
       # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
@@ -73,33 +94,44 @@ module Rdkafka
         end
       end
 
+      # Changes the time-to-live of cached values. Cached entries are kept and expire based on the
+      # new value.
+      #
+      # @param value [Integer] time-to-live in milliseconds
+      # @raise [ArgumentError] when the value is not a positive Integer
+      def ttl_ms=(value)
+        raise ArgumentError.new("ttl_ms has to be a positive Integer") unless value.is_a?(Integer) && value.positive?
+
+        @ttl_ms = value
+      end
+
       # Reads partition count for a topic with automatic refresh when expired
       #
       # This method will return the cached partition count if available and not expired.
       # If the value is expired or not available, it will execute the provided block
       # to fetch the current value from Kafka.
       #
-      # @param topic [String] Kafka topic name
+      # @param topic_name [String] Kafka topic name
       # @yield Block that returns the current partition count when cache needs refreshing
       # @yieldreturn [Integer] Current partition count retrieved from Kafka
       # @return [Integer] Partition count for the topic
+      # @raise [RdkafkaError] when the refresh fails and there is no last known count to serve, or
+      #   when it fails on a non-transient error
       #
       # @note The implementation prioritizes read performance over write consistency
       #   since partition counts typically only increase during normal operation.
-      def get(topic)
-        current_info = @counts[topic]
-
-        if current_info.nil? || expired?(current_info[0])
-          # The cached entry is missing or expired, so the block performs an authoritative metadata
-          # read. We hand it to `set`, which adopts a higher count always and a lower count once the
-          # entry has expired (e.g. the topic was recreated with fewer partitions). We then return
-          # whatever `set` settled on so a concurrent refresh that wrote a higher value still wins.
+      def get(topic_name)
+        current_info = @counts[topic_name]
+
+        if current_info.nil?
           new_count = yield
-          set(topic, new_count)
+          set(topic_name, new_count)
 
-          return @counts[topic][1]
+          return @counts[topic_name][1]
         end
 
+        return refresh(topic_name, current_info) { yield } if expired?(current_info[0])
+
         current_info[1]
       end
 
@@ -108,34 +140,34 @@ module Rdkafka
       # This method updates the partition count for a topic in the cache.
       # It uses a mutex to ensure thread-safety during updates.
       #
-      # @param topic [String] Kafka topic name
+      # @param topic_name [String] Kafka topic name
       # @param new_count [Integer] New partition count value
       #
       # @note We prioritize higher partition counts and only accept them when using
       #   a mutex to ensure consistency. This design decision is based on the fact that
       #   partition counts in Kafka only increase during normal operation.
-      def set(topic, new_count)
+      def set(topic_name, new_count)
         # First check outside mutex to avoid unnecessary locking
-        current_info = @counts[topic]
+        current_info = @counts[topic_name]
 
         # Within the TTL window a lower value is treated as a stale/racy read and ignored, since
         # partition counts only grow during normal operation. Once the entry has expired a lower
         # value is an authoritative refresh (e.g. the topic was recreated with fewer partitions),
         # so we fall through and adopt it below.
         if current_info && new_count < current_info[1] && !expired?(current_info[0])
-          refresh_timestamp(topic)
+          refresh_timestamp(topic_name)
 
           return
         end
 
         # Only lock the specific topic mutex
-        mutex_for(topic).synchronize do
+        mutex_for(topic_name).synchronize do
           # Check again inside the lock as another thread might have updated
-          current_info = @counts[topic]
+          current_info = @counts[topic_name]
 
           if current_info.nil?
             # Create new entry
-            @counts[topic] = [monotonic_now_ms, new_count]
+            @counts[topic_name] = [monotonic_now_ms, new_count]
           elsif new_count > current_info[1] || expired?(current_info[0])
             # A higher count always wins; a lower count is accepted only when the existing entry
             # has expired, so a concurrent fresh higher value (which reset the timestamp) is never
@@ -161,24 +193,24 @@ module Rdkafka
       # This method ensures that each topic has its own mutex,
       # allowing operations on different topics to proceed in parallel.
       #
-      # @param topic [String] Kafka topic name
+      # @param topic_name [String] Kafka topic name
       # @return [Mutex] Mutex for the specified topic
       #
       # @note We use a separate mutex (@mutex_for_hash) to protect the creation
       #   of new topic mutexes. This pattern allows fine-grained locking while
       #   maintaining thread-safety.
-      def mutex_for(topic)
-        mutex = @mutex_hash[topic]
+      def mutex_for(topic_name)
+        mutex = @mutex_hash[topic_name]
 
         return mutex if mutex
 
         # Use a separate mutex to protect the creation of new topic mutexes
         @mutex_for_hash.synchronize do
           # Check again in case another thread created it
-          @mutex_hash[topic] ||= Mutex.new
+          @mutex_hash[topic_name] ||= Mutex.new
         end
 
-        @mutex_hash[topic]
+        @mutex_hash[topic_name]
       end
 
       # Update the timestamp without acquiring the mutex
@@ -186,13 +218,13 @@ module Rdkafka
       # This is an optimization that allows refreshing the TTL of existing entries
       # without the overhead of mutex acquisition.
       #
-      # @param topic [String] Kafka topic name
+      # @param topic_name [String] Kafka topic name
       #
       # @note This method is safe for refreshing existing data regardless of count
       #   because it only updates the timestamp, which doesn't affect the correctness
       #   of concurrent operations.
-      def refresh_timestamp(topic)
-        current_info = @counts[topic]
+      def refresh_timestamp(topic_name)
+        current_info = @counts[topic_name]
 
         return unless current_info
 
@@ -200,6 +232,57 @@ module Rdkafka
         current_info[0] = monotonic_now_ms
       end
 
+      # Refreshes an expired entry. While the refresh runs, other callers see a fresh entry and get
+      # the known count instead of running their own metadata request.
+      #
+      # The block performs an authoritative metadata read. We hand it to `set`, which adopts a
+      # higher count always and a lower count once the entry has expired (e.g. the topic was
+      # recreated with fewer partitions). We then return whatever `set` settled on so a concurrent
+      # refresh that wrote a higher value still wins.
+      #
+      # @param topic_name [String] Kafka topic name
+      # @param current_info [Array<Integer>] cached timestamp and partition count
+      # @yieldreturn [Integer] Current partition count retrieved from Kafka
+      # @return [Integer] Partition count for the topic
+      # @raise [RdkafkaError] when the refresh fails on a non-transient error
+      def refresh(topic_name, current_info)
+        expired_at = current_info[0]
+        # Outlasts a failed refresh even with a TTL shorter than one metadata request
+        lease = monotonic_now_ms + Defaults::PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS
+        current_info[0] = lease
+
+        begin
+          new_count = yield
+        rescue RdkafkaError => e
+          raise unless STALE_FALLBACK_ERRORS.include?(e.code)
+        ensure
+          # Hand the entry back as expired, unless a concurrent write refreshed it meanwhile
+          current_info[0] = expired_at if current_info[0] == lease
+        end
+
+        if new_count.nil?
+          # A concurrent write refreshed the entry, so it keeps its full TTL
+          return current_info[1] unless current_info[0] == expired_at
+
+          return serve_stale(current_info)
+        end
+
+        set(topic_name, new_count)
+
+        @counts[topic_name][1]
+      end
+
+      # Keeps serving an expired entry after a failed refresh and schedules the next refresh
+      # after {Defaults::PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS}
+      #
+      # @param current_info [Array<Integer>] cached timestamp and partition count
+      # @return [Integer] last known partition count
+      def serve_stale(current_info)
+        current_info[0] = monotonic_now_ms - @ttl_ms + Defaults::PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS
+
+        current_info[1]
+      end
+
       # Check if a timestamp has expired based on the TTL
       #
       # @param timestamp [Integer] Monotonic timestamp in milliseconds to check

data/lib/rdkafka/producer.rb

@@ -33,6 +33,27 @@ module Rdkafka
 
     private_constant :EMPTY_HASH
 
+    # FFI types of the `rd_kafka_producev` varargs are resolved once. A variadic call resolves each
+    # type argument it gets on every call, unless it already is an `FFI::Type`
+
+    # `:int` vararg type
+    FFI_INT = Rdkafka::Bindings.find_type(:int)
+    # `:pointer` vararg type
+    FFI_POINTER = Rdkafka::Bindings.find_type(:pointer)
+    # `:buffer_in` vararg type
+    FFI_BUFFER_IN = Rdkafka::Bindings.find_type(:buffer_in)
+    # `:size_t` vararg type
+    FFI_SIZE_T = Rdkafka::Bindings.find_type(:size_t)
+    # `:int32` vararg type
+    FFI_INT32 = Rdkafka::Bindings.find_type(:int32)
+    # `:int64` vararg type
+    FFI_INT64 = Rdkafka::Bindings.find_type(:int64)
+    # `:string` vararg type
+    FFI_STRING = Rdkafka::Bindings.find_type(:string)
+
+    private_constant :FFI_INT, :FFI_POINTER, :FFI_BUFFER_IN, :FFI_SIZE_T, :FFI_INT32, :FFI_INT64,
+      :FFI_STRING
+
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
     # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
     # resolved and this is normalized in all the places
@@ -55,8 +76,9 @@ module Rdkafka
     # @param partitioner [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
     def initialize(native_kafka, partitioner)
-      @topics_refs_map = {}
-      @topics_configs = {}
+      @topics_refs_map = EMPTY_HASH
+      @topics_configs = EMPTY_HASH
+      @topics_mutex = Mutex.new
       @native_kafka = native_kafka
       @partitioner = partitioner || "consistent_random"
 
@@ -66,54 +88,27 @@ module Rdkafka
 
     # Sets alternative set of configuration details that can be set per topic
     #
-    # @param topic [String] The topic name
+    # @param topic_name [String] The topic name
     # @param config [Hash] config we want to use per topic basis
     # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
     #   because it is already computed during the retrieval attempt in the `#produce` flow.
     # @note It is not allowed to re-set the same topic config twice because of the underlying
     #   librdkafka caching
-    def set_topic_config(topic, config, config_hash)
-      # Ensure lock on topic reference just in case
+    def set_topic_config(topic_name, config, config_hash)
+      # `with_inner` only counts this as an operation in progress; it takes no lock. Concurrent
+      # first produces to the same topic are serialized by @topics_mutex, so librdkafka gets one
+      # `rd_kafka_topic_new` per topic and config.
       @native_kafka.with_inner do |inner|
-        @topics_refs_map[topic] ||= {}
-        @topics_configs[topic] ||= {}
-
-        return if @topics_configs[topic].key?(config_hash)
-
-        # If config is empty, we create an empty reference that will be used with defaults
-        rd_topic_config = if config.empty?
-          nil
-        else
-          Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
-            config.each do |key, value|
-              error_buffer = FFI::MemoryPointer.new(:char, 256)
-              result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
-                topic_config,
-                key.to_s,
-                value.to_s,
-                error_buffer,
-                256
-              )
-
-              unless result == :config_ok
-                # rd_kafka_topic_new has not been called yet, so librdkafka has not taken
-                # ownership of the config. Destroy it ourselves before raising, otherwise the
-                # partially built rd_kafka_topic_conf_t leaks. (On the rd_kafka_topic_new path
-                # librdkafka frees the conf on both success and failure, so we never destroy it
-                # there.)
-                Rdkafka::Bindings.rd_kafka_topic_conf_destroy(topic_config)
-                raise Config::ConfigError.new(error_buffer.read_string)
-              end
-            end
-          end
+        @topics_mutex.synchronize do
+          return if @topics_refs_map.dig(topic_name, config_hash)
+
+          store_topic_handle(
+            topic_name,
+            config,
+            config_hash,
+            create_topic_handle(inner, topic_name, config)
+          )
         end
-
-        topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
-
-        raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
-
-        @topics_configs[topic][config_hash] = config
-        @topics_refs_map[topic][config_hash] = topic_handle
       end
     end
 
@@ -254,14 +249,16 @@ module Rdkafka
       @native_kafka.close do
         # We need to remove the topics references objects before we destroy the producer,
         # otherwise they would leak out
-        @topics_refs_map.each_value do |refs|
-          refs.each_value do |ref|
-            Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+        @topics_mutex.synchronize do
+          @topics_refs_map.each_value do |refs|
+            refs.each_value do |ref|
+              Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+            end
           end
+
+          @topics_refs_map = EMPTY_HASH
         end
       end
-
-      @topics_refs_map.clear
     end
 
     # Whether this producer has closed
@@ -346,7 +343,7 @@ module Rdkafka
 
     # Polls for events in a non-blocking loop, yielding the count after each iteration.
     #
-    # This method processes delivery callbacks in a single GVL/mutex session, which is more
+    # This method processes delivery callbacks in a single GVL session, which is more
     # efficient than repeated individual polls. It uses non-blocking polls internally
     # (no GVL release between polls).
     #
@@ -367,8 +364,9 @@ module Rdkafka
     #   producer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other producer operations (produce, close, etc.) will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
     def events_poll_nb_each
       closed_producer_check(__method__)
@@ -384,7 +382,7 @@ module Rdkafka
 
     # Partition count for a given topic.
     #
-    # @param topic [String] The topic name.
+    # @param topic_name [String] The topic name.
     # @return [Integer] partition count for a given topic or `RD_KAFKA_PARTITION_UA (-1)` if it
     #   could not be obtained.
     #
@@ -399,14 +397,14 @@ module Rdkafka
     #   `RD_KAFKA_PARTITION_UA` too, so producing to a not-yet-created topic does not run a
     #   blocking metadata query on every message; the cache TTL still bounds how long that result
     #   is reused before we look again.
-    def partition_count(topic)
+    def partition_count(topic_name)
       closed_producer_check(__method__)
 
-      self.class.partitions_count_cache.get(topic) do
+      self.class.partitions_count_cache.get(topic_name) do
         topic_metadata = nil
 
         @native_kafka.with_inner do |inner|
-          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic_name).topics&.first
         end
 
         topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
@@ -486,6 +484,9 @@ module Rdkafka
       set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
       topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
 
+      # A NULL topic handle would crash librdkafka, so never hand one over
+      raise TopicHandleCreationError.new("No topic handle for topic #{topic}") if topic_ref.nil?
+
       if partition_key
         partition_count = partition_count(topic)
 
@@ -535,13 +536,15 @@ module Rdkafka
       # would orphan the handle in the registry forever, since nothing else will ever resolve it.
       begin
         args = [
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
-          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, FFI_POINTER, topic_ref,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE,
+          FFI_BUFFER_IN, payload, FFI_SIZE_T, payload_size,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, FFI_BUFFER_IN, key, FFI_SIZE_T, key_size,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, FFI_INT32, partition,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, FFI_INT64, raw_timestamp,
+          FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, FFI_POINTER, delivery_handle
         ]
 
         if headers && !headers.empty?
@@ -553,26 +556,26 @@ module Rdkafka
               value0.each do |v|
                 value = v.to_s
                 args.push(
-                  :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-                  :string, key,
-                  :pointer, value,
-                  :size_t, value.bytesize
+                  FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
+                  FFI_STRING, key,
+                  FFI_POINTER, value,
+                  FFI_SIZE_T, value.bytesize
                 )
               end
             else
               # Handle single value
               value = value0.to_s
               args.push(
-                :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-                :string, key,
-                :pointer, value,
-                :size_t, value.bytesize
+                FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
+                FFI_STRING, key,
+                FFI_POINTER, value,
+                FFI_SIZE_T, value.bytesize
               )
             end
           end
         end
 
-        args.push(:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END)
+        args.push(FFI_INT, Rdkafka::Bindings::RD_KAFKA_VTYPE_END)
 
         # Produce the message
         response = @native_kafka.with_inner do |inner|
@@ -636,5 +639,72 @@ module Rdkafka
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
     alias_method :closed_check, :closed_producer_check
+
+    # Creates a librdkafka topic handle. Callers hold @topics_mutex.
+    #
+    # @param inner [FFI::Pointer] native producer handle
+    # @param topic_name [String] topic name
+    # @param config [Hash] per-topic config; empty for the defaults
+    # @return [FFI::Pointer] the topic handle
+    # @raise [Config::ConfigError] when a per-topic config value is rejected
+    # @raise [TopicHandleCreationError] when librdkafka cannot create the handle
+    def create_topic_handle(inner, topic_name, config)
+      # If config is empty, we create an empty reference that will be used with defaults
+      rd_topic_config = if config.empty?
+        nil
+      else
+        Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+          config.each do |key, value|
+            error_buffer = FFI::MemoryPointer.new(:char, 256)
+            result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+              topic_config,
+              key.to_s,
+              value.to_s,
+              error_buffer,
+              256
+            )
+
+            unless result == :config_ok
+              # rd_kafka_topic_new has not been called yet, so librdkafka has not taken
+              # ownership of the config. Destroy it ourselves before raising, otherwise the
+              # partially built rd_kafka_topic_conf_t leaks. (On the rd_kafka_topic_new path
+              # librdkafka frees the conf on both success and failure, so we never destroy it
+              # there.)
+              Rdkafka::Bindings.rd_kafka_topic_conf_destroy(topic_config)
+              raise Config::ConfigError.new(error_buffer.read_string)
+            end
+          end
+        end
+      end
+
+      topic_handle = Bindings.rd_kafka_topic_new(inner, topic_name, rd_topic_config)
+
+      raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic_name}") if topic_handle.null?
+
+      topic_handle
+    end
+
+    # Publishes a topic handle and its config. Callers hold @topics_mutex. Both maps are
+    # replaced with frozen copies rather than mutated, and the config goes in first, so a
+    # lock-free reader that finds the handle also finds its config.
+    #
+    # @param topic_name [String] topic name
+    # @param config [Hash] per-topic config
+    # @param config_hash [Integer] hash of the config
+    # @param topic_handle [FFI::Pointer] the topic handle
+    def store_topic_handle(topic_name, config, config_hash, topic_handle)
+      @topics_configs = with_topic_entry(@topics_configs, topic_name, config_hash, config)
+      @topics_refs_map = with_topic_entry(@topics_refs_map, topic_name, config_hash, topic_handle)
+    end
+
+    # @param map [Hash{String => Hash{Integer => Object}}] a frozen topic map
+    # @param topic_name [String] topic name
+    # @param config_hash [Integer] hash of the config
+    # @param value [Object] value to store
+    # @return [Hash{String => Hash{Integer => Object}}] a frozen copy of the map with the value
+    def with_topic_entry(map, topic_name, config_hash, value)
+      entries = map.fetch(topic_name, EMPTY_HASH).merge(config_hash => value).freeze
+      map.merge(topic_name => entries).freeze
+    end
   end
 end