karafka-rdkafka 0.20.0.rc2 → 0.20.0

data/lib/rdkafka/bindings.rb

@@ -35,6 +35,8 @@ module Rdkafka
     RD_KAFKA_OFFSET_STORED    = -1000
     RD_KAFKA_OFFSET_INVALID   = -1001
 
+    EMPTY_HASH = {}.freeze
+
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -215,9 +217,31 @@ module Rdkafka
     StatsCallback = FFI::Function.new(
       :int, [:pointer, :string, :int, :pointer]
     ) do |_client_ptr, json, _json_len, _opaque|
-      # Pass the stats hash to callback in config
       if Rdkafka::Config.statistics_callback
         stats = JSON.parse(json)
+
+        # If user requested statistics callbacks, we can use the statistics data to get the
+        # partitions count for each topic when this data is published. That way we do not have
+        # to query this information when user is using `partition_key`. This takes around 0.02ms
+        # every statistics interval period (most likely every 5 seconds) and saves us from making
+        # any queries to the cluster for the partition count.
+        #
+        # One edge case is if user would set the `statistics.interval.ms` much higher than the
+        # default current partition count refresh (30 seconds). This is taken care of as the lack
+        # of reporting to the partitions cache will cause cache expire and blocking refresh.
+        #
+        # If user sets `topic.metadata.refresh.interval.ms` too high this is on the user.
+        #
+        # Since this cache is shared, having few consumers and/or producers in one process will
+        # automatically improve the querying times even with low refresh times.
+        (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
+          partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
+
+          next unless partitions_count.positive?
+
+          Rdkafka::Producer.partitions_count_cache.set(topic_name, partitions_count)
+        end
+
         Rdkafka::Config.statistics_callback.call(stats)
       end
 
@@ -256,7 +280,7 @@ module Rdkafka
     OAuthbearerTokenRefreshCallback = FFI::Function.new(
       :void, [:pointer, :string, :pointer]
     ) do |client_ptr, config, _opaque|
-      if Rdkafka::Config.oauthbearer_token_refresh_callback
+      if Rdkafka::Config.oauthbearer_token_refresh_callback && !client_ptr.null?
         Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
       end
     end
@@ -379,15 +403,16 @@ module Rdkafka
       hsh[name] = method_name
     end
 
-    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
+    def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
       str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
-      method_name = PARTITIONERS.fetch(partitioner_name) do
-        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      method_name = PARTITIONERS.fetch(partitioner) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
       end
-      public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
+
+      public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics
@@ -505,6 +530,7 @@ module Rdkafka
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
     RD_KAFKA_RESOURCE_BROKER = 4
+    RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
 

data/lib/rdkafka/config.rb

@@ -129,7 +129,10 @@ module Rdkafka
     end
 
     # Default config that can be overwritten.
-    DEFAULT_CONFIG = {}.freeze
+    DEFAULT_CONFIG = {
+      # Request api version so advanced features work
+      :"api.version.request" => true
+    }.freeze
 
     # Required config that cannot be overwritten.
     REQUIRED_CONFIG = {

data/lib/rdkafka/error.rb

@@ -126,7 +126,14 @@ module Rdkafka
                      else
                        ''
                      end
-      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+
+      err_str = Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)
+      base = "#{message_prefix_part}#{err_str} (#{code})"
+
+      return base if broker_message.nil?
+      return base if broker_message.empty?
+
+      "#{base}\n#{broker_message}"
     end
 
     # Whether this error indicates the partition is EOF.

data/lib/rdkafka/native_kafka.rb

@@ -126,9 +126,13 @@ module Rdkafka
         # and would continue to run, trying to destroy inner twice
         return unless @inner
 
+        yield if block_given?
+
         Rdkafka::Bindings.rd_kafka_destroy(@inner)
         @inner = nil
         @opaque = nil
+        @poll_mutex.unlock
+        @poll_mutex = nil
       end
     end
   end

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Producer
+    # Caching mechanism for Kafka topic partition counts to avoid frequent cluster queries
+    #
+    # This cache is designed to optimize the process of obtaining partition counts for topics.
+    # It uses several strategies to minimize Kafka cluster queries:
+    #
+    # @note Design considerations:
+    #
+    # 1. Statistics-based updates
+    #    When statistics callbacks are enabled (via `statistics.interval.ms`), we leverage
+    #    this data to proactively update the partition counts cache. This approach costs
+    #    approximately 0.02ms of processing time during each statistics interval (typically
+    #    every 5 seconds) but eliminates the need for explicit blocking metadata queries.
+    #
+    # 2. Edge case handling
+    #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
+    #    (30 seconds), the cache will still function correctly. When statistics updates don't
+    #    occur frequently enough, the cache entries will expire naturally, triggering a
+    #    blocking refresh when needed.
+    #
+    # 3. User configuration awareness
+    #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
+    #    set very high, the responsibility for potentially stale data falls on the user. This
+    #    is an explicit design choice to honor user configuration preferences and align with
+    #    librdkafka settings.
+    #
+    # 4. Process-wide efficiency
+    #    Since this cache is shared across all Rdkafka producers and consumers within a process,
+    #    having multiple clients improves overall efficiency. Each client contributes to keeping
+    #    the cache updated, benefiting all other clients.
+    #
+    # 5. Thread-safety approach
+    #    The implementation uses fine-grained locking with per-topic mutexes to minimize
+    #    contention in multi-threaded environments while ensuring data consistency.
+    #
+    # 6. Topic recreation handling
+    #    If a topic is deleted and recreated with fewer partitions, the cache will continue to
+    #    report the higher count until either the TTL expires or the process is restarted. This
+    #    design choice simplifies the implementation while relying on librdkafka's error handling
+    #    for edge cases. In production environments, topic recreation with different partition
+    #    counts is typically accompanied by application restarts to handle structural changes.
+    #    This also aligns with the previous cache implementation.
+    class PartitionsCountCache
+      include Helpers::Time
+
+      # Default time-to-live for cached partition counts in seconds
+      #
+      # @note This default was chosen to balance freshness of metadata with performance
+      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
+      #   seconds a reasonable compromise.
+      DEFAULT_TTL = 30
+
+      # Creates a new partition count cache
+      #
+      # @param ttl [Integer] Time-to-live in seconds for cached values
+      def initialize(ttl = DEFAULT_TTL)
+        @counts = {}
+        @mutex_hash = {}
+        # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
+        @mutex_for_hash = Mutex.new
+        @ttl = ttl
+      end
+
+      # Reads partition count for a topic with automatic refresh when expired
+      #
+      # This method will return the cached partition count if available and not expired.
+      # If the value is expired or not available, it will execute the provided block
+      # to fetch the current value from Kafka.
+      #
+      # @param topic [String] Kafka topic name
+      # @yield Block that returns the current partition count when cache needs refreshing
+      # @yieldreturn [Integer] Current partition count retrieved from Kafka
+      # @return [Integer] Partition count for the topic
+      #
+      # @note The implementation prioritizes read performance over write consistency
+      #   since partition counts typically only increase during normal operation.
+      def get(topic)
+        current_info = @counts[topic]
+
+        if current_info.nil? || expired?(current_info[0])
+          new_count = yield
+
+          if current_info.nil?
+            # No existing data, create a new entry with mutex
+            set(topic, new_count)
+
+            return new_count
+          else
+            current_count = current_info[1]
+
+            if new_count > current_count
+              # Higher value needs mutex to update both timestamp and count
+              set(topic, new_count)
+
+              return new_count
+            else
+              # Same or lower value, just update timestamp without mutex
+              refresh_timestamp(topic)
+
+              return current_count
+            end
+          end
+        end
+
+        current_info[1]
+      end
+
+      # Update partition count for a topic when needed
+      #
+      # This method updates the partition count for a topic in the cache.
+      # It uses a mutex to ensure thread-safety during updates.
+      #
+      # @param topic [String] Kafka topic name
+      # @param new_count [Integer] New partition count value
+      #
+      # @note We prioritize higher partition counts and only accept them when using
+      #   a mutex to ensure consistency. This design decision is based on the fact that
+      #   partition counts in Kafka only increase during normal operation.
+      def set(topic, new_count)
+        # First check outside mutex to avoid unnecessary locking
+        current_info = @counts[topic]
+
+        # For lower values, we don't update count but might need to refresh timestamp
+        if current_info && new_count < current_info[1]
+          refresh_timestamp(topic)
+
+          return
+        end
+
+        # Only lock the specific topic mutex
+        mutex_for(topic).synchronize do
+          # Check again inside the lock as another thread might have updated
+          current_info = @counts[topic]
+
+          if current_info.nil?
+            # Create new entry
+            @counts[topic] = [monotonic_now, new_count]
+          else
+            current_count = current_info[1]
+
+            if new_count > current_count
+              # Update to higher count value
+              current_info[0] = monotonic_now
+              current_info[1] = new_count
+            else
+              # Same or lower count, update timestamp only
+              current_info[0] = monotonic_now
+            end
+          end
+        end
+      end
+
+      # @return [Hash] hash with ttls and partitions counts array
+      def to_h
+        @counts
+      end
+
+      private
+
+      # Get or create a mutex for a specific topic
+      #
+      # This method ensures that each topic has its own mutex,
+      # allowing operations on different topics to proceed in parallel.
+      #
+      # @param topic [String] Kafka topic name
+      # @return [Mutex] Mutex for the specified topic
+      #
+      # @note We use a separate mutex (@mutex_for_hash) to protect the creation
+      #   of new topic mutexes. This pattern allows fine-grained locking while
+      #   maintaining thread-safety.
+      def mutex_for(topic)
+        mutex = @mutex_hash[topic]
+
+        return mutex if mutex
+
+        # Use a separate mutex to protect the creation of new topic mutexes
+        @mutex_for_hash.synchronize do
+          # Check again in case another thread created it
+          @mutex_hash[topic] ||= Mutex.new
+        end
+
+        @mutex_hash[topic]
+      end
+
+      # Update the timestamp without acquiring the mutex
+      #
+      # This is an optimization that allows refreshing the TTL of existing entries
+      # without the overhead of mutex acquisition.
+      #
+      # @param topic [String] Kafka topic name
+      #
+      # @note This method is safe for refreshing existing data regardless of count
+      #   because it only updates the timestamp, which doesn't affect the correctness
+      #   of concurrent operations.
+      def refresh_timestamp(topic)
+        current_info = @counts[topic]
+
+        return unless current_info
+
+        # Update the timestamp in-place
+        current_info[0] = monotonic_now
+      end
+
+      # Check if a timestamp has expired based on the TTL
+      #
+      # @param timestamp [Float] Monotonic timestamp to check
+      # @return [Boolean] true if expired, false otherwise
+      def expired?(timestamp)
+        monotonic_now - timestamp > @ttl
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -6,13 +6,31 @@ module Rdkafka
     include Helpers::Time
     include Helpers::OAuth
 
-    # Cache partitions count for 30 seconds
-    PARTITIONS_COUNT_TTL = 30
+    # @private
+    @@partitions_count_cache = PartitionsCountCache.new
+
+    # Global (process wide) partitions cache. We use it to store number of topics partitions,
+    # either from the librdkafka statistics (if enabled) or via direct inline calls every now and
+    # then. Since the partitions count can only grow and should be same for all consumers and
+    # producers, we can use a global cache as long as we ensure that updates only move up.
+    #
+    # @note It is critical to remember, that not all users may have statistics callbacks enabled,
+    #   hence we should not make assumption that this cache is always updated from the stats.
+    #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
+    def self.partitions_count_cache
+      @@partitions_count_cache
+    end
+
+    # @param partitions_count_cache [Rdkafka::Producer::PartitionsCountCache]
+    def self.partitions_count_cache=(partitions_count_cache)
+      @@partitions_count_cache = partitions_count_cache
+    end
 
     # Empty hash used as a default
     EMPTY_HASH = {}.freeze
 
-    private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
+    private_constant :EMPTY_HASH
 
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
     # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
@@ -33,35 +51,16 @@ module Rdkafka
 
     # @private
     # @param native_kafka [NativeKafka]
-    # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
+    # @param partitioner [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
-    def initialize(native_kafka, partitioner_name)
+    def initialize(native_kafka, partitioner)
       @topics_refs_map = {}
       @topics_configs = {}
       @native_kafka = native_kafka
-      @partitioner_name = partitioner_name || "consistent_random"
+      @partitioner = partitioner || "consistent_random"
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
-
-      @_partitions_count_cache = Hash.new do |cache, topic|
-        topic_metadata = nil
-
-        @native_kafka.with_inner do |inner|
-          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
-        end
-
-        partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
-
-        # This approach caches the failure to fetch only for 1 second. This will make sure, that
-        # we do not cache the failure for too long but also "buys" us a bit of time in case there
-        # would be issues in the cluster so we won't overaload it with consecutive requests
-        cache[topic] = if partition_count.positive?
-                         [monotonic_now, partition_count]
-                       else
-                         [monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
-                       end
-      end
     end
 
     # Sets alternative set of configuration details that can be set per topic
@@ -284,18 +283,31 @@ module Rdkafka
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
     #
-    # @note We cache the partition count for a given topic for given time.
+    # @note We cache the partition count for a given topic for given time. If statistics are
+    #   enabled for any producer or consumer, it will take precedence over per instance fetching.
+    #
     #   This prevents us in case someone uses `partition_key` from querying for the count with
-    #   each message. Instead we query once every 30 seconds at most if we have a valid partition
-    #   count or every 5 seconds in case we were not able to obtain number of partitions
+    #   each message. Instead we query at most once every 30 seconds at most if we have a valid
+    #   partition count or every 5 seconds in case we were not able to obtain number of partitions.
     def partition_count(topic)
       closed_producer_check(__method__)
 
-      @_partitions_count_cache.delete_if do |_, cached|
-        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
+      self.class.partitions_count_cache.get(topic) do
+        topic_metadata = nil
+
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        topic_metadata ? topic_metadata[:partition_count] : -1
       end
+    rescue Rdkafka::RdkafkaError => e
+      # If the topic does not exist, it will be created or if not allowed another error will be
+      # raised. We here return -1 so this can happen without early error happening on metadata
+      # discovery.
+      return -1 if e.code == :unknown_topic_or_part
 
-      @_partitions_count_cache[topic].last
+      raise(e)
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -325,7 +337,8 @@ module Rdkafka
       timestamp: nil,
       headers: nil,
       label: nil,
-      topic_config: EMPTY_HASH
+      topic_config: EMPTY_HASH,
+      partitioner: @partitioner
     )
       closed_producer_check(__method__)
 
@@ -357,10 +370,14 @@ module Rdkafka
 
         # Check if there are no overrides for the partitioner and use the default one only when
         # no per-topic is present.
-        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+        selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, partitioner_name) if partition_count.positive?
+        partition = Rdkafka::Bindings.partitioner(
+          topic_ref,
+          partition_key,
+          partition_count,
+          selected_partitioner) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.20.0.rc2"
-  LIBRDKAFKA_VERSION = "2.10.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "004b1cc2685d1d6d416b90b426a0a9d27327a214c6b807df6f9ea5887346ba3a"
+  VERSION = "0.20.0"
+  LIBRDKAFKA_VERSION = "2.8.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/lib/rdkafka.rb

@@ -42,6 +42,7 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/native_kafka"
+require "rdkafka/producer/partitions_count_cache"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/renovate.json

@@ -14,5 +14,79 @@
       ],
       "minimumReleaseAge": "7 days"
     }
+  ],
+  "customManagers": [
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly OPENSSL_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "openssl/openssl",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^OpenSSL_(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly CYRUS_SASL_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "cyrusimap/cyrus-sasl",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^cyrus-sasl-(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly ZLIB_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "madler/zlib",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly ZSTD_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "facebook/zstd",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly KRB5_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "krb5/krb5",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^krb5-(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly LIBRDKAFKA_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "confluentinc/librdkafka",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    }
   ]
 }