rdkafka 0.22.2 → 0.25.0

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,8 +12,8 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to or nil in case of reports with errors
-      # where topic was not reached.
+      # The name of the topic this message was produced to or nil in case delivery failed and we
+      #   we not able to get the topic reference
       #
       # @return [String, nil]
       attr_reader :topic_name
@@ -32,8 +32,12 @@ module Rdkafka
       # We do not remove the original `#topic_name` because of backwards compatibility
       alias topic topic_name
 
-      private
-
+      # @private
+      # @param partition [Integer] partition number
+      # @param offset [Integer] message offset
+      # @param topic_name [String, nil] topic name
+      # @param error [Integer, nil] error code if any
+      # @param label [Object, nil] user-defined label
       def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -17,9 +17,9 @@ module Rdkafka
     #
     # 2. Edge case handling
     #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
-    #    (30 seconds), the cache will still function correctly. When statistics updates don't
-    #    occur frequently enough, the cache entries will expire naturally, triggering a
-    #    blocking refresh when needed.
+    #    ({Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}ms), the cache will still function correctly.
+    #    When statistics updates don't occur frequently enough, the cache entries will expire
+    #    naturally, triggering a blocking refresh when needed.
     #
     # 3. User configuration awareness
     #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
@@ -46,22 +46,32 @@ module Rdkafka
     class PartitionsCountCache
       include Helpers::Time
 
-      # Default time-to-live for cached partition counts in seconds
-      #
-      # @note This default was chosen to balance freshness of metadata with performance
-      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
-      #   seconds a reasonable compromise.
-      DEFAULT_TTL = 30
-
       # Creates a new partition count cache
       #
-      # @param ttl [Integer] Time-to-live in seconds for cached values
-      def initialize(ttl = DEFAULT_TTL)
+      # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
+      #   Time-to-live in seconds for cached values. Will be removed in v1.0.0.
+      # @param ttl_ms [Integer, nil] Time-to-live in milliseconds for cached values.
+      #   Defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}.
+      def initialize(ttl = :not_provided, ttl_ms: :not_provided)
         @counts = {}
         @mutex_hash = {}
         # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
         @mutex_for_hash = Mutex.new
-        @ttl = ttl
+
+        # Determine which TTL value to use
+        if ttl != :not_provided && ttl_ms != :not_provided
+          warn "DEPRECATION WARNING: Both ttl and ttl_ms were provided to PartitionsCountCache. " \
+               "Using ttl_ms. The ttl parameter is deprecated and will be removed in v1.0.0."
+          @ttl_ms = ttl_ms
+        elsif ttl != :not_provided
+          warn "DEPRECATION WARNING: ttl (seconds) parameter for PartitionsCountCache is deprecated. " \
+               "Use ttl_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
+          @ttl_ms = (ttl * 1000).to_i
+        elsif ttl_ms == :not_provided
+          @ttl_ms = Defaults::PARTITIONS_COUNT_CACHE_TTL_MS
+        else
+          @ttl_ms = ttl_ms
+        end
       end
 
       # Reads partition count for a topic with automatic refresh when expired
@@ -209,7 +219,7 @@ module Rdkafka
       # @param timestamp [Float] Monotonic timestamp to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        monotonic_now - timestamp > @ttl
+        (monotonic_now - timestamp) * 1_000 > @ttl_ms
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -6,9 +6,6 @@ module Rdkafka
     include Helpers::Time
     include Helpers::OAuth
 
-    # Empty hash used as a default
-    EMPTY_HASH = {}.freeze
-
     # @private
     @@partitions_count_cache = PartitionsCountCache.new
 
@@ -17,10 +14,9 @@ module Rdkafka
     # then. Since the partitions count can only grow and should be same for all consumers and
     # producers, we can use a global cache as long as we ensure that updates only move up.
     #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
     # @note It is critical to remember, that not all users may have statistics callbacks enabled,
     #   hence we should not make assumption that this cache is always updated from the stats.
-    #
-    # @return [Rdkafka::Producer::PartitionsCountCache]
     def self.partitions_count_cache
       @@partitions_count_cache
     end
@@ -30,6 +26,9 @@ module Rdkafka
       @@partitions_count_cache = partitions_count_cache
     end
 
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
     private_constant :EMPTY_HASH
 
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
@@ -64,12 +63,13 @@ module Rdkafka
     end
 
     # Sets alternative set of configuration details that can be set per topic
-    # @note It is not allowed to re-set the same topic config twice because of the underlying
-    #   librdkafka caching
+    #
     # @param topic [String] The topic name
     # @param config [Hash] config we want to use per topic basis
     # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
     #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
     def set_topic_config(topic, config, config_hash)
       # Ensure lock on topic reference just in case
       @native_kafka.with_inner do |inner|
@@ -125,8 +125,7 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call] callable object to handle delivery reports
     # @return [nil]
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
@@ -168,7 +167,7 @@ module Rdkafka
     #   should be no other errors.
     #
     # @note For `timed_out` we do not raise an error to keep it backwards compatible
-    def flush(timeout_ms=5_000)
+    def flush(timeout_ms = Defaults::PRODUCER_FLUSH_TIMEOUT_MS)
       closed_producer_check(__method__)
 
       code = nil
@@ -208,11 +207,37 @@ module Rdkafka
       code.zero? || raise(Rdkafka::RdkafkaError.new(code))
 
       # Wait for the purge to affect everything
-      sleep(0.001) until flush(100)
+      sleep(Defaults::PRODUCER_PURGE_SLEEP_INTERVAL_MS / 1000.0) until flush(Defaults::PRODUCER_PURGE_FLUSH_TIMEOUT_MS)
 
       true
     end
 
+    # Returns the number of messages and requests waiting to be sent to the broker as well as
+    # delivery reports queued for the application.
+    #
+    # This provides visibility into the producer's internal queue depth, useful for:
+    # - Monitoring producer backpressure
+    # - Implementing custom flow control
+    # - Debugging message delivery issues
+    # - Graceful shutdown logic (wait until queue is empty)
+    #
+    # @return [Integer] the number of messages in the queue
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
+    #
+    # @example
+    #   producer.queue_size #=> 42
+    def queue_size
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_outq_len(inner)
+      end
+    end
+
+    alias queue_length queue_size
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
@@ -237,13 +262,13 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        topic_metadata ? topic_metadata[:partition_count] : -1
+        topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       end
     rescue Rdkafka::RdkafkaError => e
       # If the topic does not exist, it will be created or if not allowed another error will be
-      # raised. We here return -1 so this can happen without early error happening on metadata
-      # discovery.
-      return -1 if e.code == :unknown_topic_or_part
+      # raised. We here return RD_KAFKA_PARTITION_UA so this can happen without early error
+      # happening on metadata discovery.
+      return Rdkafka::Bindings::RD_KAFKA_PARTITION_UA if e.code == :unknown_topic_or_part
 
       raise(e)
     end
@@ -254,14 +279,15 @@ module Rdkafka
     # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
-    # @param payload [String,nil] The message's payload
-    # @param key [String, nil] The message's key
-    # @param partition [Integer,nil] Optional partition to produce to
+    # @param payload [String, nil]
+    # @param key [String, nil]
+    # @param partition [Integer, nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
-    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
+    # @param timestamp [Time, Integer, nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash{String => String, Array<String>}] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
+    # @param partitioner [String] name of the partitioner to use
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
@@ -318,9 +344,9 @@ module Rdkafka
           selected_partitioner) if partition_count.positive?
       end
 
-      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # If partition is nil, use RD_KAFKA_PARTITION_UA to let librdafka set the partition randomly or
       # based on the key when present.
-      partition ||= -1
+      partition ||= Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.
@@ -338,9 +364,9 @@ module Rdkafka
       delivery_handle.label = label
       delivery_handle.topic = topic
       delivery_handle[:pending] = true
-      delivery_handle[:response] = -1
-      delivery_handle[:partition] = -1
-      delivery_handle[:offset] = -1
+      delivery_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:partition] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:offset] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       DeliveryHandle.register(delivery_handle)
 
       args = [
@@ -387,7 +413,7 @@ module Rdkafka
       end
 
       # Raise error if the produce call was not successful
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
         raise RdkafkaError.new(response)
       end

data/lib/rdkafka/version.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.22.2"
-  LIBRDKAFKA_VERSION = "2.8.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
+  # Current rdkafka-ruby gem version
+  VERSION = "0.25.0"
+  # Target librdkafka version to be used
+  LIBRDKAFKA_VERSION = "2.12.1"
+  # SHA256 hash of the librdkafka source tarball for verification
+  LIBRDKAFKA_SOURCE_SHA256 = "ec103fa05cb0f251e375f6ea0b6112cfc9d0acd977dc5b69fdc54242ba38a16f"
 end

data/lib/rdkafka.rb

@@ -8,6 +8,7 @@ require "json"
 require "rdkafka/version"
 require "rdkafka/helpers/time"
 require "rdkafka/helpers/oauth"
+require "rdkafka/defaults"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"

data/rdkafka.gemspec

@@ -13,17 +13,31 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 3.1'
+  gem.required_ruby_version = '>= 3.2'
+
+  files = `git ls-files`.split($\)
+  files = files.reject do |file|
+    next true if file.start_with?('.')
+    next true if file.start_with?('spec/')
+    next true if file.start_with?('ext/README.md')
+
+    false
+  end
 
   if ENV['RUBY_PLATFORM']
     gem.platform = ENV['RUBY_PLATFORM']
-    gem.files = `git ls-files`.split($\)
 
     # Do not include the source code for librdkafka as it should be precompiled already per
     # platform. Same applies to any possible patches.
-    gem.files = gem.files.reject do |file|
-      file.match?(%r{^dist/librdkafka-.*\.tar\.gz$}) ||
-      file.match?(%r{^dist/patches/})
+    # Do not include github actions details in RubyGems releases
+    gem.files = files.reject do |file|
+      next true if file.start_with?('dist/')
+      next true if file.start_with?('ext/build_')
+      next true if file.start_with?('ext/ci_')
+      next true if file.start_with?('ext/Rakefile')
+      next true if file.start_with?('ext/generate-')
+
+      false
     end
 
     # Add the compiled extensions that exist (not in git)
@@ -36,21 +50,29 @@ Gem::Specification.new do |gem|
     end
   else
     gem.platform = Gem::Platform::RUBY
-    gem.files = `git ls-files`.split($\)
+
+    # Do not include code used for building native extensions
+    # Do not include github actions details in RubyGems releases
+    gem.files = files.reject do |file|
+      next true if file.start_with?('ext/build_')
+      next true if file.start_with?('ext/ci_')
+      next true if file.start_with?('ext/generate-')
+      next false unless file.start_with?('dist/')
+      next false if file.start_with?('dist/patches')
+      next false if file.start_with?('dist/librdkafka-')
+
+      true
+    end
+
     gem.extensions = %w(ext/Rakefile)
   end
 
-  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'ffi', '~> 1.17.1'
+  gem.add_dependency 'json', '> 2.0'
   gem.add_dependency 'logger'
   gem.add_dependency 'mini_portile2', '~> 2.6'
   gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'ostruct'
-  gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'simplecov'
-
   gem.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',

data/renovate.json

@@ -3,23 +3,16 @@
   "extends": [
     "config:recommended"
   ],
+  "minimumReleaseAge": "7 days",
   "github-actions": {
     "enabled": true,
     "pinDigests": true
   },
-  "packageRules": [
-    {
-      "matchManagers": [
-        "github-actions"
-      ],
-      "minimumReleaseAge": "7 days"
-    }
-  ],
   "customManagers": [
     {
       "customType": "regex",
       "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
+        "/ext/build_common.sh/"
       ],
       "matchStrings": [
         "readonly OPENSSL_VERSION=\"(?<currentValue>.*)\""
@@ -31,7 +24,7 @@
     {
       "customType": "regex",
       "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
+        "/ext/build_common.sh/"
       ],
       "matchStrings": [
         "readonly CYRUS_SASL_VERSION=\"(?<currentValue>.*)\""
@@ -43,7 +36,7 @@
     {
       "customType": "regex",
       "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
+        "/ext/build_common.sh/"
       ],
       "matchStrings": [
         "readonly ZLIB_VERSION=\"(?<currentValue>.*)\""
@@ -55,7 +48,7 @@
     {
       "customType": "regex",
       "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
+        "/ext/build_common.sh/"
       ],
       "matchStrings": [
         "readonly ZSTD_VERSION=\"(?<currentValue>.*)\""
@@ -67,7 +60,7 @@
     {
       "customType": "regex",
       "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
+        "/ext/build_common.sh/"
       ],
       "matchStrings": [
         "readonly KRB5_VERSION=\"(?<currentValue>.*)\""
@@ -75,18 +68,6 @@
       "depNameTemplate": "krb5/krb5",
       "datasourceTemplate": "github-releases",
       "extractVersionTemplate": "^krb5-(?<version>.*)$"
-    },
-    {
-      "customType": "regex",
-      "managerFilePatterns": [
-        "/^ext/build_common\\.sh$/"
-      ],
-      "matchStrings": [
-        "readonly LIBRDKAFKA_VERSION=\"(?<currentValue>.*)\""
-      ],
-      "depNameTemplate": "confluentinc/librdkafka",
-      "datasourceTemplate": "github-releases",
-      "extractVersionTemplate": "^v(?<version>.*)$"
     }
   ]
 }