rdkafka 0.24.2 → 0.25.0

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -17,9 +17,9 @@ module Rdkafka
     #
     # 2. Edge case handling
     #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
-    #    (30 seconds), the cache will still function correctly. When statistics updates don't
-    #    occur frequently enough, the cache entries will expire naturally, triggering a
-    #    blocking refresh when needed.
+    #    ({Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}ms), the cache will still function correctly.
+    #    When statistics updates don't occur frequently enough, the cache entries will expire
+    #    naturally, triggering a blocking refresh when needed.
     #
     # 3. User configuration awareness
     #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
@@ -46,22 +46,32 @@ module Rdkafka
     class PartitionsCountCache
       include Helpers::Time
 
-      # Default time-to-live for cached partition counts in seconds
-      #
-      # @note This default was chosen to balance freshness of metadata with performance
-      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
-      #   seconds a reasonable compromise.
-      DEFAULT_TTL = 30
-
       # Creates a new partition count cache
       #
-      # @param ttl [Integer] Time-to-live in seconds for cached values
-      def initialize(ttl = DEFAULT_TTL)
+      # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
+      #   Time-to-live in seconds for cached values. Will be removed in v1.0.0.
+      # @param ttl_ms [Integer, nil] Time-to-live in milliseconds for cached values.
+      #   Defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}.
+      def initialize(ttl = :not_provided, ttl_ms: :not_provided)
         @counts = {}
         @mutex_hash = {}
         # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
         @mutex_for_hash = Mutex.new
-        @ttl = ttl
+
+        # Determine which TTL value to use
+        if ttl != :not_provided && ttl_ms != :not_provided
+          warn "DEPRECATION WARNING: Both ttl and ttl_ms were provided to PartitionsCountCache. " \
+               "Using ttl_ms. The ttl parameter is deprecated and will be removed in v1.0.0."
+          @ttl_ms = ttl_ms
+        elsif ttl != :not_provided
+          warn "DEPRECATION WARNING: ttl (seconds) parameter for PartitionsCountCache is deprecated. " \
+               "Use ttl_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
+          @ttl_ms = (ttl * 1000).to_i
+        elsif ttl_ms == :not_provided
+          @ttl_ms = Defaults::PARTITIONS_COUNT_CACHE_TTL_MS
+        else
+          @ttl_ms = ttl_ms
+        end
       end
 
       # Reads partition count for a topic with automatic refresh when expired
@@ -209,7 +219,7 @@ module Rdkafka
       # @param timestamp [Float] Monotonic timestamp to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        monotonic_now - timestamp > @ttl
+        (monotonic_now - timestamp) * 1_000 > @ttl_ms
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -14,10 +14,9 @@ module Rdkafka
     # then. Since the partitions count can only grow and should be same for all consumers and
     # producers, we can use a global cache as long as we ensure that updates only move up.
     #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
     # @note It is critical to remember, that not all users may have statistics callbacks enabled,
     #   hence we should not make assumption that this cache is always updated from the stats.
-    #
-    # @return [Rdkafka::Producer::PartitionsCountCache]
     def self.partitions_count_cache
       @@partitions_count_cache
     end
@@ -64,12 +63,13 @@ module Rdkafka
     end
 
     # Sets alternative set of configuration details that can be set per topic
-    # @note It is not allowed to re-set the same topic config twice because of the underlying
-    #   librdkafka caching
+    #
     # @param topic [String] The topic name
     # @param config [Hash] config we want to use per topic basis
     # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
     #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
     def set_topic_config(topic, config, config_hash)
       # Ensure lock on topic reference just in case
       @native_kafka.with_inner do |inner|
@@ -125,8 +125,7 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call] callable object to handle delivery reports
     # @return [nil]
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
@@ -168,7 +167,7 @@ module Rdkafka
     #   should be no other errors.
     #
     # @note For `timed_out` we do not raise an error to keep it backwards compatible
-    def flush(timeout_ms=5_000)
+    def flush(timeout_ms = Defaults::PRODUCER_FLUSH_TIMEOUT_MS)
       closed_producer_check(__method__)
 
       code = nil
@@ -208,11 +207,37 @@ module Rdkafka
       code.zero? || raise(Rdkafka::RdkafkaError.new(code))
 
       # Wait for the purge to affect everything
-      sleep(0.001) until flush(100)
+      sleep(Defaults::PRODUCER_PURGE_SLEEP_INTERVAL_MS / 1000.0) until flush(Defaults::PRODUCER_PURGE_FLUSH_TIMEOUT_MS)
 
       true
     end
 
+    # Returns the number of messages and requests waiting to be sent to the broker as well as
+    # delivery reports queued for the application.
+    #
+    # This provides visibility into the producer's internal queue depth, useful for:
+    # - Monitoring producer backpressure
+    # - Implementing custom flow control
+    # - Debugging message delivery issues
+    # - Graceful shutdown logic (wait until queue is empty)
+    #
+    # @return [Integer] the number of messages in the queue
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
+    #
+    # @example
+    #   producer.queue_size #=> 42
+    def queue_size
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_outq_len(inner)
+      end
+    end
+
+    alias queue_length queue_size
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
@@ -237,13 +262,13 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        topic_metadata ? topic_metadata[:partition_count] : -1
+        topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       end
     rescue Rdkafka::RdkafkaError => e
       # If the topic does not exist, it will be created or if not allowed another error will be
-      # raised. We here return -1 so this can happen without early error happening on metadata
-      # discovery.
-      return -1 if e.code == :unknown_topic_or_part
+      # raised. We here return RD_KAFKA_PARTITION_UA so this can happen without early error
+      # happening on metadata discovery.
+      return Rdkafka::Bindings::RD_KAFKA_PARTITION_UA if e.code == :unknown_topic_or_part
 
       raise(e)
     end
@@ -254,14 +279,15 @@ module Rdkafka
     # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
-    # @param payload [String,nil] The message's payload
-    # @param key [String, nil] The message's key
-    # @param partition [Integer,nil] Optional partition to produce to
+    # @param payload [String, nil]
+    # @param key [String, nil]
+    # @param partition [Integer, nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
-    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
+    # @param timestamp [Time, Integer, nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash{String => String, Array<String>}] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
+    # @param partitioner [String] name of the partitioner to use
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
@@ -318,9 +344,9 @@ module Rdkafka
           selected_partitioner) if partition_count.positive?
       end
 
-      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # If partition is nil, use RD_KAFKA_PARTITION_UA to let librdafka set the partition randomly or
       # based on the key when present.
-      partition ||= -1
+      partition ||= Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.
@@ -338,9 +364,9 @@ module Rdkafka
       delivery_handle.label = label
       delivery_handle.topic = topic
       delivery_handle[:pending] = true
-      delivery_handle[:response] = -1
-      delivery_handle[:partition] = -1
-      delivery_handle[:offset] = -1
+      delivery_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:partition] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:offset] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       DeliveryHandle.register(delivery_handle)
 
       args = [
@@ -387,7 +413,7 @@ module Rdkafka
       end
 
       # Raise error if the produce call was not successful
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
         raise RdkafkaError.new(response)
       end

data/lib/rdkafka/version.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.24.2"
-  LIBRDKAFKA_VERSION = "2.11.1"
-  LIBRDKAFKA_SOURCE_SHA256 = "a2c87186b081e2705bb7d5338d5a01bc88d43273619b372ccb7bb0d264d0ca9f"
+  # Current rdkafka-ruby gem version
+  VERSION = "0.25.0"
+  # Target librdkafka version to be used
+  LIBRDKAFKA_VERSION = "2.12.1"
+  # SHA256 hash of the librdkafka source tarball for verification
+  LIBRDKAFKA_SOURCE_SHA256 = "ec103fa05cb0f251e375f6ea0b6112cfc9d0acd977dc5b69fdc54242ba38a16f"
 end

data/lib/rdkafka.rb

@@ -8,6 +8,7 @@ require "json"
 require "rdkafka/version"
 require "rdkafka/helpers/time"
 require "rdkafka/helpers/oauth"
+require "rdkafka/defaults"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"

data/rdkafka.gemspec

@@ -73,13 +73,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'mini_portile2', '~> 2.6'
   gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'ostruct'
-  gem.add_development_dependency 'pry'
-  gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'simplecov'
-  gem.add_development_dependency 'warning'
-
   gem.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',

data/renovate.json

@@ -3,18 +3,11 @@
   "extends": [
     "config:recommended"
   ],
+  "minimumReleaseAge": "7 days",
   "github-actions": {
     "enabled": true,
     "pinDigests": true
   },
-  "packageRules": [
-    {
-      "matchManagers": [
-        "github-actions"
-      ],
-      "minimumReleaseAge": "7 days"
-    }
-  ],
   "customManagers": [
     {
       "customType": "regex",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.24.2
+  version: 0.25.0
 platform: ruby
 authors:
 - Thijs Cadier
@@ -80,90 +80,6 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '12'
-- !ruby/object:Gem::Dependency
-  name: ostruct
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: warning
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - contact@karafka.io
@@ -178,7 +94,7 @@ files:
 - README.md
 - Rakefile
 - bin/verify_kafka_warnings
-- dist/librdkafka-2.11.1.tar.gz
+- dist/librdkafka-2.12.1.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose-ssl.yml
 - docker-compose.yml
@@ -215,6 +131,7 @@ files:
 - lib/rdkafka/consumer/message.rb
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb
+- lib/rdkafka/defaults.rb
 - lib/rdkafka/error.rb
 - lib/rdkafka/helpers/oauth.rb
 - lib/rdkafka/helpers/time.rb
@@ -251,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+