karafka-rdkafka 0.19.0 → 0.19.2.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a1e9fa0ca2b5dd14aed77c653fc4d154bb566113cac11c04d51cccc4e1d9fb7
-  data.tar.gz: c513b0b82bdae4d9a16251a0abcd4b73a79ba21d2833814bccddcd8f6446151a
+  metadata.gz: d274be74f9ce13d34e8b4151809a030c7965328f39929a7dd9edcda480c38b82
+  data.tar.gz: 2a31bab8e00b6b37479e65d717ba797bd1fa5588f6f22304396bd0f760e08782
 SHA512:
-  metadata.gz: 53bee0b1c513f6947ca657ca3836df05e6de31ba441aa6d85d71f523c28cad7b996ec14fae798b50bbaafb09eb00367bbc4298da5a926e3fae61cc94cb5179bb
-  data.tar.gz: 51e903bb75f34fa7f49a8ebd6cdff193b2ee916fd0f4823145aae6ee219bf45e51ce538f3c9c3e4de9ba58c659fa4ecaf494768df1b83f998c9d2509cda58074
+  metadata.gz: 21b2985059a41f125539f3921b81c2768da2b48a86e1370e81925641dfd353fa36c9f8f0f10cb5f60972eb6ad544b210e566f0f2f9f7fcc27680b64fe6ddff20
+  data.tar.gz: 26c74433fa18d9ffdd1e516bcaa9a8fa3c17cbbf1afe7efb2f749a8219d2ce5540adf440635ce1bf104d1e7776ccd01a598b2ff3e1d2570eb29de9e2b079fe08

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci.yml

@@ -6,9 +6,14 @@ concurrency:
 
 on:
   pull_request:
+    branches: [ main, master ]
   push:
+    branches: [ main, master ]
   schedule:
-    - cron:  '0 1 * * *'
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
 
 env:
   BUNDLE_RETRY: 6
@@ -26,20 +31,27 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
+          - 'jruby-10.0'
         include:
           - ruby: '3.4'
             coverage: 'true'
+          - ruby: 'jruby-10.0'
+            continue-on-error: true
+
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker compose
+      - name: Start Kafka with Docker Compose
         run: |
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@eaecf785f6a34567a6d97f686bbb7bccc1ac1e5c # v1.237.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -47,15 +59,14 @@ jobs:
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
-
+        continue-on-error: ${{ matrix.continue-on-error || false }}  # Use the matrix value if present
         run: |
           set -e
-          bundle install --path vendor/bundle
+          bundle install --jobs 4 --retry 3
           cd ext && bundle exec rake
           cd ..
           bundle exec rspec
 
-
   macos_build:
     timeout-minutes: 30
     runs-on: macos-latest
@@ -67,17 +78,22 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
+          - 'jruby-9.4'
+        include:
+          - ruby: 'jruby-10.0'
+            continue-on-error: true
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@eaecf785f6a34567a6d97f686bbb7bccc1ac1e5c # v1.237.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: false
 
       - name: Build rdkafka-ruby
+        continue-on-error: ${{ matrix.continue-on-error || false }}
         run: |
           set -e
-          bundle install --path vendor/bundle
+          bundle install --jobs 4 --retry 3
           cd ext && bundle exec rake

data/.github/workflows/verify-action-pins.yml

@@ -0,0 +1,16 @@
+name: Verify Action Pins
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check SHA pins
+        run: |
+          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
+            echo "::error::Actions should use SHA pins, not tags or branch names"
+            exit 1
+          fi

data/.ruby-version

@@ -1 +1 @@
-3.4.1
+3.4.3

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Rdkafka Changelog
 
+## 0.19.2 (Unreleased)
+- [Enhancement] Replace TTL-based partition count cache with a global cache that reuses `librdkafka` statistics data when possible.
+- [Enhancement] Roll out experimental jruby support.
+
+## 0.19.1 (2025-04-07)
+- [Enhancement] Support producing and consuming of headers with mulitple values (KIP-82).
+- [Enhancement] Allow native Kafka customization poll time.
+
 ## 0.19.0 (2025-01-20)
 - **[Breaking]** Deprecate and remove `#each_batch` due to data consistency concerns.
 - [Enhancement] Bump librdkafka to 2.8.0

data/README.md

@@ -163,14 +163,16 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
-| 0.19.0 (2025-01-20) | 2.8.0 (2025-01-07) | yes |
-| 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18) | yes |
-| 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |
-| 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10) | yes |
-| 0.16.0 (2024-06-13) | 2.4.0 (2024-05-07) | no  |
-| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) | no  |
-| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) | no  |
-| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) | no  |
-| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16) | no  |
-| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18) | no  |
-| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20) | no  |
+| 0.19.2 (Unreleased) | 2.8.0 (2025-01-07)  | yes |
+| 0.19.1 (2025-04-07) | 2.8.0 (2025-01-07)  | yes |
+| 0.19.0 (2025-01-20) | 2.8.0 (2025-01-07)  | yes |
+| 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18)  | yes |
+| 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02)  | yes |
+| 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10)  | yes |
+| 0.16.0 (2024-06-13) | 2.4.0 (2024-05-07)  | no  |
+| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25)  | no  |
+| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12)  | no  |
+| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20)  | no  |
+| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16)  | no  |
+| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18)  | no  |
+| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20)  | no  |

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.1
 
     ports:
       - 9092:9092

data/lib/rdkafka/bindings.rb

@@ -35,6 +35,8 @@ module Rdkafka
     RD_KAFKA_OFFSET_STORED    = -1000
     RD_KAFKA_OFFSET_INVALID   = -1001
 
+    EMPTY_HASH = {}.freeze
+
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -215,9 +217,31 @@ module Rdkafka
     StatsCallback = FFI::Function.new(
       :int, [:pointer, :string, :int, :pointer]
     ) do |_client_ptr, json, _json_len, _opaque|
-      # Pass the stats hash to callback in config
       if Rdkafka::Config.statistics_callback
         stats = JSON.parse(json)
+
+        # If user requested statistics callbacks, we can use the statistics data to get the
+        # partitions count for each topic when this data is published. That way we do not have
+        # to query this information when user is using `partition_key`. This takes around 0.02ms
+        # every statistics interval period (most likely every 5 seconds) and saves us from making
+        # any queries to the cluster for the partition count.
+        #
+        # One edge case is if user would set the `statistics.interval.ms` much higher than the
+        # default current partition count refresh (30 seconds). This is taken care of as the lack
+        # of reporting to the partitions cache will cause cache expire and blocking refresh.
+        #
+        # If user sets `topic.metadata.refresh.interval.ms` too high this is on the user.
+        #
+        # Since this cache is shared, having few consumers and/or producers in one process will
+        # automatically improve the querying times even with low refresh times.
+        (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
+          partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
+
+          next unless partitions_count.positive?
+
+          Rdkafka::Producer.partitions_count_cache.set(topic_name, partitions_count)
+        end
+
         Rdkafka::Config.statistics_callback.call(stats)
       end
 

data/lib/rdkafka/config.rb

@@ -233,11 +233,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer(native_kafka_auto_start: true)
+    def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -254,7 +255,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         ),
         partitioner_name
       ).tap do |producer|
@@ -266,11 +268,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin(native_kafka_auto_start: true)
+    def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
@@ -282,7 +285,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         )
       )
     end

data/lib/rdkafka/consumer/headers.rb

@@ -7,11 +7,13 @@ module Rdkafka
       EMPTY_HEADERS = {}.freeze
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      # where each key maps to either a String (single value) or Array<String> (multiple values)
+      # to support duplicate headers per KIP-82
       #
       # @private
       #
       # @param [Rdkafka::Bindings::Message] native_message
-      # @return [Hash<String, String>] headers Hash for the native_message
+      # @return [Hash<String, String|Array<String>>] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -53,10 +55,19 @@ module Rdkafka
           size = size_ptr[:value]
 
           value_ptr = value_ptrptr.read_pointer
-
           value = value_ptr.read_string(size)
 
-          headers[name] = value
+          if headers.key?(name)
+            # If we've seen this header before, convert to array if needed and append
+            if headers[name].is_a?(Array)
+              headers[name] << value
+            else
+              headers[name] = [headers[name], value]
+            end
+          else
+            # First occurrence - store as single value
+            headers[name] = value
+          end
 
           idx += 1
         end

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -30,6 +30,8 @@ module Rdkafka
 
       @run_polling_thread = run_polling_thread
 
+      @timeout_ms = timeout_ms
+
       start if auto_start
 
       @closing = false
@@ -50,7 +52,7 @@ module Rdkafka
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
-                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+                Rdkafka::Bindings.rd_kafka_poll(@inner, @timeout_ms)
               end
 
               # Exit thread if closing and the poll queue is empty

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Producer
+    # Caching mechanism for Kafka topic partition counts to avoid frequent cluster queries
+    #
+    # This cache is designed to optimize the process of obtaining partition counts for topics.
+    # It uses several strategies to minimize Kafka cluster queries:
+    #
+    # @note Design considerations:
+    #
+    # 1. Statistics-based updates
+    #    When statistics callbacks are enabled (via `statistics.interval.ms`), we leverage
+    #    this data to proactively update the partition counts cache. This approach costs
+    #    approximately 0.02ms of processing time during each statistics interval (typically
+    #    every 5 seconds) but eliminates the need for explicit blocking metadata queries.
+    #
+    # 2. Edge case handling
+    #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
+    #    (30 seconds), the cache will still function correctly. When statistics updates don't
+    #    occur frequently enough, the cache entries will expire naturally, triggering a
+    #    blocking refresh when needed.
+    #
+    # 3. User configuration awareness
+    #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
+    #    set very high, the responsibility for potentially stale data falls on the user. This
+    #    is an explicit design choice to honor user configuration preferences and align with
+    #    librdkafka settings.
+    #
+    # 4. Process-wide efficiency
+    #    Since this cache is shared across all Rdkafka producers and consumers within a process,
+    #    having multiple clients improves overall efficiency. Each client contributes to keeping
+    #    the cache updated, benefiting all other clients.
+    #
+    # 5. Thread-safety approach
+    #    The implementation uses fine-grained locking with per-topic mutexes to minimize
+    #    contention in multi-threaded environments while ensuring data consistency.
+    #
+    # 6. Topic recreation handling
+    #    If a topic is deleted and recreated with fewer partitions, the cache will continue to
+    #    report the higher count until either the TTL expires or the process is restarted. This
+    #    design choice simplifies the implementation while relying on librdkafka's error handling
+    #    for edge cases. In production environments, topic recreation with different partition
+    #    counts is typically accompanied by application restarts to handle structural changes.
+    #    This also aligns with the previous cache implementation.
+    class PartitionsCountCache
+      include Helpers::Time
+
+      # Default time-to-live for cached partition counts in seconds
+      #
+      # @note This default was chosen to balance freshness of metadata with performance
+      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
+      #   seconds a reasonable compromise.
+      DEFAULT_TTL = 30
+
+      # Creates a new partition count cache
+      #
+      # @param ttl [Integer] Time-to-live in seconds for cached values
+      def initialize(ttl = DEFAULT_TTL)
+        @counts = {}
+        @mutex_hash = {}
+        # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
+        @mutex_for_hash = Mutex.new
+        @ttl = ttl
+      end
+
+      # Reads partition count for a topic with automatic refresh when expired
+      #
+      # This method will return the cached partition count if available and not expired.
+      # If the value is expired or not available, it will execute the provided block
+      # to fetch the current value from Kafka.
+      #
+      # @param topic [String] Kafka topic name
+      # @yield Block that returns the current partition count when cache needs refreshing
+      # @yieldreturn [Integer] Current partition count retrieved from Kafka
+      # @return [Integer] Partition count for the topic
+      #
+      # @note The implementation prioritizes read performance over write consistency
+      #   since partition counts typically only increase during normal operation.
+      def get(topic)
+        current_info = @counts[topic]
+
+        if current_info.nil? || expired?(current_info[0])
+          new_count = yield
+
+          if current_info.nil?
+            # No existing data, create a new entry with mutex
+            set(topic, new_count)
+
+            return new_count
+          else
+            current_count = current_info[1]
+
+            if new_count > current_count
+              # Higher value needs mutex to update both timestamp and count
+              set(topic, new_count)
+
+              return new_count
+            else
+              # Same or lower value, just update timestamp without mutex
+              refresh_timestamp(topic)
+
+              return current_count
+            end
+          end
+        end
+
+        current_info[1]
+      end
+
+      # Update partition count for a topic when needed
+      #
+      # This method updates the partition count for a topic in the cache.
+      # It uses a mutex to ensure thread-safety during updates.
+      #
+      # @param topic [String] Kafka topic name
+      # @param new_count [Integer] New partition count value
+      #
+      # @note We prioritize higher partition counts and only accept them when using
+      #   a mutex to ensure consistency. This design decision is based on the fact that
+      #   partition counts in Kafka only increase during normal operation.
+      def set(topic, new_count)
+        # First check outside mutex to avoid unnecessary locking
+        current_info = @counts[topic]
+
+        # For lower values, we don't update count but might need to refresh timestamp
+        if current_info && new_count < current_info[1]
+          refresh_timestamp(topic)
+
+          return
+        end
+
+        # Only lock the specific topic mutex
+        mutex_for(topic).synchronize do
+          # Check again inside the lock as another thread might have updated
+          current_info = @counts[topic]
+
+          if current_info.nil?
+            # Create new entry
+            @counts[topic] = [monotonic_now, new_count]
+          else
+            current_count = current_info[1]
+
+            if new_count > current_count
+              # Update to higher count value
+              current_info[0] = monotonic_now
+              current_info[1] = new_count
+            else
+              # Same or lower count, update timestamp only
+              current_info[0] = monotonic_now
+            end
+          end
+        end
+      end
+
+      # @return [Hash] hash with ttls and partitions counts array
+      def to_h
+        @counts
+      end
+
+      private
+
+      # Get or create a mutex for a specific topic
+      #
+      # This method ensures that each topic has its own mutex,
+      # allowing operations on different topics to proceed in parallel.
+      #
+      # @param topic [String] Kafka topic name
+      # @return [Mutex] Mutex for the specified topic
+      #
+      # @note We use a separate mutex (@mutex_for_hash) to protect the creation
+      #   of new topic mutexes. This pattern allows fine-grained locking while
+      #   maintaining thread-safety.
+      def mutex_for(topic)
+        mutex = @mutex_hash[topic]
+
+        return mutex if mutex
+
+        # Use a separate mutex to protect the creation of new topic mutexes
+        @mutex_for_hash.synchronize do
+          # Check again in case another thread created it
+          @mutex_hash[topic] ||= Mutex.new
+        end
+
+        @mutex_hash[topic]
+      end
+
+      # Update the timestamp without acquiring the mutex
+      #
+      # This is an optimization that allows refreshing the TTL of existing entries
+      # without the overhead of mutex acquisition.
+      #
+      # @param topic [String] Kafka topic name
+      #
+      # @note This method is safe for refreshing existing data regardless of count
+      #   because it only updates the timestamp, which doesn't affect the correctness
+      #   of concurrent operations.
+      def refresh_timestamp(topic)
+        current_info = @counts[topic]
+
+        return unless current_info
+
+        # Update the timestamp in-place
+        current_info[0] = monotonic_now
+      end
+
+      # Check if a timestamp has expired based on the TTL
+      #
+      # @param timestamp [Float] Monotonic timestamp to check
+      # @return [Boolean] true if expired, false otherwise
+      def expired?(timestamp)
+        monotonic_now - timestamp > @ttl
+      end
+    end
+  end
+end