karafka-rdkafka 0.19.5 → 0.20.0.rc1

metadata

@@ -1,13 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.19.5
+  version: 0.20.0.rc1
 platform: ruby
 authors:
 - Thijs Cadier
 - Maciej Mensfeld
 bindir: bin
-cert_chain: []
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+  YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
+  dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
+  -----END CERTIFICATE-----
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -147,7 +174,6 @@ files:
 - ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
-- ".github/workflows/push.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -160,7 +186,8 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- dist/librdkafka-2.8.0.tar.gz
+- certs/cert.pem
+- dist/librdkafka-2.10.0.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose.yml
 - ext/README.md
@@ -206,7 +233,6 @@ files:
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
-- lib/rdkafka/producer/partitions_count_cache.rb
 - lib/rdkafka/version.rb
 - renovate.json
 - spec/rdkafka/abstract_handle_spec.rb
@@ -234,8 +260,6 @@ files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
@@ -243,7 +267,7 @@ licenses:
 metadata:
   funding_uri: https://karafka.io/#become-pro
   homepage_uri: https://karafka.io
-  changelog_uri: https://karafka.io/docs/Changelog-Karafka-Rdkafka/
+  changelog_uri: https://github.com/karafka/karafka-rdkafka/blob/master/CHANGELOG.md
   bug_tracker_uri: https://github.com/karafka/karafka-rdkafka/issues
   source_code_uri: https://github.com/karafka/karafka-rdkafka
   documentation_uri: https://karafka.io/docs
@@ -293,7 +317,5 @@ test_files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb

data/.github/workflows/push.yml

@@ -1,37 +0,0 @@
-name: Push Gem
-
-on:
-  push:
-    tags:
-      - v*
-
-permissions:
-  contents: read
-
-jobs:
-  push:
-    if: github.repository_owner == 'karafka'
-    runs-on: ubuntu-latest
-    environment: deployment
-
-    permissions:
-      contents: write
-      id-token: write
-
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
-        with:
-          bundler-cache: false
-
-      - name: Build rdkafka-ruby
-        run: |
-          set -e
-          bundle install --jobs 4 --retry 3
-          cd ext && bundle exec rake
-
-      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-module Rdkafka
-  class Producer
-    # Caching mechanism for Kafka topic partition counts to avoid frequent cluster queries
-    #
-    # This cache is designed to optimize the process of obtaining partition counts for topics.
-    # It uses several strategies to minimize Kafka cluster queries:
-    #
-    # @note Design considerations:
-    #
-    # 1. Statistics-based updates
-    #    When statistics callbacks are enabled (via `statistics.interval.ms`), we leverage
-    #    this data to proactively update the partition counts cache. This approach costs
-    #    approximately 0.02ms of processing time during each statistics interval (typically
-    #    every 5 seconds) but eliminates the need for explicit blocking metadata queries.
-    #
-    # 2. Edge case handling
-    #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
-    #    (30 seconds), the cache will still function correctly. When statistics updates don't
-    #    occur frequently enough, the cache entries will expire naturally, triggering a
-    #    blocking refresh when needed.
-    #
-    # 3. User configuration awareness
-    #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
-    #    set very high, the responsibility for potentially stale data falls on the user. This
-    #    is an explicit design choice to honor user configuration preferences and align with
-    #    librdkafka settings.
-    #
-    # 4. Process-wide efficiency
-    #    Since this cache is shared across all Rdkafka producers and consumers within a process,
-    #    having multiple clients improves overall efficiency. Each client contributes to keeping
-    #    the cache updated, benefiting all other clients.
-    #
-    # 5. Thread-safety approach
-    #    The implementation uses fine-grained locking with per-topic mutexes to minimize
-    #    contention in multi-threaded environments while ensuring data consistency.
-    #
-    # 6. Topic recreation handling
-    #    If a topic is deleted and recreated with fewer partitions, the cache will continue to
-    #    report the higher count until either the TTL expires or the process is restarted. This
-    #    design choice simplifies the implementation while relying on librdkafka's error handling
-    #    for edge cases. In production environments, topic recreation with different partition
-    #    counts is typically accompanied by application restarts to handle structural changes.
-    #    This also aligns with the previous cache implementation.
-    class PartitionsCountCache
-      include Helpers::Time
-
-      # Default time-to-live for cached partition counts in seconds
-      #
-      # @note This default was chosen to balance freshness of metadata with performance
-      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
-      #   seconds a reasonable compromise.
-      DEFAULT_TTL = 30
-
-      # Creates a new partition count cache
-      #
-      # @param ttl [Integer] Time-to-live in seconds for cached values
-      def initialize(ttl = DEFAULT_TTL)
-        @counts = {}
-        @mutex_hash = {}
-        # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
-        @mutex_for_hash = Mutex.new
-        @ttl = ttl
-      end
-
-      # Reads partition count for a topic with automatic refresh when expired
-      #
-      # This method will return the cached partition count if available and not expired.
-      # If the value is expired or not available, it will execute the provided block
-      # to fetch the current value from Kafka.
-      #
-      # @param topic [String] Kafka topic name
-      # @yield Block that returns the current partition count when cache needs refreshing
-      # @yieldreturn [Integer] Current partition count retrieved from Kafka
-      # @return [Integer] Partition count for the topic
-      #
-      # @note The implementation prioritizes read performance over write consistency
-      #   since partition counts typically only increase during normal operation.
-      def get(topic)
-        current_info = @counts[topic]
-
-        if current_info.nil? || expired?(current_info[0])
-          new_count = yield
-
-          if current_info.nil?
-            # No existing data, create a new entry with mutex
-            set(topic, new_count)
-
-            return new_count
-          else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Higher value needs mutex to update both timestamp and count
-              set(topic, new_count)
-
-              return new_count
-            else
-              # Same or lower value, just update timestamp without mutex
-              refresh_timestamp(topic)
-
-              return current_count
-            end
-          end
-        end
-
-        current_info[1]
-      end
-
-      # Update partition count for a topic when needed
-      #
-      # This method updates the partition count for a topic in the cache.
-      # It uses a mutex to ensure thread-safety during updates.
-      #
-      # @param topic [String] Kafka topic name
-      # @param new_count [Integer] New partition count value
-      #
-      # @note We prioritize higher partition counts and only accept them when using
-      #   a mutex to ensure consistency. This design decision is based on the fact that
-      #   partition counts in Kafka only increase during normal operation.
-      def set(topic, new_count)
-        # First check outside mutex to avoid unnecessary locking
-        current_info = @counts[topic]
-
-        # For lower values, we don't update count but might need to refresh timestamp
-        if current_info && new_count < current_info[1]
-          refresh_timestamp(topic)
-
-          return
-        end
-
-        # Only lock the specific topic mutex
-        mutex_for(topic).synchronize do
-          # Check again inside the lock as another thread might have updated
-          current_info = @counts[topic]
-
-          if current_info.nil?
-            # Create new entry
-            @counts[topic] = [monotonic_now, new_count]
-          else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Update to higher count value
-              current_info[0] = monotonic_now
-              current_info[1] = new_count
-            else
-              # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now
-            end
-          end
-        end
-      end
-
-      # @return [Hash] hash with ttls and partitions counts array
-      def to_h
-        @counts
-      end
-
-      private
-
-      # Get or create a mutex for a specific topic
-      #
-      # This method ensures that each topic has its own mutex,
-      # allowing operations on different topics to proceed in parallel.
-      #
-      # @param topic [String] Kafka topic name
-      # @return [Mutex] Mutex for the specified topic
-      #
-      # @note We use a separate mutex (@mutex_for_hash) to protect the creation
-      #   of new topic mutexes. This pattern allows fine-grained locking while
-      #   maintaining thread-safety.
-      def mutex_for(topic)
-        mutex = @mutex_hash[topic]
-
-        return mutex if mutex
-
-        # Use a separate mutex to protect the creation of new topic mutexes
-        @mutex_for_hash.synchronize do
-          # Check again in case another thread created it
-          @mutex_hash[topic] ||= Mutex.new
-        end
-
-        @mutex_hash[topic]
-      end
-
-      # Update the timestamp without acquiring the mutex
-      #
-      # This is an optimization that allows refreshing the TTL of existing entries
-      # without the overhead of mutex acquisition.
-      #
-      # @param topic [String] Kafka topic name
-      #
-      # @note This method is safe for refreshing existing data regardless of count
-      #   because it only updates the timestamp, which doesn't affect the correctness
-      #   of concurrent operations.
-      def refresh_timestamp(topic)
-        current_info = @counts[topic]
-
-        return unless current_info
-
-        # Update the timestamp in-place
-        current_info[0] = monotonic_now
-      end
-
-      # Check if a timestamp has expired based on the TTL
-      #
-      # @param timestamp [Float] Monotonic timestamp to check
-      # @return [Boolean] true if expired, false otherwise
-      def expired?(timestamp)
-        monotonic_now - timestamp > @ttl
-      end
-    end
-  end
-end