karafka-rdkafka 0.20.0.rc4-x86_64-linux-musl → 0.20.0-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd99c2c62c306653ed04cde903454408ae3ee4c59316ab4e2fa8da495307a98b
-  data.tar.gz: 007d9fe5481f9ebfee55fbcbe6a5d5656e6dc6cc44a2095ecaef7d8d336b52fa
+  metadata.gz: d68434b9c41f257de1f8570bbfa645f8c49f97c59f83f0b63faba8c2ecb2eb1b
+  data.tar.gz: 66d47fb1e0b186d5c39d4e29950cde5f9931028e3656c7e15f8d75d719b4caa1
 SHA512:
-  metadata.gz: fc8d968ad84f4cf5a0253cd1a68d7128ba95c7fe7348f0e01be6a468ce737ad0fea15cb4f008e9bc1a802a013a85be098f01a51271908fc1152b6d2e0b08c09d
-  data.tar.gz: 1cc9b177099d3426e065510ab46ae4e988b2977d657146a7c551fa016b81a4c4b60b8c7427b36fcbfcefafba45ec59b781970a444f9a22bd532dcf806e33ec86
+  metadata.gz: ccd11e549995c0fe8c5a82eb63871de11689d81f6a51770169fb0fc41bd42a17946d22432a9e0197ec8c84b6c9d5019cc104f8bbe7a0089f4d767005fad5903e
+  data.tar.gz: 0ddae6bc6d8a0f379c6bd3d3c7366d787d653594a5743803534bd5ddd252bc4d28b04beec8d28be1ccd31f8e2d8caee456b996812b37ef835b363f5376aa7ccb

data/.github/workflows/ci_linux_x86_64_musl.yml

@@ -24,7 +24,7 @@ jobs:
     timeout-minutes: 30
     runs-on: ubuntu-latest
     container:
-      image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
+      image: alpine:3.22@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
     steps:
       - name: Install dependencies
         run: |
@@ -123,7 +123,7 @@ jobs:
     timeout-minutes: 45
     runs-on: ubuntu-latest
     container:
-      image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
+      image: alpine:3.22@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
     steps:
       - name: Install dependencies
         run: |

data/.github/workflows/push_linux_x86_64_musl.yml

@@ -13,8 +13,9 @@ jobs:
     if: github.repository_owner == 'karafka'
     timeout-minutes: 30
     runs-on: ubuntu-latest
+    environment: deployment
     container:
-      image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
+      image: alpine:3.22@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
     steps:
       - name: Install dependencies
         run: |
@@ -43,7 +44,6 @@ jobs:
           name: librdkafka-precompiled-musl
           path: ext/
           retention-days: 1
-
   push:
     if: github.repository_owner == 'karafka'
     timeout-minutes: 30

data/.github/workflows/push_ruby.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@472790540115ce5bd69d399a020189a8c87d641f # v1.247.0
         with:
           bundler-cache: false
 

data/.ruby-version

@@ -1 +1 @@
-3.4.4
+3.4.5

data/CHANGELOG.md

@@ -1,10 +1,15 @@
 # Rdkafka Changelog
 
-## 0.20.0 (Unreleased)
+## 0.20.0 (2025-07-17)
 - **[Feature]** Add precompiled `x86_64-linux-gnu` setup.
 - **[Feature]** Add precompiled `x86_64-linux-musl` setup.
 - **[Feature]** Add precompiled `macos_arm64` setup.
 - [Enhancement] Run all specs on each of the platforms with and without precompilation.
+- [Enhancement] Support transactional id in the ACL API.
+- [Fix] Fix a case where using empty key on the `musl` architecture would cause a segfault.
+- [Fix] Fix for null pointer reference bypass on empty string being too wide causing segfault.
+
+**Note**: Precompiled extensions are a new feature in this release. While they significantly improve installation speed and reduce build dependencies, they should be thoroughly tested in your staging environment before deploying to production. If you encounter any issues with precompiled extensions, you can fall back to building from sources. For more information, see the [Native Extensions documentation](https://karafka.io/docs/Development-Native-Extensions/).
 
 ## 0.19.5 (2025-05-30)
 - [Enhancement] Allow for producing to non-existing topics with `key` and `partition_key` present.

data/README.md

@@ -163,7 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
-| 0.20.x (Unreleased) | 2.8.0 (2025-01-07)  | yes |
+| 0.20.x (2025-07-17) | 2.8.0 (2025-01-07)  | yes |
 | 0.19.x (2025-01-20) | 2.8.0 (2025-01-07)  | yes |
 | 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18)  | yes |
 | 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02)  | yes |

data/karafka-rdkafka.gemspec

@@ -40,12 +40,37 @@ Gem::Specification.new do |gem|
     gem.extensions = %w(ext/Rakefile)
   end
 
+  if ENV['RUBY_PLATFORM']
+    gem.platform = ENV['RUBY_PLATFORM']
+    gem.files = `git ls-files`.split($\)
+
+    # Do not include the source code for librdkafka as it should be precompiled already per
+    # platform. Same applies to any possible patches.
+    gem.files = gem.files.reject do |file|
+      file.match?(%r{^dist/librdkafka-.*\.tar\.gz$}) ||
+      file.match?(%r{^dist/patches/})
+    end
+
+    # Add the compiled extensions that exist (not in git)
+    if File.exist?('ext/librdkafka.so')
+      gem.files << 'ext/librdkafka.so'
+    end
+
+    if File.exist?('ext/librdkafka.dylib')
+      gem.files << 'ext/librdkafka.dylib'
+    end
+  else
+    gem.platform = Gem::Platform::RUBY
+    gem.files = `git ls-files`.split($\)
+    gem.extensions = %w(ext/Rakefile)
+  end
+
   gem.add_dependency 'ffi', '~> 1.15'
   gem.add_dependency 'logger'
   gem.add_dependency 'mini_portile2', '~> 2.6'
-  gem.add_dependency 'ostruct'
   gem.add_dependency 'rake', '> 12'
 
+  gem.add_development_dependency 'ostruct'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
   gem.add_development_dependency 'rake'

data/lib/rdkafka/bindings.rb

@@ -403,18 +403,16 @@ module Rdkafka
       hsh[name] = method_name
     end
 
-    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
+    def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
-      # musl architecture crashes with empty string
-      return 0 if str.empty?
 
-      str_ptr = FFI::MemoryPointer.from_string(str)
-      method_name = PARTITIONERS.fetch(partitioner_name) do
-        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
+      method_name = PARTITIONERS.fetch(partitioner) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
       end
 
-      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
+      public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics
@@ -532,6 +530,7 @@ module Rdkafka
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
     RD_KAFKA_RESOURCE_BROKER = 4
+    RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
 

data/lib/rdkafka/producer.rb

@@ -51,13 +51,13 @@ module Rdkafka
 
     # @private
     # @param native_kafka [NativeKafka]
-    # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
+    # @param partitioner [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
-    def initialize(native_kafka, partitioner_name)
+    def initialize(native_kafka, partitioner)
       @topics_refs_map = {}
       @topics_configs = {}
       @native_kafka = native_kafka
-      @partitioner_name = partitioner_name || "consistent_random"
+      @partitioner = partitioner || "consistent_random"
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
@@ -337,7 +337,8 @@ module Rdkafka
       timestamp: nil,
       headers: nil,
       label: nil,
-      topic_config: EMPTY_HASH
+      topic_config: EMPTY_HASH,
+      partitioner: @partitioner
     )
       closed_producer_check(__method__)
 
@@ -369,10 +370,14 @@ module Rdkafka
 
         # Check if there are no overrides for the partitioner and use the default one only when
         # no per-topic is present.
-        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+        selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, partitioner_name) if partition_count.positive?
+        partition = Rdkafka::Bindings.partitioner(
+          topic_ref,
+          partition_key,
+          partition_count,
+          selected_partitioner) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.20.0.rc4"
+  VERSION = "0.20.0"
   LIBRDKAFKA_VERSION = "2.8.0"
   LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/spec/rdkafka/admin_spec.rb

@@ -513,7 +513,7 @@ describe Rdkafka::Admin do
     end
   end
 
-  describe "#ACL tests" do
+  describe "#ACL tests for topic resource" do
     let(:non_existing_resource_name) {"non-existing-topic"}
     before do
       #create topic for testing acl
@@ -615,6 +615,207 @@ describe Rdkafka::Admin do
     end
   end
 
+  describe "#ACL tests for transactional_id" do
+    let(:transactional_id_resource_name) {"test-transactional-id"}
+    let(:non_existing_transactional_id) {"non-existing-transactional-id"}
+    let(:transactional_id_resource_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID }
+    let(:transactional_id_resource_pattern_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL }
+    let(:transactional_id_principal) { "User:test-user" }
+    let(:transactional_id_host) { "*" }
+    let(:transactional_id_operation) { Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_WRITE }
+    let(:transactional_id_permission_type) { Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW }
+
+    after do
+      # Clean up any ACLs that might have been created during tests
+      begin
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_handle.wait(max_wait_timeout: 15.0)
+      rescue
+        # Ignore cleanup errors
+      end
+    end
+
+    describe "#create_acl" do
+      it "creates acl for a transactional_id" do
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: transactional_id_resource_name,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+      end
+
+      it "creates acl for a non-existing transactional_id" do
+        # ACL creation for transactional_ids that don't exist will still get created successfully
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Clean up the ACL that was created for the non-existing transactional_id
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(1)
+      end
+    end
+
+    describe "#describe_acl" do
+      it "describes acl of a transactional_id that does not exist" do
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.size).to eq(0)
+      end
+
+      it "creates acls and describes the newly created transactional_id acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
+        # Describe ACLs - filter by transactional_id resource type
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.length).to eq(2)
+      end
+    end
+
+    describe "#delete_acl" do
+      it "deletes acl of a transactional_id that does not exist" do
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(0)
+      end
+
+      it "creates transactional_id acls and deletes the newly created acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Delete ACLs - resource_name nil to delete all ACLs with any resource name and matching all other filters
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.length).to eq(2)
+      end
+    end
+  end
+
   describe('Group tests') do
     describe "#delete_group" do
       describe("with an existing group") do

data/spec/rdkafka/bindings_spec.rb

@@ -77,30 +77,6 @@ describe Rdkafka::Bindings do
     end
   end
 
-  describe "partitioner" do
-    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
-    let(:partition_count) { rand(50) + 1 }
-
-    it "should return the same partition for a similar string and the same partition count" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should match the old partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = (Zlib.crc32(partition_key) % partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should return the partition calculated by the specified partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
-      ptr = FFI::MemoryPointer.from_string(partition_key)
-      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
-      expect(result_1).to eq(result_2)
-    end
-  end
-
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/rdkafka/consumer_spec.rb

@@ -1291,9 +1291,6 @@ describe Rdkafka::Consumer do
       end
 
       expect(eof_error.code).to eq(:partition_eof)
-      expect(eof_error.details[:topic]).to eq('consume_test_topic')
-      expect(eof_error.details[:partition]).to be_a(Integer)
-      expect(eof_error.details[:offset]).to be_a(Integer)
     end
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -340,7 +340,7 @@ describe Rdkafka::Producer do
       )
     end
 
-    expect(messages[0].partition).to eq 0
+    expect(messages[0].partition).to be >= 0
     expect(messages[0].key).to eq 'a'
   end
 
@@ -1231,4 +1231,298 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
+
+  describe "partitioner behavior through producer API" do
+    context "testing all partitioners with same key" do
+      it "should not return partition 0 for all partitioners" do
+        test_key = "test-key-123"
+        results = {}
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          results[partitioner] = report.partition
+        end
+
+        # Should not all be the same partition (especially not all 0)
+        unique_partitions = results.values.uniq
+        expect(unique_partitions.size).to be > 1
+      end
+    end
+
+    context "empty string partition key" do
+      it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: "",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+        end
+      end
+    end
+
+    context "nil partition key" do
+      it "should handle nil partition key gracefully" do
+        handle = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload",
+          key: "test-key",
+          partition_key: nil
+        )
+
+        report = handle.wait(max_wait_timeout: 5)
+        expect(report.partition).to be >= 0
+        expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+      end
+    end
+
+    context "various key types and lengths with different partitioners" do
+      it "should handle very short keys with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "a",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle very long keys with all partitioners" do
+        long_key = "a" * 1000
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: long_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle unicode keys with all partitioners" do
+        unicode_key = "测试键值🚀"
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: unicode_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "consistency testing for deterministic partitioners" do
+      %w(consistent murmur2 fnv1a).each do |partitioner|
+        it "should consistently route same partition key to same partition with #{partitioner}" do
+          partition_key = "consistent-test-key"
+
+          # Produce multiple messages with same partition key
+          reports = 5.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          # All should go to same partition
+          partitions = reports.map(&:partition).uniq
+          expect(partitions.size).to eq(1)
+        end
+      end
+    end
+
+    context "randomness testing for random partitioners" do
+      %w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
+        it "should potentially distribute across partitions with #{partitioner}" do
+          # Note: random partitioners might still return same value by chance
+          partition_key = "random-test-key"
+
+          reports = 10.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition)
+
+          # Just ensure they're valid partitions
+          partitions.each do |partition|
+            expect(partition).to be >= 0
+            expect(partition).to be < producer.partition_count("partitioner_test_topic")
+          end
+        end
+      end
+    end
+
+    context "comparing different partitioners with same key" do
+      it "should route different partition keys to potentially different partitions" do
+        keys = ["key1", "key2", "key3", "key4", "key5"]
+
+        all_partitioners.each do |partitioner|
+          reports = keys.map do |key|
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload",
+              partition_key: key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition).uniq
+
+          # Should distribute across multiple partitions for most partitioners
+          # (though some might hash all keys to same partition by chance)
+          expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
+        end
+      end
+    end
+
+    context "partition key vs regular key behavior" do
+      it "should use partition key for partitioning when both key and partition_key are provided" do
+        # Use keys that would hash to different partitions
+        regular_key = "regular-key-123"
+        partition_key = "partition-key-456"
+
+        # Message with both keys
+        handle1 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 1",
+          key: regular_key,
+          partition_key: partition_key
+        )
+
+        # Message with only partition key (should go to same partition)
+        handle2 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 2",
+          partition_key: partition_key
+        )
+
+        # Message with only regular key (should go to different partition)
+        handle3 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 3",
+          key: regular_key
+        )
+
+        report1 = handle1.wait(max_wait_timeout: 5)
+        report2 = handle2.wait(max_wait_timeout: 5)
+        report3 = handle3.wait(max_wait_timeout: 5)
+
+        # Messages 1 and 2 should go to same partition (both use partition_key)
+        expect(report1.partition).to eq(report2.partition)
+
+        # Message 3 should potentially go to different partition (uses regular key)
+        expect(report3.partition).not_to eq(report1.partition)
+      end
+    end
+
+    context "edge case combinations with different partitioners" do
+      it "should handle nil partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: nil,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle whitespace-only partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "   ",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle newline characters in partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "key\nwith\nnewlines",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "debugging partitioner issues" do
+      it "should show if all partitioners return 0 (indicating a problem)" do
+        test_key = "debug-test-key"
+        zero_count = 0
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "debug payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          zero_count += 1 if report.partition == 0
+        end
+
+        expect(zero_count).to be < all_partitioners.size
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -78,18 +78,32 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = consumer.nil?
-  consumer ||= rdkafka_consumer_config.consumer
+  consumer ||= rdkafka_consumer_config('allow.auto.create.topics': true).consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
+  retry_count = 0
+  max_retries = 10
+
   loop do
     if timeout <= Time.now.to_i
       raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
     end
-    message = consumer.poll(100)
-    if message &&
-        message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset
-      return message
+
+    begin
+      message = consumer.poll(100)
+      if message &&
+          message.partition == delivery_report.partition &&
+          message.offset == delivery_report.offset
+        return message
+      end
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :unknown_topic_or_part && retry_count < max_retries
+        retry_count += 1
+        sleep(0.1) # Small delay before retry
+        next
+      else
+        raise
+      end
     end
   end
 ensure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.20.0.rc4
+  version: 0.20.0
 platform: x86_64-linux-musl
 authors:
 - Thijs Cadier
@@ -53,33 +53,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.6'
 - !ruby/object:Gem::Dependency
-  name: ostruct
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -241,7 +241,6 @@ files:
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:

data/spec/rdkafka/producer/partitions_count_spec.rb

@@ -1,359 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Rdkafka::Producer::PartitionsCountCache do
-  let(:default_ttl) { 1 } # Reduced from 30 to speed up tests
-  let(:custom_ttl) { 0.5 } # Half the default TTL
-  let(:cache) { described_class.new(default_ttl) }
-  let(:custom_ttl_cache) { described_class.new(custom_ttl) }
-  let(:topic) { "test_topic" }
-  let(:topic2) { "test_topic2" }
-  let(:partition_count) { 5 }
-  let(:higher_partition_count) { 10 }
-  let(:lower_partition_count) { 3 }
-  let(:even_higher_partition_count) { 15 }
-
-  describe "#initialize" do
-    it "creates a cache with default TTL when no TTL is specified" do
-      standard_cache = described_class.new
-      expect(standard_cache).to be_a(described_class)
-    end
-
-    it "creates a cache with custom TTL when specified" do
-      expect(custom_ttl_cache).to be_a(described_class)
-    end
-  end
-
-  describe "#get" do
-    context "when cache is empty" do
-      it "yields to get the value and caches it" do
-        block_called = false
-        result = cache.get(topic) do
-          block_called = true
-          partition_count
-        end
-
-        expect(block_called).to be true
-        expect(result).to eq(partition_count)
-
-        # Verify caching by checking if block is called again
-        second_block_called = false
-        second_result = cache.get(topic) do
-          second_block_called = true
-          partition_count + 1 # Different value to ensure we get cached value
-        end
-
-        expect(second_block_called).to be false
-        expect(second_result).to eq(partition_count)
-      end
-    end
-
-    context "when cache has a value" do
-      before do
-        # Seed the cache with a value
-        cache.get(topic) { partition_count }
-      end
-
-      it "returns cached value without yielding if not expired" do
-        block_called = false
-        result = cache.get(topic) do
-          block_called = true
-          partition_count + 1 # Different value to ensure we get cached one
-        end
-
-        expect(block_called).to be false
-        expect(result).to eq(partition_count)
-      end
-
-      it "yields to get new value when TTL has expired" do
-        # Wait for TTL to expire
-        sleep(default_ttl + 0.1)
-
-        block_called = false
-        new_count = partition_count + 1
-        result = cache.get(topic) do
-          block_called = true
-          new_count
-        end
-
-        expect(block_called).to be true
-        expect(result).to eq(new_count)
-
-        # Verify the new value is cached
-        second_block_called = false
-        second_result = cache.get(topic) do
-          second_block_called = true
-          new_count + 1 # Different value again
-        end
-
-        expect(second_block_called).to be false
-        expect(second_result).to eq(new_count)
-      end
-
-      it "respects a custom TTL" do
-        # Seed the custom TTL cache with a value
-        custom_ttl_cache.get(topic) { partition_count }
-
-        # Wait for custom TTL to expire but not default TTL
-        sleep(custom_ttl + 0.1)
-
-        # Custom TTL cache should refresh
-        custom_block_called = false
-        custom_result = custom_ttl_cache.get(topic) do
-          custom_block_called = true
-          higher_partition_count
-        end
-
-        expect(custom_block_called).to be true
-        expect(custom_result).to eq(higher_partition_count)
-
-        # Default TTL cache should not refresh yet
-        default_block_called = false
-        default_result = cache.get(topic) do
-          default_block_called = true
-          higher_partition_count
-        end
-
-        expect(default_block_called).to be false
-        expect(default_result).to eq(partition_count)
-      end
-    end
-
-    context "when new value is obtained" do
-      before do
-        # Seed the cache with initial value
-        cache.get(topic) { partition_count }
-      end
-
-      it "updates cache when new value is higher than cached value" do
-        # Wait for TTL to expire
-        sleep(default_ttl + 0.1)
-
-        # Get higher value
-        result = cache.get(topic) { higher_partition_count }
-        expect(result).to eq(higher_partition_count)
-
-        # Verify it was cached
-        second_result = cache.get(topic) { fail "Should not be called" }
-        expect(second_result).to eq(higher_partition_count)
-      end
-
-      it "preserves higher cached value when new value is lower" do
-        # First update to higher value
-        sleep(default_ttl + 0.1)
-        cache.get(topic) { higher_partition_count }
-
-        # Then try to update to lower value
-        sleep(default_ttl + 0.1)
-        result = cache.get(topic) { lower_partition_count }
-
-        expect(result).to eq(higher_partition_count)
-
-        # and subsequent gets should return the previously cached higher value
-        second_result = cache.get(topic) { fail "Should not be called" }
-        expect(second_result).to eq(higher_partition_count)
-      end
-
-      it "handles multiple topics independently" do
-        # Set up both topics with different values
-        cache.get(topic) { partition_count }
-        cache.get(topic2) { higher_partition_count }
-
-        # Wait for TTL to expire
-        sleep(default_ttl + 0.1)
-
-        # Update first topic
-        first_result = cache.get(topic) { even_higher_partition_count }
-        expect(first_result).to eq(even_higher_partition_count)
-
-        # Update second topic independently
-        second_updated = higher_partition_count + 3
-        second_result = cache.get(topic2) { second_updated }
-        expect(second_result).to eq(second_updated)
-
-        # Both topics should have their updated values
-        expect(cache.get(topic) { fail "Should not be called" }).to eq(even_higher_partition_count)
-        expect(cache.get(topic2) { fail "Should not be called" }).to eq(second_updated)
-      end
-    end
-  end
-
-  describe "#set" do
-    context "when cache is empty" do
-      it "adds a new entry to the cache" do
-        cache.set(topic, partition_count)
-
-        # Verify through get
-        result = cache.get(topic) { fail "Should not be called" }
-        expect(result).to eq(partition_count)
-      end
-    end
-
-    context "when cache already has a value" do
-      before do
-        cache.set(topic, partition_count)
-      end
-
-      it "updates cache when new value is higher" do
-        cache.set(topic, higher_partition_count)
-
-        result = cache.get(topic) { fail "Should not be called" }
-        expect(result).to eq(higher_partition_count)
-      end
-
-      it "keeps original value when new value is lower" do
-        cache.set(topic, lower_partition_count)
-
-        result = cache.get(topic) { fail "Should not be called" }
-        expect(result).to eq(partition_count)
-      end
-
-      it "updates the timestamp even when keeping original value" do
-        # Set initial value
-        cache.set(topic, partition_count)
-
-        # Wait until close to TTL expiring
-        sleep(default_ttl - 0.2)
-
-        # Set lower value (should update timestamp but not value)
-        cache.set(topic, lower_partition_count)
-
-        # Wait a bit more, but still under the full TTL if timestamp was refreshed
-        sleep(0.3)
-
-        # Should still be valid due to timestamp refresh
-        result = cache.get(topic) { fail "Should not be called" }
-        expect(result).to eq(partition_count)
-      end
-    end
-
-    context "with concurrent access" do
-      it "correctly handles simultaneous updates to the same topic" do
-        # This test focuses on the final value after concurrent updates
-        threads = []
-
-        # Create 5 threads that all try to update the same topic with increasing values
-        5.times do |i|
-          threads << Thread.new do
-            value = 10 + i  # Start at 10 to ensure all are higher than initial value
-            cache.set(topic, value)
-          end
-        end
-
-        # Wait for all threads to complete
-        threads.each(&:join)
-
-        # The highest value (14) should be stored and accessible through get
-        result = cache.get(topic) { fail "Should not be called" }
-        expect(result).to eq(14)
-      end
-    end
-  end
-
-  describe "TTL behavior" do
-    it "treats entries as expired when they exceed TTL" do
-      # Set initial value
-      cache.get(topic) { partition_count }
-
-      # Wait just under TTL
-      sleep(default_ttl - 0.2)
-
-      # Value should still be cached (block should not be called)
-      result = cache.get(topic) { fail "Should not be called when cache is valid" }
-      expect(result).to eq(partition_count)
-
-      # Now wait to exceed TTL
-      sleep(0.2) # Total sleep is now default_ttl + 0.1
-
-      # Cache should be expired, block should be called
-      block_called = false
-      new_value = partition_count + 3
-      result = cache.get(topic) do
-        block_called = true
-        new_value
-      end
-
-      expect(block_called).to be true
-      expect(result).to eq(new_value)
-    end
-  end
-
-  describe "comprehensive scenarios" do
-    it "handles a full lifecycle of cache operations" do
-      # 1. Initial cache miss, fetch and store
-      result1 = cache.get(topic) { partition_count }
-      expect(result1).to eq(partition_count)
-
-      # 2. Cache hit
-      result2 = cache.get(topic) { fail "Should not be called" }
-      expect(result2).to eq(partition_count)
-
-      # 3. Attempt to set lower value
-      cache.set(topic, lower_partition_count)
-      result3 = cache.get(topic) { fail "Should not be called" }
-      # Should still return the higher original value
-      expect(result3).to eq(partition_count)
-
-      # 4. Set higher value
-      cache.set(topic, higher_partition_count)
-      result4 = cache.get(topic) { fail "Should not be called" }
-      expect(result4).to eq(higher_partition_count)
-
-      # 5. TTL expires, new value provided is lower
-      sleep(default_ttl + 0.1)
-      result5 = cache.get(topic) { lower_partition_count }
-      # This returns the highest value
-      expect(result5).to eq(higher_partition_count)
-
-      # 6. But subsequent get should return the higher cached value
-      result6 = cache.get(topic) { fail "Should not be called" }
-      expect(result6).to eq(higher_partition_count)
-
-      # 7. Set new highest value directly
-      even_higher = higher_partition_count + 5
-      cache.set(topic, even_higher)
-      result7 = cache.get(topic) { fail "Should not be called" }
-      expect(result7).to eq(even_higher)
-    end
-
-    it "handles multiple topics with different TTLs correctly" do
-      # Set up initial values
-      cache.get(topic) { partition_count }
-      custom_ttl_cache.get(topic) { partition_count }
-
-      # Wait past custom TTL but not default TTL
-      sleep(custom_ttl + 0.1)
-
-      # Default cache should NOT refresh (still within default TTL)
-      default_result = cache.get(topic) { fail "Should not be called for default cache" }
-      # Original value should be maintained
-      expect(default_result).to eq(partition_count)
-
-      # Custom TTL cache SHOULD refresh (past custom TTL)
-      custom_cache_value = partition_count + 8
-      custom_block_called = false
-      custom_result = custom_ttl_cache.get(topic) do
-        custom_block_called = true
-        custom_cache_value
-      end
-
-      expect(custom_block_called).to be true
-      expect(custom_result).to eq(custom_cache_value)
-
-      # Now wait past default TTL
-      sleep(default_ttl - custom_ttl + 0.1)
-
-      # Now default cache should also refresh
-      default_block_called = false
-      new_default_value = partition_count + 10
-      new_default_result = cache.get(topic) do
-        default_block_called = true
-        new_default_value
-      end
-
-      expect(default_block_called).to be true
-      expect(new_default_result).to eq(new_default_value)
-    end
-  end
-end