karafka-rdkafka 0.19.5 → 0.20.1

data/spec/rdkafka/producer_spec.rb

@@ -53,7 +53,7 @@ describe Rdkafka::Producer do
         let(:producer) do
           rdkafka_producer_config(
             'message.timeout.ms': 1_000_000,
-            :"bootstrap.servers" => "localhost:9094",
+            :"bootstrap.servers" => "127.0.0.1:9094",
           ).producer
         end
 
@@ -340,7 +340,7 @@ describe Rdkafka::Producer do
       )
     end
 
-    expect(messages[0].partition).to eq 0
+    expect(messages[0].partition).to be >= 0
     expect(messages[0].key).to eq 'a'
   end
 
@@ -665,7 +665,7 @@ describe Rdkafka::Producer do
   context "when not being able to deliver the message" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "localhost:9093",
+        "bootstrap.servers": "127.0.0.1:9093",
         "message.timeout.ms": 100
       ).producer
     end
@@ -682,7 +682,7 @@ describe Rdkafka::Producer do
   context "when topic does not exist and allow.auto.create.topics is false" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "localhost:9092",
+        "bootstrap.servers": "127.0.0.1:9092",
         "message.timeout.ms": 100,
         "allow.auto.create.topics": false
       ).producer
@@ -781,7 +781,7 @@ describe Rdkafka::Producer do
     context 'when it cannot flush due to a timeout' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -828,7 +828,7 @@ describe Rdkafka::Producer do
     context 'when there are outgoing things in the queue' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -1231,4 +1231,298 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
+
+  describe "partitioner behavior through producer API" do
+    context "testing all partitioners with same key" do
+      it "should not return partition 0 for all partitioners" do
+        test_key = "test-key-123"
+        results = {}
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          results[partitioner] = report.partition
+        end
+
+        # Should not all be the same partition (especially not all 0)
+        unique_partitions = results.values.uniq
+        expect(unique_partitions.size).to be > 1
+      end
+    end
+
+    context "empty string partition key" do
+      it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: "",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+        end
+      end
+    end
+
+    context "nil partition key" do
+      it "should handle nil partition key gracefully" do
+        handle = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload",
+          key: "test-key",
+          partition_key: nil
+        )
+
+        report = handle.wait(max_wait_timeout: 5)
+        expect(report.partition).to be >= 0
+        expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+      end
+    end
+
+    context "various key types and lengths with different partitioners" do
+      it "should handle very short keys with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "a",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle very long keys with all partitioners" do
+        long_key = "a" * 1000
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: long_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle unicode keys with all partitioners" do
+        unicode_key = "测试键值🚀"
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: unicode_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "consistency testing for deterministic partitioners" do
+      %w(consistent murmur2 fnv1a).each do |partitioner|
+        it "should consistently route same partition key to same partition with #{partitioner}" do
+          partition_key = "consistent-test-key"
+
+          # Produce multiple messages with same partition key
+          reports = 5.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          # All should go to same partition
+          partitions = reports.map(&:partition).uniq
+          expect(partitions.size).to eq(1)
+        end
+      end
+    end
+
+    context "randomness testing for random partitioners" do
+      %w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
+        it "should potentially distribute across partitions with #{partitioner}" do
+          # Note: random partitioners might still return same value by chance
+          partition_key = "random-test-key"
+
+          reports = 10.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition)
+
+          # Just ensure they're valid partitions
+          partitions.each do |partition|
+            expect(partition).to be >= 0
+            expect(partition).to be < producer.partition_count("partitioner_test_topic")
+          end
+        end
+      end
+    end
+
+    context "comparing different partitioners with same key" do
+      it "should route different partition keys to potentially different partitions" do
+        keys = ["key1", "key2", "key3", "key4", "key5"]
+
+        all_partitioners.each do |partitioner|
+          reports = keys.map do |key|
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload",
+              partition_key: key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition).uniq
+
+          # Should distribute across multiple partitions for most partitioners
+          # (though some might hash all keys to same partition by chance)
+          expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
+        end
+      end
+    end
+
+    context "partition key vs regular key behavior" do
+      it "should use partition key for partitioning when both key and partition_key are provided" do
+        # Use keys that would hash to different partitions
+        regular_key = "regular-key-123"
+        partition_key = "partition-key-456"
+
+        # Message with both keys
+        handle1 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 1",
+          key: regular_key,
+          partition_key: partition_key
+        )
+
+        # Message with only partition key (should go to same partition)
+        handle2 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 2",
+          partition_key: partition_key
+        )
+
+        # Message with only regular key (should go to different partition)
+        handle3 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 3",
+          key: regular_key
+        )
+
+        report1 = handle1.wait(max_wait_timeout: 5)
+        report2 = handle2.wait(max_wait_timeout: 5)
+        report3 = handle3.wait(max_wait_timeout: 5)
+
+        # Messages 1 and 2 should go to same partition (both use partition_key)
+        expect(report1.partition).to eq(report2.partition)
+
+        # Message 3 should potentially go to different partition (uses regular key)
+        expect(report3.partition).not_to eq(report1.partition)
+      end
+    end
+
+    context "edge case combinations with different partitioners" do
+      it "should handle nil partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: nil,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle whitespace-only partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "   ",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle newline characters in partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "key\nwith\nnewlines",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "debugging partitioner issues" do
+      it "should show if all partitioners return 0 (indicating a problem)" do
+        test_key = "debug-test-key"
+        zero_count = 0
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "debug payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          zero_count += 1 if report.partition == 0
+        end
+
+        expect(zero_count).to be < all_partitioners.size
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -17,7 +17,7 @@ def rdkafka_base_config
   {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
-    :"bootstrap.servers" => "localhost:9092",
+    :"bootstrap.servers" => "127.0.0.1:9092",
     # Display statistics and refresh often just to cover those in specs
     :'statistics.interval.ms' => 1_000,
     :'topic.metadata.refresh.interval.ms' => 1_000
@@ -78,18 +78,32 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = consumer.nil?
-  consumer ||= rdkafka_consumer_config.consumer
+  consumer ||= rdkafka_consumer_config('allow.auto.create.topics': true).consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
+  retry_count = 0
+  max_retries = 10
+
   loop do
     if timeout <= Time.now.to_i
       raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
     end
-    message = consumer.poll(100)
-    if message &&
-        message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset
-      return message
+
+    begin
+      message = consumer.poll(100)
+      if message &&
+          message.partition == delivery_report.partition &&
+          message.offset == delivery_report.offset
+        return message
+      end
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :unknown_topic_or_part && retry_count < max_retries
+        retry_count += 1
+        sleep(0.1) # Small delay before retry
+        next
+      else
+        raise
+      end
     end
   end
 ensure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.19.5
+  version: 0.20.1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mini_portile2
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '12'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,21 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-- !ruby/object:Gem::Dependency
-  name: rake
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,21 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -146,8 +146,13 @@ extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
-- ".github/workflows/ci.yml"
-- ".github/workflows/push.yml"
+- ".github/workflows/ci_linux_x86_64_gnu.yml"
+- ".github/workflows/ci_linux_x86_64_musl.yml"
+- ".github/workflows/ci_macos_arm64.yml"
+- ".github/workflows/push_linux_x86_64_gnu.yml"
+- ".github/workflows/push_linux_x86_64_musl.yml"
+- ".github/workflows/push_macos_arm64.yml"
+- ".github/workflows/push_ruby.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -156,7 +161,6 @@ files:
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
-- Guardfile
 - MIT-LICENSE
 - README.md
 - Rakefile
@@ -165,6 +169,10 @@ files:
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
+- ext/build_common.sh
+- ext/build_linux_x86_64_gnu.sh
+- ext/build_linux_x86_64_musl.sh
+- ext/build_macos_arm64.sh
 - karafka-rdkafka.gemspec
 - lib/rdkafka.rb
 - lib/rdkafka/abstract_handle.rb
@@ -235,7 +243,6 @@ files:
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
@@ -262,38 +269,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
   and Ruby 2.7+.
-test_files:
-- spec/rdkafka/abstract_handle_spec.rb
-- spec/rdkafka/admin/create_acl_handle_spec.rb
-- spec/rdkafka/admin/create_acl_report_spec.rb
-- spec/rdkafka/admin/create_topic_handle_spec.rb
-- spec/rdkafka/admin/create_topic_report_spec.rb
-- spec/rdkafka/admin/delete_acl_handle_spec.rb
-- spec/rdkafka/admin/delete_acl_report_spec.rb
-- spec/rdkafka/admin/delete_topic_handle_spec.rb
-- spec/rdkafka/admin/delete_topic_report_spec.rb
-- spec/rdkafka/admin/describe_acl_handle_spec.rb
-- spec/rdkafka/admin/describe_acl_report_spec.rb
-- spec/rdkafka/admin_spec.rb
-- spec/rdkafka/bindings_spec.rb
-- spec/rdkafka/callbacks_spec.rb
-- spec/rdkafka/config_spec.rb
-- spec/rdkafka/consumer/headers_spec.rb
-- spec/rdkafka/consumer/message_spec.rb
-- spec/rdkafka/consumer/partition_spec.rb
-- spec/rdkafka/consumer/topic_partition_list_spec.rb
-- spec/rdkafka/consumer_spec.rb
-- spec/rdkafka/error_spec.rb
-- spec/rdkafka/metadata_spec.rb
-- spec/rdkafka/native_kafka_spec.rb
-- spec/rdkafka/producer/delivery_handle_spec.rb
-- spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_cache_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
-- spec/rdkafka/producer_spec.rb
-- spec/spec_helper.rb
+test_files: []

data/.github/workflows/ci.yml

@@ -1,99 +0,0 @@
-name: CI
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ main, master ]
-  push:
-    branches: [ main, master ]
-  schedule:
-    - cron: '0 1 * * *'
-
-permissions:
-  contents: read
-
-env:
-  BUNDLE_RETRY: 6
-  BUNDLE_JOBS: 4
-
-jobs:
-  specs:
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby:
-          - '3.4'
-          - '3.3'
-          - '3.2'
-          - '3.1'
-          - 'jruby-10.0'
-        include:
-          - ruby: '3.4'
-            coverage: 'true'
-          - ruby: 'jruby-10.0'
-            continue-on-error: true
-
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-
-      - name: Install package dependencies
-        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
-
-      - name: Start Kafka with Docker Compose
-        run: |
-          docker compose up -d || (sleep 5 && docker compose up -d)
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
-        with:
-          ruby-version: ${{matrix.ruby}}
-          bundler-cache: true
-
-      - name: Run all specs
-        env:
-          GITHUB_COVERAGE: ${{matrix.coverage}}
-        continue-on-error: ${{ matrix.continue-on-error || false }}  # Use the matrix value if present
-        run: |
-          set -e
-          bundle install --jobs 4 --retry 3
-          cd ext && bundle exec rake
-          cd ..
-          bundle exec rspec
-
-  macos_build:
-    timeout-minutes: 30
-    runs-on: macos-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby:
-          - '3.4'
-          - '3.3'
-          - '3.2'
-          - '3.1'
-          - 'jruby-9.4'
-        include:
-          - ruby: 'jruby-10.0'
-            continue-on-error: true
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
-        with:
-          ruby-version: ${{matrix.ruby}}
-          bundler-cache: false
-
-      - name: Build rdkafka-ruby
-        continue-on-error: ${{ matrix.continue-on-error || false }}
-        run: |
-          set -e
-          bundle install --jobs 4 --retry 3
-          cd ext && bundle exec rake

data/Guardfile

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-logger level: :error
-
-guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
-  require "guard/rspec/dsl"
-  dsl = Guard::RSpec::Dsl.new(self)
-
-  # Ruby files
-  ruby = dsl.ruby
-  dsl.watch_spec_files_for(ruby.lib_files)
-  watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
-
-  # RSpec files
-  rspec = dsl.rspec
-  watch(rspec.spec_helper) { rspec.spec_dir }
-  watch(rspec.spec_support) { rspec.spec_dir }
-  watch(rspec.spec_files)
-end