rdkafka 0.21.1.alpha2 → 0.22.0

data/spec/rdkafka/producer_spec.rb

@@ -53,7 +53,7 @@ describe Rdkafka::Producer do
         let(:producer) do
           rdkafka_producer_config(
             'message.timeout.ms': 1_000_000,
-            :"bootstrap.servers" => "localhost:9094",
+            :"bootstrap.servers" => "127.0.0.1:9094",
           ).producer
         end
 
@@ -340,7 +340,7 @@ describe Rdkafka::Producer do
       )
     end
 
-    expect(messages[0].partition).to eq 0
+    expect(messages[0].partition).to be >= 0
     expect(messages[0].key).to eq 'a'
   end
 
@@ -364,6 +364,48 @@ describe Rdkafka::Producer do
     expect(message.key).to eq "key utf8"
   end
 
+  it "should produce a message to a non-existing topic with key and partition key" do
+    new_topic = "it-#{SecureRandom.uuid}"
+
+    handle = producer.produce(
+      # Needs to be a new topic each time
+      topic:   new_topic,
+      payload: "payload",
+      key:     "key",
+      partition_key: "partition_key",
+      label:   "label"
+    )
+
+    # Should be pending at first
+    expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
+
+    # Check delivery handle and report
+    report = handle.wait(max_wait_timeout: 5)
+    expect(handle.pending?).to be false
+    expect(report).not_to be_nil
+    expect(report.partition).to eq 0
+    expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
+
+    # Flush and close producer
+    producer.flush
+    producer.close
+
+    # Consume message and verify its content
+    message = wait_for_message(
+      topic: new_topic,
+      delivery_report: report,
+      consumer: consumer
+    )
+    expect(message.partition).to eq 0
+    expect(message.payload).to eq "payload"
+    expect(message.key).to eq "key"
+    # Since api.version.request is on by default we will get
+    # the message creation timestamp if it's not set.
+    expect(message.timestamp).to be_within(10).of(Time.now)
+  end
+
   context "timestamp" do
     it "should raise a type error if not nil, integer or time" do
       expect {
@@ -623,7 +665,7 @@ describe Rdkafka::Producer do
   context "when not being able to deliver the message" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "localhost:9093",
+        "bootstrap.servers": "127.0.0.1:9093",
         "message.timeout.ms": 100
       ).producer
     end
@@ -637,6 +679,25 @@ describe Rdkafka::Producer do
     end
   end
 
+  context "when topic does not exist and allow.auto.create.topics is false" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "127.0.0.1:9092",
+        "message.timeout.ms": 100,
+        "allow.auto.create.topics": false
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: "it-#{SecureRandom.uuid}", payload: nil, label: 'na')
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.error.code).to eq(:msg_timed_out)
+      expect(handler.create_result.label).to eq('na')
+    end
+  end
+
   describe '#partition_count' do
     it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
 
@@ -693,7 +754,7 @@ describe Rdkafka::Producer do
     context 'when it cannot flush due to a timeout' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -740,7 +801,7 @@ describe Rdkafka::Producer do
     context 'when there are outgoing things in the queue' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -859,7 +920,6 @@ describe Rdkafka::Producer do
     end
   end
 
-
   describe 'with active statistics callback' do
     let(:producer) do
       rdkafka_producer_config('statistics.interval.ms': 1_000).producer
@@ -988,4 +1048,298 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
+
+  describe "partitioner behavior through producer API" do
+    context "testing all partitioners with same key" do
+      it "should not return partition 0 for all partitioners" do
+        test_key = "test-key-123"
+        results = {}
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          results[partitioner] = report.partition
+        end
+
+        # Should not all be the same partition (especially not all 0)
+        unique_partitions = results.values.uniq
+        expect(unique_partitions.size).to be > 1
+      end
+    end
+
+    context "empty string partition key" do
+      it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: "",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+        end
+      end
+    end
+
+    context "nil partition key" do
+      it "should handle nil partition key gracefully" do
+        handle = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload",
+          key: "test-key",
+          partition_key: nil
+        )
+
+        report = handle.wait(max_wait_timeout: 5)
+        expect(report.partition).to be >= 0
+        expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+      end
+    end
+
+    context "various key types and lengths with different partitioners" do
+      it "should handle very short keys with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "a",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle very long keys with all partitioners" do
+        long_key = "a" * 1000
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: long_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle unicode keys with all partitioners" do
+        unicode_key = "测试键值🚀"
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: unicode_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "consistency testing for deterministic partitioners" do
+      %w(consistent murmur2 fnv1a).each do |partitioner|
+        it "should consistently route same partition key to same partition with #{partitioner}" do
+          partition_key = "consistent-test-key"
+
+          # Produce multiple messages with same partition key
+          reports = 5.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          # All should go to same partition
+          partitions = reports.map(&:partition).uniq
+          expect(partitions.size).to eq(1)
+        end
+      end
+    end
+
+    context "randomness testing for random partitioners" do
+      %w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
+        it "should potentially distribute across partitions with #{partitioner}" do
+          # Note: random partitioners might still return same value by chance
+          partition_key = "random-test-key"
+
+          reports = 10.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition)
+
+          # Just ensure they're valid partitions
+          partitions.each do |partition|
+            expect(partition).to be >= 0
+            expect(partition).to be < producer.partition_count("partitioner_test_topic")
+          end
+        end
+      end
+    end
+
+    context "comparing different partitioners with same key" do
+      it "should route different partition keys to potentially different partitions" do
+        keys = ["key1", "key2", "key3", "key4", "key5"]
+
+        all_partitioners.each do |partitioner|
+          reports = keys.map do |key|
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload",
+              partition_key: key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition).uniq
+
+          # Should distribute across multiple partitions for most partitioners
+          # (though some might hash all keys to same partition by chance)
+          expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
+        end
+      end
+    end
+
+    context "partition key vs regular key behavior" do
+      it "should use partition key for partitioning when both key and partition_key are provided" do
+        # Use keys that would hash to different partitions
+        regular_key = "regular-key-123"
+        partition_key = "partition-key-456"
+
+        # Message with both keys
+        handle1 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 1",
+          key: regular_key,
+          partition_key: partition_key
+        )
+
+        # Message with only partition key (should go to same partition)
+        handle2 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 2",
+          partition_key: partition_key
+        )
+
+        # Message with only regular key (should go to different partition)
+        handle3 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 3",
+          key: regular_key
+        )
+
+        report1 = handle1.wait(max_wait_timeout: 5)
+        report2 = handle2.wait(max_wait_timeout: 5)
+        report3 = handle3.wait(max_wait_timeout: 5)
+
+        # Messages 1 and 2 should go to same partition (both use partition_key)
+        expect(report1.partition).to eq(report2.partition)
+
+        # Message 3 should potentially go to different partition (uses regular key)
+        expect(report3.partition).not_to eq(report1.partition)
+      end
+    end
+
+    context "edge case combinations with different partitioners" do
+      it "should handle nil partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: nil,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle whitespace-only partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "   ",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle newline characters in partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "key\nwith\nnewlines",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "debugging partitioner issues" do
+      it "should show if all partitioners return 0 (indicating a problem)" do
+        test_key = "debug-test-key"
+        zero_count = 0
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "debug payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          zero_count += 1 if report.partition == 0
+        end
+
+        expect(zero_count).to be < all_partitioners.size
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -17,7 +17,7 @@ def rdkafka_base_config
   {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
-    :"bootstrap.servers" => "localhost:9092",
+    :"bootstrap.servers" => "127.0.0.1:9092",
     # Display statistics and refresh often just to cover those in specs
     :'statistics.interval.ms' => 1_000,
     :'topic.metadata.refresh.interval.ms' => 1_000
@@ -78,18 +78,32 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = consumer.nil?
-  consumer ||= rdkafka_consumer_config.consumer
+  consumer ||= rdkafka_consumer_config('allow.auto.create.topics': true).consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
+  retry_count = 0
+  max_retries = 10
+
   loop do
     if timeout <= Time.now.to_i
       raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
     end
-    message = consumer.poll(100)
-    if message &&
-        message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset
-      return message
+
+    begin
+      message = consumer.poll(100)
+      if message &&
+          message.partition == delivery_report.partition &&
+          message.offset == delivery_report.offset
+        return message
+      end
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :unknown_topic_or_part && retry_count < max_retries
+        retry_count += 1
+        sleep(0.1) # Small delay before retry
+        next
+      else
+        raise
+      end
     end
   end
 ensure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.21.1.alpha2
+  version: 0.22.0
 platform: ruby
 authors:
 - Thijs Cadier
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mini_portile2
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '12'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,21 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.5'
-- !ruby/object:Gem::Dependency
-  name: rake
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,21 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -146,8 +146,13 @@ extra_rdoc_files: []
 files:
 - ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
-- ".github/workflows/ci.yml"
-- ".github/workflows/push.yml"
+- ".github/workflows/ci_linux_x86_64_gnu.yml"
+- ".github/workflows/ci_linux_x86_64_musl.yml"
+- ".github/workflows/ci_macos_arm64.yml"
+- ".github/workflows/push_linux_x86_64_gnu.yml"
+- ".github/workflows/push_linux_x86_64_musl.yml"
+- ".github/workflows/push_macos_arm64.yml"
+- ".github/workflows/push_ruby.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -156,7 +161,6 @@ files:
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
-- Guardfile
 - MIT-LICENSE
 - README.md
 - Rakefile
@@ -165,6 +169,10 @@ files:
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
+- ext/build_common.sh
+- ext/build_linux_x86_64_gnu.sh
+- ext/build_linux_x86_64_musl.sh
+- ext/build_macos_arm64.sh
 - lib/rdkafka.rb
 - lib/rdkafka/abstract_handle.rb
 - lib/rdkafka/admin.rb
@@ -234,7 +242,7 @@ files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
+- spec/rdkafka/producer/partitions_count_cache_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
@@ -261,37 +269,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
   and Ruby 2.7+.
-test_files:
-- spec/rdkafka/abstract_handle_spec.rb
-- spec/rdkafka/admin/create_acl_handle_spec.rb
-- spec/rdkafka/admin/create_acl_report_spec.rb
-- spec/rdkafka/admin/create_topic_handle_spec.rb
-- spec/rdkafka/admin/create_topic_report_spec.rb
-- spec/rdkafka/admin/delete_acl_handle_spec.rb
-- spec/rdkafka/admin/delete_acl_report_spec.rb
-- spec/rdkafka/admin/delete_topic_handle_spec.rb
-- spec/rdkafka/admin/delete_topic_report_spec.rb
-- spec/rdkafka/admin/describe_acl_handle_spec.rb
-- spec/rdkafka/admin/describe_acl_report_spec.rb
-- spec/rdkafka/admin_spec.rb
-- spec/rdkafka/bindings_spec.rb
-- spec/rdkafka/callbacks_spec.rb
-- spec/rdkafka/config_spec.rb
-- spec/rdkafka/consumer/headers_spec.rb
-- spec/rdkafka/consumer/message_spec.rb
-- spec/rdkafka/consumer/partition_spec.rb
-- spec/rdkafka/consumer/topic_partition_list_spec.rb
-- spec/rdkafka/consumer_spec.rb
-- spec/rdkafka/error_spec.rb
-- spec/rdkafka/metadata_spec.rb
-- spec/rdkafka/native_kafka_spec.rb
-- spec/rdkafka/producer/delivery_handle_spec.rb
-- spec/rdkafka/producer/delivery_report_spec.rb
-- spec/rdkafka/producer/partitions_count_spec.rb
-- spec/rdkafka/producer_spec.rb
-- spec/spec_helper.rb
+test_files: []