rdkafka 0.21.0 → 0.22.2

data/spec/rdkafka/producer_spec.rb

@@ -53,7 +53,7 @@ describe Rdkafka::Producer do
         let(:producer) do
           rdkafka_producer_config(
             'message.timeout.ms': 1_000_000,
-            :"bootstrap.servers" => "localhost:9094",
+            :"bootstrap.servers" => "127.0.0.1:9094",
           ).producer
         end
 
@@ -340,7 +340,7 @@ describe Rdkafka::Producer do
       )
     end
 
-    expect(messages[0].partition).to eq 0
+    expect(messages[0].partition).to be >= 0
     expect(messages[0].key).to eq 'a'
   end
 
@@ -364,6 +364,48 @@ describe Rdkafka::Producer do
     expect(message.key).to eq "key utf8"
   end
 
+  it "should produce a message to a non-existing topic with key and partition key" do
+    new_topic = "it-#{SecureRandom.uuid}"
+
+    handle = producer.produce(
+      # Needs to be a new topic each time
+      topic:   new_topic,
+      payload: "payload",
+      key:     "key",
+      partition_key: "partition_key",
+      label:   "label"
+    )
+
+    # Should be pending at first
+    expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
+
+    # Check delivery handle and report
+    report = handle.wait(max_wait_timeout: 5)
+    expect(handle.pending?).to be false
+    expect(report).not_to be_nil
+    expect(report.partition).to eq 0
+    expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
+
+    # Flush and close producer
+    producer.flush
+    producer.close
+
+    # Consume message and verify its content
+    message = wait_for_message(
+      topic: new_topic,
+      delivery_report: report,
+      consumer: consumer
+    )
+    expect(message.partition).to eq 0
+    expect(message.payload).to eq "payload"
+    expect(message.key).to eq "key"
+    # Since api.version.request is on by default we will get
+    # the message creation timestamp if it's not set.
+    expect(message.timestamp).to be_within(10).of(Time.now)
+  end
+
   context "timestamp" do
     it "should raise a type error if not nil, integer or time" do
       expect {
@@ -623,7 +665,7 @@ describe Rdkafka::Producer do
   context "when not being able to deliver the message" do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "localhost:9093",
+        "bootstrap.servers": "127.0.0.1:9093",
         "message.timeout.ms": 100
       ).producer
     end
@@ -637,6 +679,25 @@ describe Rdkafka::Producer do
     end
   end
 
+  context "when topic does not exist and allow.auto.create.topics is false" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "127.0.0.1:9092",
+        "message.timeout.ms": 100,
+        "allow.auto.create.topics": false
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: "it-#{SecureRandom.uuid}", payload: nil, label: 'na')
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.error.code).to eq(:msg_timed_out)
+      expect(handler.create_result.label).to eq('na')
+    end
+  end
+
   describe '#partition_count' do
     it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
 
@@ -654,12 +715,11 @@ describe Rdkafka::Producer do
 
     context 'when the partition count value was cached but time expired' do
       before do
-        allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
-        producer.partition_count('consume_test_topic')
+        ::Rdkafka::Producer.partitions_count_cache = Rdkafka::Producer::PartitionsCountCache.new
         allow(::Rdkafka::Metadata).to receive(:new).and_call_original
       end
 
-      it 'expect not to query it again' do
+      it 'expect to query it again' do
         producer.partition_count('consume_test_topic')
         expect(::Rdkafka::Metadata).to have_received(:new)
       end
@@ -694,7 +754,7 @@ describe Rdkafka::Producer do
     context 'when it cannot flush due to a timeout' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -741,7 +801,7 @@ describe Rdkafka::Producer do
     context 'when there are outgoing things in the queue' do
       let(:producer) do
         rdkafka_producer_config(
-          "bootstrap.servers": "localhost:9093",
+          "bootstrap.servers": "127.0.0.1:9093",
           "message.timeout.ms": 2_000
         ).producer
       end
@@ -819,4 +879,467 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe "#produce with headers" do
+    it "should produce a message with array headers" do
+      headers = {
+        "version" => ["2.1.3", "2.1.4"],
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it "should produce a message with single value headers" do
+      headers = {
+        "version" => "2.1.3",
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq('2.1.3')
+    end
+  end
+
+  describe 'with active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to be < post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This will anyhow be populated from statistic
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via blocking but via use stats' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).not_to be_nil
+      end
+    end
+  end
+
+  describe 'without active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to eq post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This should not be populated because stats are not in use
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via anything' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).to be_nil
+      end
+    end
+  end
+
+  describe 'with other fiber closing' do
+    context 'when we create many fibers and close producer in some of them' do
+      it 'expect not to crash ruby' do
+        10.times do |i|
+          producer = rdkafka_producer_config.producer
+
+          Fiber.new do
+            GC.start
+            producer.close
+          end.resume
+        end
+      end
+    end
+  end
+
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
+
+  describe "partitioner behavior through producer API" do
+    context "testing all partitioners with same key" do
+      it "should not return partition 0 for all partitioners" do
+        test_key = "test-key-123"
+        results = {}
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          results[partitioner] = report.partition
+        end
+
+        # Should not all be the same partition (especially not all 0)
+        unique_partitions = results.values.uniq
+        expect(unique_partitions.size).to be > 1
+      end
+    end
+
+    context "empty string partition key" do
+      it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: "",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+        end
+      end
+    end
+
+    context "nil partition key" do
+      it "should handle nil partition key gracefully" do
+        handle = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload",
+          key: "test-key",
+          partition_key: nil
+        )
+
+        report = handle.wait(max_wait_timeout: 5)
+        expect(report.partition).to be >= 0
+        expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+      end
+    end
+
+    context "various key types and lengths with different partitioners" do
+      it "should handle very short keys with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "a",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle very long keys with all partitioners" do
+        long_key = "a" * 1000
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: long_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle unicode keys with all partitioners" do
+        unicode_key = "测试键值🚀"
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: unicode_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "consistency testing for deterministic partitioners" do
+      %w(consistent murmur2 fnv1a).each do |partitioner|
+        it "should consistently route same partition key to same partition with #{partitioner}" do
+          partition_key = "consistent-test-key"
+
+          # Produce multiple messages with same partition key
+          reports = 5.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          # All should go to same partition
+          partitions = reports.map(&:partition).uniq
+          expect(partitions.size).to eq(1)
+        end
+      end
+    end
+
+    context "randomness testing for random partitioners" do
+      %w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
+        it "should potentially distribute across partitions with #{partitioner}" do
+          # Note: random partitioners might still return same value by chance
+          partition_key = "random-test-key"
+
+          reports = 10.times.map do
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload #{Time.now.to_f}",
+              partition_key: partition_key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition)
+
+          # Just ensure they're valid partitions
+          partitions.each do |partition|
+            expect(partition).to be >= 0
+            expect(partition).to be < producer.partition_count("partitioner_test_topic")
+          end
+        end
+      end
+    end
+
+    context "comparing different partitioners with same key" do
+      it "should route different partition keys to potentially different partitions" do
+        keys = ["key1", "key2", "key3", "key4", "key5"]
+
+        all_partitioners.each do |partitioner|
+          reports = keys.map do |key|
+            handle = producer.produce(
+              topic: "partitioner_test_topic",
+              payload: "test payload",
+              partition_key: key,
+              partitioner: partitioner
+            )
+            handle.wait(max_wait_timeout: 5)
+          end
+
+          partitions = reports.map(&:partition).uniq
+
+          # Should distribute across multiple partitions for most partitioners
+          # (though some might hash all keys to same partition by chance)
+          expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
+        end
+      end
+    end
+
+    context "partition key vs regular key behavior" do
+      it "should use partition key for partitioning when both key and partition_key are provided" do
+        # Use keys that would hash to different partitions
+        regular_key = "regular-key-123"
+        partition_key = "partition-key-456"
+
+        # Message with both keys
+        handle1 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 1",
+          key: regular_key,
+          partition_key: partition_key
+        )
+
+        # Message with only partition key (should go to same partition)
+        handle2 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 2",
+          partition_key: partition_key
+        )
+
+        # Message with only regular key (should go to different partition)
+        handle3 = producer.produce(
+          topic: "partitioner_test_topic",
+          payload: "test payload 3",
+          key: regular_key
+        )
+
+        report1 = handle1.wait(max_wait_timeout: 5)
+        report2 = handle2.wait(max_wait_timeout: 5)
+        report3 = handle3.wait(max_wait_timeout: 5)
+
+        # Messages 1 and 2 should go to same partition (both use partition_key)
+        expect(report1.partition).to eq(report2.partition)
+
+        # Message 3 should potentially go to different partition (uses regular key)
+        expect(report3.partition).not_to eq(report1.partition)
+      end
+    end
+
+    context "edge case combinations with different partitioners" do
+      it "should handle nil partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            key: "test-key",
+            partition_key: nil,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle whitespace-only partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "   ",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+
+      it "should handle newline characters in partition key with all partitioners" do
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "test payload",
+            partition_key: "key\nwith\nnewlines",
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          expect(report.partition).to be >= 0
+          expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
+        end
+      end
+    end
+
+    context "debugging partitioner issues" do
+      it "should show if all partitioners return 0 (indicating a problem)" do
+        test_key = "debug-test-key"
+        zero_count = 0
+
+        all_partitioners.each do |partitioner|
+          handle = producer.produce(
+            topic: "partitioner_test_topic",
+            payload: "debug payload",
+            partition_key: test_key,
+            partitioner: partitioner
+          )
+
+          report = handle.wait(max_wait_timeout: 5)
+          zero_count += 1 if report.partition == 0
+        end
+
+        expect(zero_count).to be < all_partitioners.size
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -17,7 +17,10 @@ def rdkafka_base_config
   {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
-    :"bootstrap.servers" => "localhost:9092",
+    :"bootstrap.servers" => "127.0.0.1:9092",
+    # Display statistics and refresh often just to cover those in specs
+    :'statistics.interval.ms' => 1_000,
+    :'topic.metadata.refresh.interval.ms' => 1_000
   }
 end
 
@@ -75,18 +78,32 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = consumer.nil?
-  consumer ||= rdkafka_consumer_config.consumer
+  consumer ||= rdkafka_consumer_config('allow.auto.create.topics': true).consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
+  retry_count = 0
+  max_retries = 10
+
   loop do
     if timeout <= Time.now.to_i
       raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
     end
-    message = consumer.poll(100)
-    if message &&
-        message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset
-      return message
+
+    begin
+      message = consumer.poll(100)
+      if message &&
+          message.partition == delivery_report.partition &&
+          message.offset == delivery_report.offset
+        return message
+      end
+    rescue Rdkafka::RdkafkaError => e
+      if e.code == :unknown_topic_or_part && retry_count < max_retries
+        retry_count += 1
+        sleep(0.1) # Small delay before retry
+        next
+      else
+        raise
+      end
     end
   end
 ensure
@@ -125,6 +142,12 @@ RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true
 
+  config.before(:each) do
+    Rdkafka::Config.statistics_callback = nil
+    # We need to clear it so state does not leak between specs
+    Rdkafka::Producer.partitions_count_cache.to_h.clear
+  end
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {