rdkafka 0.21.0 → 0.22.0.beta1

data/spec/rdkafka/config_spec.rb

@@ -33,23 +33,25 @@ describe Rdkafka::Config do
       expect(log.string).to include "FATAL -- : I love testing"
     end
 
-    it "expect to start new logger thread after fork and work" do
-      reader, writer = IO.pipe
-
-      pid = fork do
-        $stdout.reopen(writer)
-        Rdkafka::Config.logger = Logger.new($stdout)
-        reader.close
-        producer = rdkafka_producer_config(debug: 'all').producer
-        producer.close
+    unless RUBY_PLATFORM == 'java'
+      it "expect to start new logger thread after fork and work" do
+        reader, writer = IO.pipe
+
+        pid = fork do
+          $stdout.reopen(writer)
+          Rdkafka::Config.logger = Logger.new($stdout)
+          reader.close
+          producer = rdkafka_producer_config(debug: 'all').producer
+          producer.close
+          writer.close
+          sleep(1)
+        end
+
         writer.close
-        sleep(1)
+        Process.wait(pid)
+        output = reader.read
+        expect(output.split("\n").size).to be >= 20
       end
-
-      writer.close
-      Process.wait(pid)
-      output = reader.read
-      expect(output.split("\n").size).to be >= 20
     end
   end
 

data/spec/rdkafka/consumer/headers_spec.rb

@@ -3,7 +3,7 @@
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!
-      "version" => "2.1.3",
+      "version" => ["2.1.3", "2.1.4"],
       "type" => "String"
     }
   end
@@ -17,27 +17,39 @@ describe Rdkafka::Consumer::Headers do
         Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
       end
 
+      # First version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers["version"][0]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Second version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers["version"][1]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Single type header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
-          .with(headers_ptr, 2, anything, anything, anything)
+          .with(headers_ptr, 2, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 2", read_string_to_null: "type"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["type"].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 2", read_string: headers["type"]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 3, anything, anything, anything)
           .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
     end
 
@@ -46,8 +58,12 @@ describe Rdkafka::Consumer::Headers do
     it { is_expected.to eq(headers) }
     it { is_expected.to be_frozen }
 
-    it 'allows String key' do
-      expect(subject['version']).to eq("2.1.3")
+    it 'returns array for duplicate headers' do
+      expect(subject['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it 'returns string for single headers' do
+      expect(subject['type']).to eq("String")
     end
 
     it 'does not support symbols mappings' do

data/spec/rdkafka/consumer_spec.rb

@@ -170,8 +170,16 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek" do
+    let(:topic) { "it-#{SecureRandom.uuid}" }
+
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
-      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+      fake_msg = OpenStruct.new(topic: topic, partition: 0, offset: 0)
 
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -181,9 +189,12 @@ describe Rdkafka::Consumer do
 
     context "subscription" do
       let(:timeout) { 1000 }
+      # Some specs here test the manual offset commit hence we want to ensure, that we have some
+      # offsets in-memory that we can manually commit
+      let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
 
       before do
-        consumer.subscribe("consume_test_topic")
+        consumer.subscribe(topic)
 
         # 1. partitions are assigned
         wait_for_assignment(consumer)
@@ -196,7 +207,7 @@ describe Rdkafka::Consumer do
 
       def send_one_message(val)
         producer.produce(
-          topic:     "consume_test_topic",
+          topic:     topic,
           payload:   "payload #{val}",
           key:       "key 1",
           partition: 0
@@ -211,7 +222,7 @@ describe Rdkafka::Consumer do
 
         # 4. pause the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.pause(tpl)
 
         # 5. seek to previous message
@@ -219,7 +230,7 @@ describe Rdkafka::Consumer do
 
         # 6. resume the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.resume(tpl)
 
         # 7. ensure same message is read again
@@ -227,7 +238,7 @@ describe Rdkafka::Consumer do
 
         # This is needed because `enable.auto.offset.store` is true but when running in CI that
         # is overloaded, offset store lags
-        sleep(2)
+        sleep(1)
 
         consumer.commit
         expect(message1.offset).to eq message2.offset
@@ -259,10 +270,17 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek_by" do
-    let(:topic) { "consume_test_topic" }
+    let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
+    let(:topic) { "it-#{SecureRandom.uuid}" }
     let(:partition) { 0 }
     let(:offset) { 0 }
 
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -588,12 +606,18 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+        let(:metadata) { SecureRandom.uuid }
+        let(:group_id) { SecureRandom.uuid }
+        let(:base_config) do
+          {
+            'group.id': group_id,
+            'enable.auto.offset.store': false,
+            'enable.auto.commit': false
+          }
+        end
 
         before do
-          config = {}
-          config[:'enable.auto.offset.store'] = false
-          config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_consumer_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(base_config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -795,12 +819,14 @@ describe Rdkafka::Consumer do
     end
 
     it "should return a message if there is one" do
+      topic = "it-#{SecureRandom.uuid}"
+
       producer.produce(
-        topic:     "consume_test_topic",
+        topic:     topic,
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-      consumer.subscribe("consume_test_topic")
+      consumer.subscribe(topic)
       message = consumer.each {|m| break m}
 
       expect(message).to be_a Rdkafka::Consumer::Message
@@ -1000,7 +1026,7 @@ describe Rdkafka::Consumer do
     after { Rdkafka::Config.statistics_callback = nil }
 
     let(:consumer) do
-      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config = rdkafka_consumer_config('statistics.interval.ms': 500)
       config.consumer_poll_set = false
       config.consumer
     end
@@ -1097,7 +1123,8 @@ describe Rdkafka::Consumer do
         :assign                  => [ nil ],
         :assignment              => nil,
         :committed               => [],
-        :query_watermark_offsets => [ nil, nil ]
+        :query_watermark_offsets => [ nil, nil ],
+        :assignment_lost?        => []
     }.each do |method, args|
       it "raises an exception if #{method} is called" do
         expect {
@@ -1212,4 +1239,36 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  describe "when reaching eof on a topic and eof reporting enabled" do
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
+
+    it "should return proper details" do
+      (0..2).each do |i|
+        producer.produce(
+          topic:     "consume_test_topic",
+          key:       "key lag #{i}",
+          partition: i
+        ).wait
+      end
+
+      # Consume to the end
+      consumer.subscribe("consume_test_topic")
+      eof_count = 0
+      eof_error = nil
+
+      loop do
+        begin
+          consumer.poll(100)
+        rescue Rdkafka::RdkafkaError => error
+          if error.is_partition_eof?
+            eof_error = error
+          end
+          break if eof_error
+        end
+      end
+
+      expect(eof_error.code).to eq(:partition_eof)
+    end
+  end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -30,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+        expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -53,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+      expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/producer/partitions_count_cache_spec.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Rdkafka::Producer::PartitionsCountCache do
+  let(:default_ttl) { 1 } # Reduced from 30 to speed up tests
+  let(:custom_ttl) { 0.5 } # Half the default TTL
+  let(:cache) { described_class.new(default_ttl) }
+  let(:custom_ttl_cache) { described_class.new(custom_ttl) }
+  let(:topic) { "test_topic" }
+  let(:topic2) { "test_topic2" }
+  let(:partition_count) { 5 }
+  let(:higher_partition_count) { 10 }
+  let(:lower_partition_count) { 3 }
+  let(:even_higher_partition_count) { 15 }
+
+  describe "#initialize" do
+    it "creates a cache with default TTL when no TTL is specified" do
+      standard_cache = described_class.new
+      expect(standard_cache).to be_a(described_class)
+    end
+
+    it "creates a cache with custom TTL when specified" do
+      expect(custom_ttl_cache).to be_a(described_class)
+    end
+  end
+
+  describe "#get" do
+    context "when cache is empty" do
+      it "yields to get the value and caches it" do
+        block_called = false
+        result = cache.get(topic) do
+          block_called = true
+          partition_count
+        end
+
+        expect(block_called).to be true
+        expect(result).to eq(partition_count)
+
+        # Verify caching by checking if block is called again
+        second_block_called = false
+        second_result = cache.get(topic) do
+          second_block_called = true
+          partition_count + 1 # Different value to ensure we get cached value
+        end
+
+        expect(second_block_called).to be false
+        expect(second_result).to eq(partition_count)
+      end
+    end
+
+    context "when cache has a value" do
+      before do
+        # Seed the cache with a value
+        cache.get(topic) { partition_count }
+      end
+
+      it "returns cached value without yielding if not expired" do
+        block_called = false
+        result = cache.get(topic) do
+          block_called = true
+          partition_count + 1 # Different value to ensure we get cached one
+        end
+
+        expect(block_called).to be false
+        expect(result).to eq(partition_count)
+      end
+
+      it "yields to get new value when TTL has expired" do
+        # Wait for TTL to expire
+        sleep(default_ttl + 0.1)
+
+        block_called = false
+        new_count = partition_count + 1
+        result = cache.get(topic) do
+          block_called = true
+          new_count
+        end
+
+        expect(block_called).to be true
+        expect(result).to eq(new_count)
+
+        # Verify the new value is cached
+        second_block_called = false
+        second_result = cache.get(topic) do
+          second_block_called = true
+          new_count + 1 # Different value again
+        end
+
+        expect(second_block_called).to be false
+        expect(second_result).to eq(new_count)
+      end
+
+      it "respects a custom TTL" do
+        # Seed the custom TTL cache with a value
+        custom_ttl_cache.get(topic) { partition_count }
+
+        # Wait for custom TTL to expire but not default TTL
+        sleep(custom_ttl + 0.1)
+
+        # Custom TTL cache should refresh
+        custom_block_called = false
+        custom_result = custom_ttl_cache.get(topic) do
+          custom_block_called = true
+          higher_partition_count
+        end
+
+        expect(custom_block_called).to be true
+        expect(custom_result).to eq(higher_partition_count)
+
+        # Default TTL cache should not refresh yet
+        default_block_called = false
+        default_result = cache.get(topic) do
+          default_block_called = true
+          higher_partition_count
+        end
+
+        expect(default_block_called).to be false
+        expect(default_result).to eq(partition_count)
+      end
+    end
+
+    context "when new value is obtained" do
+      before do
+        # Seed the cache with initial value
+        cache.get(topic) { partition_count }
+      end
+
+      it "updates cache when new value is higher than cached value" do
+        # Wait for TTL to expire
+        sleep(default_ttl + 0.1)
+
+        # Get higher value
+        result = cache.get(topic) { higher_partition_count }
+        expect(result).to eq(higher_partition_count)
+
+        # Verify it was cached
+        second_result = cache.get(topic) { fail "Should not be called" }
+        expect(second_result).to eq(higher_partition_count)
+      end
+
+      it "preserves higher cached value when new value is lower" do
+        # First update to higher value
+        sleep(default_ttl + 0.1)
+        cache.get(topic) { higher_partition_count }
+
+        # Then try to update to lower value
+        sleep(default_ttl + 0.1)
+        result = cache.get(topic) { lower_partition_count }
+
+        expect(result).to eq(higher_partition_count)
+
+        # and subsequent gets should return the previously cached higher value
+        second_result = cache.get(topic) { fail "Should not be called" }
+        expect(second_result).to eq(higher_partition_count)
+      end
+
+      it "handles multiple topics independently" do
+        # Set up both topics with different values
+        cache.get(topic) { partition_count }
+        cache.get(topic2) { higher_partition_count }
+
+        # Wait for TTL to expire
+        sleep(default_ttl + 0.1)
+
+        # Update first topic
+        first_result = cache.get(topic) { even_higher_partition_count }
+        expect(first_result).to eq(even_higher_partition_count)
+
+        # Update second topic independently
+        second_updated = higher_partition_count + 3
+        second_result = cache.get(topic2) { second_updated }
+        expect(second_result).to eq(second_updated)
+
+        # Both topics should have their updated values
+        expect(cache.get(topic) { fail "Should not be called" }).to eq(even_higher_partition_count)
+        expect(cache.get(topic2) { fail "Should not be called" }).to eq(second_updated)
+      end
+    end
+  end
+
+  describe "#set" do
+    context "when cache is empty" do
+      it "adds a new entry to the cache" do
+        cache.set(topic, partition_count)
+
+        # Verify through get
+        result = cache.get(topic) { fail "Should not be called" }
+        expect(result).to eq(partition_count)
+      end
+    end
+
+    context "when cache already has a value" do
+      before do
+        cache.set(topic, partition_count)
+      end
+
+      it "updates cache when new value is higher" do
+        cache.set(topic, higher_partition_count)
+
+        result = cache.get(topic) { fail "Should not be called" }
+        expect(result).to eq(higher_partition_count)
+      end
+
+      it "keeps original value when new value is lower" do
+        cache.set(topic, lower_partition_count)
+
+        result = cache.get(topic) { fail "Should not be called" }
+        expect(result).to eq(partition_count)
+      end
+
+      it "updates the timestamp even when keeping original value" do
+        # Set initial value
+        cache.set(topic, partition_count)
+
+        # Wait until close to TTL expiring
+        sleep(default_ttl - 0.2)
+
+        # Set lower value (should update timestamp but not value)
+        cache.set(topic, lower_partition_count)
+
+        # Wait a bit more, but still under the full TTL if timestamp was refreshed
+        sleep(0.3)
+
+        # Should still be valid due to timestamp refresh
+        result = cache.get(topic) { fail "Should not be called" }
+        expect(result).to eq(partition_count)
+      end
+    end
+
+    context "with concurrent access" do
+      it "correctly handles simultaneous updates to the same topic" do
+        # This test focuses on the final value after concurrent updates
+        threads = []
+
+        # Create 5 threads that all try to update the same topic with increasing values
+        5.times do |i|
+          threads << Thread.new do
+            value = 10 + i  # Start at 10 to ensure all are higher than initial value
+            cache.set(topic, value)
+          end
+        end
+
+        # Wait for all threads to complete
+        threads.each(&:join)
+
+        # The highest value (14) should be stored and accessible through get
+        result = cache.get(topic) { fail "Should not be called" }
+        expect(result).to eq(14)
+      end
+    end
+  end
+
+  describe "TTL behavior" do
+    it "treats entries as expired when they exceed TTL" do
+      # Set initial value
+      cache.get(topic) { partition_count }
+
+      # Wait just under TTL
+      sleep(default_ttl - 0.2)
+
+      # Value should still be cached (block should not be called)
+      result = cache.get(topic) { fail "Should not be called when cache is valid" }
+      expect(result).to eq(partition_count)
+
+      # Now wait to exceed TTL
+      sleep(0.2) # Total sleep is now default_ttl + 0.1
+
+      # Cache should be expired, block should be called
+      block_called = false
+      new_value = partition_count + 3
+      result = cache.get(topic) do
+        block_called = true
+        new_value
+      end
+
+      expect(block_called).to be true
+      expect(result).to eq(new_value)
+    end
+  end
+
+  describe "comprehensive scenarios" do
+    it "handles a full lifecycle of cache operations" do
+      # 1. Initial cache miss, fetch and store
+      result1 = cache.get(topic) { partition_count }
+      expect(result1).to eq(partition_count)
+
+      # 2. Cache hit
+      result2 = cache.get(topic) { fail "Should not be called" }
+      expect(result2).to eq(partition_count)
+
+      # 3. Attempt to set lower value
+      cache.set(topic, lower_partition_count)
+      result3 = cache.get(topic) { fail "Should not be called" }
+      # Should still return the higher original value
+      expect(result3).to eq(partition_count)
+
+      # 4. Set higher value
+      cache.set(topic, higher_partition_count)
+      result4 = cache.get(topic) { fail "Should not be called" }
+      expect(result4).to eq(higher_partition_count)
+
+      # 5. TTL expires, new value provided is lower
+      sleep(default_ttl + 0.1)
+      result5 = cache.get(topic) { lower_partition_count }
+      # This returns the highest value
+      expect(result5).to eq(higher_partition_count)
+
+      # 6. But subsequent get should return the higher cached value
+      result6 = cache.get(topic) { fail "Should not be called" }
+      expect(result6).to eq(higher_partition_count)
+
+      # 7. Set new highest value directly
+      even_higher = higher_partition_count + 5
+      cache.set(topic, even_higher)
+      result7 = cache.get(topic) { fail "Should not be called" }
+      expect(result7).to eq(even_higher)
+    end
+
+    it "handles multiple topics with different TTLs correctly" do
+      # Set up initial values
+      cache.get(topic) { partition_count }
+      custom_ttl_cache.get(topic) { partition_count }
+
+      # Wait past custom TTL but not default TTL
+      sleep(custom_ttl + 0.1)
+
+      # Default cache should NOT refresh (still within default TTL)
+      default_result = cache.get(topic) { fail "Should not be called for default cache" }
+      # Original value should be maintained
+      expect(default_result).to eq(partition_count)
+
+      # Custom TTL cache SHOULD refresh (past custom TTL)
+      custom_cache_value = partition_count + 8
+      custom_block_called = false
+      custom_result = custom_ttl_cache.get(topic) do
+        custom_block_called = true
+        custom_cache_value
+      end
+
+      expect(custom_block_called).to be true
+      expect(custom_result).to eq(custom_cache_value)
+
+      # Now wait past default TTL
+      sleep(default_ttl - custom_ttl + 0.1)
+
+      # Now default cache should also refresh
+      default_block_called = false
+      new_default_value = partition_count + 10
+      new_default_result = cache.get(topic) do
+        default_block_called = true
+        new_default_value
+      end
+
+      expect(default_block_called).to be true
+      expect(new_default_result).to eq(new_default_value)
+    end
+  end
+end