karafka-rdkafka 0.19.1 → 0.19.2.rc1

data/spec/rdkafka/producer_spec.rb

@@ -654,12 +654,11 @@ describe Rdkafka::Producer do
 
     context 'when the partition count value was cached but time expired' do
       before do
-        allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
-        producer.partition_count('example_topic')
+        ::Rdkafka::Producer.partitions_count_cache = Rdkafka::Producer::PartitionsCountCache.new
         allow(::Rdkafka::Metadata).to receive(:new).and_call_original
       end
 
-      it 'expect not to query it again' do
+      it 'expect to query it again' do
         producer.partition_count('example_topic')
         expect(::Rdkafka::Metadata).to have_received(:new)
       end
@@ -1042,4 +1041,118 @@ describe Rdkafka::Producer do
       expect(message.headers['version']).to eq('2.1.3')
     end
   end
+
+  describe 'with active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to be < post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This will anyhow be populated from statistic
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via blocking but via use stats' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).not_to be_nil
+      end
+    end
+  end
+
+  describe 'without active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to eq post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This should not be populated because stats are not in use
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via anything' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).to be_nil
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -18,6 +18,9 @@ def rdkafka_base_config
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
+    # Display statistics and refresh often just to cover those in specs
+    :'statistics.interval.ms' => 1_000,
+    :'topic.metadata.refresh.interval.ms' => 1_000
   }
 end
 
@@ -125,6 +128,12 @@ RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true
 
+  config.before(:each) do
+    Rdkafka::Config.statistics_callback = nil
+    # We need to clear it so state does not leak between specs
+    Rdkafka::Producer.partitions_count_cache.to_h.clear
+  end
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {

data.tar.gz.sig

@@ -1,3 +1 @@
-��E�d/�;��u'q�������ˏn)��}-^ �V�l-���^@M��%b���s��{i�l�ý��-��?�L���1�l�s��c��Z$xɪW�Ƚ��l(��D�|[����)�5V̚��k%�X~�Ÿ��%
-���1}kڃ��29���ؔ�1!T�ޯ[��8��	���^�Ԥ�gk�	3�kP��o�&)�=`
]c|u(�$
-!%������,Ȓ �������`��V�m+Y.�бt�M�������z�hc,���
+%��;��l�x�_9|�e�5�Bv77)��N^lӷ�o<`x���5��jYKFdJ��"/V5��p�\

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.19.1
+  version: 0.19.2.rc1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2025-04-07 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -233,6 +233,7 @@ files:
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
+- lib/rdkafka/producer/partitions_count_cache.rb
 - lib/rdkafka/version.rb
 - renovate.json
 - spec/rdkafka/abstract_handle_spec.rb
@@ -260,6 +261,8 @@ files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
+- spec/rdkafka/producer/partitions_count_cache_spec.rb
+- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
@@ -286,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
@@ -317,5 +320,7 @@ test_files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
+- spec/rdkafka/producer/partitions_count_cache_spec.rb
+- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb