karafka-rdkafka 0.19.0 → 0.19.2.rc1

data/spec/rdkafka/producer_spec.rb

@@ -654,12 +654,11 @@ describe Rdkafka::Producer do
 
     context 'when the partition count value was cached but time expired' do
       before do
-        allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
-        producer.partition_count('example_topic')
+        ::Rdkafka::Producer.partitions_count_cache = Rdkafka::Producer::PartitionsCountCache.new
         allow(::Rdkafka::Metadata).to receive(:new).and_call_original
       end
 
-      it 'expect not to query it again' do
+      it 'expect to query it again' do
         producer.partition_count('example_topic')
         expect(::Rdkafka::Metadata).to have_received(:new)
       end
@@ -1002,4 +1001,158 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe "#produce with headers" do
+    it "should produce a message with array headers" do
+      headers = {
+        "version" => ["2.1.3", "2.1.4"],
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it "should produce a message with single value headers" do
+      headers = {
+        "version" => "2.1.3",
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq('2.1.3')
+    end
+  end
+
+  describe 'with active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to be < post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        Rdkafka::Config.statistics_callback = ->(*) {}
+
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This will anyhow be populated from statistic
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via blocking but via use stats' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).not_to be_nil
+      end
+    end
+  end
+
+  describe 'without active statistics callback' do
+    let(:producer) do
+      rdkafka_producer_config('statistics.interval.ms': 1_000).producer
+    end
+
+    let(:count_cache_hash) { described_class.partitions_count_cache.to_h }
+    let(:pre_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+    let(:post_statistics_ttl) { count_cache_hash.fetch('produce_test_topic', [])[0] }
+
+    context "when using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers",
+          partition_key: "test"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via statistics' do
+        expect(pre_statistics_ttl).to eq post_statistics_ttl
+      end
+    end
+
+    context "when not using partition key" do
+      before do
+        # This call will make a blocking request to the metadata cache
+        producer.produce(
+          topic:         "produce_test_topic",
+          payload:       "payload headers"
+        ).wait
+
+        pre_statistics_ttl
+
+        # We wait to make sure that statistics are triggered and that there is a refresh
+        sleep(1.5)
+
+        # This should not be populated because stats are not in use
+        post_statistics_ttl
+      end
+
+      it 'expect not to update ttl on the partitions count cache via anything' do
+        expect(pre_statistics_ttl).to be_nil
+        expect(post_statistics_ttl).to be_nil
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -18,6 +18,9 @@ def rdkafka_base_config
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
+    # Display statistics and refresh often just to cover those in specs
+    :'statistics.interval.ms' => 1_000,
+    :'topic.metadata.refresh.interval.ms' => 1_000
   }
 end
 
@@ -125,6 +128,12 @@ RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true
 
+  config.before(:each) do
+    Rdkafka::Config.statistics_callback = nil
+    # We need to clear it so state does not leak between specs
+    Rdkafka::Producer.partitions_count_cache.to_h.clear
+  end
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {

data.tar.gz.sig

@@ -1,2 +1 @@
-<����T��HT9wKo�s<��
-��Y@�J�w�vr���|���%W��6^[�G����q�3�D���ڟ0�:d6����#�ă�C*�Ƌ{����Lw�~��A�m�;6Ey��$0�D��@����`�-5@�	�V�%'��׏
󸤆L��'$��s����X|�!b*�[�,�����6�$Zli�Ek��¹g�}M��fi�0:���?����UЖJ��ki>i�$u*�����R&�܏�Q�P"š՚|$�cyJ��U"Ф�]QB�L4��|�������R�vq�ۃW'F�17(��ǻX���@]J�~�F�"��q�Ŝ���OJ�09�D�1Pq��>Q�/��4�d��O��oa/$/C��'h��Lr�(��\
+%��;��l�x�_9|�e�5�Bv77)��N^lӷ�o<`x���5��jYKFdJ��"/V5��p�\

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.19.2.rc1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2025-01-20 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -171,8 +171,10 @@ extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -231,6 +233,7 @@ files:
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
+- lib/rdkafka/producer/partitions_count_cache.rb
 - lib/rdkafka/version.rb
 - renovate.json
 - spec/rdkafka/abstract_handle_spec.rb
@@ -258,6 +261,8 @@ files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
+- spec/rdkafka/producer/partitions_count_cache_spec.rb
+- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
 licenses:
@@ -284,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
@@ -315,5 +320,7 @@ test_files:
 - spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
+- spec/rdkafka/producer/partitions_count_cache_spec.rb
+- spec/rdkafka/producer/partitions_count_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb