rdkafka 0.20.0 → 0.21.1.alpha2

data/lib/rdkafka/producer.rb

@@ -6,13 +6,31 @@ module Rdkafka
     include Helpers::Time
     include Helpers::OAuth
 
-    # Cache partitions count for 30 seconds
-    PARTITIONS_COUNT_TTL = 30
-
     # Empty hash used as a default
     EMPTY_HASH = {}.freeze
 
-    private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
+    # @private
+    @@partitions_count_cache = PartitionsCountCache.new
+
+    # Global (process wide) partitions cache. We use it to store number of topics partitions,
+    # either from the librdkafka statistics (if enabled) or via direct inline calls every now and
+    # then. Since the partitions count can only grow and should be same for all consumers and
+    # producers, we can use a global cache as long as we ensure that updates only move up.
+    #
+    # @note It is critical to remember, that not all users may have statistics callbacks enabled,
+    #   hence we should not make assumption that this cache is always updated from the stats.
+    #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
+    def self.partitions_count_cache
+      @@partitions_count_cache
+    end
+
+    # @param partitions_count_cache [Rdkafka::Producer::PartitionsCountCache]
+    def self.partitions_count_cache=(partitions_count_cache)
+      @@partitions_count_cache = partitions_count_cache
+    end
+
+    private_constant :EMPTY_HASH
 
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
     # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
@@ -43,25 +61,6 @@ module Rdkafka
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
-
-      @_partitions_count_cache = Hash.new do |cache, topic|
-        topic_metadata = nil
-
-        @native_kafka.with_inner do |inner|
-          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
-        end
-
-        partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
-
-        # This approach caches the failure to fetch only for 1 second. This will make sure, that
-        # we do not cache the failure for too long but also "buys" us a bit of time in case there
-        # would be issues in the cluster so we won't overaload it with consecutive requests
-        cache[topic] = if partition_count.positive?
-                         [monotonic_now, partition_count]
-                       else
-                         [monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
-                       end
-      end
     end
 
     # Sets alternative set of configuration details that can be set per topic
@@ -222,18 +221,24 @@ module Rdkafka
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
     #
-    # @note We cache the partition count for a given topic for given time.
+    # @note We cache the partition count for a given topic for given time. If statistics are
+    #   enabled for any producer or consumer, it will take precedence over per instance fetching.
+    #
     #   This prevents us in case someone uses `partition_key` from querying for the count with
-    #   each message. Instead we query once every 30 seconds at most if we have a valid partition
-    #   count or every 5 seconds in case we were not able to obtain number of partitions
+    #   each message. Instead we query at most once every 30 seconds at most if we have a valid
+    #   partition count or every 5 seconds in case we were not able to obtain number of partitions.
     def partition_count(topic)
       closed_producer_check(__method__)
 
-      @_partitions_count_cache.delete_if do |_, cached|
-        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
-      end
+      self.class.partitions_count_cache.get(topic) do
+        topic_metadata = nil
 
-      @_partitions_count_cache[topic].last
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        topic_metadata ? topic_metadata[:partition_count] : -1
+      end
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -247,7 +252,7 @@ module Rdkafka
     # @param partition [Integer,nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String>] Optional message headers
+    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
@@ -339,11 +344,23 @@ module Rdkafka
       if headers
         headers.each do |key0, value0|
           key = key0.to_s
-          value = value0.to_s
-          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
-          args << :string << key
-          args << :pointer << value
-          args << :size_t << value.bytesize
+          if value0.is_a?(Array)
+            # Handle array of values per KIP-82
+            value0.each do |value|
+              value = value.to_s
+              args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+              args << :string << key
+              args << :pointer << value
+              args << :size_t << value.bytesize
+            end
+          else
+            # Handle single value
+            value = value0.to_s
+            args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+            args << :string << key
+            args << :pointer << value
+            args << :size_t << value.bytesize
+          end
         end
       end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.20.0"
-  LIBRDKAFKA_VERSION = "2.6.1"
-  LIBRDKAFKA_SOURCE_SHA256 = "0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b"
+  VERSION = "0.21.1.alpha2"
+  LIBRDKAFKA_VERSION = "2.8.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/lib/rdkafka.rb

@@ -42,6 +42,7 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/native_kafka"
+require "rdkafka/producer/partitions_count_cache"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -17,11 +17,6 @@ Gem::Specification.new do |gem|
   gem.version = Rdkafka::VERSION
   gem.required_ruby_version = '>= 3.1'
   gem.extensions = %w(ext/Rakefile)
-  gem.cert_chain = %w[certs/cert.pem]
-
-  if $PROGRAM_NAME.end_with?('gem')
-    gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
-  end
 
   gem.add_dependency 'ffi', '~> 1.15'
   gem.add_dependency 'mini_portile2', '~> 2.6'
@@ -37,10 +32,10 @@ Gem::Specification.new do |gem|
   gem.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/CHANGELOG.md',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-Rdkafka',
     'bug_tracker_uri' => 'https://github.com/karafka/rdkafka-ruby/issues',
     'source_code_uri' => 'https://github.com/karafka/rdkafka-ruby',
-    'documentation_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/README.md',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/renovate.json

@@ -1,6 +1,18 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base"
+    "config:recommended"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

data/spec/rdkafka/admin_spec.rb

@@ -738,17 +738,19 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
     end
   end
 
-  context "when operating from a fork" do
-    # @see https://github.com/ffi/ffi/issues/1114
-    it 'expect to be able to create topics and run other admin operations without hanging' do
-      # If the FFI issue is not mitigated, this will hang forever
-      pid = fork do
-        admin
-          .create_topic(topic_name, topic_partition_count, topic_replication_factor)
-          .wait
-      end
+  unless RUBY_PLATFORM == 'java'
+    context "when operating from a fork" do
+      # @see https://github.com/ffi/ffi/issues/1114
+      it 'expect to be able to create topics and run other admin operations without hanging' do
+        # If the FFI issue is not mitigated, this will hang forever
+        pid = fork do
+          admin
+            .create_topic(topic_name, topic_partition_count, topic_replication_factor)
+            .wait
+        end
 
-      Process.wait(pid)
+        Process.wait(pid)
+      end
     end
   end
 end

data/spec/rdkafka/bindings_spec.rb

@@ -149,15 +149,6 @@ describe Rdkafka::Bindings do
   end
 
   describe "oauthbearer set token" do
-
-    context "without args" do
-      it "should raise argument error" do
-        expect {
-          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token
-        }.to raise_error(ArgumentError)
-      end
-    end
-
     context "with args" do
       before do
         DEFAULT_TOKEN_EXPIRY_SECONDS = 900

data/spec/rdkafka/config_spec.rb

@@ -33,23 +33,25 @@ describe Rdkafka::Config do
       expect(log.string).to include "FATAL -- : I love testing"
     end
 
-    it "expect to start new logger thread after fork and work" do
-      reader, writer = IO.pipe
-
-      pid = fork do
-        $stdout.reopen(writer)
-        Rdkafka::Config.logger = Logger.new($stdout)
-        reader.close
-        producer = rdkafka_producer_config(debug: 'all').producer
-        producer.close
+    unless RUBY_PLATFORM == 'java'
+      it "expect to start new logger thread after fork and work" do
+        reader, writer = IO.pipe
+
+        pid = fork do
+          $stdout.reopen(writer)
+          Rdkafka::Config.logger = Logger.new($stdout)
+          reader.close
+          producer = rdkafka_producer_config(debug: 'all').producer
+          producer.close
+          writer.close
+          sleep(1)
+        end
+
         writer.close
-        sleep(1)
+        Process.wait(pid)
+        output = reader.read
+        expect(output.split("\n").size).to be >= 20
       end
-
-      writer.close
-      Process.wait(pid)
-      output = reader.read
-      expect(output.split("\n").size).to be >= 20
     end
   end
 

data/spec/rdkafka/consumer/headers_spec.rb

@@ -3,7 +3,7 @@
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!
-      "version" => "2.1.3",
+      "version" => ["2.1.3", "2.1.4"],
       "type" => "String"
     }
   end
@@ -17,27 +17,39 @@ describe Rdkafka::Consumer::Headers do
         Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
       end
 
+      # First version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers["version"][0]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Second version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers["version"][1]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Single type header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
-          .with(headers_ptr, 2, anything, anything, anything)
+          .with(headers_ptr, 2, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 2", read_string_to_null: "type"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["type"].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 2", read_string: headers["type"]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 3, anything, anything, anything)
           .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
     end
 
@@ -46,8 +58,12 @@ describe Rdkafka::Consumer::Headers do
     it { is_expected.to eq(headers) }
     it { is_expected.to be_frozen }
 
-    it 'allows String key' do
-      expect(subject['version']).to eq("2.1.3")
+    it 'returns array for duplicate headers' do
+      expect(subject['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it 'returns string for single headers' do
+      expect(subject['type']).to eq("String")
     end
 
     it 'does not support symbols mappings' do