rdkafka 0.15.1 → 0.16.0.rc1

data/lib/rdkafka/config.rb

@@ -15,13 +15,13 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-
-    Thread.start do
-      loop do
-        severity, msg = @@log_queue.pop
-        @@logger.add(severity, msg)
-      end
-    end
+    # We memoize thread on the first log flush
+    # This allows us also to restart logger thread on forks
+    @@log_thread = nil
+    # @private
+    @@log_mutex = Mutex.new
+    # @private
+    @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -30,6 +30,24 @@ module Rdkafka
       @@logger
     end
 
+    # Makes sure that there is a thread for consuming logs
+    # We do not spawn thread immediately and we need to check if it operates to support forking
+    def self.ensure_log_thread
+      return if @@log_thread && @@log_thread.alive?
+
+      @@log_mutex.synchronize do
+        # Restart if dead (fork, crash)
+        @@log_thread = nil if @@log_thread && !@@log_thread.alive?
+
+        @@log_thread ||= Thread.start do
+          loop do
+            severity, msg = @@log_queue.pop
+            @@logger.add(severity, msg)
+          end
+        end
+      end
+    end
+
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -87,6 +105,24 @@ module Rdkafka
       @@error_callback
     end
 
+    # Sets the SASL/OAUTHBEARER token refresh callback.
+    # This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.oauthbearer_token_refresh_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      @@oauthbearer_token_refresh_callback = callback
+    end
+
+    # Returns the current oauthbearer_token_refresh_callback callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.oauthbearer_token_refresh_callback
+      @@oauthbearer_token_refresh_callback
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -159,11 +195,13 @@ module Rdkafka
 
     # Creates a consumer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Consumer] The created consumer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def consumer
+    def consumer(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
 
@@ -183,18 +221,21 @@ module Rdkafka
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: false,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
 
     # Create a producer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer
+    def producer(native_kafka_auto_start: true)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -203,11 +244,15 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Producer.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         ),
         partitioner_name
       ).tap do |producer|
@@ -217,19 +262,25 @@ module Rdkafka
 
     # Creates an admin instance with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin
+    def admin(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Admin.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
@@ -283,6 +334,9 @@ module Rdkafka
 
         # Set error callback
         Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+
+        # Set oauth callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -13,12 +13,19 @@ module Rdkafka
   class Consumer
     include Enumerable
     include Helpers::Time
+    include Helpers::OAuth
 
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] consumer name
     def name
       @name ||= @native_kafka.with_inner do |inner|

data/lib/rdkafka/helpers/oauth.rb

@@ -0,0 +1,58 @@
+module Rdkafka
+  module Helpers
+
+    module OAuth
+
+      # Set the OAuthBearer token
+      #
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param principal_name [String] the mandatory Kafka principal name associated with the token.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @return [Integer] 0 on success
+      def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
+          )
+        end
+
+        return response if response.zero?
+
+        oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
+
+        response
+      end
+
+      # Marks failed oauth token acquire in librdkafka
+      #
+      # @param reason [String] human readable error reason for failing to acquire token
+      def oauthbearer_set_token_failure(reason)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
+            inner,
+            reason
+          )
+        end
+      end
+
+      private
+
+      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
+      def flatten_extensions(extensions)
+        return nil unless extensions
+        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      end
+
+      # extension_size is the number of keys + values which should be a non-negative even number
+      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      def extension_size(extensions)
+        return 0 unless extensions
+        extensions.size * 2
+      end
+    end
+  end
+end

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -28,30 +28,43 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
-      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
-      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+      @run_polling_thread = run_polling_thread
 
-      if run_polling_thread
-        # Start thread to poll client for delivery callbacks,
-        # not used in consumer.
-        @polling_thread = Thread.new do
-          loop do
-            @poll_mutex.synchronize do
-              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
-            end
+      start if auto_start
 
-            # Exit thread if closing and the poll queue is empty
-            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-              break
+      @closing = false
+    end
+
+    def start
+      synchronize do
+        return if @started
+
+        @started = true
+
+        # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+        Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
+
+        if @run_polling_thread
+          # Start thread to poll client for delivery callbacks,
+          # not used in consumer.
+          @polling_thread = Thread.new do
+            loop do
+              @poll_mutex.synchronize do
+                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+              end
+
+              # Exit thread if closing and the poll queue is empty
+              if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
+                break
+              end
             end
           end
-        end
 
-        @polling_thread.abort_on_exception = true
-        @polling_thread[:closing] = false
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.abort_on_exception = true
+          @polling_thread[:closing] = false
+        end
       end
-
-      @closing = false
     end
 
     def with_inner

data/lib/rdkafka/producer.rb

@@ -4,11 +4,20 @@ module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
     include Helpers::Time
+    include Helpers::OAuth
 
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
 
-    private_constant :PARTITIONS_COUNT_TTL
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
+    private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
+
+    # Raised when there was a critical issue when invoking rd_kafka_topic_new
+    # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
+    # resolved and this is normalized in all the places
+    class TopicHandleCreationError < RuntimeError; end
 
     # @private
     # Returns the current delivery callback, by default this is nil.
@@ -27,6 +36,8 @@ module Rdkafka
     # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
     def initialize(native_kafka, partitioner_name)
+      @topics_refs_map = {}
+      @topics_configs = {}
       @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
@@ -53,6 +64,58 @@ module Rdkafka
       end
     end
 
+    # Sets alternative set of configuration details that can be set per topic
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
+    # @param topic [String] The topic name
+    # @param config [Hash] config we want to use per topic basis
+    # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
+    #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    def set_topic_config(topic, config, config_hash)
+      # Ensure lock on topic reference just in case
+      @native_kafka.with_inner do |inner|
+        @topics_refs_map[topic] ||= {}
+        @topics_configs[topic] ||= {}
+
+        return if @topics_configs[topic].key?(config_hash)
+
+        # If config is empty, we create an empty reference that will be used with defaults
+        rd_topic_config = if config.empty?
+                            nil
+                          else
+                            Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+                              config.each do |key, value|
+                                error_buffer = FFI::MemoryPointer.new(:char, 256)
+                                result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+                                  topic_config,
+                                  key.to_s,
+                                  value.to_s,
+                                  error_buffer,
+                                  256
+                                )
+
+                                unless result == :config_ok
+                                  raise Config::ConfigError.new(error_buffer.read_string)
+                                end
+                              end
+                            end
+                          end
+
+        topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
+
+        raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
+
+        @topics_configs[topic][config_hash] = config
+        @topics_refs_map[topic][config_hash] = topic_handle
+      end
+    end
+
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] producer name
     def name
       @name ||= @native_kafka.with_inner do |inner|
@@ -76,7 +139,18 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.close
+
+      @native_kafka.close do
+        # We need to remove the topics references objects before we destroy the producer,
+        # otherwise they would leak out
+        @topics_refs_map.each_value do |refs|
+          refs.each_value do |ref|
+            Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+          end
+        end
+      end
+
+      @topics_refs_map.clear
     end
 
     # Whether this producer has closed
@@ -175,11 +249,22 @@ module Rdkafka
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
+    # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
+    def produce(
+      topic:,
+      payload: nil,
+      key: nil,
+      partition: nil,
+      partition_key: nil,
+      timestamp: nil,
+      headers: nil,
+      label: nil,
+      topic_config: EMPTY_HASH
+    )
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -198,8 +283,20 @@ module Rdkafka
                    key.bytesize
                  end
 
+      topic_config_hash = topic_config.hash
+
+      # Checks if we have the rdkafka topic reference object ready. It saves us on object
+      # allocation and allows to use custom config on demand.
+      set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
+      topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
+
       if partition_key
         partition_count = partition_count(topic)
+
+        # Check if there are no overrides for the partitioner and use the default one only when
+        # no per-topic is present.
+        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+
         # If the topic is not present, set to -1
         partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
@@ -229,7 +326,7 @@ module Rdkafka
       DeliveryHandle.register(delivery_handle)
 
       args = [
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.1"
+  VERSION = "0.16.0.rc1"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -7,6 +7,7 @@ require "json"
 
 require "rdkafka/version"
 require "rdkafka/helpers/time"
+require "rdkafka/helpers/oauth"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -23,7 +24,13 @@ require "rdkafka/admin/delete_acl_handle"
 require "rdkafka/admin/delete_acl_report"
 require "rdkafka/admin/describe_acl_handle"
 require "rdkafka/admin/describe_acl_report"
+require "rdkafka/admin/describe_configs_handle"
+require "rdkafka/admin/describe_configs_report"
+require "rdkafka/admin/incremental_alter_configs_handle"
+require "rdkafka/admin/incremental_alter_configs_report"
 require "rdkafka/admin/acl_binding_result"
+require "rdkafka/admin/config_binding_result"
+require "rdkafka/admin/config_resource_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"

data/rdkafka.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.7'
+  gem.required_ruby_version = '>= 3.0'
   gem.extensions = %w(ext/Rakefile)
   gem.cert_chain = %w[certs/cert_chain.pem]
 

data/spec/rdkafka/abstract_handle_spec.rb

@@ -76,37 +76,50 @@ describe Rdkafka::AbstractHandle do
   end
 
   describe "#wait" do
-    let(:pending_handle) { true }
+    context 'when pending_handle true' do
+      let(:pending_handle) { true }
 
-    it "should wait until the timeout and then raise an error" do
-      expect {
-        subject.wait(max_wait_timeout: 0.1)
-      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      it "should wait until the timeout and then raise an error" do
+        expect(Kernel).not_to receive(:warn)
+        expect {
+          subject.wait(max_wait_timeout: 0.1)
+        }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      end
     end
 
-    context "when not pending anymore and no error" do
+    context 'when pending_handle false' do
       let(:pending_handle) { false }
-      let(:result) { 1 }
 
-      it "should return a result" do
-        wait_result = subject.wait
-        expect(wait_result).to eq(result)
+      it 'should show a deprecation warning when wait_timeout is set' do
+        expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
+        subject.wait(wait_timeout: 0.1)
       end
 
-      it "should wait without a timeout" do
-        wait_result = subject.wait(max_wait_timeout: nil)
-        expect(wait_result).to eq(result)
+      context "without error" do
+        let(:result) { 1 }
+
+        it "should return a result" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait
+          expect(wait_result).to eq(result)
+        end
+
+        it "should wait without a timeout" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait(max_wait_timeout: nil)
+          expect(wait_result).to eq(result)
+        end
       end
-    end
 
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
+      context "with error" do
+        let(:response) { 20 }
 
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
+        it "should raise an rdkafka error" do
+          expect(Kernel).not_to receive(:warn)
+          expect {
+            subject.wait
+          }.to raise_error Rdkafka::RdkafkaError
+        end
       end
     end
   end