rdkafka 0.15.0 → 0.16.0.beta1

data/lib/rdkafka/callbacks.rb

@@ -156,7 +156,8 @@ module Rdkafka
           create_topic_handle[:response] = create_topic_results[0].result_error
           create_topic_handle[:error_string] = create_topic_results[0].error_string
           create_topic_handle[:result_name] = create_topic_results[0].result_name
-          create_topic_handle[:pending] = false
+
+          create_topic_handle.unlock
         end
       end
 
@@ -173,7 +174,8 @@ module Rdkafka
           delete_group_handle[:response] = delete_group_results[0].result_error
           delete_group_handle[:error_string] = delete_group_results[0].error_string
           delete_group_handle[:result_name] = delete_group_results[0].result_name
-          delete_group_handle[:pending] = false
+
+          delete_group_handle.unlock
         end
       end
 
@@ -190,7 +192,8 @@ module Rdkafka
           delete_topic_handle[:response] = delete_topic_results[0].result_error
           delete_topic_handle[:error_string] = delete_topic_results[0].error_string
           delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-          delete_topic_handle[:pending] = false
+
+          delete_topic_handle.unlock
         end
       end
 
@@ -207,7 +210,8 @@ module Rdkafka
           create_partitions_handle[:response] = create_partitions_results[0].result_error
           create_partitions_handle[:error_string] = create_partitions_results[0].error_string
           create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-          create_partitions_handle[:pending] = false
+
+          create_partitions_handle.unlock
         end
       end
 
@@ -223,7 +227,8 @@ module Rdkafka
         if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
           create_acl_handle[:response] = create_acl_results[0].result_error
           create_acl_handle[:response_string] = create_acl_results[0].error_string
-          create_acl_handle[:pending] = false
+
+          create_acl_handle.unlock
         end
       end
 
@@ -239,11 +244,13 @@ module Rdkafka
         if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
           delete_acl_handle[:response] = delete_acl_results[0].result_error
           delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-          delete_acl_handle[:pending] = false
+
           if delete_acl_results[0].result_error == 0
             delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
             delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
           end
+
+          delete_acl_handle.unlock
         end
       end
 
@@ -254,18 +261,18 @@ module Rdkafka
         if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
           describe_acl_handle[:response] = describe_acl.result_error
           describe_acl_handle[:response_string] = describe_acl.error_string
-          describe_acl_handle[:pending] = false
+
           if describe_acl.result_error == 0
             describe_acl_handle[:acls]       = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end
+
+          describe_acl_handle.unlock
         end
       end
-
     end
 
     # FFI Function used for Message Delivery callbacks
-
     DeliveryCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -285,15 +292,24 @@ module Rdkafka
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
           delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-          delivery_handle[:pending] = false
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
+            opaque.call_delivery_callback(
+              Rdkafka::Producer::DeliveryReport.new(
+                message[:partition],
+                message[:offset],
+                topic_name,
+                message[:err],
+                delivery_handle.label
+              ),
+              delivery_handle
+            )
           end
+
+          delivery_handle.unlock
         end
       end
     end
-
   end
 end

data/lib/rdkafka/config.rb

@@ -15,13 +15,13 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-
-    Thread.start do
-      loop do
-        severity, msg = @@log_queue.pop
-        @@logger.add(severity, msg)
-      end
-    end
+    # We memoize thread on the first log flush
+    # This allows us also to restart logger thread on forks
+    @@log_thread = nil
+    # @private
+    @@log_mutex = Mutex.new
+    # @private
+    @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -30,6 +30,24 @@ module Rdkafka
       @@logger
     end
 
+    # Makes sure that there is a thread for consuming logs
+    # We do not spawn thread immediately and we need to check if it operates to support forking
+    def self.ensure_log_thread
+      return if @@log_thread && @@log_thread.alive?
+
+      @@log_mutex.synchronize do
+        # Restart if dead (fork, crash)
+        @@log_thread = nil if @@log_thread && !@@log_thread.alive?
+
+        @@log_thread ||= Thread.start do
+          loop do
+            severity, msg = @@log_queue.pop
+            @@logger.add(severity, msg)
+          end
+        end
+      end
+    end
+
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -87,6 +105,24 @@ module Rdkafka
       @@error_callback
     end
 
+    # Sets the SASL/OAUTHBEARER token refresh callback.
+    # This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.oauthbearer_token_refresh_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      @@oauthbearer_token_refresh_callback = callback
+    end
+
+    # Returns the current oauthbearer_token_refresh_callback callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.oauthbearer_token_refresh_callback
+      @@oauthbearer_token_refresh_callback
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -159,11 +195,13 @@ module Rdkafka
 
     # Creates a consumer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Consumer] The created consumer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def consumer
+    def consumer(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
 
@@ -183,18 +221,21 @@ module Rdkafka
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: false,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
 
     # Create a producer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer
+    def producer(native_kafka_auto_start: true)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -203,11 +244,15 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Producer.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         ),
         partitioner_name
       ).tap do |producer|
@@ -217,19 +262,25 @@ module Rdkafka
 
     # Creates an admin instance with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin
+    def admin(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+
+      kafka = native_kafka(config, :rd_kafka_producer)
+
       Rdkafka::Admin.new(
         Rdkafka::NativeKafka.new(
-          native_kafka(config, :rd_kafka_producer),
+          kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
@@ -283,6 +334,9 @@ module Rdkafka
 
         # Set error callback
         Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+
+        # Set oauth callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -13,14 +13,17 @@ module Rdkafka
   class Consumer
     include Enumerable
     include Helpers::Time
+    include Helpers::OAuth
 
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
 
-    def finalizer
-      ->(_) { close }
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
     end
 
     # @return [String] consumer name
@@ -30,6 +33,10 @@ module Rdkafka
       end
     end
 
+    def finalizer
+      ->(_) { close }
+    end
+
     # Close this consumer
     # @return [nil]
     def close
@@ -239,7 +246,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=1200)
+    def committed(list=nil, timeout_ms=2000)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -387,27 +394,26 @@ module Rdkafka
     def store_offset(message)
       closed_consumer_check(__method__)
 
-      # rd_kafka_offset_store is one of the few calls that does not support
-      # a string as the topic, so create a native topic for it.
-      native_topic = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_topic_new(
+      list = TopicPartitionList.new
+      list.add_topic_and_partitions_with_offsets(
+        message.topic,
+        message.partition => message.offset + 1
+      )
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_store(
           inner,
-          message.topic,
-          nil
+          tpl
         )
       end
-      response = Rdkafka::Bindings.rd_kafka_offset_store(
-        native_topic,
-        message.partition,
-        message.offset
-      )
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
-      if native_topic && !native_topic.null?
-        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
-      end
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Seek to a particular message. The next poll on the topic/partition will return the
@@ -673,6 +679,22 @@ module Rdkafka
       end
     end
 
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
+    end
+
     private
 
     def closed_consumer_check(method)

data/lib/rdkafka/helpers/oauth.rb

@@ -0,0 +1,58 @@
+module Rdkafka
+  module Helpers
+
+    module OAuth
+
+      # Set the OAuthBearer token
+      #
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param principal_name [String] the mandatory Kafka principal name associated with the token.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @return [Integer] 0 on success
+      def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
+          )
+        end
+
+        return response if response.zero?
+
+        oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
+
+        response
+      end
+
+      # Marks failed oauth token acquire in librdkafka
+      #
+      # @param reason [String] human readable error reason for failing to acquire token
+      def oauthbearer_set_token_failure(reason)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
+            inner,
+            reason
+          )
+        end
+      end
+
+      private
+
+      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
+      def flatten_extensions(extensions)
+        return nil unless extensions
+        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      end
+
+      # extension_size is the number of keys + values which should be a non-negative even number
+      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      def extension_size(extensions)
+        return 0 unless extensions
+        extensions.size * 2
+      end
+    end
+  end
+end

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -28,30 +28,43 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
-      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
-      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+      @run_polling_thread = run_polling_thread
 
-      if run_polling_thread
-        # Start thread to poll client for delivery callbacks,
-        # not used in consumer.
-        @polling_thread = Thread.new do
-          loop do
-            @poll_mutex.synchronize do
-              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
-            end
+      start if auto_start
 
-            # Exit thread if closing and the poll queue is empty
-            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-              break
+      @closing = false
+    end
+
+    def start
+      synchronize do
+        return if @started
+
+        @started = true
+
+        # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+        Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
+
+        if @run_polling_thread
+          # Start thread to poll client for delivery callbacks,
+          # not used in consumer.
+          @polling_thread = Thread.new do
+            loop do
+              @poll_mutex.synchronize do
+                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+              end
+
+              # Exit thread if closing and the poll queue is empty
+              if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
+                break
+              end
             end
           end
-        end
 
-        @polling_thread.abort_on_exception = true
-        @polling_thread[:closing] = false
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.abort_on_exception = true
+          @polling_thread[:closing] = false
+        end
       end
-
-      @closing = false
     end
 
     def with_inner

data/lib/rdkafka/producer/delivery_handle.rb

@@ -11,6 +11,9 @@ module Rdkafka
              :offset, :int64,
              :topic_name, :pointer
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -18,7 +21,15 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
+        DeliveryReport.new(
+          self[:partition],
+          self[:offset],
+          # For part of errors, we will not get a topic name reference and in cases like this
+          # we should not return it
+          self[:topic_name].null? ? nil : self[:topic_name].read_string,
+          self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
+          label
+        )
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,21 +12,34 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to.
-      # @return [String]
+      # The name of the topic this message was produced to or nil in case of reports with errors
+      # where topic was not reached.
+      #
+      # @return [String, nil]
       attr_reader :topic_name
 
       # Error in case happen during produce.
       # @return [Integer]
       attr_reader :error
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
+      # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
+      # where the topic name is under `#topic` method. That way we have a consistent name that
+      # is present in both places
+      #
+      # We do not remove the original `#topic_name` because of backwards compatibility
+      alias topic topic_name
+
       private
 
-      def initialize(partition, offset, topic_name = nil, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
+        @label = label
       end
     end
   end