karafka-rdkafka 0.20.0.rc3-x86_64-linux-gnu

data/lib/rdkafka/error.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Base error class.
+  class BaseError < RuntimeError; end
+
+  # Error returned by the underlying rdkafka library.
+  class RdkafkaError < BaseError
+    # Empty hash for details default allocation
+    EMPTY_HASH = {}.freeze
+
+    # The underlying raw error response
+    # @return [Integer]
+    attr_reader :rdkafka_response
+
+    # Prefix to be used for human readable representation
+    # @return [String]
+    attr_reader :message_prefix
+
+    # Error message sent by the broker
+    # @return [String]
+    attr_reader :broker_message
+
+    # Optional details hash specific to a given error or empty hash if none or not supported
+    # @return [Hash]
+    attr_reader :details
+
+    class << self
+      def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
+        code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
+
+        return false if code.zero?
+
+        message = broker_message || Rdkafka::Bindings.rd_kafka_err2str(code)
+        fatal = !Rdkafka::Bindings.rd_kafka_error_is_fatal(response_ptr).zero?
+        retryable = !Rdkafka::Bindings.rd_kafka_error_is_retriable(response_ptr).zero?
+        abortable = !Rdkafka::Bindings.rd_kafka_error_txn_requires_abort(response_ptr).zero?
+
+        Rdkafka::Bindings.rd_kafka_error_destroy(response_ptr)
+
+        new(
+          code,
+          message_prefix,
+          broker_message: message,
+          fatal: fatal,
+          retryable: retryable,
+          abortable: abortable
+        )
+      end
+
+      def build(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        case response_ptr_or_code
+        when Integer
+          return false if response_ptr_or_code.zero?
+
+          new(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        when Bindings::Message
+          return false if response_ptr_or_code[:err].zero?
+
+          unless response_ptr_or_code[:payload].null?
+            message_prefix ||= response_ptr_or_code[:payload].read_string(response_ptr_or_code[:len])
+          end
+
+          details = if response_ptr_or_code[:rkt].null?
+                      EMPTY_HASH
+                    else
+                      {
+                        partition: response_ptr_or_code[:partition],
+                        offset: response_ptr_or_code[:offset],
+                        topic: Bindings.rd_kafka_topic_name(response_ptr_or_code[:rkt])
+                      }.freeze
+                    end
+          new(
+            response_ptr_or_code[:err],
+            message_prefix,
+            broker_message: broker_message,
+            details: details
+          )
+        else
+          build_from_c(response_ptr_or_code, message_prefix)
+        end
+      end
+
+      def validate!(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        error = build(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        error ? raise(error) : false
+      end
+    end
+
+    # @private
+    def initialize(
+      response,
+      message_prefix=nil,
+      broker_message: nil,
+      fatal: false,
+      retryable: false,
+      abortable: false,
+      details: EMPTY_HASH
+    )
+      raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
+      @rdkafka_response = response
+      @message_prefix = message_prefix
+      @broker_message = broker_message
+      @fatal = fatal
+      @retryable = retryable
+      @abortable = abortable
+      @details = details
+    end
+
+    # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
+    # @return [Symbol]
+    def code
+      code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
+      if code[0] == "_"
+        code[1..-1].to_sym
+      else
+        code.to_sym
+      end
+    end
+
+    # Human readable representation of this error.
+    # @return [String]
+    def to_s
+      message_prefix_part = if message_prefix
+                       "#{message_prefix} - "
+                     else
+                       ''
+                     end
+
+      err_str = Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)
+      base = "#{message_prefix_part}#{err_str} (#{code})"
+
+      return base if broker_message.nil?
+      return base if broker_message.empty?
+
+      "#{base}\n#{broker_message}"
+    end
+
+    # Whether this error indicates the partition is EOF.
+    # @return [Boolean]
+    def is_partition_eof?
+      code == :partition_eof
+    end
+
+    # Error comparison
+    def ==(another_error)
+       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    end
+
+    def fatal?
+      @fatal
+    end
+
+    def retryable?
+      @retryable
+    end
+
+    def abortable?
+      @abortable
+    end
+  end
+
+  # Error with topic partition list returned by the underlying rdkafka library.
+  class RdkafkaTopicPartitionListError < RdkafkaError
+    # @return [TopicPartitionList]
+    attr_reader :topic_partition_list
+
+    # @private
+    def initialize(response, topic_partition_list, message_prefix=nil)
+      super(response, message_prefix)
+      @topic_partition_list = topic_partition_list
+    end
+  end
+
+  # Error class for public consumer method calls on a closed consumer.
+  class ClosedConsumerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed consumer")
+    end
+  end
+
+  # Error class for public producer method calls on a closed producer.
+  class ClosedProducerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed producer")
+    end
+  end
+
+  # Error class for public consumer method calls on a closed admin.
+  class ClosedAdminError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed admin")
+    end
+  end
+
+  class ClosedInnerError < BaseError
+    def initialize
+      super("Illegal call to a closed inner librdkafka instance")
+    end
+  end
+end

data/lib/rdkafka/helpers/oauth.rb

@@ -0,0 +1,58 @@
+module Rdkafka
+  module Helpers
+
+    module OAuth
+
+      # Set the OAuthBearer token
+      #
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param principal_name [String] the mandatory Kafka principal name associated with the token.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @return [Integer] 0 on success
+      def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
+          )
+        end
+
+        return response if response.zero?
+
+        oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
+
+        response
+      end
+
+      # Marks failed oauth token acquire in librdkafka
+      #
+      # @param reason [String] human readable error reason for failing to acquire token
+      def oauthbearer_set_token_failure(reason)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
+            inner,
+            reason
+          )
+        end
+      end
+
+      private
+
+      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
+      def flatten_extensions(extensions)
+        return nil unless extensions
+        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      end
+
+      # extension_size is the number of keys + values which should be a non-negative even number
+      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      def extension_size(extensions)
+        return 0 unless extensions
+        extensions.size * 2
+      end
+    end
+  end
+end

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/metadata.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Metadata
+    attr_reader :brokers, :topics
+
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
+      native_topic = if topic_name
+        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      end
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+
+      # If topic_flag is 1, we request info about *all* topics in the cluster.  If topic_flag is 0,
+      # we only request info about locally known topics (or a single topic if one is passed in).
+      topic_flag = topic_name.nil? ? 1 : 0
+
+      # Retrieve the Metadata
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
+
+      Rdkafka::RdkafkaError.validate!(result)
+
+      metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+    end
+
+    private
+
+    def metadata_from_native(ptr)
+      metadata = Metadata.new(ptr)
+      @brokers = Array.new(metadata[:brokers_count]) do |i|
+        BrokerMetadata.new(metadata[:brokers_metadata] + (i * BrokerMetadata.size)).to_h
+      end
+
+      @topics = Array.new(metadata[:topics_count]) do |i|
+        topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
+
+        RdkafkaError.validate!(topic[:rd_kafka_resp_err])
+
+        partitions = Array.new(topic[:partition_count]) do |j|
+          partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
+          RdkafkaError.validate!(partition[:rd_kafka_resp_err])
+          partition.to_h
+        end
+        topic.to_h.merge!(partitions: partitions)
+      end
+    end
+
+    class CustomFFIStruct < FFI::Struct
+      def to_h
+        members.each_with_object({}) do |mem, hsh|
+          val = self.[](mem)
+          next if val.is_a?(FFI::Pointer) || mem == :rd_kafka_resp_err
+
+          hsh[mem] = self.[](mem)
+        end
+      end
+    end
+
+    class Metadata < CustomFFIStruct
+      layout :brokers_count, :int,
+             :brokers_metadata, :pointer,
+             :topics_count, :int,
+             :topics_metadata, :pointer,
+             :broker_id, :int32,
+             :broker_name, :string
+    end
+
+    class BrokerMetadata < CustomFFIStruct
+      layout :broker_id, :int32,
+             :broker_name, :string,
+             :broker_port, :int
+    end
+
+    class TopicMetadata < CustomFFIStruct
+      layout :topic_name, :string,
+             :partition_count, :int,
+             :partitions_metadata, :pointer,
+             :rd_kafka_resp_err, :int
+    end
+
+    class PartitionMetadata < CustomFFIStruct
+      layout :partition_id, :int32,
+             :rd_kafka_resp_err, :int,
+             :leader, :int32,
+             :replica_count, :int,
+             :replicas, :pointer,
+             :in_sync_replica_brokers, :int,
+             :isrs, :pointer
+    end
+  end
+end

data/lib/rdkafka/native_kafka.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # @private
+  # A wrapper around a native kafka that polls and cleanly exits
+  class NativeKafka
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
+      @inner = inner
+      @opaque = opaque
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
+      # Lock around decrementing the operations in progress counter
+      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
+      # because they serve different purposes:
+      #
+      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
+      #      the `#synchronize` is the only one running and that there are no other running
+      #      operations.
+      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
+      #     implementation.
+      #
+      # We do not use the same mutex, because it could create a deadlock when an already
+      # incremented operation cannot decrement because `@access_lock` is now owned by a different
+      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
+      @decrement_mutex = Mutex.new
+      # counter for operations in progress using inner
+      @operations_in_progress = 0
+
+      @run_polling_thread = run_polling_thread
+
+      @timeout_ms = timeout_ms
+
+      start if auto_start
+
+      @closing = false
+    end
+
+    def start
+      synchronize do
+        return if @started
+
+        @started = true
+
+        # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+        Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
+
+        if @run_polling_thread
+          # Start thread to poll client for delivery callbacks,
+          # not used in consumer.
+          @polling_thread = Thread.new do
+            loop do
+              @poll_mutex.synchronize do
+                Rdkafka::Bindings.rd_kafka_poll(@inner, @timeout_ms)
+              end
+
+              # Exit thread if closing and the poll queue is empty
+              if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
+                break
+              end
+            end
+          end
+
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.abort_on_exception = true
+          @polling_thread[:closing] = false
+        end
+      end
+    end
+
+    def with_inner
+      if @access_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @access_mutex.synchronize { @operations_in_progress += 1 }
+      end
+
+      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+    ensure
+      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+    end
+
+    def synchronize(&block)
+      @access_mutex.synchronize do
+        # Wait for any commands using the inner to finish
+        # This can take a while on blocking operations like polling but is essential not to proceed
+        # with certain types of operations like resources destruction as it can cause the process
+        # to hang or crash
+        sleep(0.01) until @operations_in_progress.zero?
+
+        with_inner(&block)
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    def closed?
+      @closing || @inner.nil?
+    end
+
+    def close(object_id=nil)
+      return if closed?
+
+      synchronize do
+        # Indicate to the outside world that we are closing
+        @closing = true
+
+        if @polling_thread
+          # Indicate to polling thread that we're closing
+          @polling_thread[:closing] = true
+
+          # Wait for the polling thread to finish up,
+          # this can be aborted in practice if this
+          # code runs from a finalizer.
+          @polling_thread.join
+        end
+
+        # Destroy the client after locking both mutexes
+        @poll_mutex.lock
+
+        # This check prevents a race condition, where we would enter the close in two threads
+        # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+        # and would continue to run, trying to destroy inner twice
+        return unless @inner
+
+        yield if block_given?
+
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+        @opaque = nil
+        @poll_mutex.unlock
+        @poll_mutex = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Producer
+    # Handle to wait for a delivery report which is returned when
+    # producing a message.
+    class DeliveryHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :partition, :int,
+             :offset, :int64,
+             :topic_name, :pointer
+
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
+      # @return [String] topic where we are trying to send the message
+      # We use this instead of reading from `topic_name` pointer to save on memory allocations
+      attr_accessor :topic
+
+      # @return [String] the name of the operation (e.g. "delivery")
+      def operation_name
+        "delivery"
+      end
+
+      # @return [DeliveryReport] a report on the delivery of the message
+      def create_result
+        if self[:response] == 0
+          DeliveryReport.new(
+            self[:partition],
+            self[:offset],
+            topic,
+            nil,
+            label
+          )
+        else
+          DeliveryReport.new(
+            self[:partition],
+            self[:offset],
+            topic,
+            Rdkafka::RdkafkaError.build(self[:response]),
+            label
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_report.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Producer
+    # Delivery report for a successfully produced message.
+    class DeliveryReport
+      # The partition this message was produced to.
+      # @return [Integer]
+      attr_reader :partition
+
+      # The offset of the produced message.
+      # @return [Integer]
+      attr_reader :offset
+
+      # The name of the topic this message was produced to or nil in case delivery failed and we
+      #   we not able to get the topic reference
+      # @return [String, nil]
+      attr_reader :topic_name
+
+      # Error in case happen during produce.
+      # @return [Integer]
+      attr_reader :error
+
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
+      # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
+      # where the topic name is under `#topic` method. That way we have a consistent name that
+      # is present in both places
+      #
+      # We do not remove the original `#topic_name` because of backwards compatibility
+      alias topic topic_name
+
+      private
+
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
+        @partition = partition
+        @offset = offset
+        @topic_name = topic_name
+        @error = error
+        @label = label
+      end
+    end
+  end
+end