rdkafka 0.13.0 → 0.14.0.rc1

data/lib/rdkafka/config.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require "logger"
-
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
@@ -14,7 +12,7 @@ module Rdkafka
     # @private
     @@error_callback = nil
     # @private
-    @@opaques = {}
+    @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
 
@@ -53,13 +51,13 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
       @@statistics_callback = callback
     end
 
@@ -142,12 +140,12 @@ module Rdkafka
       @consumer_rebalance_listener = listener
     end
 
-    # Create a consumer with this configuration.
+    # Creates a consumer with this configuration.
+    #
+    # @return [Consumer] The created consumer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Consumer] The created consumer
     def consumer
       opaque = Opaque.new
       config = native_config(opaque)
@@ -164,15 +162,21 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
+      Rdkafka::Consumer.new(
+        Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: false,
+          opaque: opaque
+        )
+      )
     end
 
     # Create a producer with this configuration.
     #
+    # @return [Producer] The created producer
+    #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Producer] The created producer
     def producer
       # Create opaque
       opaque = Opaque.new
@@ -182,22 +186,31 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(
+        Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true, opaque: opaque),
+        partitioner_name
+      ).tap do |producer|
         opaque.producer = producer
       end
     end
 
-    # Create an admin instance with this configuration.
+    # Creates an admin instance with this configuration.
+    #
+    # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Admin] The created admin instance
     def admin
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
+      Rdkafka::Admin.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        )
+      )
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/consumer/headers.rb

@@ -18,13 +18,11 @@ module Rdkafka
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
-      # @param [librdkakfa message] native_message
+      # @private
       #
+      # @param [librdkakfa message] native_message
       # @return [Hash<String, String>] headers Hash for the native_message
-      #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
-      #
-      # @private
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -142,11 +142,13 @@ module Rdkafka
               )
 
               if p.offset
+                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
                   topic,
                   p.partition,
-                  p.offset
+                  offset
                 )
               end
             end

data/lib/rdkafka/consumer.rb

@@ -12,6 +12,7 @@ module Rdkafka
   # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
+    include Helpers::Time
 
     # @private
     def initialize(native_kafka)
@@ -22,6 +23,13 @@ module Rdkafka
       ->(_) { close }
     end
 
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     # Close this consumer
     # @return [nil]
     def close
@@ -40,13 +48,11 @@ module Rdkafka
       @native_kafka.closed?
     end
 
-    # Subscribe to one or more topics letting Kafka handle partition assignments.
+    # Subscribes to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
-    #
-    # @raise [RdkafkaError] When subscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When subscribing fails
     def subscribe(*topics)
       closed_consumer_check(__method__)
 
@@ -70,9 +76,8 @@ module Rdkafka
 
     # Unsubscribe from all subscribed topics.
     #
-    # @raise [RdkafkaError] When unsubscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe
       closed_consumer_check(__method__)
 
@@ -87,10 +92,8 @@ module Rdkafka
     # Pause producing or consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
     def pause(list)
       closed_consumer_check(__method__)
 
@@ -114,13 +117,11 @@ module Rdkafka
       end
     end
 
-    # Resume producing consumption for the provided list of partitions
+    # Resumes producing consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaError] When resume subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When resume subscription fails.
     def resume(list)
       closed_consumer_check(__method__)
 
@@ -142,11 +143,10 @@ module Rdkafka
       end
     end
 
-    # Return the current subscription to topics and partitions
-    #
-    # @raise [RdkafkaError] When getting the subscription fails.
+    # Returns the current subscription to topics and partitions
     #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the subscription fails.
     def subscription
       closed_consumer_check(__method__)
 
@@ -171,7 +171,6 @@ module Rdkafka
     # Atomic assignment of partitions to consume
     #
     # @param list [TopicPartitionList] The topic with partitions to assign
-    #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
       closed_consumer_check(__method__)
@@ -196,9 +195,8 @@ module Rdkafka
 
     # Returns the current partition assignment.
     #
-    # @raise [RdkafkaError] When getting the assignment fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the assignment fails.
     def assignment
       closed_consumer_check(__method__)
 
@@ -224,14 +222,14 @@ module Rdkafka
     end
 
     # Return the current committed offset per partition for this consumer group.
-    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    # The offset field of each requested partition will either be set to stored offset or to -1001
+    # in case there was no stored offset for that partition.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
-    #
-    # @raise [RdkafkaError] When getting the committed positions fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the committed positions fails.
     def committed(list=nil, timeout_ms=1200)
       closed_consumer_check(__method__)
 
@@ -256,15 +254,41 @@ module Rdkafka
       end
     end
 
+    # Return the current positions (offsets) for topics and partitions.
+    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    #
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    #
+    # @raise [RdkafkaError] When getting the positions fails.
+    #
+    # @return [TopicPartitionList]
+    def position(list=nil)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    end
+
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
     # @param topic [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
-    #
-    # @raise [RdkafkaError] When querying the broker fails.
-    #
     # @return [Integer] The low and high watermark
+    # @raise [RdkafkaError] When querying the broker fails.
     def query_watermark_offsets(topic, partition, timeout_ms=200)
       closed_consumer_check(__method__)
 
@@ -298,10 +322,9 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    #
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
 
@@ -350,10 +373,8 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    #
-    # @raise [RdkafkaError] When storing the offset fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message)
       closed_consumer_check(__method__)
 
@@ -384,10 +405,8 @@ module Rdkafka
     # message at the given offset.
     #
     # @param message [Rdkafka::Consumer::Message] The message to which to seek
-    #
-    # @raise [RdkafkaError] When seeking fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
     def seek(message)
       closed_consumer_check(__method__)
 
@@ -415,6 +434,39 @@ module Rdkafka
       end
     end
 
+    # Lookup offset for the given partitions by timestamp.
+    #
+    # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
+    #
+    # @raise [RdKafkaError] When the OffsetForTimes lookup fails
+    #
+    # @return [TopicPartitionList]
+    def offsets_for_times(list, timeout_ms = 1000)
+      closed_consumer_check(__method__)
+
+      if !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_for_times(
+          inner,
+          tpl,
+          timeout_ms # timeout
+        )
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
+    end
+
     # Manually commit the current offsets of this consumer.
     #
     # To use this set `enable.auto.commit`to `false` to disable automatic triggering
@@ -426,10 +478,8 @@ module Rdkafka
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
-    #
-    # @raise [RdkafkaError] When committing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When committing fails
     def commit(list=nil, async=false)
       closed_consumer_check(__method__)
 
@@ -454,10 +504,8 @@ module Rdkafka
     # Poll for the next message on one of the subscribed topics
     #
     # @param timeout_ms [Integer] Timeout of this poll
-    #
-    # @raise [RdkafkaError] When polling fails
-    #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
@@ -486,14 +534,11 @@ module Rdkafka
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
-    # If `enable.partition.eof` is turned on in the config this will raise an
-    # error when an eof is reached, so you probably want to disable that when
-    # using this method of iteration.
+    # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
+    # reached, so you probably want to disable that when using this method of iteration.
     #
     # @raise [RdkafkaError] When polling fails
-    #
     # @yieldparam message [Message] Received message
-    #
     # @return [nil]
     def each
       loop do
@@ -546,9 +591,7 @@ module Rdkafka
     # that you may or may not see again.
     #
     # @param max_items [Integer] Maximum size of the yielded array of messages
-    #
     # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    #
     # @param timeout_ms [Integer] max time to wait for up to max_items
     #
     # @raise [RdkafkaError] When polling fails
@@ -595,10 +638,6 @@ module Rdkafka
     end
 
     private
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
 
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/metadata.rb

@@ -4,7 +4,18 @@ module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil, timeout_ms = 250)
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -22,6 +33,16 @@ module Rdkafka
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
     ensure
       Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
       Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)

data/lib/rdkafka/native_kafka.rb

@@ -4,8 +4,9 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:)
+    def initialize(inner, run_polling_thread:, opaque:)
       @inner = inner
+      @opaque = opaque
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -27,6 +28,9 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
+      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
         # not used in consumer.
@@ -109,6 +113,7 @@ module Rdkafka
 
         Rdkafka::Bindings.rd_kafka_destroy(@inner)
         @inner = nil
+        @opaque = nil
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    include Helpers::Time
+
+    # Cache partitions count for 30 seconds
+    PARTITIONS_COUNT_TTL = 30
+
+    private_constant :PARTITIONS_COUNT_TTL
+
     # @private
     # Returns the current delivery callback, by default this is nil.
     #
@@ -24,6 +29,26 @@ module Rdkafka
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+
+      @_partitions_count_cache = Hash.new do |cache, topic|
+        topic_metadata = nil
+
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        cache[topic] = [
+          monotonic_now,
+          topic_metadata ? topic_metadata[:partition_count] : nil
+        ]
+      end
+    end
+
+    # @return [String] producer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -54,25 +79,77 @@ module Rdkafka
     # in seconds. Call this before closing a producer to ensure delivery of all messages.
     #
     # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    # @return [Boolean] true if no more data and all was flushed, false in case there are still
+    #   outgoing messages after the timeout
+    #
+    # @note We raise an exception for other errors because based on the librdkafka docs, there
+    #   should be no other errors.
+    #
+    # @note For `timed_out` we do not raise an error to keep it backwards compatible
     def flush(timeout_ms=5_000)
       closed_producer_check(__method__)
 
+      code = nil
+
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        code = Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
       end
+
+      # Early skip not to build the error message
+      return true if code.zero?
+
+      error = Rdkafka::RdkafkaError.new(code)
+
+      return false if error.code == :timed_out
+
+      raise(error)
+    end
+
+    # Purges the outgoing queue and releases all resources.
+    #
+    # Useful when closing the producer with outgoing messages to unstable clusters or when for
+    # any other reasons waiting cannot go on anymore. This purges both the queue and all the
+    # inflight requests + updates the delivery handles statuses so they can be materialized into
+    # `purge_queue` errors.
+    def purge
+      closed_producer_check(__method__)
+
+      code = nil
+
+      @native_kafka.with_inner do |inner|
+        code = Bindings.rd_kafka_purge(
+          inner,
+          Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
+        )
+      end
+
+      code.zero? || raise(Rdkafka::RdkafkaError.new(code))
+
+      # Wait for the purge to affect everything
+      sleep(0.001) until flush(100)
+
+      true
     end
 
     # Partition count for a given topic.
-    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
     # @param topic [String] The topic name.
+    # @return [Integer] partition count for a given topic
+    #
+    # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
+    #   auto-created after returning nil.
     #
-    # @return partition count [Integer,nil]
+    # @note We cache the partition count for a given topic for given time.
+    #   This prevents us in case someone uses `partition_key` from querying for the count with
+    #   each message. Instead we query once every 30 seconds at most
     def partition_count(topic)
       closed_producer_check(__method__)
-      @native_kafka.with_inner do |inner|
-        Rdkafka::Metadata.new(inner, topic).topics&.first[:partition_count]
+
+      @_partitions_count_cache.delete_if do |_, cached|
+        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
       end
+
+      @_partitions_count_cache[topic].last
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -193,6 +270,7 @@ module Rdkafka
     end
 
     private
+
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0"
-  LIBRDKAFKA_VERSION = "2.0.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
+  VERSION = "0.14.0.rc1"
+  LIBRDKAFKA_VERSION = "2.2.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end