rdkafka 0.13.0 → 0.15.1

data/lib/rdkafka/consumer.rb

@@ -12,12 +12,20 @@ module Rdkafka
   # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
+    include Helpers::Time
 
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
 
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     def finalizer
       ->(_) { close }
     end
@@ -40,13 +48,11 @@ module Rdkafka
       @native_kafka.closed?
     end
 
-    # Subscribe to one or more topics letting Kafka handle partition assignments.
+    # Subscribes to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
-    #
-    # @raise [RdkafkaError] When subscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When subscribing fails
     def subscribe(*topics)
       closed_consumer_check(__method__)
 
@@ -70,9 +76,8 @@ module Rdkafka
 
     # Unsubscribe from all subscribed topics.
     #
-    # @raise [RdkafkaError] When unsubscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe
       closed_consumer_check(__method__)
 
@@ -87,10 +92,8 @@ module Rdkafka
     # Pause producing or consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
     def pause(list)
       closed_consumer_check(__method__)
 
@@ -114,13 +117,11 @@ module Rdkafka
       end
     end
 
-    # Resume producing consumption for the provided list of partitions
+    # Resumes producing consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaError] When resume subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When resume subscription fails.
     def resume(list)
       closed_consumer_check(__method__)
 
@@ -142,11 +143,10 @@ module Rdkafka
       end
     end
 
-    # Return the current subscription to topics and partitions
-    #
-    # @raise [RdkafkaError] When getting the subscription fails.
+    # Returns the current subscription to topics and partitions
     #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the subscription fails.
     def subscription
       closed_consumer_check(__method__)
 
@@ -171,7 +171,6 @@ module Rdkafka
     # Atomic assignment of partitions to consume
     #
     # @param list [TopicPartitionList] The topic with partitions to assign
-    #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
       closed_consumer_check(__method__)
@@ -196,9 +195,8 @@ module Rdkafka
 
     # Returns the current partition assignment.
     #
-    # @raise [RdkafkaError] When getting the assignment fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the assignment fails.
     def assignment
       closed_consumer_check(__method__)
 
@@ -223,16 +221,25 @@ module Rdkafka
       ptr.free unless ptr.nil?
     end
 
+    # @return [Boolean] true if our current assignment has been lost involuntarily.
+    def assignment_lost?
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        !Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
+      end
+    end
+
     # Return the current committed offset per partition for this consumer group.
-    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    # The offset field of each requested partition will either be set to stored offset or to -1001
+    # in case there was no stored offset for that partition.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
-    #
-    # @raise [RdkafkaError] When getting the committed positions fails.
-    #
     # @return [TopicPartitionList]
-    def committed(list=nil, timeout_ms=1200)
+    # @raise [RdkafkaError] When getting the committed positions fails.
+    def committed(list=nil, timeout_ms=2000)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -256,16 +263,42 @@ module Rdkafka
       end
     end
 
+    # Return the current positions (offsets) for topics and partitions.
+    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    #
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    #
+    # @return [TopicPartitionList]
+    #
+    # @raise [RdkafkaError] When getting the positions fails.
+    def position(list=nil)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    end
+
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
     # @param topic [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
-    #
-    # @raise [RdkafkaError] When querying the broker fails.
-    #
     # @return [Integer] The low and high watermark
-    def query_watermark_offsets(topic, partition, timeout_ms=200)
+    # @raise [RdkafkaError] When querying the broker fails.
+    def query_watermark_offsets(topic, partition, timeout_ms=1000)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
@@ -298,11 +331,10 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    #
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
-    def lag(topic_partition_list, watermark_timeout_ms=100)
+    def lag(topic_partition_list, watermark_timeout_ms=1000)
       out = {}
 
       topic_partition_list.to_h.each do |topic, partitions|
@@ -350,44 +382,39 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    #
-    # @raise [RdkafkaError] When storing the offset fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message)
       closed_consumer_check(__method__)
 
-      # rd_kafka_offset_store is one of the few calls that does not support
-      # a string as the topic, so create a native topic for it.
-      native_topic = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_topic_new(
+      list = TopicPartitionList.new
+      list.add_topic_and_partitions_with_offsets(
+        message.topic,
+        message.partition => message.offset + 1
+      )
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_store(
           inner,
-          message.topic,
-          nil
+          tpl
         )
       end
-      response = Rdkafka::Bindings.rd_kafka_offset_store(
-        native_topic,
-        message.partition,
-        message.offset
-      )
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
-      if native_topic && !native_topic.null?
-        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
-      end
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Seek to a particular message. The next poll on the topic/partition will return the
     # message at the given offset.
     #
     # @param message [Rdkafka::Consumer::Message] The message to which to seek
-    #
-    # @raise [RdkafkaError] When seeking fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
     def seek(message)
       closed_consumer_check(__method__)
 
@@ -415,6 +442,39 @@ module Rdkafka
       end
     end
 
+    # Lookup offset for the given partitions by timestamp.
+    #
+    # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
+    #
+    # @return [TopicPartitionList]
+    #
+    # @raise [RdKafkaError] When the OffsetForTimes lookup fails
+    def offsets_for_times(list, timeout_ms = 1000)
+      closed_consumer_check(__method__)
+
+      if !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_for_times(
+          inner,
+          tpl,
+          timeout_ms # timeout
+        )
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
+    end
+
     # Manually commit the current offsets of this consumer.
     #
     # To use this set `enable.auto.commit`to `false` to disable automatic triggering
@@ -426,10 +486,8 @@ module Rdkafka
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
-    #
-    # @raise [RdkafkaError] When committing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When committing fails
     def commit(list=nil, async=false)
       closed_consumer_check(__method__)
 
@@ -454,10 +512,8 @@ module Rdkafka
     # Poll for the next message on one of the subscribed topics
     #
     # @param timeout_ms [Integer] Timeout of this poll
-    #
-    # @raise [RdkafkaError] When polling fails
-    #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
@@ -483,18 +539,41 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
+    #   was set to `true`.
+    #
+    # Events will cause application-provided callbacks to be called.
+    #
+    # Events (in the context of the consumer):
+    #   - error callbacks
+    #   - stats callbacks
+    #   - any other callbacks supported by librdkafka that are not part of the consumer_poll, that
+    #     would have a callback configured and activated.
+    #
+    # This method needs to be called at regular intervals to serve any queued callbacks waiting to
+    # be called. When in use, does **NOT** replace `#poll` but needs to run complementary with it.
+    #
+    # @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
+    #   block until any events available.
+    #
+    # @note This method technically should be called `#poll` and the current `#poll` should be
+    #   called `#consumer_poll` though we keep the current naming convention to make it backward
+    #   compatible.
+    def events_poll(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
-    # If `enable.partition.eof` is turned on in the config this will raise an
-    # error when an eof is reached, so you probably want to disable that when
-    # using this method of iteration.
-    #
-    # @raise [RdkafkaError] When polling fails
+    # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
+    # reached, so you probably want to disable that when using this method of iteration.
     #
     # @yieldparam message [Message] Received message
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When polling fails
     def each
       loop do
         message = poll(250)
@@ -546,19 +625,18 @@ module Rdkafka
     # that you may or may not see again.
     #
     # @param max_items [Integer] Maximum size of the yielded array of messages
-    #
     # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    #
     # @param timeout_ms [Integer] max time to wait for up to max_items
     #
-    # @raise [RdkafkaError] When polling fails
-    #
-    # @yield [messages, pending_exception]
     # @yieldparam messages [Array] An array of received Message
     # @yieldparam pending_exception [Exception] normally nil, or an exception
+    #
+    # @yield [messages, pending_exception]
     # which will be propagated after processing of the partial batch is complete.
     #
     # @return [nil]
+    #
+    # @raise [RdkafkaError] When polling fails
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
       closed_consumer_check(__method__)
       slice = []
@@ -594,12 +672,24 @@ module Rdkafka
       end
     end
 
-    private
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
     end
 
+    private
+
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/metadata.rb

@@ -4,7 +4,18 @@ module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil, timeout_ms = 250)
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -22,6 +33,16 @@ module Rdkafka
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
     ensure
       Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
       Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)

data/lib/rdkafka/native_kafka.rb

@@ -4,8 +4,9 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:)
+    def initialize(inner, run_polling_thread:, opaque:)
       @inner = inner
+      @opaque = opaque
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -27,6 +28,9 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
+      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
         # not used in consumer.
@@ -109,6 +113,7 @@ module Rdkafka
 
         Rdkafka::Bindings.rd_kafka_destroy(@inner)
         @inner = nil
+        @opaque = nil
       end
     end
   end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -11,6 +11,9 @@ module Rdkafka
              :offset, :int64,
              :topic_name, :pointer
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -18,7 +21,15 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
+        DeliveryReport.new(
+          self[:partition],
+          self[:offset],
+          # For part of errors, we will not get a topic name reference and in cases like this
+          # we should not return it
+          self[:topic_name].null? ? nil : self[:topic_name].read_string,
+          self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
+          label
+        )
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,21 +12,34 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to.
-      # @return [String]
+      # The name of the topic this message was produced to or nil in case of reports with errors
+      # where topic was not reached.
+      #
+      # @return [String, nil]
       attr_reader :topic_name
 
       # Error in case happen during produce.
       # @return [Integer]
       attr_reader :error
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
+      # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
+      # where the topic name is under `#topic` method. That way we have a consistent name that
+      # is present in both places
+      #
+      # We do not remove the original `#topic_name` because of backwards compatibility
+      alias topic topic_name
+
       private
 
-      def initialize(partition, offset, topic_name = nil, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
+        @label = label
       end
     end
   end