rdkafka 0.12.0 → 0.15.0

data/lib/rdkafka/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
   # brokers automatically assign partitions and load balance partitions over consumers that
@@ -10,31 +12,47 @@ module Rdkafka
   # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
+    include Helpers::Time
 
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
     end
 
     # Close this consumer
     # @return [nil]
     def close
-      return unless @native_kafka
+      return if closed?
+      ObjectSpace.undefine_finalizer(self)
+
+      @native_kafka.synchronize do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+      end
+
+      @native_kafka.close
+    end
 
-      @closing = true
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+    # Whether this consumer has closed
+    def closed?
+      @native_kafka.closed?
     end
 
-    # Subscribe to one or more topics letting Kafka handle partition assignments.
+    # Subscribes to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
-    #
-    # @raise [RdkafkaError] When subscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When subscribing fails
     def subscribe(*topics)
       closed_consumer_check(__method__)
 
@@ -46,7 +64,9 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -56,13 +76,14 @@ module Rdkafka
 
     # Unsubscribe from all subscribed topics.
     #
-    # @raise [RdkafkaError] When unsubscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -71,10 +92,8 @@ module Rdkafka
     # Pause producing or consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
     def pause(list)
       closed_consumer_check(__method__)
 
@@ -85,7 +104,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
+        end
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -96,13 +117,11 @@ module Rdkafka
       end
     end
 
-    # Resume producing consumption for the provided list of partitions
+    # Resumes producing consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaError] When resume subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When resume subscription fails.
     def resume(list)
       closed_consumer_check(__method__)
 
@@ -113,7 +132,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -122,16 +143,17 @@ module Rdkafka
       end
     end
 
-    # Return the current subscription to topics and partitions
-    #
-    # @raise [RdkafkaError] When getting the subscription fails.
+    # Returns the current subscription to topics and partitions
     #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the subscription fails.
     def subscription
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      end
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -149,7 +171,6 @@ module Rdkafka
     # Atomic assignment of partitions to consume
     #
     # @param list [TopicPartitionList] The topic with partitions to assign
-    #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
       closed_consumer_check(__method__)
@@ -161,7 +182,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -172,14 +195,15 @@ module Rdkafka
 
     # Returns the current partition assignment.
     #
-    # @raise [RdkafkaError] When getting the assignment fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the assignment fails.
     def assignment
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -197,15 +221,24 @@ module Rdkafka
       ptr.free unless ptr.nil?
     end
 
+    # @return [Boolean] true if our current assignment has been lost involuntarily.
+    def assignment_lost?
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        !Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
+      end
+    end
+
     # Return the current committed offset per partition for this consumer group.
-    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    # The offset field of each requested partition will either be set to stored offset or to -1001
+    # in case there was no stored offset for that partition.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
-    #
-    # @raise [RdkafkaError] When getting the committed positions fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the committed positions fails.
     def committed(list=nil, timeout_ms=1200)
       closed_consumer_check(__method__)
 
@@ -218,7 +251,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -228,29 +263,57 @@ module Rdkafka
       end
     end
 
+    # Return the current positions (offsets) for topics and partitions.
+    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    #
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    #
+    # @return [TopicPartitionList]
+    #
+    # @raise [RdkafkaError] When getting the positions fails.
+    def position(list=nil)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    end
+
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
     # @param topic [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
-    #
-    # @raise [RdkafkaError] When querying the broker fails.
-    #
     # @return [Integer] The low and high watermark
-    def query_watermark_offsets(topic, partition, timeout_ms=200)
+    # @raise [RdkafkaError] When querying the broker fails.
+    def query_watermark_offsets(topic, partition, timeout_ms=1000)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka,
-        topic,
-        partition,
-        low,
-        high,
-        timeout_ms,
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+          inner,
+          topic,
+          partition,
+          low,
+          high,
+          timeout_ms,
+        )
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
@@ -268,11 +331,10 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    #
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
-    def lag(topic_partition_list, watermark_timeout_ms=100)
+    def lag(topic_partition_list, watermark_timeout_ms=1000)
       out = {}
 
       topic_partition_list.to_h.each do |topic, partitions|
@@ -298,7 +360,9 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+      end
     end
 
     # Returns this client's broker-assigned group member id
@@ -308,7 +372,9 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_memberid(inner)
+      end
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -316,20 +382,20 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    #
-    # @raise [RdkafkaError] When storing the offset fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_offset_store(
         native_topic,
         message.partition,
@@ -348,20 +414,20 @@ module Rdkafka
     # message at the given offset.
     #
     # @param message [Rdkafka::Consumer::Message] The message to which to seek
-    #
-    # @raise [RdkafkaError] When seeking fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
     def seek(message)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         message.partition,
@@ -377,6 +443,39 @@ module Rdkafka
       end
     end
 
+    # Lookup offset for the given partitions by timestamp.
+    #
+    # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
+    #
+    # @return [TopicPartitionList]
+    #
+    # @raise [RdKafkaError] When the OffsetForTimes lookup fails
+    def offsets_for_times(list, timeout_ms = 1000)
+      closed_consumer_check(__method__)
+
+      if !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_for_times(
+          inner,
+          tpl,
+          timeout_ms # timeout
+        )
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
+    end
+
     # Manually commit the current offsets of this consumer.
     #
     # To use this set `enable.auto.commit`to `false` to disable automatic triggering
@@ -388,10 +487,8 @@ module Rdkafka
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
-    #
-    # @raise [RdkafkaError] When committing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When committing fails
     def commit(list=nil, async=false)
       closed_consumer_check(__method__)
 
@@ -402,7 +499,9 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -414,14 +513,14 @@ module Rdkafka
     # Poll for the next message on one of the subscribed topics
     #
     # @param timeout_ms [Integer] Timeout of this poll
-    #
-    # @raise [RdkafkaError] When polling fails
-    #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
+      end
       if message_ptr.null?
         nil
       else
@@ -436,30 +535,53 @@ module Rdkafka
       end
     ensure
       # Clean up rdkafka message if there is one
-      if !message_ptr.nil? && !message_ptr.null?
+      if message_ptr && !message_ptr.null?
         Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end
 
+    # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
+    #   was set to `true`.
+    #
+    # Events will cause application-provided callbacks to be called.
+    #
+    # Events (in the context of the consumer):
+    #   - error callbacks
+    #   - stats callbacks
+    #   - any other callbacks supported by librdkafka that are not part of the consumer_poll, that
+    #     would have a callback configured and activated.
+    #
+    # This method needs to be called at regular intervals to serve any queued callbacks waiting to
+    # be called. When in use, does **NOT** replace `#poll` but needs to run complementary with it.
+    #
+    # @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
+    #   block until any events available.
+    #
+    # @note This method technically should be called `#poll` and the current `#poll` should be
+    #   called `#consumer_poll` though we keep the current naming convention to make it backward
+    #   compatible.
+    def events_poll(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
-    # If `enable.partition.eof` is turned on in the config this will raise an
-    # error when an eof is reached, so you probably want to disable that when
-    # using this method of iteration.
-    #
-    # @raise [RdkafkaError] When polling fails
+    # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
+    # reached, so you probably want to disable that when using this method of iteration.
     #
     # @yieldparam message [Message] Received message
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When polling fails
     def each
       loop do
         message = poll(250)
         if message
           yield(message)
         else
-          if @closing
+          if closed?
             break
           else
             next
@@ -468,10 +590,6 @@ module Rdkafka
       end
     end
 
-    def closed_consumer_check(method)
-      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
-    end
-
     # Poll for new messages and yield them in batches that may contain
     # messages from more than one partition.
     #
@@ -508,26 +626,25 @@ module Rdkafka
     # that you may or may not see again.
     #
     # @param max_items [Integer] Maximum size of the yielded array of messages
-    #
     # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    #
     # @param timeout_ms [Integer] max time to wait for up to max_items
     #
-    # @raise [RdkafkaError] When polling fails
-    #
-    # @yield [messages, pending_exception]
     # @yieldparam messages [Array] An array of received Message
     # @yieldparam pending_exception [Exception] normally nil, or an exception
+    #
+    # @yield [messages, pending_exception]
     # which will be propagated after processing of the partial batch is complete.
     #
     # @return [nil]
+    #
+    # @raise [RdkafkaError] When polling fails
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
       closed_consumer_check(__method__)
       slice = []
       bytes = 0
       end_time = monotonic_now + timeout_ms / 1000.0
       loop do
-        break if @closing
+        break if closed?
         max_wait = end_time - monotonic_now
         max_wait_ms = if max_wait <= 0
                         0  # should not block, but may retrieve a message
@@ -545,7 +662,7 @@ module Rdkafka
         end
         if message
           slice << message
-          bytes += message.payload.bytesize
+          bytes += message.payload.bytesize if message.payload
         end
         if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
           yield slice.dup, nil
@@ -557,9 +674,9 @@ module Rdkafka
     end
 
     private
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # Base error class.
   class BaseError < RuntimeError; end
@@ -83,4 +85,17 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed producer")
     end
   end
+
+  # Error class for public consumer method calls on a closed admin.
+  class ClosedAdminError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed admin")
+    end
+  end
+
+  class ClosedInnerError < BaseError
+    def initialize
+      super("Illegal call to a closed inner librdkafka instance")
+    end
+  end
 end

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/metadata.rb

@@ -1,8 +1,21 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil)
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -14,12 +27,22 @@ module Rdkafka
       topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
-      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
     ensure
       Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
       Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)