karafka-rdkafka 0.13.2 → 0.13.9

data/lib/rdkafka/consumer.rb

@@ -12,6 +12,7 @@ module Rdkafka
   # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
+    include Helpers::Time
 
     # @private
     def initialize(native_kafka)
@@ -29,6 +30,13 @@ module Rdkafka
       ->(_) { close }
     end
 
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     # Close this consumer
     # @return [nil]
     def close
@@ -47,13 +55,11 @@ module Rdkafka
       @native_kafka.closed?
     end
 
-    # Subscribe to one or more topics letting Kafka handle partition assignments.
+    # Subscribes to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
-    #
-    # @raise [RdkafkaError] When subscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When subscribing fails
     def subscribe(*topics)
       closed_consumer_check(__method__)
 
@@ -68,36 +74,33 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
-      end
+
+      Rdkafka::RdkafkaError.validate!(response, "Error subscribing to '#{topics.join(', ')}'")
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
     end
 
     # Unsubscribe from all subscribed topics.
     #
-    # @raise [RdkafkaError] When unsubscribing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe
       closed_consumer_check(__method__)
 
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     end
 
     # Pause producing or consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaTopicPartitionListError] When pausing subscription fails.
     def pause(list)
       closed_consumer_check(__method__)
 
@@ -121,13 +124,11 @@ module Rdkafka
       end
     end
 
-    # Resume producing consumption for the provided list of partitions
+    # Resumes producing consumption for the provided list of partitions
     #
     # @param list [TopicPartitionList] The topic with partitions to pause
-    #
-    # @raise [RdkafkaError] When resume subscription fails.
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When resume subscription fails.
     def resume(list)
       closed_consumer_check(__method__)
 
@@ -141,19 +142,19 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
-        end
+
+        Rdkafka::RdkafkaError.validate!(response, "Error resume '#{list.to_h}'")
+
+        nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
-    # Return the current subscription to topics and partitions
-    #
-    # @raise [RdkafkaError] When getting the subscription fails.
+    # Returns the current subscription to topics and partitions
     #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the subscription fails.
     def subscription
       closed_consumer_check(__method__)
 
@@ -162,9 +163,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
       end
 
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
 
       native = ptr.read_pointer
 
@@ -178,7 +177,6 @@ module Rdkafka
     # Atomic assignment of partitions to consume
     #
     # @param list [TopicPartitionList] The topic with partitions to assign
-    #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
       closed_consumer_check(__method__)
@@ -193,9 +191,8 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
-        end
+
+        Rdkafka::RdkafkaError.validate!(response, "Error assigning '#{list.to_h}'")
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
@@ -203,9 +200,8 @@ module Rdkafka
 
     # Returns the current partition assignment.
     #
-    # @raise [RdkafkaError] When getting the assignment fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the assignment fails.
     def assignment
       closed_consumer_check(__method__)
 
@@ -213,9 +209,8 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
 
       tpl = ptr.read_pointer
 
@@ -240,14 +235,14 @@ module Rdkafka
     end
 
     # Return the current committed offset per partition for this consumer group.
-    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    # The offset field of each requested partition will either be set to stored offset or to -1001
+    # in case there was no stored offset for that partition.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
-    #
-    # @raise [RdkafkaError] When getting the committed positions fails.
-    #
     # @return [TopicPartitionList]
+    # @raise [RdkafkaError] When getting the committed positions fails.
     def committed(list=nil, timeout_ms=1200)
       closed_consumer_check(__method__)
 
@@ -263,24 +258,48 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response)
-        end
+
+        Rdkafka::RdkafkaError.validate!(response)
+
         TopicPartitionList.from_native_tpl(tpl)
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
+    # Return the current positions (offsets) for topics and partitions.
+    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    #
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    #
+    # @raise [RdkafkaError] When getting the positions fails.
+    #
+    # @return [TopicPartitionList]
+    def position(list=nil)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      TopicPartitionList.from_native_tpl(tpl)
+    end
+
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
     # @param topic [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
-    #
-    # @raise [RdkafkaError] When querying the broker fails.
-    #
     # @return [Integer] The low and high watermark
+    # @raise [RdkafkaError] When querying the broker fails.
     def query_watermark_offsets(topic, partition, timeout_ms=200)
       closed_consumer_check(__method__)
 
@@ -297,9 +316,8 @@ module Rdkafka
           timeout_ms,
         )
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
-      end
+
+      Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
 
       return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
     ensure
@@ -314,10 +332,9 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    #
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
 
@@ -327,7 +344,7 @@ module Rdkafka
         topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          low, high = query_watermark_offsets(
+          _, high = query_watermark_offsets(
             topic,
             p.partition,
             watermark_timeout_ms
@@ -366,10 +383,8 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    #
-    # @raise [RdkafkaError] When storing the offset fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message)
       closed_consumer_check(__method__)
 
@@ -387,9 +402,10 @@ module Rdkafka
         message.partition,
         message.offset
       )
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     ensure
       if native_topic && !native_topic.null?
         Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
@@ -400,10 +416,8 @@ module Rdkafka
     # message at the given offset.
     #
     # @param message [Rdkafka::Consumer::Message] The message to which to seek
-    #
-    # @raise [RdkafkaError] When seeking fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
     def seek(message)
       closed_consumer_check(__method__)
 
@@ -422,9 +436,9 @@ module Rdkafka
         message.offset,
         0 # timeout
       )
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     ensure
       if native_topic && !native_topic.null?
         Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
@@ -455,9 +469,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
 
       TopicPartitionList.from_native_tpl(tpl)
     ensure
@@ -475,10 +487,8 @@ module Rdkafka
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
-    #
-    # @raise [RdkafkaError] When committing fails
-    #
     # @return [nil]
+    # @raise [RdkafkaError] When committing fails
     def commit(list=nil, async=false)
       closed_consumer_check(__method__)
 
@@ -492,9 +502,10 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response)
-        end
+
+        Rdkafka::RdkafkaError.validate!(response)
+
+        nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
@@ -503,10 +514,8 @@ module Rdkafka
     # Poll for the next message on one of the subscribed topics
     #
     # @param timeout_ms [Integer] Timeout of this poll
-    #
-    # @raise [RdkafkaError] When polling fails
-    #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
@@ -519,9 +528,9 @@ module Rdkafka
         # Create struct wrapper
         native_message = Rdkafka::Bindings::Message.new(message_ptr)
         # Raise error if needed
-        if native_message[:err] != 0
-          raise Rdkafka::RdkafkaError.new(native_message[:err])
-        end
+
+        Rdkafka::RdkafkaError.validate!(native_message[:err])
+
         # Create a message to pass out
         Rdkafka::Consumer::Message.new(native_message)
       end
@@ -532,17 +541,40 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
+    #   was set to `true`.
+    #
+    # Events will cause application-provided callbacks to be called.
+    #
+    # Events (in the context of the consumer):
+    #   - error callbacks
+    #   - stats callbacks
+    #   - any other callbacks supported by librdkafka that are not part of the consumer_poll, that
+    #     would have a callback configured and activated.
+    #
+    # This method needs to be called at regular intervals to serve any queued callbacks waiting to
+    # be called. When in use, does **NOT** replace `#poll` but needs to run complementary with it.
+    #
+    # @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
+    #   block until any events available.
+    #
+    # @note This method technically should be called `#poll` and the current `#poll` should be
+    #   called `#consumer_poll` though we keep the current naming convention to make it backward
+    #   compatible.
+    def events_poll(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
-    # If `enable.partition.eof` is turned on in the config this will raise an
-    # error when an eof is reached, so you probably want to disable that when
-    # using this method of iteration.
+    # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
+    # reached, so you probably want to disable that when using this method of iteration.
     #
     # @raise [RdkafkaError] When polling fails
-    #
     # @yieldparam message [Message] Received message
-    #
     # @return [nil]
     def each
       loop do
@@ -595,9 +627,7 @@ module Rdkafka
     # that you may or may not see again.
     #
     # @param max_items [Integer] Maximum size of the yielded array of messages
-    #
     # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    #
     # @param timeout_ms [Integer] max time to wait for up to max_items
     #
     # @raise [RdkafkaError] When polling fails
@@ -644,10 +674,6 @@ module Rdkafka
     end
 
     private
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
 
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?

data/lib/rdkafka/error.rb

@@ -18,12 +18,59 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    class << self
+      def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
+        code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
+
+        return false if code.zero?
+
+        message = broker_message || Rdkafka::Bindings.rd_kafka_err2str(code)
+        fatal = !Rdkafka::Bindings.rd_kafka_error_is_fatal(response_ptr).zero?
+        retryable = !Rdkafka::Bindings.rd_kafka_error_is_retriable(response_ptr).zero?
+        abortable = !Rdkafka::Bindings.rd_kafka_error_txn_requires_abort(response_ptr).zero?
+
+        Rdkafka::Bindings.rd_kafka_error_destroy(response_ptr)
+
+        new(
+          code,
+          message_prefix,
+          broker_message: message,
+          fatal: fatal,
+          retryable: retryable,
+          abortable: abortable
+        )
+      end
+
+      def build(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        if response_ptr_or_code.is_a?(Integer)
+          response_ptr_or_code.zero? ? false : new(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        else
+          build_from_c(response_ptr_or_code, message_prefix)
+        end
+      end
+
+      def validate!(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        error = build(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        error ? raise(error) : false
+      end
+    end
+
     # @private
-    def initialize(response, message_prefix=nil, broker_message: nil)
+    def initialize(
+      response,
+      message_prefix=nil,
+      broker_message: nil,
+      fatal: false,
+      retryable: false,
+      abortable: false
+    )
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
       @broker_message = broker_message
+      @fatal = fatal
+      @retryable = retryable
+      @abortable = abortable
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -58,6 +105,18 @@ module Rdkafka
     def ==(another_error)
        another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
     end
+
+    def fatal?
+      @fatal
+    end
+
+    def retryable?
+      @retryable
+    end
+
+    def abortable?
+      @abortable
+    end
   end
 
   # Error with topic partition list returned by the underlying rdkafka library.

data/lib/rdkafka/helpers/time.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Namespace for some small utilities used in multiple components
+  module Helpers
+    # Time related methods used across Karafka
+    module Time
+      # @return [Float] current monotonic time in seconds with microsecond precision
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rdkafka/metadata.rb

@@ -29,8 +29,7 @@ module Rdkafka
       # Retrieve the Metadata
       result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
-      # Error Handling
-      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
+      Rdkafka::RdkafkaError.validate!(result)
 
       metadata_from_native(ptr.read_pointer)
     rescue ::Rdkafka::RdkafkaError => e
@@ -58,11 +57,12 @@ module Rdkafka
 
       @topics = Array.new(metadata[:topics_count]) do |i|
         topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
-        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        RdkafkaError.validate!(topic[:rd_kafka_resp_err])
 
         partitions = Array.new(topic[:partition_count]) do |j|
           partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
-          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          RdkafkaError.validate!(partition[:rd_kafka_resp_err])
           partition.to_h
         end
         topic.to_h.merge!(partitions: partitions)

data/lib/rdkafka/native_kafka.rb

@@ -4,8 +4,9 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:)
+    def initialize(inner, run_polling_thread:, opaque:)
       @inner = inner
+      @opaque = opaque
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -27,6 +28,9 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
+      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
         # not used in consumer.
@@ -109,6 +113,7 @@ module Rdkafka
 
         Rdkafka::Bindings.rd_kafka_destroy(@inner)
         @inner = nil
+        @opaque = nil
       end
     end
   end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -18,7 +18,22 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
+        if self[:response] == 0
+          DeliveryReport.new(
+            self[:partition],
+            self[:offset],
+            self[:topic_name].read_string
+          )
+        else
+          DeliveryReport.new(
+            self[:partition],
+            self[:offset],
+            # For part of errors, we will not get a topic name reference and in cases like this
+            # we should not return it
+            self[:topic_name].null? ? nil : self[:topic_name].read_string,
+            Rdkafka::RdkafkaError.build(self[:response])
+          )
+        end
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,8 +12,9 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to.
-      # @return [String]
+      # The name of the topic this message was produced to or nil in case delivery failed and we
+      #   we not able to get the topic reference
+      # @return [String, nil]
       attr_reader :topic_name
 
       # Error in case happen during produce.