karafka-rdkafka 0.12.2 → 0.13.0.beta1

data/lib/rdkafka/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "logger"
 
 module Rdkafka
@@ -30,7 +32,6 @@ module Rdkafka
       @@logger
     end
 
-
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -47,7 +48,7 @@ module Rdkafka
     # @return [nil]
     def self.logger=(logger)
       raise NoLoggerError if logger.nil?
-      @@logger=logger
+      @@logger = logger
     end
 
     # Set a callback that will be called every time the underlying client emits statistics.
@@ -156,13 +157,14 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(kafka)
+      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
     end
 
     # Create a producer with this configuration.
@@ -179,7 +181,8 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
+      partitioner_name = self[:partitioner] || self["partitioner"]
+      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -194,7 +197,7 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
@@ -210,7 +213,7 @@ module Rdkafka
 
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
-    def native_config(opaque=nil)
+    def native_config(opaque = nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
         # Create config
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
@@ -282,18 +285,18 @@ module Rdkafka
       producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
-    def call_on_partitions_assigned(consumer, list)
+    def call_on_partitions_assigned(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
 
-      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+      consumer_rebalance_listener.on_partitions_assigned(list)
     end
 
-    def call_on_partitions_revoked(consumer, list)
+    def call_on_partitions_revoked(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
 
-      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+      consumer_rebalance_listener.on_partitions_revoked(list)
     end
   end
 end

data/lib/rdkafka/consumer/headers.rb

@@ -1,10 +1,26 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
-    # A message headers
-    class Headers
-      # Reads a native kafka's message header into ruby's hash
+    # Interface to return headers for a consumer message
+    module Headers
+      class HashWithSymbolKeysTreatedLikeStrings < Hash
+        def [](key)
+          if key.is_a?(Symbol)
+            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
+                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+            super(key.to_s)
+          else
+            super
+          end
+        end
+      end
+
+      # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      #
+      # @param [librdkakfa message] native_message
       #
-      # @return [Hash<String, String>] a message headers
+      # @return [Hash<String, String>] headers Hash for the native_message
       #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       #
@@ -24,7 +40,8 @@ module Rdkafka
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
-        headers = {}
+
+        headers = HashWithSymbolKeysTreatedLikeStrings.new
 
         idx = 0
         loop do
@@ -51,12 +68,12 @@ module Rdkafka
 
           value = value_ptr.read_string(size)
 
-          headers[name.to_sym] = value
+          headers[name] = value
 
           idx += 1
         end
 
-        headers
+        headers.freeze
       end
     end
   end

data/lib/rdkafka/consumer/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A message that was consumed from a topic.
@@ -18,7 +20,7 @@ module Rdkafka
       # @return [String, nil]
       attr_reader :key
 
-      # This message's offset in it's partition
+      # This message's offset in its partition
       # @return [Integer]
       attr_reader :offset
 

data/lib/rdkafka/consumer/partition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # Information about a partition, used in {TopicPartitionList}.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Consumer
     # A list of topics with their partition information

data/lib/rdkafka/consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
   # brokers automatically assign partitions and load balance partitions over consumers that
@@ -14,18 +16,33 @@ module Rdkafka
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
+    end
+
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
     end
 
     # Close this consumer
     # @return [nil]
     def close
-      return unless @native_kafka
+      return if closed?
+      ObjectSpace.undefine_finalizer(self)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+      end
+      @native_kafka.close
+    end
 
-      @closing = true
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+    # Whether this consumer has closed
+    def closed?
+      @native_kafka.closed?
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -46,7 +63,9 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -62,7 +81,9 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -85,7 +106,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
+        end
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -113,7 +136,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -131,7 +156,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      end
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -161,7 +188,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -179,7 +208,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -197,6 +228,15 @@ module Rdkafka
       ptr.free unless ptr.nil?
     end
 
+    # @return [Boolean] true if our current assignment has been lost involuntarily.
+    def assignment_lost?
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        !Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
+      end
+    end
+
     # Return the current committed offset per partition for this consumer group.
     # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
     #
@@ -218,7 +258,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -243,14 +285,16 @@ module Rdkafka
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka,
-        topic,
-        partition,
-        low,
-        high,
-        timeout_ms,
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+          inner,
+          topic,
+          partition,
+          low,
+          high,
+          timeout_ms,
+        )
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
@@ -298,7 +342,9 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+      end
     end
 
     # Returns this client's broker-assigned group member id
@@ -308,7 +354,9 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_memberid(inner)
+      end
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -325,11 +373,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_offset_store(
         native_topic,
         message.partition,
@@ -357,11 +407,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         message.partition,
@@ -402,7 +454,9 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -421,7 +475,9 @@ module Rdkafka
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
+      end
       if message_ptr.null?
         nil
       else
@@ -436,7 +492,7 @@ module Rdkafka
       end
     ensure
       # Clean up rdkafka message if there is one
-      if !message_ptr.nil? && !message_ptr.null?
+      if message_ptr && !message_ptr.null?
         Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end
@@ -459,7 +515,7 @@ module Rdkafka
         if message
           yield(message)
         else
-          if @closing
+          if closed?
             break
           else
             next
@@ -468,10 +524,6 @@ module Rdkafka
       end
     end
 
-    def closed_consumer_check(method)
-      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
-    end
-
     # Poll for new messages and yield them in batches that may contain
     # messages from more than one partition.
     #
@@ -527,7 +579,7 @@ module Rdkafka
       bytes = 0
       end_time = monotonic_now + timeout_ms / 1000.0
       loop do
-        break if @closing
+        break if closed?
         max_wait = end_time - monotonic_now
         max_wait_ms = if max_wait <= 0
                         0  # should not block, but may retrieve a message
@@ -545,7 +597,7 @@ module Rdkafka
         end
         if message
           slice << message
-          bytes += message.payload.bytesize
+          bytes += message.payload.bytesize if message.payload
         end
         if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
           yield slice.dup, nil
@@ -561,5 +613,9 @@ module Rdkafka
       # needed because Time.now can go backwards
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if closed?
+    end
   end
 end

data/lib/rdkafka/error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # Base error class.
   class BaseError < RuntimeError; end
@@ -83,4 +85,11 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed producer")
     end
   end
+
+  # Error class for public consumer method calls on a closed admin.
+  class ClosedAdminError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed admin")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -1,8 +1,21 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil)
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -14,12 +27,22 @@ module Rdkafka
       topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
-      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 2_000)
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
     ensure
       Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
       Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)

data/lib/rdkafka/native_kafka.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # @private
+  # A wrapper around a native kafka that polls and cleanly exits
+  class NativeKafka
+    def initialize(inner, run_polling_thread:)
+      @inner = inner
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
+
+      if run_polling_thread
+        # Start thread to poll client for delivery callbacks,
+        # not used in consumer.
+        @polling_thread = Thread.new do
+          loop do
+            @poll_mutex.synchronize do
+              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
+            end
+
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+              break
+            end
+          end
+        end
+
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      @closing = false
+    end
+
+    def with_inner
+      return if @inner.nil?
+
+      @access_mutex.synchronize do
+        yield @inner
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    def closed?
+      @closing || @inner.nil?
+    end
+
+    def close(object_id=nil)
+      return if closed?
+
+      @access_mutex.lock
+
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        @polling_thread.join
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      return unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      @inner = nil
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
@@ -6,7 +8,8 @@ module Rdkafka
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
-             :offset, :int64
+             :offset, :int64,
+             :topic_name, :pointer
 
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
@@ -15,7 +18,7 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset])
+        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Delivery report for a successfully produced message.
@@ -10,15 +12,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # Error in case happen during produce.
+      # The name of the topic this message was produced to.
       # @return [String]
+      attr_reader :topic_name
+
+      # Error in case happen during produce.
+      # @return [Integer]
       attr_reader :error
 
       private
 
-      def initialize(partition, offset, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil)
         @partition = partition
         @offset = offset
+        @topic_name = topic_name
         @error = error
       end
     end