rdkafka 0.8.0 → 0.8.1

data/lib/rdkafka/callbacks.rb

@@ -0,0 +1,106 @@
+module Rdkafka
+  module Callbacks
+
+    # Extracts attributes of a rd_kafka_topic_result_t
+    #
+    # @private
+    class TopicResult
+      attr_reader :result_error, :error_string, :result_name
+
+      def initialize(topic_result_pointer)
+        @result_error = Rdkafka::Bindings.rd_kafka_topic_result_error(topic_result_pointer)
+        @error_string = Rdkafka::Bindings.rd_kafka_topic_result_error_string(topic_result_pointer)
+        @result_name = Rdkafka::Bindings.rd_kafka_topic_result_name(topic_result_pointer)
+      end
+
+      def self.create_topic_results_from_array(count, array_pointer)
+        (1..count).map do |index|
+          result_pointer = (array_pointer + (index - 1)).read_pointer
+          new(result_pointer)
+        end
+      end
+    end
+
+    # FFI Function used for Create Topic and Delete Topic callbacks
+    BackgroundEventCallbackFunction = FFI::Function.new(
+        :void, [:pointer, :pointer, :pointer]
+    ) do |client_ptr, event_ptr, opaque_ptr|
+      BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
+    end
+
+    # @private
+    class BackgroundEventCallback
+      def self.call(_, event_ptr, _)
+        event_type = Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        if event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
+          process_create_topic(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
+          process_delete_topic(event_ptr)
+        end
+      end
+
+      private
+
+      def self.process_create_topic(event_ptr)
+        create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
+
+        # Get the number of create topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
+        create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
+        create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
+          create_topic_handle[:response] = create_topic_results[0].result_error
+          create_topic_handle[:error_string] = create_topic_results[0].error_string
+          create_topic_handle[:result_name] = create_topic_results[0].result_name
+          create_topic_handle[:pending] = false
+        end
+      end
+
+      def self.process_delete_topic(event_ptr)
+        delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
+
+        # Get the number of topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
+        delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
+        delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
+          delete_topic_handle[:response] = delete_topic_results[0].result_error
+          delete_topic_handle[:error_string] = delete_topic_results[0].error_string
+          delete_topic_handle[:result_name] = delete_topic_results[0].result_name
+          delete_topic_handle[:pending] = false
+        end
+      end
+    end
+
+    # FFI Function used for Message Delivery callbacks
+
+    DeliveryCallbackFunction = FFI::Function.new(
+        :void, [:pointer, :pointer, :pointer]
+    ) do |client_ptr, message_ptr, opaque_ptr|
+      DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
+    end
+
+    # @private
+    class DeliveryCallback
+      def self.call(_, message_ptr, opaque_ptr)
+        message = Rdkafka::Bindings::Message.new(message_ptr)
+        delivery_handle_ptr_address = message[:_private].address
+        if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
+          # Update delivery handle
+          delivery_handle[:response] = message[:err]
+          delivery_handle[:partition] = message[:partition]
+          delivery_handle[:offset] = message[:offset]
+          delivery_handle[:pending] = false
+          # Call delivery callback on opaque
+          if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/config.rb

@@ -137,13 +137,26 @@ module Rdkafka
       # Create Kafka config
       config = native_config(opaque)
       # Set callback to receive delivery reports on config
-      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Bindings::DeliveryCallback)
+      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
         opaque.producer = producer
       end
     end
 
+    # Create an admin instance with this configuration.
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @return [Admin] The created admin instance
+    def admin
+      opaque = Opaque.new
+      config = native_config(opaque)
+      Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 

data/lib/rdkafka/consumer.rb

@@ -36,6 +36,8 @@ module Rdkafka
     #
     # @return [nil]
     def subscribe(*topics)
+      closed_consumer_check(__method__)
+
       # Create topic partition list with topics and no partition set
       tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
@@ -49,7 +51,7 @@ module Rdkafka
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
     ensure
-      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
     end
 
     # Unsubscribe from all subscribed topics.
@@ -58,6 +60,8 @@ module Rdkafka
     #
     # @return [nil]
     def unsubscribe
+      closed_consumer_check(__method__)
+
       response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -72,6 +76,8 @@ module Rdkafka
     #
     # @return [nil]
     def pause(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
@@ -98,6 +104,8 @@ module Rdkafka
     #
     # @return [nil]
     def resume(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
@@ -120,6 +128,8 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def subscription
+      closed_consumer_check(__method__)
+
       ptr = FFI::MemoryPointer.new(:pointer)
       response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
 
@@ -142,6 +152,8 @@ module Rdkafka
     #
     # @raise [RdkafkaError] When assigning fails
     def assign(list)
+      closed_consumer_check(__method__)
+
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
@@ -164,6 +176,8 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def assignment
+      closed_consumer_check(__method__)
+
       ptr = FFI::MemoryPointer.new(:pointer)
       response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
       if response != 0
@@ -180,7 +194,7 @@ module Rdkafka
         end
       end
     ensure
-      ptr.free
+      ptr.free unless ptr.nil?
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -193,6 +207,8 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def committed(list=nil, timeout_ms=1200)
+      closed_consumer_check(__method__)
+
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -222,6 +238,8 @@ module Rdkafka
     #
     # @return [Integer] The low and high watermark
     def query_watermark_offsets(topic, partition, timeout_ms=200)
+      closed_consumer_check(__method__)
+
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
@@ -239,8 +257,8 @@ module Rdkafka
 
       return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
     ensure
-      low.free
-      high.free
+      low.free   unless low.nil?
+      high.free  unless high.nil?
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -279,6 +297,7 @@ module Rdkafka
     #
     # @return [String, nil]
     def cluster_id
+      closed_consumer_check(__method__)
       Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
     end
 
@@ -288,6 +307,7 @@ module Rdkafka
     #
     # @return [String, nil]
     def member_id
+      closed_consumer_check(__method__)
       Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
     end
 
@@ -301,6 +321,8 @@ module Rdkafka
     #
     # @return [nil]
     def store_offset(message)
+      closed_consumer_check(__method__)
+
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
@@ -331,6 +353,8 @@ module Rdkafka
     #
     # @return [nil]
     def seek(message)
+      closed_consumer_check(__method__)
+
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
@@ -369,6 +393,8 @@ module Rdkafka
     #
     # @return [nil]
     def commit(list=nil, async=false)
+      closed_consumer_check(__method__)
+
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
@@ -393,7 +419,7 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
-      return unless @native_kafka
+      closed_consumer_check(__method__)
 
       message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
@@ -441,5 +467,9 @@ module Rdkafka
         end
       end
     end
+
+    def closed_consumer_check(method)
+      raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
+    end
   end
 end

data/lib/rdkafka/error.rb

@@ -1,15 +1,27 @@
 module Rdkafka
+  # Base error class.
+  class BaseError < RuntimeError; end
+
   # Error returned by the underlying rdkafka library.
-  class RdkafkaError < RuntimeError
+  class RdkafkaError < BaseError
     # The underlying raw error response
     # @return [Integer]
-    attr_reader :rdkafka_response, :message_prefix
+    attr_reader :rdkafka_response
+
+    # Prefix to be used for human readable representation
+    # @return [String]
+    attr_reader :message_prefix
+
+    # Error message sent by the broker
+    # @return [String]
+    attr_reader :broker_message
 
     # @private
-    def initialize(response, message_prefix=nil)
+    def initialize(response, message_prefix=nil, broker_message: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
+      @broker_message = broker_message
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -57,4 +69,18 @@ module Rdkafka
       @topic_partition_list = topic_partition_list
     end
   end
+
+  # Error class for public consumer method calls on a closed consumer.
+  class ClosedConsumerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed consumer")
+    end
+  end
+
+  # Error class for public producer method calls on a closed producer.
+  class ClosedProducerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed producer")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -9,14 +9,15 @@ module Rdkafka
 
       ptr = FFI::MemoryPointer.new(:pointer)
 
-      # Retrieve metadata flag is 0/1 for single/multiple topics.
-      topic_flag = topic_name ? 1 : 0
+      # If topic_flag is 1, we request info about *all* topics in the cluster.  If topic_flag is 0,
+      # we only request info about locally known topics (or a single topic if one is passed in).
+      topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
       result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
 
       # Error Handling
-      Rdkafka::Error.new(result) unless result.zero?
+      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
     ensure
@@ -34,11 +35,11 @@ module Rdkafka
 
       @topics = Array.new(metadata[:topics_count]) do |i|
         topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
-        Rdkafka::Error.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
 
         partitions = Array.new(topic[:partition_count]) do |j|
           partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
-          Rdkafka::Error.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
           partition.to_h
         end
         topic.to_h.merge!(partitions: partitions)

data/lib/rdkafka/producer.rb

@@ -11,6 +11,10 @@ module Rdkafka
     def initialize(native_kafka)
       @closing = false
       @native_kafka = native_kafka
+
+      # Makes sure, that the producer gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, proc { close })
+
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
         loop do
@@ -55,7 +59,8 @@ module Rdkafka
     # @return partition count [Integer,nil]
     #
     def partition_count(topic)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.select { |x| x[:topic_name] == topic }&.dig(0, :partition_count)
+      closed_producer_check(__method__)
+      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -74,6 +79,8 @@ module Rdkafka
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+      closed_producer_check(__method__)
+
       # Start by checking and converting the input
 
       # Get payload length
@@ -117,7 +124,7 @@ module Rdkafka
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
       delivery_handle[:offset] = -1
-      DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
+      DeliveryHandle.register(delivery_handle)
 
       args = [
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
@@ -163,5 +170,9 @@ module Rdkafka
     def call_delivery_callback(delivery_handle)
       @delivery_callback.call(delivery_handle) if @delivery_callback
     end
+
+    def closed_producer_check(method)
+      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+    end
   end
 end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,67 +2,21 @@ module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
     # producing a message.
-    class DeliveryHandle < FFI::Struct
+    class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
              :offset, :int64
 
-      REGISTRY = {}
-
-      CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
-
-      private_constant :CURRENT_TIME
-
-      def self.register(address, handle)
-        REGISTRY[address] = handle
-      end
-
-      def self.remove(address)
-        REGISTRY.delete(address)
-      end
-
-      # Whether the delivery handle is still pending.
-      #
-      # @return [Boolean]
-      def pending?
-        self[:pending]
+      # @return [String] the name of the operation (e.g. "delivery")
+      def operation_name
+        "delivery"
       end
 
-      # Wait for the delivery report or raise an error if this takes longer than the timeout.
-      # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
-      # In this case it is possible to call wait again.
-      #
-      # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
-      # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
-      #
-      # @raise [RdkafkaError] When delivering the message failed
-      # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-      #
-      # @return [DeliveryReport]
-      def wait(max_wait_timeout: 60, wait_timeout: 0.1)
-        timeout = if max_wait_timeout
-                    CURRENT_TIME.call + max_wait_timeout
-                  else
-                    nil
-                  end
-        loop do
-          if pending?
-            if timeout && timeout <= CURRENT_TIME.call
-              raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
-            end
-            sleep wait_timeout
-          elsif self[:response] != 0
-            raise RdkafkaError.new(self[:response])
-          else
-            return DeliveryReport.new(self[:partition], self[:offset])
-          end
-        end
+      # @return [DeliveryReport] a report on the delivery of the message
+      def create_result
+        DeliveryReport.new(self[:partition], self[:offset])
       end
-
-      # Error that is raised when waiting for a delivery handle to complete
-      # takes longer than the specified timeout.
-      class WaitTimeoutError < RuntimeError; end
     end
   end
 end