rdkafka 0.6.0 → 0.9.0

data/lib/rdkafka/consumer/headers.rb

@@ -19,7 +19,7 @@ module Rdkafka
           raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
         end
 
-        headers_ptr = headers_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+        headers_ptr = headers_ptrptr.read_pointer
 
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -42,12 +42,14 @@ module Rdkafka
             raise Rdkafka::RdkafkaError.new(err, "Error reading a message header at index #{idx}")
           end
 
-          name = name_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          name = name.read_string_to_null
+          name_ptr = name_ptrptr.read_pointer
+          name = name_ptr.respond_to?(:read_string_to_null) ? name_ptr.read_string_to_null : name_ptr.read_string
 
           size = size_ptr[:value]
-          value = value_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          value = value.read_string(size)
+
+          value_ptr = value_ptrptr.read_pointer
+
+          value = value_ptr.read_string(size)
 
           headers[name.to_sym] = value
 

data/lib/rdkafka/consumer/partition.rb

@@ -11,7 +11,7 @@ module Rdkafka
       attr_reader :offset
 
       # Partition's error code
-      # @retuen [Integer]
+      # @return [Integer]
       attr_reader :err
 
       # @private

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -4,7 +4,7 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash<String => [nil,Partition]>] The topic and partion data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
       #
       # @return [TopicPartitionList]
       def initialize(data=nil)
@@ -71,7 +71,7 @@ module Rdkafka
 
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
       #
-      # @return [Hash<String, [Array<Partition>, nil]>]
+      # @return [Hash{String => Array<Partition>,nil}]
       def to_h
         @data
       end
@@ -106,7 +106,7 @@ module Rdkafka
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []
-            offset = if elem[:offset] == -1001
+            offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
                        nil
                      else
                        elem[:offset]
@@ -125,10 +125,10 @@ module Rdkafka
       #
       # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
       #
-      # @return [FFI::AutoPointer]
+      # @return [FFI::Pointer]
       # @private
       def to_native_tpl
-        tpl = TopicPartitionList.new_native_tpl(count)
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
         @data.each do |topic, partitions|
           if partitions
@@ -138,6 +138,7 @@ module Rdkafka
                 topic,
                 p.partition
               )
+
               if p.offset
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
@@ -158,17 +159,6 @@ module Rdkafka
 
         tpl
       end
-
-      # Creates a new native tpl and wraps it into FFI::AutoPointer which in turn calls
-      # `rd_kafka_topic_partition_list_destroy` when a pointer will be cleaned by GC
-      #
-      # @param count [Integer] an initial capacity of partitions list
-      # @return [FFI::AutoPointer]
-      # @private
-      def self.new_native_tpl(count)
-        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
-        FFI::AutoPointer.new(tpl, Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy))
-      end
     end
   end
 end

data/lib/rdkafka/error.rb

@@ -1,15 +1,27 @@
 module Rdkafka
+  # Base error class.
+  class BaseError < RuntimeError; end
+
   # Error returned by the underlying rdkafka library.
-  class RdkafkaError < RuntimeError
+  class RdkafkaError < BaseError
     # The underlying raw error response
     # @return [Integer]
-    attr_reader :rdkafka_response, :message_prefix
+    attr_reader :rdkafka_response
+
+    # Prefix to be used for human readable representation
+    # @return [String]
+    attr_reader :message_prefix
+
+    # Error message sent by the broker
+    # @return [String]
+    attr_reader :broker_message
 
     # @private
-    def initialize(response, message_prefix=nil)
+    def initialize(response, message_prefix=nil, broker_message: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
+      @broker_message = broker_message
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -39,9 +51,14 @@ module Rdkafka
     def is_partition_eof?
       code == :partition_eof
     end
+
+    # Error comparison
+    def ==(another_error)
+       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    end
   end
 
-  # Error with potic partition list returned by the underlying rdkafka library.
+  # Error with topic partition list returned by the underlying rdkafka library.
   class RdkafkaTopicPartitionListError < RdkafkaError
     # @return [TopicPartitionList]
     attr_reader :topic_partition_list
@@ -52,4 +69,18 @@ module Rdkafka
       @topic_partition_list = topic_partition_list
     end
   end
+
+  # Error class for public consumer method calls on a closed consumer.
+  class ClosedConsumerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed consumer")
+    end
+  end
+
+  # Error class for public producer method calls on a closed producer.
+  class ClosedProducerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed producer")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -0,0 +1,92 @@
+module Rdkafka
+  class Metadata
+    attr_reader :brokers, :topics
+
+    def initialize(native_client, topic_name = nil)
+      native_topic = if topic_name
+        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      end
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+
+      # If topic_flag is 1, we request info about *all* topics in the cluster.  If topic_flag is 0,
+      # we only request info about locally known topics (or a single topic if one is passed in).
+      topic_flag = topic_name.nil? ? 1 : 0
+
+      # Retrieve the Metadata
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+
+      # Error Handling
+      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
+
+      metadata_from_native(ptr.read_pointer)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+    end
+
+    private
+
+    def metadata_from_native(ptr)
+      metadata = Metadata.new(ptr)
+      @brokers = Array.new(metadata[:brokers_count]) do |i|
+        BrokerMetadata.new(metadata[:brokers_metadata] + (i * BrokerMetadata.size)).to_h
+      end
+
+      @topics = Array.new(metadata[:topics_count]) do |i|
+        topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
+        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        partitions = Array.new(topic[:partition_count]) do |j|
+          partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
+          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          partition.to_h
+        end
+        topic.to_h.merge!(partitions: partitions)
+      end
+    end
+
+    class CustomFFIStruct < FFI::Struct
+      def to_h
+        members.each_with_object({}) do |mem, hsh|
+          val = self.[](mem)
+          next if val.is_a?(FFI::Pointer) || mem == :rd_kafka_resp_err
+
+          hsh[mem] = self.[](mem)
+        end
+      end
+    end
+
+    class Metadata < CustomFFIStruct
+      layout :brokers_count, :int,
+             :brokers_metadata, :pointer,
+             :topics_count, :int,
+             :topics_metadata, :pointer,
+             :broker_id, :int32,
+             :broker_name, :string
+    end
+
+    class BrokerMetadata < CustomFFIStruct
+      layout :broker_id, :int32,
+             :broker_name, :string,
+             :broker_port, :int
+    end
+
+    class TopicMetadata < CustomFFIStruct
+      layout :topic_name, :string,
+             :partition_count, :int,
+             :partitions_metadata, :pointer,
+             :rd_kafka_resp_err, :int
+    end
+
+    class PartitionMetadata < CustomFFIStruct
+      layout :partition_id, :int32,
+             :rd_kafka_resp_err, :int,
+             :leader, :int32,
+             :replica_count, :int,
+             :replicas, :pointer,
+             :in_sync_replica_brokers, :int,
+             :isrs, :pointer
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -2,12 +2,19 @@ module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
     # @private
-    @delivery_callback = nil
+    # Returns the current delivery callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    attr_reader :delivery_callback
 
     # @private
     def initialize(native_kafka)
       @closing = false
       @native_kafka = native_kafka
+
+      # Makes sure, that the producer gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, proc { close })
+
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
         loop do
@@ -24,33 +31,42 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport}
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def delivery_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
     end
 
-    # Returns the current delivery callback, by default this is nil.
-    #
-    # @return [Proc, nil]
-    def delivery_callback
-      @delivery_callback
-    end
-
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return unless @native_kafka
+
       # Indicate to polling thread that we're closing
       @closing = true
       # Wait for the polling thread to finish up
       @polling_thread.join
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
+    end
+
+    # Partition count for a given topic.
+    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
+    #
+    # @param topic [String] The topic name.
+    #
+    # @return partition count [Integer,nil]
+    #
+    def partition_count(topic)
+      closed_producer_check(__method__)
+      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
     #
     # When no partition is specified the underlying Kafka library picks a partition based on the key. If no key is specified, a random partition will be used.
-    # When a timestamp is provided this is used instead of the autogenerated timestamp.
+    # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
@@ -62,7 +78,9 @@ module Rdkafka
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
-    def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil, headers: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+      closed_producer_check(__method__)
+
       # Start by checking and converting the input
 
       # Get payload length
@@ -79,9 +97,15 @@ module Rdkafka
                    key.bytesize
                  end
 
-      # If partition is nil use -1 to let Kafka set the partition based
-      # on the key/randomly if there is no key
-      partition = -1 if partition.nil?
+      if partition_key
+        partition_count = partition_count(topic)
+        # If the topic is not present, set to -1
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+      end
+
+      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # based on the key when present.
+      partition ||= -1
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.
@@ -100,7 +124,7 @@ module Rdkafka
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
       delivery_handle[:offset] = -1
-      DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
+      DeliveryHandle.register(delivery_handle)
 
       args = [
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
@@ -116,16 +140,14 @@ module Rdkafka
         headers.each do |key0, value0|
           key = key0.to_s
           value = value0.to_s
-          args += [
-            :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-            :string, key,
-            :pointer, value,
-            :size_t, value.bytes.size
-          ]
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytes.size
         end
       end
 
-      args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
+      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
@@ -133,7 +155,7 @@ module Rdkafka
         *args
       )
 
-      # Raise error if the produce call was not successfull
+      # Raise error if the produce call was not successful
       if response != 0
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
         raise RdkafkaError.new(response)
@@ -146,5 +168,9 @@ module Rdkafka
     def call_delivery_callback(delivery_handle)
       @delivery_callback.call(delivery_handle) if @delivery_callback
     end
+
+    def closed_producer_check(method)
+      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+    end
   end
 end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,63 +2,21 @@ module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
     # producing a message.
-    class DeliveryHandle < FFI::Struct
+    class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
              :offset, :int64
 
-      REGISTRY = {}
-
-      def self.register(address, handle)
-        REGISTRY[address] = handle
-      end
-
-      def self.remove(address)
-        REGISTRY.delete(address)
+      # @return [String] the name of the operation (e.g. "delivery")
+      def operation_name
+        "delivery"
       end
 
-      # Whether the delivery handle is still pending.
-      #
-      # @return [Boolean]
-      def pending?
-        self[:pending]
+      # @return [DeliveryReport] a report on the delivery of the message
+      def create_result
+        DeliveryReport.new(self[:partition], self[:offset])
       end
-
-      # Wait for the delivery report or raise an error if this takes longer than the timeout.
-      # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
-      # In this case it is possible to call wait again.
-      #
-      # @param timeout_in_seconds [Integer, nil] Number of seconds to wait before timing out. If this is nil it does not time out.
-      #
-      # @raise [RdkafkaError] When delivering the message failed
-      # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-      #
-      # @return [DeliveryReport]
-      def wait(timeout_in_seconds=60)
-        timeout = if timeout_in_seconds
-                    Time.now.to_i + timeout_in_seconds
-                  else
-                    nil
-                  end
-        loop do
-          if pending?
-            if timeout && timeout <= Time.now.to_i
-              raise WaitTimeoutError.new("Waiting for delivery timed out after #{timeout_in_seconds} seconds")
-            end
-            sleep 0.1
-            next
-          elsif self[:response] != 0
-            raise RdkafkaError.new(self[:response])
-          else
-            return DeliveryReport.new(self[:partition], self[:offset])
-          end
-        end
-      end
-
-      # Error that is raised when waiting for a delivery handle to complete
-      # takes longer than the specified timeout.
-      class WaitTimeoutError < RuntimeError; end
     end
   end
 end