rdkafka 0.8.0.beta.1 → 0.10.0

data/lib/rdkafka/error.rb

@@ -1,15 +1,27 @@
 module Rdkafka
+  # Base error class.
+  class BaseError < RuntimeError; end
+
   # Error returned by the underlying rdkafka library.
-  class RdkafkaError < RuntimeError
+  class RdkafkaError < BaseError
     # The underlying raw error response
     # @return [Integer]
-    attr_reader :rdkafka_response, :message_prefix
+    attr_reader :rdkafka_response
+
+    # Prefix to be used for human readable representation
+    # @return [String]
+    attr_reader :message_prefix
+
+    # Error message sent by the broker
+    # @return [String]
+    attr_reader :broker_message
 
     # @private
-    def initialize(response, message_prefix=nil)
+    def initialize(response, message_prefix=nil, broker_message: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
+      @broker_message = broker_message
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -57,4 +69,18 @@ module Rdkafka
       @topic_partition_list = topic_partition_list
     end
   end
+
+  # Error class for public consumer method calls on a closed consumer.
+  class ClosedConsumerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed consumer")
+    end
+  end
+
+  # Error class for public producer method calls on a closed producer.
+  class ClosedProducerError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed producer")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -9,14 +9,15 @@ module Rdkafka
 
       ptr = FFI::MemoryPointer.new(:pointer)
 
-      # Retrieve metadata flag is 0/1 for single/multiple topics.
-      topic_flag = topic_name ? 1 : 0
+      # If topic_flag is 1, we request info about *all* topics in the cluster.  If topic_flag is 0,
+      # we only request info about locally known topics (or a single topic if one is passed in).
+      topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
       result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
 
       # Error Handling
-      Rdkafka::Error.new(result) unless result.zero?
+      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
     ensure
@@ -34,11 +35,11 @@ module Rdkafka
 
       @topics = Array.new(metadata[:topics_count]) do |i|
         topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
-        Rdkafka::Error.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
 
         partitions = Array.new(topic[:partition_count]) do |j|
           partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
-          Rdkafka::Error.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
           partition.to_h
         end
         topic.to_h.merge!(partitions: partitions)

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,67 +2,21 @@ module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
     # producing a message.
-    class DeliveryHandle < FFI::Struct
+    class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
              :offset, :int64
 
-      REGISTRY = {}
-
-      CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
-
-      private_constant :CURRENT_TIME
-
-      def self.register(address, handle)
-        REGISTRY[address] = handle
-      end
-
-      def self.remove(address)
-        REGISTRY.delete(address)
-      end
-
-      # Whether the delivery handle is still pending.
-      #
-      # @return [Boolean]
-      def pending?
-        self[:pending]
+      # @return [String] the name of the operation (e.g. "delivery")
+      def operation_name
+        "delivery"
       end
 
-      # Wait for the delivery report or raise an error if this takes longer than the timeout.
-      # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
-      # In this case it is possible to call wait again.
-      #
-      # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
-      # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
-      #
-      # @raise [RdkafkaError] When delivering the message failed
-      # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-      #
-      # @return [DeliveryReport]
-      def wait(max_wait_timeout: 60, wait_timeout: 0.1)
-        timeout = if max_wait_timeout
-                    CURRENT_TIME.call + max_wait_timeout
-                  else
-                    nil
-                  end
-        loop do
-          if pending?
-            if timeout && timeout <= CURRENT_TIME.call
-              raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
-            end
-            sleep wait_timeout
-          elsif self[:response] != 0
-            raise RdkafkaError.new(self[:response])
-          else
-            return DeliveryReport.new(self[:partition], self[:offset])
-          end
-        end
+      # @return [DeliveryReport] a report on the delivery of the message
+      def create_result
+        DeliveryReport.new(self[:partition], self[:offset])
       end
-
-      # Error that is raised when waiting for a delivery handle to complete
-      # takes longer than the specified timeout.
-      class WaitTimeoutError < RuntimeError; end
     end
   end
 end

data/lib/rdkafka/producer.rb

@@ -1,3 +1,5 @@
+require "securerandom"
+
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
@@ -9,8 +11,13 @@ module Rdkafka
 
     # @private
     def initialize(native_kafka)
+      @id = SecureRandom.uuid
       @closing = false
       @native_kafka = native_kafka
+
+      # Makes sure, that the producer gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(@id, proc { close })
+
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
         loop do
@@ -27,16 +34,18 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport}
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def delivery_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
     end
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      ObjectSpace.undefine_finalizer(@id)
+
       return unless @native_kafka
 
       # Indicate to polling thread that we're closing
@@ -55,7 +64,8 @@ module Rdkafka
     # @return partition count [Integer,nil]
     #
     def partition_count(topic)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.select { |x| x[:topic_name] == topic }&.dig(0, :partition_count)
+      closed_producer_check(__method__)
+      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -74,6 +84,8 @@ module Rdkafka
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+      closed_producer_check(__method__)
+
       # Start by checking and converting the input
 
       # Get payload length
@@ -117,7 +129,7 @@ module Rdkafka
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
       delivery_handle[:offset] = -1
-      DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
+      DeliveryHandle.register(delivery_handle)
 
       args = [
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
@@ -133,16 +145,14 @@ module Rdkafka
         headers.each do |key0, value0|
           key = key0.to_s
           value = value0.to_s
-          args += [
-            :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-            :string, key,
-            :pointer, value,
-            :size_t, value.bytes.size
-          ]
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytes.size
         end
       end
 
-      args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
+      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
@@ -163,5 +173,9 @@ module Rdkafka
     def call_delivery_callback(delivery_handle)
       @delivery_callback.call(delivery_handle) if @delivery_callback
     end
+
+    def closed_producer_check(method)
+      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.8.0.beta.1"
-  LIBRDKAFKA_VERSION = "1.4.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
+  VERSION = "0.10.0"
+  LIBRDKAFKA_VERSION = "1.5.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "f7fee59fdbf1286ec23ef0b35b2dfb41031c8727c90ced6435b8cf576f23a656"
 end

data/lib/rdkafka.rb

@@ -1,6 +1,13 @@
 require "rdkafka/version"
 
+require "rdkafka/abstract_handle"
+require "rdkafka/admin"
+require "rdkafka/admin/create_topic_handle"
+require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_topic_handle"
+require "rdkafka/admin/delete_topic_report"
 require "rdkafka/bindings"
+require "rdkafka/callbacks"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"

data/spec/rdkafka/abstract_handle_spec.rb

@@ -0,0 +1,114 @@
+require "spec_helper"
+
+describe Rdkafka::AbstractHandle do
+  let(:response) { 0 }
+  let(:result) { -1 }
+
+  context "A subclass that does not implement the required methods" do
+
+    class BadTestHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int
+    end
+
+    it "raises an exception if operation_name is called" do
+      expect {
+        BadTestHandle.new.operation_name
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+
+    it "raises an exception if create_result is called" do
+      expect {
+        BadTestHandle.new.create_result
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+  end
+
+  class TestHandle < Rdkafka::AbstractHandle
+    layout :pending, :bool,
+           :response, :int,
+           :result, :int
+
+    def operation_name
+      "test_operation"
+    end
+
+    def create_result
+      self[:result]
+    end
+  end
+
+  subject do
+    TestHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:result] = result
+    end
+  end
+
+  describe ".register and .remove" do
+    let(:pending_handle) { true }
+
+    it "should register and remove a delivery handle" do
+      Rdkafka::AbstractHandle.register(subject)
+      removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
+      expect(removed).to eq subject
+      expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
+    end
+  end
+
+  describe "#pending?" do
+    context "when true" do
+      let(:pending_handle) { true }
+
+      it "should be true" do
+        expect(subject.pending?).to be true
+      end
+    end
+
+    context "when not true" do
+      let(:pending_handle) { false }
+
+      it "should be false" do
+        expect(subject.pending?).to be false
+      end
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+      let(:result) { 1 }
+
+      it "should return a result" do
+        wait_result = subject.wait
+        expect(wait_result).to eq(result)
+      end
+
+      it "should wait without a timeout" do
+        wait_result = subject.wait(max_wait_timeout: nil)
+        expect(wait_result).to eq(result)
+      end
+    end
+
+    context "when not pending anymore and there was an error" do
+      let(:pending_handle) { false }
+      let(:response) { 20 }
+
+      it "should raise an rdkafka error" do
+        expect {
+          subject.wait
+        }.to raise_error Rdkafka::RdkafkaError
+      end
+    end
+  end
+end
+

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end