rdkafka 0.8.0 → 0.11.0

data/lib/rdkafka/producer.rb

@@ -1,3 +1,5 @@
+require "securerandom"
+
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
@@ -9,8 +11,13 @@ module Rdkafka
 
     # @private
     def initialize(native_kafka)
+      @id = SecureRandom.uuid
       @closing = false
       @native_kafka = native_kafka
+
+      # Makes sure, that the producer gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(@id, proc { close })
+
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
         loop do
@@ -27,16 +34,18 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport}
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def delivery_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
     end
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      ObjectSpace.undefine_finalizer(@id)
+
       return unless @native_kafka
 
       # Indicate to polling thread that we're closing
@@ -55,7 +64,8 @@ module Rdkafka
     # @return partition count [Integer,nil]
     #
     def partition_count(topic)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.select { |x| x[:topic_name] == topic }&.dig(0, :partition_count)
+      closed_producer_check(__method__)
+      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -65,8 +75,9 @@ module Rdkafka
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
-    # @param key [String] The message's key
+    # @param key [String, nil] The message's key
     # @param partition [Integer,nil] Optional partition to produce to
+    # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     #
@@ -74,6 +85,8 @@ module Rdkafka
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+      closed_producer_check(__method__)
+
       # Start by checking and converting the input
 
       # Get payload length
@@ -117,7 +130,7 @@ module Rdkafka
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
       delivery_handle[:offset] = -1
-      DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
+      DeliveryHandle.register(delivery_handle)
 
       args = [
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
@@ -133,16 +146,14 @@ module Rdkafka
         headers.each do |key0, value0|
           key = key0.to_s
           value = value0.to_s
-          args += [
-            :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-            :string, key,
-            :pointer, value,
-            :size_t, value.bytes.size
-          ]
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytes.size
         end
       end
 
-      args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
+      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
@@ -163,5 +174,9 @@ module Rdkafka
     def call_delivery_callback(delivery_handle)
       @delivery_callback.call(delivery_handle) if @delivery_callback
     end
+
+    def closed_producer_check(method)
+      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.8.0"
-  LIBRDKAFKA_VERSION = "1.4.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
+  VERSION = "0.11.0"
+  LIBRDKAFKA_VERSION = "1.8.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
 end

data/lib/rdkafka.rb

@@ -1,6 +1,13 @@
 require "rdkafka/version"
 
+require "rdkafka/abstract_handle"
+require "rdkafka/admin"
+require "rdkafka/admin/create_topic_handle"
+require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_topic_handle"
+require "rdkafka/admin/delete_topic_report"
 require "rdkafka/bindings"
+require "rdkafka/callbacks"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"

data/rdkafka.gemspec

@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.6'
   gem.extensions = %w(ext/Rakefile)
 
-  gem.add_dependency 'ffi', '~> 1.9'
-  gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '>= 12.3'
+  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'mini_portile2', '~> 2.7'
+  gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake', '~> 12.0'
-  gem.add_development_dependency 'simplecov', '~> 0.15'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -0,0 +1,113 @@
+require "spec_helper"
+
+describe Rdkafka::AbstractHandle do
+  let(:response) { 0 }
+  let(:result) { -1 }
+
+  context "A subclass that does not implement the required methods" do
+
+    class BadTestHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int
+    end
+
+    it "raises an exception if operation_name is called" do
+      expect {
+        BadTestHandle.new.operation_name
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+
+    it "raises an exception if create_result is called" do
+      expect {
+        BadTestHandle.new.create_result
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+  end
+
+  class TestHandle < Rdkafka::AbstractHandle
+    layout :pending, :bool,
+           :response, :int,
+           :result, :int
+
+    def operation_name
+      "test_operation"
+    end
+
+    def create_result
+      self[:result]
+    end
+  end
+
+  subject do
+    TestHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:result] = result
+    end
+  end
+
+  describe ".register and .remove" do
+    let(:pending_handle) { true }
+
+    it "should register and remove a delivery handle" do
+      Rdkafka::AbstractHandle.register(subject)
+      removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
+      expect(removed).to eq subject
+      expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
+    end
+  end
+
+  describe "#pending?" do
+    context "when true" do
+      let(:pending_handle) { true }
+
+      it "should be true" do
+        expect(subject.pending?).to be true
+      end
+    end
+
+    context "when not true" do
+      let(:pending_handle) { false }
+
+      it "should be false" do
+        expect(subject.pending?).to be false
+      end
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+      let(:result) { 1 }
+
+      it "should return a result" do
+        wait_result = subject.wait
+        expect(wait_result).to eq(result)
+      end
+
+      it "should wait without a timeout" do
+        wait_result = subject.wait(max_wait_timeout: nil)
+        expect(wait_result).to eq(result)
+      end
+    end
+
+    context "when not pending anymore and there was an error" do
+      let(:pending_handle) { false }
+      let(:response) { 20 }
+
+      it "should raise an rdkafka error" do
+        expect {
+          subject.wait
+        }.to raise_error Rdkafka::RdkafkaError
+      end
+    end
+  end
+end

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateTopicReport do
+  subject { Rdkafka::Admin::CreateTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::DeleteTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DeleteTopicHandle::WaitTimeoutError, /delete topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a delete topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteTopicReport do
+  subject { Rdkafka::Admin::DeleteTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end

data/spec/rdkafka/admin_spec.rb

@@ -0,0 +1,203 @@
+require "spec_helper"
+require "ostruct"
+
+describe Rdkafka::Admin do
+  let(:config)   { rdkafka_config }
+  let(:admin)    { config.admin }
+
+  after do
+    # Registry should always end up being empty
+    expect(Rdkafka::Admin::CreateTopicHandle::REGISTRY).to be_empty
+    admin.close
+  end
+
+  let(:topic_name)               { "test-topic-#{Random.new.rand(0..1_000_000)}" }
+  let(:topic_partition_count)    { 3 }
+  let(:topic_replication_factor) { 1 }
+  let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
+  let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
+
+  describe "#create_topic" do
+    describe "called with invalid input" do
+      describe "with an invalid topic name" do
+        # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
+        # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
+        let(:topic_name) { "[!@#]" }
+
+        it "raises an exception" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
+            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+          }
+        end
+      end
+
+      describe "with the name of a topic that already exists" do
+        let(:topic_name) { "empty_test_topic" } # created in spec_helper.rb
+
+        it "raises an exception" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Topic already exists \(topic_already_exists\)/)
+            expect(ex.broker_message).to match(/Topic 'empty_test_topic' already exists/)
+          }
+        end
+      end
+
+      describe "with an invalid partition count" do
+        let(:topic_partition_count) { -999 }
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /num_partitions out of expected range/
+        end
+      end
+
+      describe "with an invalid replication factor" do
+        let(:topic_replication_factor) { -2  }
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /replication_factor out of expected range/
+        end
+      end
+
+      describe "with an invalid topic configuration" do
+        it "doesn't create the topic" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, invalid_topic_config)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_error Rdkafka::RdkafkaError, /Broker: Configuration is invalid \(invalid_config\)/
+        end
+      end
+    end
+
+    context "edge case" do
+      context "where we are unable to get the background queue" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
+        end
+      end
+
+      context "where rd_kafka_CreateTopics raises an exception" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_CreateTopics).and_raise(RuntimeError.new("oops"))
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error RuntimeError, /oops/
+        end
+      end
+    end
+
+    it "creates a topic" do
+      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, topic_config)
+      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      expect(create_topic_report.error_string).to be_nil
+      expect(create_topic_report.result_name).to eq(topic_name)
+    end
+  end
+
+  describe "#delete_topic" do
+    describe "called with invalid input" do
+      # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
+      # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
+      describe "with an invalid topic name" do
+        let(:topic_name) { "[!@#]" }
+
+        it "raises an exception" do
+          delete_topic_handle = admin.delete_topic(topic_name)
+          expect {
+            delete_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
+            expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
+          }
+        end
+      end
+
+      describe "with the name of a topic that does not exist" do
+        it "raises an exception" do
+          delete_topic_handle = admin.delete_topic(topic_name)
+          expect {
+            delete_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
+            expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
+          }
+        end
+      end
+    end
+
+    context "edge case" do
+      context "where we are unable to get the background queue" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.delete_topic(topic_name)
+          }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
+        end
+      end
+
+      context "where rd_kafka_DeleteTopics raises an exception" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_DeleteTopics).and_raise(RuntimeError.new("oops"))
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.delete_topic(topic_name)
+          }.to raise_error RuntimeError, /oops/
+        end
+      end
+    end
+
+
+    it "deletes a topic that was newly created" do
+      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      expect(create_topic_report.error_string).to be_nil
+      expect(create_topic_report.result_name).to eq(topic_name)
+
+      # Retry topic deletion a few times. On CI Kafka seems to not
+      # always be ready for it immediately
+      delete_topic_report = nil
+      10.times do |i|
+        begin
+          delete_topic_handle = admin.delete_topic(topic_name)
+          delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
+          break
+        rescue Rdkafka::RdkafkaError => ex
+          if i > 3
+            raise ex
+          end
+        end
+      end
+
+      expect(delete_topic_report.error_string).to be_nil
+      expect(delete_topic_report.result_name).to eq(topic_name)
+    end
+  end
+end