rdkafka 0.12.0 → 0.15.1

data/spec/rdkafka/error_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::RdkafkaError do
   it "should raise a type error for a nil response" do

data/spec/rdkafka/metadata_spec.rb

@@ -1,4 +1,5 @@
-require "spec_helper"
+# frozen_string_literal: true
+
 require "securerandom"
 
 describe Rdkafka::Metadata do
@@ -29,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -52,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb}

@@ -1,17 +1,15 @@
-require "spec_helper"
+# frozen_string_literal: true
 
-describe Rdkafka::Producer::Client do
+describe Rdkafka::NativeKafka do
   let(:config) { rdkafka_producer_config }
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
   let(:thread) { double(Thread) }
+  let(:opaque) { Rdkafka::Opaque.new }
 
-  subject(:client) { described_class.new(native) }
+  subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
 
   before do
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
     allow(Thread).to receive(:new).and_return(thread)
 
     allow(thread).to receive(:[]=).with(:closing, anything)
@@ -19,6 +17,8 @@ describe Rdkafka::Producer::Client do
     allow(thread).to receive(:abort_on_exception=).with(anything)
   end
 
+  after { client.close }
+
   context "defaults" do
     it "sets the thread to abort on exception" do
       expect(thread).to receive(:abort_on_exception=).with(true)
@@ -39,32 +39,12 @@ describe Rdkafka::Producer::Client do
 
       client
     end
-
-    it "polls the native with default 250ms timeout" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
-      end
-    end
-
-    it "check the out queue of native client" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
-      end
-    end
-  end
-
-  def polling_loop_expects(&block)
-    Thread.current[:closing] = true # this forces the loop break with line #12
-
-    allow(Thread).to receive(:new).and_yield do |_|
-      block.call
-    end.and_return(thread)
-
-    client
   end
 
-  it "exposes `native` client" do
-    expect(client.native).to eq(native)
+  it "exposes the inner client" do
+    client.with_inner do |inner|
+      expect(inner).to eq(native)
+    end
   end
 
   context "when client was not yet closed (`nil`)" do
@@ -74,7 +54,7 @@ describe Rdkafka::Producer::Client do
 
     context "and attempt to close" do
       it "calls the `destroy` binding" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
 
         client.close
       end
@@ -94,7 +74,6 @@ describe Rdkafka::Producer::Client do
       it "closes and unassign the native client" do
         client.close
 
-        expect(client.native).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
@@ -109,7 +88,7 @@ describe Rdkafka::Producer::Client do
 
     context "and attempt to close again" do
       it "does not call the `destroy` binding" do
-        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
 
         client.close
       end
@@ -129,13 +108,12 @@ describe Rdkafka::Producer::Client do
       it "does not close and unassign the native client again" do
         client.close
 
-        expect(client.native).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
   end
 
-  it "provide a finalizer Proc that closes the `native` client" do
+  it "provides a finalizer that closes the native kafka client" do
     expect(client.closed?).to eq(false)
 
     client.finalizer.call("some-ignored-object-id")

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Producer::DeliveryHandle do
   let(:response) { 0 }
@@ -9,6 +9,7 @@ describe Rdkafka::Producer::DeliveryHandle do
       handle[:response] = response
       handle[:partition] = 2
       handle[:offset] = 100
+      handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
     end
   end
 
@@ -29,6 +30,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
 
       it "should wait without a timeout" do
@@ -36,6 +38,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
     end
   end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,7 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Producer::DeliveryReport do
-  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
+  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
 
   it "should get the partition" do
     expect(subject.partition).to eq 2
@@ -11,7 +11,15 @@ describe Rdkafka::Producer::DeliveryReport do
     expect(subject.offset).to eq 100
   end
 
+  it "should get the topic_name" do
+    expect(subject.topic_name).to eq "topic"
+  end
+
+  it "should get the same topic name under topic alias" do
+    expect(subject.topic).to eq "topic"
+  end
+
   it "should get the error" do
-    expect(subject.error).to eq "error"
+    expect(subject.error).to eq -1
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -1,4 +1,5 @@
-require "spec_helper"
+# frozen_string_literal: true
+
 require "zlib"
 
 describe Rdkafka::Producer do
@@ -7,11 +8,16 @@ describe Rdkafka::Producer do
 
   after do
     # Registry should always end up being empty
-    expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
+    registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
+    expect(registry).to be_empty, registry.inspect
     producer.close
     consumer.close
   end
 
+  describe '#name' do
+    it { expect(producer.name).to include('rdkafka#producer-') }
+  end
+
   context "delivery callback" do
     context "with a proc/lambda" do
       it "should set the callback" do
@@ -28,8 +34,10 @@ describe Rdkafka::Producer do
 
         producer.delivery_callback = lambda do |report|
           expect(report).not_to be_nil
+          expect(report.label).to eq "label"
           expect(report.partition).to eq 1
           expect(report.offset).to be >= 0
+          expect(report.topic_name).to eq "produce_test_topic"
           @callback_called = true
         end
 
@@ -37,9 +45,12 @@ describe Rdkafka::Producer do
         handle = producer.produce(
           topic:   "produce_test_topic",
           payload: "payload",
-          key:     "key"
+          key:     "key",
+          label:   "label"
         )
 
+        expect(handle.label).to eq "label"
+
         # Wait for it to be delivered
         handle.wait(max_wait_timeout: 15)
 
@@ -113,6 +124,7 @@ describe Rdkafka::Producer do
         expect(called_report.first).not_to be_nil
         expect(called_report.first.partition).to eq 1
         expect(called_report.first.offset).to be >= 0
+        expect(called_report.first.topic_name).to eq "produce_test_topic"
       end
 
       it "should provide handle" do
@@ -167,11 +179,13 @@ describe Rdkafka::Producer do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "payload",
-      key:     "key"
+      key:     "key",
+      label:   "label"
     )
 
     # Should be pending at first
     expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
 
     # Check delivery handle and report
     report = handle.wait(max_wait_timeout: 5)
@@ -179,11 +193,13 @@ describe Rdkafka::Producer do
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
 
-    # Close producer
+    # Flush and close producer
+    producer.flush
     producer.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -207,7 +223,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -251,6 +267,28 @@ describe Rdkafka::Producer do
     expect(messages[2].key).to eq key
   end
 
+  it "should produce a message with empty string without crashing" do
+    messages = [{key: 'a', partition_key: ''}]
+
+    messages = messages.map do |m|
+      handle = producer.produce(
+        topic:     "partitioner_test_topic",
+        payload:   "payload partition",
+        key:       m[:key],
+        partition_key: m[:partition_key]
+      )
+      report = handle.wait(max_wait_timeout: 5)
+
+      wait_for_message(
+        topic: "partitioner_test_topic",
+        delivery_report: report,
+      )
+    end
+
+    expect(messages[0].partition).to eq 0
+    expect(messages[0].key).to eq 'a'
+  end
+
   it "should produce a message with utf-8 encoding" do
     handle = producer.produce(
       topic:   "produce_test_topic",
@@ -259,7 +297,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -292,7 +330,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -313,7 +351,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -333,7 +371,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -351,7 +389,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -371,7 +409,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -380,9 +418,9 @@ describe Rdkafka::Producer do
 
     expect(message.payload).to eq "payload headers"
     expect(message.key).to eq "key headers"
-    expect(message.headers[:foo]).to eq "bar"
-    expect(message.headers[:baz]).to eq "foobar"
-    expect(message.headers[:foobar]).to be_nil
+    expect(message.headers["foo"]).to eq "bar"
+    expect(message.headers["baz"]).to eq "foobar"
+    expect(message.headers["foobar"]).to be_nil
   end
 
   it "should produce a message with empty headers" do
@@ -394,7 +432,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -432,10 +470,10 @@ describe Rdkafka::Producer do
     # wait for and check the message in the main process.
     reader, writer = IO.pipe
 
-    fork do
+    pid = fork do
       reader.close
 
-      # Avoids sharing the socket between processes.
+      # Avoid sharing the client between processes.
       producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
@@ -448,24 +486,28 @@ describe Rdkafka::Producer do
 
       report_json = JSON.generate(
         "partition" => report.partition,
-        "offset" => report.offset
+        "offset" => report.offset,
+        "topic_name" => report.topic_name
       )
 
       writer.write(report_json)
       writer.close
+      producer.flush
       producer.close
     end
+    Process.wait(pid)
 
     writer.close
     report_hash = JSON.parse(reader.read)
     report = Rdkafka::Producer::DeliveryReport.new(
       report_hash["partition"],
-      report_hash["offset"]
+      report_hash["offset"],
+      report_hash["topic_name"]
     )
 
     reader.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -522,4 +564,174 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  context "when not being able to deliver the message" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "localhost:9093",
+        "message.timeout.ms": 100
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil, label: 'na')
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.label).to eq('na')
+    end
+  end
+
+  describe '#partition_count' do
+    it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
+
+    context 'when the partition count value is already cached' do
+      before do
+        producer.partition_count('consume_test_topic')
+        allow(::Rdkafka::Metadata).to receive(:new).and_call_original
+      end
+
+      it 'expect not to query it again' do
+        producer.partition_count('consume_test_topic')
+        expect(::Rdkafka::Metadata).not_to have_received(:new)
+      end
+    end
+
+    context 'when the partition count value was cached but time expired' do
+      before do
+        allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
+        producer.partition_count('consume_test_topic')
+        allow(::Rdkafka::Metadata).to receive(:new).and_call_original
+      end
+
+      it 'expect not to query it again' do
+        producer.partition_count('consume_test_topic')
+        expect(::Rdkafka::Metadata).to have_received(:new)
+      end
+    end
+
+    context 'when the partition count value was cached and time did not expire' do
+      before do
+        allow(::Process).to receive(:clock_gettime).and_return(0, 29.001)
+        producer.partition_count('consume_test_topic')
+        allow(::Rdkafka::Metadata).to receive(:new).and_call_original
+      end
+
+      it 'expect not to query it again' do
+        producer.partition_count('consume_test_topic')
+        expect(::Rdkafka::Metadata).not_to have_received(:new)
+      end
+    end
+  end
+
+  describe '#flush' do
+    it "should return flush when it can flush all outstanding messages or when no messages" do
+      producer.produce(
+        topic:     "produce_test_topic",
+        payload:   "payload headers",
+        key:       "key headers",
+        headers:   {}
+      )
+
+      expect(producer.flush(5_000)).to eq(true)
+    end
+
+    context 'when it cannot flush due to a timeout' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      after do
+        # Allow rdkafka to evict message preventing memory-leak
+        sleep(2)
+      end
+
+      it "should return false on flush when cannot deliver and beyond timeout" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers",
+          key:       "key headers",
+          headers:   {}
+        )
+
+        expect(producer.flush(1_000)).to eq(false)
+      end
+    end
+
+    context 'when there is a different error' do
+      before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
+
+      it 'should raise it' do
+        expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
+      end
+    end
+  end
+
+  describe '#purge' do
+    context 'when no outgoing messages' do
+      it { expect(producer.purge).to eq(true) }
+    end
+
+    context 'when librdkafka purge returns an error' do
+      before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
+
+      it 'expect to raise an error' do
+        expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
+      end
+    end
+
+    context 'when there are outgoing things in the queue' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      it "should should purge and move forward" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+        expect(producer.flush(1_000)).to eq(true)
+      end
+
+      it "should materialize the delivery handles" do
+        handle = producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+
+        expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
+      end
+
+      context "when using delivery_callback" do
+        let(:delivery_reports) { [] }
+
+        let(:delivery_callback) do
+          ->(delivery_report) { delivery_reports << delivery_report }
+        end
+
+        before { producer.delivery_callback = delivery_callback }
+
+        it "should run the callback" do
+          handle = producer.produce(
+            topic:     "produce_test_topic",
+            payload:   "payload headers"
+          )
+
+          expect(producer.purge).to eq(true)
+          # queue purge
+          expect(delivery_reports[0].error).to eq(-152)
+        end
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 unless ENV["CI"] == "true"
   require "simplecov"
   SimpleCov.start do
@@ -9,6 +11,7 @@ require "pry"
 require "rspec"
 require "rdkafka"
 require "timeout"
+require "securerandom"
 
 def rdkafka_base_config
   {
@@ -33,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
   # Add consumer specific fields to it
   config[:"auto.offset.reset"] = "earliest"
   config[:"enable.partition.eof"] = false
-  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
   # Enable debug mode if required
   if ENV["DEBUG_CONSUMER"]
     config[:debug] = "cgrp,topic,fetch"
@@ -71,7 +74,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
 end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
-  new_consumer = !!consumer
+  new_consumer = consumer.nil?
   consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
@@ -104,6 +107,20 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def notify_listener(listener, &block)
+  # 1. subscribe and poll
+  consumer.subscribe("consume_test_topic")
+  wait_for_assignment(consumer)
+  consumer.poll(100)
+
+  block.call if block
+
+  # 2. unsubscribe
+  consumer.unsubscribe
+  wait_for_unassignment(consumer)
+  consumer.close
+end
+
 RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true
@@ -118,6 +135,7 @@ RSpec.configure do |config|
         rake_test_topic:         3,
         watermarks_test_topic:   3,
         partitioner_test_topic: 25,
+        example_topic:           1
     }.each do |topic, partitions|
       create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
       begin