rdkafka 0.10.0 → 0.12.0

data/spec/rdkafka/consumer_spec.rb

@@ -3,9 +3,8 @@ require "ostruct"
 require 'securerandom'
 
 describe Rdkafka::Consumer do
-  let(:config) { rdkafka_config }
-  let(:consumer) { config.consumer }
-  let(:producer) { config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
 
   after { consumer.close }
   after { producer.close }
@@ -242,7 +241,7 @@ describe Rdkafka::Consumer do
 
     it "should return the assignment when subscribed" do
       # Make sure there's a message
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -273,7 +272,7 @@ describe Rdkafka::Consumer do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
       100.times do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           payload:   "payload #{i}",
           key:       "key #{i}",
@@ -290,7 +289,7 @@ describe Rdkafka::Consumer do
   describe "#commit, #committed and #store_offset" do
     # Make sure there's a stored offset
     let!(:report) do
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -328,7 +327,7 @@ describe Rdkafka::Consumer do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_config.producer
+        producer = rdkafka_producer_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -404,7 +403,7 @@ describe Rdkafka::Consumer do
           config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -459,13 +458,13 @@ describe Rdkafka::Consumer do
   end
 
   describe "#lag" do
-    let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
 
     it "should calculate the consumer lag" do
       # Make sure there's a message in every partition and
       # wait for the message to make sure everything is committed.
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -508,7 +507,7 @@ describe Rdkafka::Consumer do
 
       # Produce message on every topic again
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -824,11 +823,14 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is true" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0
@@ -864,11 +866,14 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is false" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0
@@ -902,51 +907,64 @@ describe Rdkafka::Consumer do
   end
 
   describe "a rebalance listener" do
-    it "should get notifications" do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          collect(:assign, list)
-        end
+    let(:consumer) do
+      config = rdkafka_consumer_config
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
 
-        def on_partitions_revoked(consumer, list)
-          collect(:revoke, list)
-        end
+    context "with a working listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            collect(:assign, list)
+          end
 
-        def collect(name, list)
-          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
-          queue << ([name] + partitions)
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            collect(:revoke, list)
+          end
 
-      notify_listener(listener)
+          def collect(name, list)
+            partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+            queue << ([name] + partitions)
+          end
+        end.new([])
+      end
 
-      expect(listener.queue).to eq([
-        [:assign, "consume_test_topic", 0, 1, 2],
-        [:revoke, "consume_test_topic", 0, 1, 2]
-      ])
+      it "should get notifications" do
+        notify_listener(listener)
+
+        expect(listener.queue).to eq([
+          [:assign, "consume_test_topic", 0, 1, 2],
+          [:revoke, "consume_test_topic", 0, 1, 2]
+        ])
+      end
     end
 
-    it 'should handle callback exceptions' do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          queue << :assigned
-          raise 'boom'
-        end
+    context "with a broken listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            queue << :assigned
+            raise 'boom'
+          end
 
-        def on_partitions_revoked(consumer, list)
-          queue << :revoked
-          raise 'boom'
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            queue << :revoked
+            raise 'boom'
+          end
+        end.new([])
+      end
 
-      notify_listener(listener)
+      it 'should handle callback exceptions' do
+        notify_listener(listener)
 
-      expect(listener.queue).to eq([:assigned, :revoked])
+        expect(listener.queue).to eq([:assigned, :revoked])
+      end
     end
 
     def notify_listener(listener)
       # 1. subscribe and poll
-      config.consumer_rebalance_listener = listener
       consumer.subscribe("consume_test_topic")
       wait_for_assignment(consumer)
       consumer.poll(100)

data/spec/rdkafka/metadata_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do
-  let(:config)        { rdkafka_config }
+  let(:config)        { rdkafka_consumer_config }
   let(:native_config) { config.send(:native_config) }
   let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
 

data/spec/rdkafka/producer/client_spec.rb

@@ -0,0 +1,145 @@
+require "spec_helper"
+
+describe Rdkafka::Producer::Client do
+  let(:config) { rdkafka_producer_config }
+  let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
+  let(:closing) { false }
+  let(:thread) { double(Thread) }
+
+  subject(:client) { described_class.new(native) }
+
+  before do
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
+    allow(Thread).to receive(:new).and_return(thread)
+
+    allow(thread).to receive(:[]=).with(:closing, anything)
+    allow(thread).to receive(:join)
+    allow(thread).to receive(:abort_on_exception=).with(anything)
+  end
+
+  context "defaults" do
+    it "sets the thread to abort on exception" do
+      expect(thread).to receive(:abort_on_exception=).with(true)
+
+      client
+    end
+
+    it "sets the thread `closing` flag to false" do
+      expect(thread).to receive(:[]=).with(:closing, false)
+
+      client
+    end
+  end
+
+  context "the polling thread" do
+    it "is created" do
+      expect(Thread).to receive(:new)
+
+      client
+    end
+
+    it "polls the native with default 250ms timeout" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
+      end
+    end
+
+    it "check the out queue of native client" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
+      end
+    end
+  end
+
+  def polling_loop_expects(&block)
+    Thread.current[:closing] = true # this forces the loop break with line #12
+
+    allow(Thread).to receive(:new).and_yield do |_|
+      block.call
+    end.and_return(thread)
+
+    client
+  end
+
+  it "exposes `native` client" do
+    expect(client.native).to eq(native)
+  end
+
+  context "when client was not yet closed (`nil`)" do
+    it "is not closed" do
+      expect(client.closed?).to eq(false)
+    end
+
+    context "and attempt to close" do
+      it "calls the `destroy` binding" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+
+        client.close
+      end
+
+      it "indicates to the polling thread that it is closing" do
+        expect(thread).to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "joins the polling thread" do
+        expect(thread).to receive(:join)
+
+        client.close
+      end
+
+      it "closes and unassign the native client" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  context "when client was already closed" do
+    before { client.close }
+
+    it "is closed" do
+      expect(client.closed?).to eq(true)
+    end
+
+    context "and attempt to close again" do
+      it "does not call the `destroy` binding" do
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+
+        client.close
+      end
+
+      it "does not indicate to the polling thread that it is closing" do
+        expect(thread).not_to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "does not join the polling thread" do
+        expect(thread).not_to receive(:join)
+
+        client.close
+      end
+
+      it "does not close and unassign the native client again" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  it "provide a finalizer Proc that closes the `native` client" do
+    expect(client.closed?).to eq(false)
+
+    client.finalizer.call("some-ignored-object-id")
+
+    expect(client.closed?).to eq(true)
+  end
+end

data/spec/rdkafka/producer_spec.rb

@@ -1,8 +1,9 @@
 require "spec_helper"
+require "zlib"
 
 describe Rdkafka::Producer do
-  let(:producer) { rdkafka_config.producer }
-  let(:consumer) { rdkafka_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
 
   after do
     # Registry should always end up being empty
@@ -48,6 +49,27 @@ describe Rdkafka::Producer do
         # Callback should have been called
         expect(@callback_called).to be true
       end
+
+      it "should provide handle" do
+        @callback_handle = nil
+
+        producer.delivery_callback = lambda { |_, handle| @callback_handle = handle }
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        expect(handle).to be @callback_handle
+      end
     end
 
     context "with a callable object" do
@@ -92,6 +114,36 @@ describe Rdkafka::Producer do
         expect(called_report.first.partition).to eq 1
         expect(called_report.first.offset).to be >= 0
       end
+
+      it "should provide handle" do
+        callback_handles = []
+        callback = Class.new do
+          def initialize(callback_handles)
+            @callback_handles = callback_handles
+          end
+
+          def call(_, handle)
+            @callback_handles << handle
+          end
+        end
+        producer.delivery_callback = callback.new(callback_handles)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(handle).to be callback_handles.first
+      end
     end
 
     it "should not accept a callback that's not callable" do
@@ -375,20 +427,16 @@ describe Rdkafka::Producer do
     end
   end
 
-  it "should produce a message in a forked process" do
+  it "should produce a message in a forked process", skip: defined?(JRUBY_VERSION) && "Kernel#fork is not available" do
     # Fork, produce a message, send the report over a pipe and
     # wait for and check the message in the main process.
-
-    # Kernel#fork is not available in JRuby
-    skip if defined?(JRUBY_VERSION)
-
     reader, writer = IO.pipe
 
     fork do
       reader.close
 
       # Avoids sharing the socket between processes.
-      producer = rdkafka_config.producer
+      producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
         topic:   "produce_test_topic",

data/spec/spec_helper.rb

@@ -8,27 +8,57 @@ end
 require "pry"
 require "rspec"
 require "rdkafka"
+require "timeout"
 
-def rdkafka_config(config_overrides={})
-  config = {
+def rdkafka_base_config
+  {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
-    :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
-    :"auto.offset.reset" => "earliest",
-    :"enable.partition.eof" => false
   }
+end
+
+def rdkafka_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_consumer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Add consumer specific fields to it
+  config[:"auto.offset.reset"] = "earliest"
+  config[:"enable.partition.eof"] = false
+  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  # Enable debug mode if required
+  if ENV["DEBUG_CONSUMER"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_producer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Enable debug mode if required
   if ENV["DEBUG_PRODUCER"]
     config[:debug] = "broker,topic,msg"
-  elsif ENV["DEBUG_CONSUMER"]
-    config[:debug] = "cgrp,topic,fetch"
   end
+  # Merge overrides
   config.merge!(config_overrides)
+  # Return it
   Rdkafka::Config.new(config)
 end
 
 def new_native_client
-  config = rdkafka_config
+  config = rdkafka_consumer_config
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
@@ -42,7 +72,7 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = !!consumer
-  consumer ||= rdkafka_config.consumer
+  consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -75,6 +105,9 @@ def wait_for_unassignment(consumer)
 end
 
 RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {
@@ -95,4 +128,12 @@ RSpec.configure do |config|
     end
     admin.close
   end
+
+  config.around(:each) do |example|
+    # Timeout specs after a minute. If they take longer
+    # they are probably stuck
+    Timeout::timeout(60) do
+      example.run
+    end
+  end
 end