rdkafka 0.12.0 → 0.15.1

data/spec/rdkafka/consumer/headers_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Consumer::Headers do
+  let(:headers) do
+    { # Note String keys!
+      "version" => "2.1.3",
+      "type" => "String"
+    }
+  end
+  let(:native_message) { double('native message') }
+  let(:headers_ptr) { double('headers pointer') }
+
+  describe '.from_native' do
+    before do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(native_message, anything) do |_, headers_ptrptr|
+        expect(headers_ptrptr).to receive(:read_pointer).and_return(headers_ptr)
+        Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+      end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 2, anything, anything, anything)
+          .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
+    end
+
+    subject { described_class.from_native(native_message) }
+
+    it { is_expected.to eq(headers) }
+    it { is_expected.to be_frozen }
+
+    it 'allows String key' do
+      expect(subject['version']).to eq("2.1.3")
+    end
+
+    it 'allows Symbol key, but warns' do
+      expect(Kernel).to \
+        receive(:warn).with("rdkafka deprecation warning: header access with Symbol key :version treated as a String. " \
+                            "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+      expect(subject[:version]).to eq("2.1.3")
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Consumer::Message do
   let(:native_client) { new_native_client }

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Consumer::TopicPartitionList do
   it "should create a new list and add unassigned topics" do
@@ -219,5 +219,24 @@ describe Rdkafka::Consumer::TopicPartitionList do
 
       expect(list).to eq other
     end
+
+    it "should create a native list with timetamp offsets if offsets are Time" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => Time.at(1505069646, 250_000))
+      end
+
+      tpl = list.to_native_tpl
+
+      compare_list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets(
+          "topic",
+          0 => (Time.at(1505069646, 250_000).to_f * 1000).floor
+        )
+      end
+
+      native_list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(native_list).to eq compare_list
+    end
   end
 end

data/spec/rdkafka/consumer_spec.rb

@@ -1,4 +1,5 @@
-require "spec_helper"
+# frozen_string_literal: true
+
 require "ostruct"
 require 'securerandom'
 
@@ -9,6 +10,10 @@ describe Rdkafka::Consumer do
   after { consumer.close }
   after { producer.close }
 
+  describe '#name' do
+    it { expect(consumer.name).to include('rdkafka#consumer-') }
+  end
+
   describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
@@ -49,11 +54,35 @@ describe Rdkafka::Consumer do
         consumer.subscription
       }.to raise_error(Rdkafka::RdkafkaError)
     end
+
+    context "when using consumer without the poll set" do
+      let(:consumer) do
+        config = rdkafka_consumer_config
+        config.consumer_poll_set = false
+        config.consumer
+      end
+
+      it "should subscribe, unsubscribe and return the subscription" do
+        expect(consumer.subscription).to be_empty
+
+        consumer.subscribe("consume_test_topic")
+
+        expect(consumer.subscription).not_to be_empty
+        expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic("consume_test_topic")
+        end
+        expect(consumer.subscription).to eq expected_subscription
+
+        consumer.unsubscribe
+
+        expect(consumer.subscription).to be_empty
+      end
+    end
   end
 
   describe "#pause and #resume" do
     context "subscription" do
-      let(:timeout) { 1000 }
+      let(:timeout) { 2000 }
 
       before { consumer.subscribe("consume_test_topic") }
       after { consumer.unsubscribe }
@@ -268,6 +297,28 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe '#assignment_lost?' do
+    it "should not return true as we do have an assignment" do
+      consumer.subscribe("consume_test_topic")
+      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("consume_test_topic")
+      end
+
+      expect(consumer.assignment_lost?).to eq false
+      consumer.unsubscribe
+    end
+
+    it "should not return true after voluntary unsubscribing" do
+      consumer.subscribe("consume_test_topic")
+      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("consume_test_topic")
+      end
+
+      consumer.unsubscribe
+      expect(consumer.assignment_lost?).to eq false
+    end
+  end
+
   describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
@@ -284,10 +335,34 @@ describe Rdkafka::Consumer do
         consumer.poll(100)
       }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
+
+    context 'when there are outgoing operations in other threads' do
+      it 'should wait and not crash' do
+        times = []
+
+        # Run a long running poll
+        thread = Thread.new do
+          times << Time.now
+          consumer.subscribe("empty_test_topic")
+          times << Time.now
+          consumer.poll(1_000)
+          times << Time.now
+        end
+
+        # Make sure it starts before we close
+        sleep(0.1)
+        consumer.close
+        close_time = Time.now
+        thread.join
+
+        times.each { |op_time| expect(op_time).to be < close_time }
+      end
+    end
   end
 
-  describe "#commit, #committed and #store_offset" do
-    # Make sure there's a stored offset
+
+  describe "#position, #commit, #committed and #store_offset" do
+    # Make sure there are messages to work with
     let!(:report) do
       producer.produce(
         topic:     "consume_test_topic",
@@ -305,29 +380,33 @@ describe Rdkafka::Consumer do
       )
     end
 
-    it "should only accept a topic partition list in committed" do
-      expect {
-        consumer.committed("list")
-      }.to raise_error TypeError
+    describe "#position" do
+      it "should only accept a topic partition list in position if not nil" do
+        expect {
+          consumer.position("list")
+        }.to raise_error TypeError
+      end
     end
 
-    it "should commit in sync mode" do
-      expect {
-        consumer.commit(nil, true)
-      }.not_to raise_error
-    end
+    describe "#committed" do
+      it "should only accept a topic partition list in commit if not nil" do
+        expect {
+          consumer.commit("list")
+        }.to raise_error TypeError
+      end
 
-    it "should only accept a topic partition list in commit if not nil" do
-      expect {
-        consumer.commit("list")
-      }.to raise_error TypeError
+      it "should commit in sync mode" do
+        expect {
+          consumer.commit(nil, true)
+        }.not_to raise_error
+      end
     end
 
     context "with a committed consumer" do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_producer_config.producer
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -371,34 +450,38 @@ describe Rdkafka::Consumer do
         }.to raise_error(Rdkafka::RdkafkaError)
       end
 
-      it "should fetch the committed offsets for the current assignment" do
-        partitions = consumer.committed.to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-      end
+      describe "#committed" do
+        it "should fetch the committed offsets for the current assignment" do
+          partitions = consumer.committed.to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+        end
 
-      it "should fetch the committed offsets for a specified topic partition list" do
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should fetch the committed offsets for a specified topic partition list" do
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+          expect(partitions[1].offset).to eq 1
+          expect(partitions[2].offset).to eq 1
         end
-        partitions = consumer.committed(list).to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-        expect(partitions[1].offset).to eq 1
-        expect(partitions[2].offset).to eq 1
-      end
 
-      it "should raise an error when getting committed fails" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should raise an error when getting committed fails" do
+          expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          expect {
+            consumer.committed(list)
+          }.to raise_error Rdkafka::RdkafkaError
         end
-        expect {
-          consumer.committed(list)
-        }.to raise_error Rdkafka::RdkafkaError
       end
 
       describe "#store_offset" do
+        let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+
         before do
           config = {}
           config[:'enable.auto.offset.store'] = false
@@ -416,6 +499,8 @@ describe Rdkafka::Consumer do
           @new_consumer.store_offset(message)
           @new_consumer.commit
 
+          # TODO use position here, should be at offset
+
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
@@ -430,6 +515,43 @@ describe Rdkafka::Consumer do
             @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
+
+        describe "#position" do
+          it "should fetch the positions for the current assignment" do
+            consumer.store_offset(message)
+
+            partitions = consumer.position.to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should fetch the positions for a specified assignment" do
+            consumer.store_offset(message)
+
+            list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+              list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
+            end
+            partitions = consumer.position(list).to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should raise an error when getting the position fails" do
+            expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
+
+            expect {
+              consumer.position
+            }.to raise_error(Rdkafka::RdkafkaError)
+          end
+        end
+
+        context "when trying to use with enable.auto.offset.store set to true" do
+          let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': true).consumer }
+
+          it "expect to raise invalid configuration error" do
+            expect { consumer.store_offset(message) }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
+          end
+        end
       end
     end
   end
@@ -593,7 +715,7 @@ describe Rdkafka::Consumer do
   end
 
   describe "#poll with headers" do
-    it "should return message with headers" do
+    it "should return message with headers using string keys (when produced with symbol keys)" do
       report = producer.produce(
         topic:     "consume_test_topic",
         key:       "key headers",
@@ -603,7 +725,20 @@ describe Rdkafka::Consumer do
       message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
       expect(message).to be
       expect(message.key).to eq('key headers')
-      expect(message.headers).to include(foo: 'bar')
+      expect(message.headers).to include('foo' => 'bar')
+    end
+
+    it "should return message with headers using string keys (when produced with string keys)" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   { 'foo' => 'bar' }
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers).to include('foo' => 'bar')
     end
 
     it "should return message with no headers" do
@@ -698,7 +833,7 @@ describe Rdkafka::Consumer do
       n.times do |i|
         handles << producer.produce(
           topic:     topic_name,
-          payload:   Time.new.to_f.to_s,
+          payload:   i % 10 == 0 ? nil : Time.new.to_f.to_s,
           key:       i.to_s,
           partition: 0
         )
@@ -723,7 +858,8 @@ describe Rdkafka::Consumer do
       #
       # This is, in effect, an integration test and the subsequent specs are
       # unit tests.
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       produce_n 42
@@ -736,6 +872,7 @@ describe Rdkafka::Consumer do
       expect(all_yields.flatten.size).to eq 42
       expect(all_yields.size).to be > 4
       expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+      admin.close
     end
 
     it "should batch poll results and yield arrays of messages" do
@@ -778,13 +915,15 @@ describe Rdkafka::Consumer do
     end
 
     it "should yield [] if nothing is received before the timeout" do
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       consumer.each_batch do |batch|
         expect(batch).to eq([])
         break
       end
+      admin.close
     end
 
     it "should yield batchs of max_items in size if messages are already fetched" do
@@ -861,6 +1000,7 @@ describe Rdkafka::Consumer do
         expect(batches_yielded.first.size).to eq 2
         expect(exceptions_yielded.flatten.size).to eq 1
         expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+        consumer.close
       end
     end
 
@@ -902,10 +1042,107 @@ describe Rdkafka::Consumer do
         expect(each_batch_iterations).to eq 0
         expect(batches_yielded.size).to eq 0
         expect(exceptions_yielded.size).to eq 0
+        consumer.close
+      end
+    end
+  end
+
+  describe "#offsets_for_times" do
+    it "should raise when not TopicPartitionList" do
+      expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
+    end
+
+    it "should raise an error when offsets_for_times fails" do
+      tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
+
+      expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
+    context "when subscribed" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 0",
+          partition: 0
+        ).wait
+      end
+
+      it "returns a TopicParticionList with updated offsets" do
+        send_one_message("a")
+        send_one_message("b")
+        send_one_message("c")
+
+        consumer.poll(timeout)
+        message = consumer.poll(timeout)
+        consumer.poll(timeout)
+
+        tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets(
+            "consume_test_topic",
+            [
+              [0, message.timestamp]
+            ]
+          )
+        end
+
+        tpl_response = consumer.offsets_for_times(tpl)
+
+        expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
       end
     end
   end
 
+  # Only relevant in case of a consumer with separate queues
+  describe '#events_poll' do
+    let(:stats) { [] }
+
+    before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
+
+    after { Rdkafka::Config.statistics_callback = nil }
+
+    let(:consumer) do
+      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config.consumer_poll_set = false
+      config.consumer
+    end
+
+    it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
+      consumer.subscribe("consume_test_topic")
+      consumer.poll(1_000)
+      expect(stats).to be_empty
+      consumer.events_poll(-1)
+      expect(stats).not_to be_empty
+    end
+  end
+
+  describe '#consumer_group_metadata_pointer' do
+    let(:pointer) { consumer.consumer_group_metadata_pointer }
+
+    after { Rdkafka::Bindings.rd_kafka_consumer_group_metadata_destroy(pointer) }
+
+    it 'expect to return a pointer' do
+      expect(pointer).to be_a(FFI::Pointer)
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config
@@ -916,11 +1153,11 @@ describe Rdkafka::Consumer do
     context "with a working listener" do
       let(:listener) do
         Struct.new(:queue) do
-          def on_partitions_assigned(consumer, list)
+          def on_partitions_assigned(list)
             collect(:assign, list)
           end
 
-          def on_partitions_revoked(consumer, list)
+          def on_partitions_revoked(list)
             collect(:revoke, list)
           end
 
@@ -944,12 +1181,12 @@ describe Rdkafka::Consumer do
     context "with a broken listener" do
       let(:listener) do
         Struct.new(:queue) do
-          def on_partitions_assigned(consumer, list)
+          def on_partitions_assigned(list)
             queue << :assigned
             raise 'boom'
           end
 
-          def on_partitions_revoked(consumer, list)
+          def on_partitions_revoked(list)
             queue << :revoked
             raise 'boom'
           end
@@ -962,18 +1199,6 @@ describe Rdkafka::Consumer do
         expect(listener.queue).to eq([:assigned, :revoked])
       end
     end
-
-    def notify_listener(listener)
-      # 1. subscribe and poll
-      consumer.subscribe("consume_test_topic")
-      wait_for_assignment(consumer)
-      consumer.poll(100)
-
-      # 2. unsubscribe
-      consumer.unsubscribe
-      wait_for_unassignment(consumer)
-      consumer.close
-    end
   end
 
   context "methods that should not be called after a consumer has been closed" do
@@ -992,7 +1217,7 @@ describe Rdkafka::Consumer do
         :assign                  => [ nil ],
         :assignment              => nil,
         :committed               => [],
-        :query_watermark_offsets => [ nil, nil ],
+        :query_watermark_offsets => [ nil, nil ]
     }.each do |method, args|
       it "raises an exception if #{method} is called" do
         expect {
@@ -1005,4 +1230,70 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  it "provides a finalizer that closes the native kafka client" do
+    expect(consumer.closed?).to eq(false)
+
+    consumer.finalizer.call("some-ignored-object-id")
+
+    expect(consumer.closed?).to eq(true)
+  end
+
+  context "when the rebalance protocol is cooperative" do
+    let(:consumer) do
+      config = rdkafka_consumer_config(
+        {
+          :"partition.assignment.strategy" => "cooperative-sticky",
+          :"debug" => "consumer",
+        }
+      )
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
+
+    let(:listener) do
+      Struct.new(:queue) do
+        def on_partitions_assigned(list)
+          collect(:assign, list)
+        end
+
+        def on_partitions_revoked(list)
+          collect(:revoke, list)
+        end
+
+        def collect(name, list)
+          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+          queue << ([name] + partitions)
+        end
+      end.new([])
+    end
+
+    it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
+      notify_listener(listener) do
+        handles = []
+        10.times do
+          handles << producer.produce(
+            topic:     "consume_test_topic",
+            payload:   "payload 1",
+            key:       "key 1",
+            partition: 0
+          )
+        end
+        handles.each(&:wait)
+
+        consumer.subscribe("consume_test_topic")
+        # Check the first 10 messages. Then close the consumer, which
+        # should break the each loop.
+        consumer.each_with_index do |message, i|
+          expect(message).to be_a Rdkafka::Consumer::Message
+          break if i == 10
+        end
+      end
+
+      expect(listener.queue).to eq([
+        [:assign, "consume_test_topic", 0, 1, 2],
+        [:revoke, "consume_test_topic", 0, 1, 2]
+      ])
+    end
+  end
 end