rdkafka 0.4.2 → 0.8.0

data/lib/rdkafka/producer/delivery_handle.rb

@@ -10,6 +10,10 @@ module Rdkafka
 
       REGISTRY = {}
 
+      CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
+
+      private_constant :CURRENT_TIME
+
       def self.register(address, handle)
         REGISTRY[address] = handle
       end
@@ -29,25 +33,25 @@ module Rdkafka
       # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
       # In this case it is possible to call wait again.
       #
-      # @param timeout_in_seconds [Integer, nil] Number of seconds to wait before timing out. If this is nil it does not time out.
+      # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
+      # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
       #
       # @raise [RdkafkaError] When delivering the message failed
       # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
       #
       # @return [DeliveryReport]
-      def wait(timeout_in_seconds=60)
-        timeout = if timeout_in_seconds
-                    Time.now.to_i + timeout_in_seconds
+      def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+        timeout = if max_wait_timeout
+                    CURRENT_TIME.call + max_wait_timeout
                   else
                     nil
                   end
         loop do
           if pending?
-            if timeout && timeout <= Time.now.to_i
-              raise WaitTimeoutError.new("Waiting for delivery timed out after #{timeout_in_seconds} seconds")
+            if timeout && timeout <= CURRENT_TIME.call
+              raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
             end
-            sleep 0.1
-            next
+            sleep wait_timeout
           elsif self[:response] != 0
             raise RdkafkaError.new(self[:response])
           else

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,6 +1,6 @@
 module Rdkafka
   class Producer
-    # Delivery report for a succesfully produced message.
+    # Delivery report for a successfully produced message.
     class DeliveryReport
       # The partition this message was produced to.
       # @return [Integer]
@@ -10,11 +10,16 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
+      # Error in case happen during produce.
+      # @return [string]
+      attr_reader :error
+
       private
 
-      def initialize(partition, offset)
+      def initialize(partition, offset, error = nil)
         @partition = partition
         @offset = offset
+        @error = error
       end
     end
   end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.4.2"
-  LIBRDKAFKA_VERSION = "0.11.6"
-  LIBRDKAFKA_SOURCE_SHA256 = '9c0afb8b53779d968225edf1e79da48a162895ad557900f75e7978f65e642032'
+  VERSION = "0.8.0"
+  LIBRDKAFKA_VERSION = "1.4.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
 end

data/rdkafka.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.authors = ['Thijs Cadier']
   gem.email = ["thijs@appsignal.com"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
-  gem.summary = "Kafka client library wrapping librdkafka using the ffi gem and futures from concurrent-ruby for Kafka 0.10+"
+  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
   gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
 
@@ -14,12 +14,12 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.1'
+  gem.required_ruby_version = '>= 2.4'
   gem.extensions = %w(ext/Rakefile)
 
   gem.add_dependency 'ffi', '~> 1.9'
   gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '~> 12.3'
+  gem.add_dependency 'rake', '>= 12.3'
 
   gem.add_development_dependency 'pry', '~> 0.10'
   gem.add_development_dependency 'rspec', '~> 3.5'

data/spec/rdkafka/bindings_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require 'zlib'
 
 describe Rdkafka::Bindings do
   it "should load librdkafka" do
@@ -7,12 +8,12 @@ describe Rdkafka::Bindings do
 
   describe ".lib_extension" do
     it "should know the lib extension for darwin" do
-      expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
       expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
     end
 
     it "should know the lib extension for linux" do
-      expect(Gem::Platform.local).to receive(:os).and_return("linux")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
       expect(Rdkafka::Bindings.lib_extension).to eq "so"
     end
   end
@@ -60,6 +61,23 @@ describe Rdkafka::Bindings do
     end
   end
 
+  describe "partitioner" do
+    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
+    let(:partition_count) { rand(50) + 1 }
+
+    it "should return the same partition for a similar string and the same partition count" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should match the old partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = (Zlib.crc32(partition_key) % partition_count)
+      expect(result_1).to eq(result_2)
+    end
+  end
+
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/rdkafka/config_spec.rb

@@ -50,7 +50,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      expect(rdkafka_config.consumer).to be_a Rdkafka::Consumer
+      consumer = rdkafka_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
     end
 
     it "should raise an error when creating a consumer with invalid config" do
@@ -76,7 +78,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      expect(rdkafka_config.producer).to be_a Rdkafka::Producer
+      producer = rdkafka_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
     end
 
     it "should raise an error when creating a producer with invalid config" do

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,8 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
-  let(:native_topic) { new_native_topic }
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do
@@ -23,8 +24,29 @@ describe Rdkafka::Consumer::Message do
       end
     end
   end
+
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
   subject { Rdkafka::Consumer::Message.new(native_message) }
 
+  before do
+    # mock headers, because it produces 'segmentation fault' while settings or reading headers for
+    # a message which is created from scratch
+    #
+    # Code dump example:
+    #
+    # ```
+    # frame #7: 0x000000010dacf5ab librdkafka.dylib`rd_list_destroy + 11
+    # frame #8: 0x000000010dae5a7e librdkafka.dylib`rd_kafka_headers_destroy + 14
+    # frame #9: 0x000000010da9ab40 librdkafka.dylib`rd_kafka_message_set_headers + 32
+    # ```
+    expect( Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) do
+      Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+    end
+  end
+
   it "should have a topic" do
     expect(subject.topic).to eq "topic_name"
   end

data/spec/rdkafka/consumer/partition_spec.rb

@@ -2,7 +2,8 @@ require "spec_helper"
 
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
-  subject { Rdkafka::Consumer::Partition.new(1, offset) }
+  let(:err) { 0 }
+  subject { Rdkafka::Consumer::Partition.new(1, offset, err) }
 
   it "should have a partition" do
     expect(subject.partition).to eq 1
@@ -12,22 +13,34 @@ describe Rdkafka::Consumer::Partition do
     expect(subject.offset).to eq 100
   end
 
+  it "should have an err code" do
+    expect(subject.err).to eq 0
+  end
+
   describe "#to_s" do
     it "should return a human readable representation" do
-      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+      expect(subject.to_s).to eq "<Partition 1 offset=100>"
     end
   end
 
   describe "#inspect" do
     it "should return a human readable representation" do
-      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+      expect(subject.to_s).to eq "<Partition 1 offset=100>"
     end
 
     context "without offset" do
       let(:offset) { nil }
 
       it "should return a human readable representation" do
-        expect(subject.to_s).to eq "<Partition 1 without offset>"
+        expect(subject.to_s).to eq "<Partition 1>"
+      end
+    end
+
+    context "with err code" do
+      let(:err) { 1 }
+
+      it "should return a human readable representation" do
+        expect(subject.to_s).to eq "<Partition 1 offset=100 err=1>"
       end
     end
   end

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -118,7 +118,7 @@ describe Rdkafka::Consumer::TopicPartitionList do
       list = Rdkafka::Consumer::TopicPartitionList.new
       list.add_topic("topic1", [0, 1])
 
-      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0 without offset>, <Partition 1 without offset>]}>"
+      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0>, <Partition 1>]}>"
 
       expect(list.to_s).to eq expected
     end

data/spec/rdkafka/consumer_spec.rb

@@ -1,11 +1,15 @@
 require "spec_helper"
+require "ostruct"
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
   let(:consumer) { config.consumer }
   let(:producer) { config.producer }
 
-  describe "#subscripe, #unsubscribe and #subscription" do
+  after { consumer.close }
+  after { producer.close }
+
+  describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
 
@@ -47,6 +51,166 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#pause and #resume" do
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before { consumer.subscribe("consume_test_topic") }
+      after { consumer.unsubscribe }
+
+      it "should pause and then resume" do
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. send a first message
+        send_one_message
+
+        # 3. ensure that message is successfully consumed
+        records = consumer.poll(timeout)
+        expect(records).not_to be_nil
+        consumer.commit
+
+        # 4. send a second message
+        send_one_message
+
+        # 5. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", (0..2))
+        consumer.pause(tpl)
+
+        # 6. ensure that messages are not available
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+
+        # 7. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", (0..2))
+        consumer.resume(tpl)
+
+        # 8. ensure that message is successfully consumed
+        records = consumer.poll(timeout)
+        expect(records).not_to be_nil
+      end
+    end
+
+    it "should raise when not TopicPartitionList" do
+      expect { consumer.pause(true) }.to raise_error(TypeError)
+      expect { consumer.resume(true) }.to raise_error(TypeError)
+    end
+
+    it "should raise an error when pausing fails" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap { |tpl| tpl.add_topic('topic', (0..1)) }
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_pause_partitions).and_return(20)
+      expect {
+        consumer.pause(list)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaTopicPartitionListError)
+        expect(err.topic_partition_list).to be
+      end
+    end
+
+    it "should raise an error when resume fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_resume_partitions).and_return(20)
+      expect {
+        consumer.resume(Rdkafka::Consumer::TopicPartitionList.new)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    def send_one_message
+      producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1"
+      ).wait
+    end
+  end
+
+  describe "#seek" do
+    it "should raise an error when seeking fails" do
+      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
+      expect {
+        consumer.seek(fake_msg)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      it "works when a partition is paused" do
+        # 3. get reference message
+        send_one_message(:a)
+        message1 = consumer.poll(timeout)
+        expect(message1&.payload).to eq "payload a"
+
+        # 4. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.pause(tpl)
+
+        # 5. seek to previous message
+        consumer.seek(message1)
+
+        # 6. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.resume(tpl)
+
+        # 7. ensure same message is read again
+        message2 = consumer.poll(timeout)
+        consumer.commit
+        expect(message1.offset).to eq message2.offset
+        expect(message1.payload).to eq message2.payload
+      end
+
+      it "allows skipping messages" do
+        # 3. send messages
+        send_one_message(:a)
+        send_one_message(:b)
+        send_one_message(:c)
+
+        # 4. get reference message
+        message = consumer.poll(timeout)
+        expect(message&.payload).to eq "payload a"
+
+        # 5. seek over one message
+        fake_msg = message.dup
+        fake_msg.instance_variable_set(:@offset, fake_msg.offset + 2)
+        consumer.seek(fake_msg)
+
+        # 6. ensure that only one message is available
+        records = consumer.poll(timeout)
+        expect(records&.payload).to eq "payload c"
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+      end
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty
@@ -149,11 +313,11 @@ describe Rdkafka::Consumer do
       }.to raise_error TypeError
     end
 
-    context "with a commited consumer" do
+    context "with a committed consumer" do
       before :all do
-        # Make sure there are some message
-        producer = rdkafka_config.producer
+        # Make sure there are some messages.
         handles = []
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -165,6 +329,7 @@ describe Rdkafka::Consumer do
           end
         end
         handles.each(&:wait)
+        producer.close
       end
 
       before do
@@ -225,20 +390,26 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         before do
+          config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          consumer.subscribe("consume_test_topic")
-          wait_for_assignment(consumer)
+          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(@new_consumer)
+        end
+
+        after do
+          @new_consumer.close
         end
 
         it "should store the offset for a message" do
-          consumer.store_offset(message)
-          consumer.commit
+          @new_consumer.store_offset(message)
+          @new_consumer.commit
 
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
-          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          partitions = @new_consumer.committed(list).to_h["consume_test_topic"]
           expect(partitions).not_to be_nil
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
@@ -246,7 +417,7 @@ describe Rdkafka::Consumer do
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {
-            consumer.store_offset(message)
+            @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
       end
@@ -257,13 +428,13 @@ describe Rdkafka::Consumer do
     it "should return the watermark offsets" do
       # Make sure there's a message
       producer.produce(
-        topic:     "consume_test_topic",
+        topic:     "watermarks_test_topic",
         payload:   "payload 1",
         key:       "key 1",
         partition: 0
       ).wait
 
-      low, high = consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
+      low, high = consumer.query_watermark_offsets("watermarks_test_topic", 0, 5000)
       expect(low).to eq 0
       expect(high).to be > 0
     end
@@ -358,6 +529,22 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#cluster_id" do
+    it 'should return the current ClusterId' do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      expect(consumer.cluster_id).not_to be_empty
+    end
+  end
+
+  describe "#member_id" do
+    it 'should return the current MemberId' do
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      expect(consumer.member_id).to start_with('rdkafka-')
+    end
+  end
+
   describe "#poll" do
     it "should return nil if there is no subscription" do
       expect(consumer.poll(1000)).to be_nil
@@ -374,12 +561,12 @@ describe Rdkafka::Consumer do
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-
       consumer.subscribe("consume_test_topic")
-      message = consumer.poll(5000)
-      expect(message).to be_a Rdkafka::Consumer::Message
+      message = consumer.each {|m| break m}
 
-      # Message content is tested in producer spec
+      expect(message).to be_a Rdkafka::Consumer::Message
+      expect(message.payload).to eq('payload 1')
+      expect(message.key).to eq('key 1')
     end
 
     it "should raise an error when polling fails" do
@@ -395,6 +582,68 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#poll with headers" do
+    it "should return message with headers" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   { foo: 'bar' }
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers).to include(foo: 'bar')
+    end
+
+    it "should return message with no headers" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key no headers",
+        headers:   nil
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key no headers')
+      expect(message.headers).to be_empty
+    end
+
+    it "should raise an error when message headers aren't readable" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) { 1 }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key err headers",
+        headers:   nil
+      ).wait
+
+      expect {
+        wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaError)
+        expect(err.message).to start_with("Error reading message headers")
+      end
+    end
+
+    it "should raise an error when the first message header aren't readable" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_header_get_all).with(any_args) { 1 }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key err headers",
+        headers:   { foo: 'bar' }
+      ).wait
+
+      expect {
+        wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      }.to raise_error do |err|
+        expect(err).to be_instance_of(Rdkafka::RdkafkaError)
+        expect(err.message).to start_with("Error reading a message header at index 0")
+      end
+    end
+  end
+
   describe "#each" do
     it "should yield messages" do
       handles = []
@@ -417,4 +666,61 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  describe "a rebalance listener" do
+    it "should get notifications" do
+      listener = Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          collect(:assign, list)
+        end
+
+        def on_partitions_revoked(consumer, list)
+          collect(:revoke, list)
+        end
+
+        def collect(name, list)
+          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+          queue << ([name] + partitions)
+        end
+      end.new([])
+
+      notify_listener(listener)
+
+      expect(listener.queue).to eq([
+        [:assign, "consume_test_topic", 0, 1, 2],
+        [:revoke, "consume_test_topic", 0, 1, 2]
+      ])
+    end
+
+    it 'should handle callback exceptions' do
+      listener = Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          queue << :assigned
+          raise 'boom'
+        end
+
+        def on_partitions_revoked(consumer, list)
+          queue << :revoked
+          raise 'boom'
+        end
+      end.new([])
+
+      notify_listener(listener)
+
+      expect(listener.queue).to eq([:assigned, :revoked])
+    end
+
+    def notify_listener(listener)
+      # 1. subscribe and poll
+      config.consumer_rebalance_listener = listener
+      consumer.subscribe("consume_test_topic")
+      wait_for_assignment(consumer)
+      consumer.poll(100)
+
+      # 2. unsubscribe
+      consumer.unsubscribe
+      wait_for_unassignment(consumer)
+      consumer.close
+    end
+  end
 end