rdkafka 0.12.0 → 0.13.0.beta.2

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Delivery report for a successfully produced message.
@@ -10,15 +12,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # Error in case happen during produce.
+      # The name of the topic this message was produced to.
       # @return [String]
+      attr_reader :topic_name
+
+      # Error in case happen during produce.
+      # @return [Integer]
       attr_reader :error
 
       private
 
-      def initialize(partition, offset, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil)
         @partition = partition
         @offset = offset
+        @topic_name = topic_name
         @error = error
       end
     end

data/lib/rdkafka/producer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "objspace"
 
 module Rdkafka
@@ -16,12 +18,12 @@ module Rdkafka
     attr_reader :delivery_callback_arity
 
     # @private
-    def initialize(client, partitioner_name)
-      @client = client
+    def initialize(native_kafka, partitioner_name)
+      @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
-      # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, client.finalizer)
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -40,7 +42,7 @@ module Rdkafka
     def close
       ObjectSpace.undefine_finalizer(self)
 
-      @client.close
+      @native_kafka.close
     end
 
     # Partition count for a given topic.
@@ -49,10 +51,9 @@ module Rdkafka
     # @param topic [String] The topic name.
     #
     # @return partition count [Integer,nil]
-    #
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
+      Rdkafka::Metadata.new(@native_kafka.inner, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -144,7 +145,7 @@ module Rdkafka
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
-        @client.native,
+        @native_kafka.inner,
         *args
       )
 
@@ -157,7 +158,6 @@ module Rdkafka
       delivery_handle
     end
 
-    # @private
     def call_delivery_callback(delivery_report, delivery_handle)
       return unless @delivery_callback
 
@@ -171,8 +171,9 @@ module Rdkafka
       callback.method(:call).arity
     end
 
+    private
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
+      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Rdkafka
-  VERSION = "0.12.0"
-  LIBRDKAFKA_VERSION = "1.9.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
+  VERSION = "0.13.0.beta.2"
+  LIBRDKAFKA_VERSION = "1.9.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "3fba157a9f80a0889c982acdd44608be8a46142270a389008b22d921be1198ad"
 end

data/lib/rdkafka.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rdkafka/version"
 
 require "rdkafka/abstract_handle"
@@ -16,7 +18,7 @@ require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
+require "rdkafka/native_kafka"
 require "rdkafka/producer"
-require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::AbstractHandle do

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::CreateTopicHandle do

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::CreateTopicReport do

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::DeleteTopicHandle do

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::DeleteTopicReport do

data/spec/rdkafka/admin_spec.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do
-  let(:config)   { rdkafka_config }
-  let(:admin)    { config.admin }
+  let(:config) { rdkafka_config }
+  let(:admin)  { config.admin }
 
   after do
     # Registry should always end up being empty
@@ -174,7 +176,6 @@ describe Rdkafka::Admin do
       end
     end
 
-
     it "deletes a topic that was newly created" do
       create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
       create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)

data/spec/rdkafka/bindings_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require 'zlib'
 

data/spec/rdkafka/callbacks_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Callbacks do

data/spec/rdkafka/config_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Config do
@@ -148,6 +150,18 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "allows string partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      config = Rdkafka::Config.new("partitioner" => "murmur2")
+      config.producer
+    end
+
+    it "allows symbol partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      config = Rdkafka::Config.new(:partitioner => "murmur2")
+      config.producer
+    end
+
     it "should allow configuring zstd compression" do
       config = Rdkafka::Config.new('compression.codec' => 'zstd')
       begin

data/spec/rdkafka/consumer/headers_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Consumer::Headers do
+  let(:headers) do
+    { # Note String keys!
+      "version" => "2.1.3",
+      "type" => "String"
+    }
+  end
+  let(:native_message) { double('native message') }
+  let(:headers_ptr) { double('headers pointer') }
+
+  describe '.from_native' do
+    before do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(native_message, anything) do |_, headers_ptrptr|
+        expect(headers_ptrptr).to receive(:read_pointer).and_return(headers_ptr)
+        Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+      end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 2, anything, anything, anything)
+          .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
+    end
+
+    subject { described_class.from_native(native_message) }
+
+    it { is_expected.to eq(headers) }
+    it { is_expected.to be_frozen }
+
+    it 'allows String key' do
+      expect(subject['version']).to eq("2.1.3")
+    end
+
+    it 'allows Symbol key, but warns' do
+      expect(Kernel).to \
+        receive(:warn).with("rdkafka deprecation warning: header access with Symbol key :version treated as a String. " \
+                            "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+      expect(subject[:version]).to eq("2.1.3")
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::Partition do

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::TopicPartitionList do

data/spec/rdkafka/consumer_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "ostruct"
 require 'securerandom'
@@ -593,7 +595,7 @@ describe Rdkafka::Consumer do
   end
 
   describe "#poll with headers" do
-    it "should return message with headers" do
+    it "should return message with headers using string keys (when produced with symbol keys)" do
       report = producer.produce(
         topic:     "consume_test_topic",
         key:       "key headers",
@@ -603,7 +605,20 @@ describe Rdkafka::Consumer do
       message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
       expect(message).to be
       expect(message.key).to eq('key headers')
-      expect(message.headers).to include(foo: 'bar')
+      expect(message.headers).to include('foo' => 'bar')
+    end
+
+    it "should return message with headers using string keys (when produced with string keys)" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   { 'foo' => 'bar' }
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers).to include('foo' => 'bar')
     end
 
     it "should return message with no headers" do
@@ -698,7 +713,7 @@ describe Rdkafka::Consumer do
       n.times do |i|
         handles << producer.produce(
           topic:     topic_name,
-          payload:   Time.new.to_f.to_s,
+          payload:   i % 10 == 0 ? nil : Time.new.to_f.to_s,
           key:       i.to_s,
           partition: 0
         )
@@ -962,18 +977,6 @@ describe Rdkafka::Consumer do
         expect(listener.queue).to eq([:assigned, :revoked])
       end
     end
-
-    def notify_listener(listener)
-      # 1. subscribe and poll
-      consumer.subscribe("consume_test_topic")
-      wait_for_assignment(consumer)
-      consumer.poll(100)
-
-      # 2. unsubscribe
-      consumer.unsubscribe
-      wait_for_unassignment(consumer)
-      consumer.close
-    end
   end
 
   context "methods that should not be called after a consumer has been closed" do
@@ -1005,4 +1008,70 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  it "provides a finalizer that closes the native kafka client" do
+    expect(consumer.closed?).to eq(false)
+
+    consumer.finalizer.call("some-ignored-object-id")
+
+    expect(consumer.closed?).to eq(true)
+  end
+
+  context "when the rebalance protocol is cooperative" do
+    let(:consumer) do
+      config = rdkafka_consumer_config(
+        {
+          :"partition.assignment.strategy" => "cooperative-sticky",
+          :"debug" => "consumer",
+        }
+      )
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
+
+    let(:listener) do
+      Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          collect(:assign, list)
+        end
+
+        def on_partitions_revoked(consumer, list)
+          collect(:revoke, list)
+        end
+
+        def collect(name, list)
+          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+          queue << ([name] + partitions)
+        end
+      end.new([])
+    end
+
+    it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
+      notify_listener(listener) do
+        handles = []
+        10.times do
+          handles << producer.produce(
+            topic:     "consume_test_topic",
+            payload:   "payload 1",
+            key:       "key 1",
+            partition: 0
+          )
+        end
+        handles.each(&:wait)
+
+        consumer.subscribe("consume_test_topic")
+        # Check the first 10 messages. Then close the consumer, which
+        # should break the each loop.
+        consumer.each_with_index do |message, i|
+          expect(message).to be_a Rdkafka::Consumer::Message
+          break if i == 10
+        end
+      end
+
+      expect(listener.queue).to eq([
+        [:assign, "consume_test_topic", 0, 1, 2],
+        [:revoke, "consume_test_topic", 0, 1, 2]
+      ])
+    end
+  end
 end

data/spec/rdkafka/error_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::RdkafkaError do

data/spec/rdkafka/metadata_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "securerandom"
 

data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb}

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
-describe Rdkafka::Producer::Client do
+describe Rdkafka::NativeKafka do
   let(:config) { rdkafka_producer_config }
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
@@ -63,8 +65,8 @@ describe Rdkafka::Producer::Client do
     client
   end
 
-  it "exposes `native` client" do
-    expect(client.native).to eq(native)
+  it "exposes inner client" do
+    expect(client.inner).to eq(native)
   end
 
   context "when client was not yet closed (`nil`)" do
@@ -94,7 +96,7 @@ describe Rdkafka::Producer::Client do
       it "closes and unassign the native client" do
         client.close
 
-        expect(client.native).to eq(nil)
+        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
@@ -129,13 +131,13 @@ describe Rdkafka::Producer::Client do
       it "does not close and unassign the native client again" do
         client.close
 
-        expect(client.native).to eq(nil)
+        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
   end
 
-  it "provide a finalizer Proc that closes the `native` client" do
+  it "provides a finalizer that closes the native kafka client" do
     expect(client.closed?).to eq(false)
 
     client.finalizer.call("some-ignored-object-id")

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryHandle do
@@ -9,6 +11,7 @@ describe Rdkafka::Producer::DeliveryHandle do
       handle[:response] = response
       handle[:partition] = 2
       handle[:offset] = 100
+      handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
     end
   end
 
@@ -29,6 +32,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
 
       it "should wait without a timeout" do
@@ -36,6 +40,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
     end
   end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryReport do
-  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
+  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
 
   it "should get the partition" do
     expect(subject.partition).to eq 2
@@ -11,7 +13,11 @@ describe Rdkafka::Producer::DeliveryReport do
     expect(subject.offset).to eq 100
   end
 
+  it "should get the topic_name" do
+    expect(subject.topic_name).to eq "topic"
+  end
+
   it "should get the error" do
-    expect(subject.error).to eq "error"
+    expect(subject.error).to eq -1
   end
 end