rdkafka 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55566df04f391a8ecbd2a51e7c2e1d91dc31397f90fcb2777b26d90b4774a290
-  data.tar.gz: 93407bbc327703e231bc9da7e2ebb74ca52089f14a50c6491b20d98e98ec7b07
+  metadata.gz: 821523c304fc7a1fbb2c7be2b58d98d56600b645b89fdb4093f976418650035d
+  data.tar.gz: '039b8e345fd8be5f295a293d64466071dbefd77d81b01460abb0fcf343a6bed3'
 SHA512:
-  metadata.gz: 1ad2f39610e53f27b9e192e36d5420e15b7d2807ca628cd80a48ba5a9e60ab6c2601a2c10303767fc048f5be812656e40b8fa6decbdf2fd249cf56854dd28dae
-  data.tar.gz: d27ef3a56b024f2c120eb0b348111e4d4f0fac19e5b818bfaa12503b7af033d1fa1f72543b3f306b07d91ced1de9aea487fff48df482cb4e75cd207f075f8c7a
+  metadata.gz: 2c7ac2199a63aacd3b1420890981ed5d953ae5cdadb874886cc4e396fa1fd8f69333633319beef35a05a002d75d22335a526a126e518cc3fbbb877a1c11ef2f7
+  data.tar.gz: 5d23c6beec3759877013b040018111453e05c41238014b07a27c1a9d8b96e8af3bc037aacd1ebe89f856435cf0afb8e34a9f89443f87cf1a3682736efb79b4bd

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.12.0
+* Bumps librdkafka to 1.9.0
+
 # 0.11.0
 * Upgrade librdkafka to 1.8.2
 * Bump supported minimum Ruby version to 2.6

data/lib/rdkafka/admin.rb

@@ -90,7 +90,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
       ensure
@@ -140,7 +140,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise
       ensure

data/lib/rdkafka/bindings.rb

@@ -246,14 +246,21 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
     # Partitioner
-    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+    PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
+      method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
+      attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+      hsh[name] = method_name
+    end
 
-    def self.partitioner(str, partition_count)
+    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
       str_ptr = FFI::MemoryPointer.from_string(str)
-      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+      method_name = PARTITIONERS.fetch(partitioner_name) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      end
+      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics

data/lib/rdkafka/callbacks.rb

@@ -97,7 +97,7 @@ module Rdkafka
           delivery_handle[:pending] = false
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]), delivery_handle)
           end
         end
       end

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -278,8 +278,8 @@ module Rdkafka
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
 
-    def call_delivery_callback(delivery_handle)
-      producer.call_delivery_callback(delivery_handle) if producer
+    def call_delivery_callback(delivery_report, delivery_handle)
+      producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
     def call_on_partitions_assigned(consumer, list)

data/lib/rdkafka/producer/client.rb

@@ -0,0 +1,47 @@
+module Rdkafka
+  class Producer
+    class Client
+      def initialize(native)
+        @native = native
+
+        # Start thread to poll client for delivery callbacks
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(native, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
+              break
+            end
+          end
+        end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      def native
+        @native
+      end
+
+      def finalizer
+        ->(_) { close }
+      end
+
+      def closed?
+        @native.nil?
+      end
+
+      def close(object_id=nil)
+        return unless @native
+
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+
+        Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+        @native = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -1,4 +1,4 @@
-require "securerandom"
+require "objspace"
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
@@ -10,29 +10,22 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(native_kafka)
-      @id = SecureRandom.uuid
-      @closing = false
-      @native_kafka = native_kafka
+    # Returns the number of arguments accepted by the callback, by default this is nil.
+    #
+    # @return [Integer, nil]
+    attr_reader :delivery_callback_arity
+
+    # @private
+    def initialize(client, partitioner_name)
+      @client = client
+      @partitioner_name = partitioner_name || "consistent_random"
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(@id, proc { close })
-
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+      ObjectSpace.define_finalizer(self, client.finalizer)
     end
 
     # Set a callback that will be called every time a message is successfully produced.
-    # The callback is called with a {DeliveryReport}
+    # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
     # @param callback [Proc, #call] The callback
     #
@@ -40,20 +33,14 @@ module Rdkafka
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
+      @delivery_callback_arity = arity(callback)
     end
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
-      ObjectSpace.undefine_finalizer(@id)
+      ObjectSpace.undefine_finalizer(self)
 
-      return unless @native_kafka
-
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      @client.close
     end
 
     # Partition count for a given topic.
@@ -65,7 +52,7 @@ module Rdkafka
     #
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
+      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -106,7 +93,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or
@@ -157,7 +144,7 @@ module Rdkafka
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka,
+        @client.native,
         *args
       )
 
@@ -171,12 +158,21 @@ module Rdkafka
     end
 
     # @private
-    def call_delivery_callback(delivery_handle)
-      @delivery_callback.call(delivery_handle) if @delivery_callback
+    def call_delivery_callback(delivery_report, delivery_handle)
+      return unless @delivery_callback
+
+      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
+      @delivery_callback.call(*args)
+    end
+
+    def arity(callback)
+      return callback.arity if callback.respond_to?(:arity)
+
+      callback.method(:call).arity
     end
 
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.11.0"
-  LIBRDKAFKA_VERSION = "1.8.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
+  VERSION = "0.12.0"
+  LIBRDKAFKA_VERSION = "1.9.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
 end

data/lib/rdkafka.rb

@@ -17,5 +17,6 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/producer"
+require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.extensions = %w(ext/Rakefile)
 
   gem.add_dependency 'ffi', '~> 1.15'
-  gem.add_dependency 'mini_portile2', '~> 2.7'
+  gem.add_dependency 'mini_portile2', '~> 2.6'
   gem.add_dependency 'rake', '> 12'
 
   gem.add_development_dependency 'pry'

data/spec/rdkafka/bindings_spec.rb

@@ -76,6 +76,13 @@ describe Rdkafka::Bindings do
       result_2 = (Zlib.crc32(partition_key) % partition_count)
       expect(result_1).to eq(result_2)
     end
+
+    it "should return the partition calculated by the specified partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
+      ptr = FFI::MemoryPointer.from_string(partition_key)
+      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
+      expect(result_1).to eq(result_2)
+    end
   end
 
   describe "stats callback" do

data/spec/rdkafka/consumer_spec.rb

@@ -241,7 +241,7 @@ describe Rdkafka::Consumer do
 
     it "should return the assignment when subscribed" do
       # Make sure there's a message
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -272,7 +272,7 @@ describe Rdkafka::Consumer do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
       100.times do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           payload:   "payload #{i}",
           key:       "key #{i}",
@@ -289,7 +289,7 @@ describe Rdkafka::Consumer do
   describe "#commit, #committed and #store_offset" do
     # Make sure there's a stored offset
     let!(:report) do
-      report = producer.produce(
+      producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
@@ -831,7 +831,6 @@ describe Rdkafka::Consumer do
         )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0
@@ -875,7 +874,6 @@ describe Rdkafka::Consumer do
         )
         consumer = config.consumer
         consumer.subscribe(topic_name)
-        loop_count = 0
         batches_yielded = []
         exceptions_yielded = []
         each_batch_iterations = 0

data/spec/rdkafka/producer/client_spec.rb

@@ -0,0 +1,145 @@
+require "spec_helper"
+
+describe Rdkafka::Producer::Client do
+  let(:config) { rdkafka_producer_config }
+  let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
+  let(:closing) { false }
+  let(:thread) { double(Thread) }
+
+  subject(:client) { described_class.new(native) }
+
+  before do
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
+    allow(Thread).to receive(:new).and_return(thread)
+
+    allow(thread).to receive(:[]=).with(:closing, anything)
+    allow(thread).to receive(:join)
+    allow(thread).to receive(:abort_on_exception=).with(anything)
+  end
+
+  context "defaults" do
+    it "sets the thread to abort on exception" do
+      expect(thread).to receive(:abort_on_exception=).with(true)
+
+      client
+    end
+
+    it "sets the thread `closing` flag to false" do
+      expect(thread).to receive(:[]=).with(:closing, false)
+
+      client
+    end
+  end
+
+  context "the polling thread" do
+    it "is created" do
+      expect(Thread).to receive(:new)
+
+      client
+    end
+
+    it "polls the native with default 250ms timeout" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
+      end
+    end
+
+    it "check the out queue of native client" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
+      end
+    end
+  end
+
+  def polling_loop_expects(&block)
+    Thread.current[:closing] = true # this forces the loop break with line #12
+
+    allow(Thread).to receive(:new).and_yield do |_|
+      block.call
+    end.and_return(thread)
+
+    client
+  end
+
+  it "exposes `native` client" do
+    expect(client.native).to eq(native)
+  end
+
+  context "when client was not yet closed (`nil`)" do
+    it "is not closed" do
+      expect(client.closed?).to eq(false)
+    end
+
+    context "and attempt to close" do
+      it "calls the `destroy` binding" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+
+        client.close
+      end
+
+      it "indicates to the polling thread that it is closing" do
+        expect(thread).to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "joins the polling thread" do
+        expect(thread).to receive(:join)
+
+        client.close
+      end
+
+      it "closes and unassign the native client" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  context "when client was already closed" do
+    before { client.close }
+
+    it "is closed" do
+      expect(client.closed?).to eq(true)
+    end
+
+    context "and attempt to close again" do
+      it "does not call the `destroy` binding" do
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+
+        client.close
+      end
+
+      it "does not indicate to the polling thread that it is closing" do
+        expect(thread).not_to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "does not join the polling thread" do
+        expect(thread).not_to receive(:join)
+
+        client.close
+      end
+
+      it "does not close and unassign the native client again" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  it "provide a finalizer Proc that closes the `native` client" do
+    expect(client.closed?).to eq(false)
+
+    client.finalizer.call("some-ignored-object-id")
+
+    expect(client.closed?).to eq(true)
+  end
+end

data/spec/rdkafka/producer_spec.rb

@@ -49,6 +49,27 @@ describe Rdkafka::Producer do
         # Callback should have been called
         expect(@callback_called).to be true
       end
+
+      it "should provide handle" do
+        @callback_handle = nil
+
+        producer.delivery_callback = lambda { |_, handle| @callback_handle = handle }
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        expect(handle).to be @callback_handle
+      end
     end
 
     context "with a callable object" do
@@ -93,6 +114,36 @@ describe Rdkafka::Producer do
         expect(called_report.first.partition).to eq 1
         expect(called_report.first.offset).to be >= 0
       end
+
+      it "should provide handle" do
+        callback_handles = []
+        callback = Class.new do
+          def initialize(callback_handles)
+            @callback_handles = callback_handles
+          end
+
+          def call(_, handle)
+            @callback_handles << handle
+          end
+        end
+        producer.delivery_callback = callback.new(callback_handles)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(handle).to be callback_handles.first
+      end
     end
 
     it "should not accept a callback that's not callable" do
@@ -376,13 +427,9 @@ describe Rdkafka::Producer do
     end
   end
 
-  it "should produce a message in a forked process" do
+  it "should produce a message in a forked process", skip: defined?(JRUBY_VERSION) && "Kernel#fork is not available" do
     # Fork, produce a message, send the report over a pipe and
     # wait for and check the message in the main process.
-
-    # Kernel#fork is not available in JRuby
-    skip if defined?(JRUBY_VERSION)
-
     reader, writer = IO.pipe
 
     fork do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Thijs Cadier
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-17 00:00:00.000000000 Z
+date: 2022-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -177,6 +177,7 @@ files:
 - lib/rdkafka/error.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/producer.rb
+- lib/rdkafka/producer/client.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
@@ -196,6 +197,7 @@ files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -204,7 +206,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -219,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
@@ -241,6 +243,7 @@ test_files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb