rdkafka 0.11.1 → 0.12.0.beta.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b7042f241be5aad91c403f61aabb8e8ba87afa46674db8ba58c487fe01f88c9
-  data.tar.gz: 5cb00dc3dc8b4069a8e62cda9271e290d1371434332767270426488dedcff641
+  metadata.gz: a58b7e8b03dec32027bf3d9ec513a87e6bb0a8abf2f6598cd34a965bebce2dbb
+  data.tar.gz: 603c985bd19a2213671b600680a748279e21d7b4aeb3f23a7f1efa16a5e1229c
 SHA512:
-  metadata.gz: 2b287a0d81aca7909702969778fb64585b6808860971d50e1c5540eb0c43cf73617f9e701605b816e8f52a81e39d9909512e1d113f6a550555e6ca503dc5ff58
-  data.tar.gz: 7924ab06f52646168b52d0478b605484e4aa1a3674ace4f97af5b56588aea6dec8e614d3258b0670f92f18a905509ae6982d2c8999fa719f8ea7451e515ecb3d
+  metadata.gz: 9e5a5bd4c911f93572d0f8be2e5f576b08ee0ff86ab773e9e7d9db1182002e62c1c57b03cfe6e7b686c9890c35f6fca811fe10daad3a0d4d57bd67a5340c7fa1
+  data.tar.gz: 5fd3c57d9797f3b1f7c636f8190cee0b8cfc56f14daa03a2be21da6ff11fff1b7e00e233ab10a37b280b06ac2a197c155ebf1fdd6dfc8a97af9ce8bd38a341e1

data/CHANGELOG.md

@@ -1,6 +1,3 @@
-# 0.11.1
-* Use mini_portile2 2.6, otherwise you can't run nokogiri and rdkafka at the same time.
-
 # 0.11.0
 * Upgrade librdkafka to 1.8.2
 * Bump supported minimum Ruby version to 2.6

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer))).tap do |producer|
         opaque.producer = producer
       end
     end

data/lib/rdkafka/producer/client.rb

@@ -0,0 +1,47 @@
+module Rdkafka
+  class Producer
+    class Client
+      def initialize(native)
+        @native = native
+
+        # Start thread to poll client for delivery callbacks
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(native, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
+              break
+            end
+          end
+        end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      def native
+        @native
+      end
+
+      def finalizer
+        ->(_) { close }
+      end
+
+      def closed?
+        @native.nil?
+      end
+
+      def close(object_id=nil)
+        return unless @native
+
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+
+        Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+        @native = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -1,4 +1,4 @@
-require "securerandom"
+require "objspace"
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
@@ -10,25 +10,11 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(native_kafka)
-      @id = SecureRandom.uuid
-      @closing = false
-      @native_kafka = native_kafka
+    def initialize(client)
+      @client = client
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(@id, proc { close })
-
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+      ObjectSpace.define_finalizer(self, client.finalizer)
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -44,16 +30,9 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
-      ObjectSpace.undefine_finalizer(@id)
-
-      return unless @native_kafka
+      ObjectSpace.undefine_finalizer(self)
 
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      @client.close
     end
 
     # Partition count for a given topic.
@@ -65,7 +44,7 @@ module Rdkafka
     #
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
+      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -157,7 +136,7 @@ module Rdkafka
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka,
+        @client.native,
         *args
       )
 
@@ -176,7 +155,7 @@ module Rdkafka
     end
 
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.11.1"
+  VERSION = "0.12.0.beta.0"
   LIBRDKAFKA_VERSION = "1.8.2"
   LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
 end

data/lib/rdkafka.rb

@@ -17,5 +17,6 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/producer"
+require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/spec/rdkafka/producer/client_spec.rb

@@ -0,0 +1,144 @@
+require "spec_helper"
+
+describe Rdkafka::Producer::Client do
+  let(:native) { double }
+  let(:closing) { false }
+  let(:thread) { double(Thread) }
+
+  subject(:client) { described_class.new(native) }
+
+  before do
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).and_return(0)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
+    allow(Thread).to receive(:new).and_return(thread)
+
+    allow(thread).to receive(:[]=).with(:closing, anything)
+    allow(thread).to receive(:join)
+    allow(thread).to receive(:abort_on_exception=).with(anything)
+  end
+
+  context "defaults" do
+    it "sets the thread to abort on exception" do
+      expect(thread).to receive(:abort_on_exception=).with(true)
+
+      client
+    end
+
+    it "sets the thread `closing` flag to false" do
+      expect(thread).to receive(:[]=).with(:closing, false)
+
+      client
+    end
+  end
+
+  context "the polling thread" do
+    it "is created" do
+      expect(Thread).to receive(:new)
+
+      client
+    end
+
+    it "polls the native with default 250ms timeout" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
+      end
+    end
+
+    it "check the out queue of native client" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native)
+      end
+    end
+  end
+
+  def polling_loop_expects(&block)
+    Thread.current[:closing] = true # this forces the loop break with line #12
+
+    allow(Thread).to receive(:new).and_yield do |_|
+      block.call
+    end.and_return(thread)
+
+    client
+  end
+
+  it "exposes `native` client" do
+    expect(client.native).to eq(native)
+  end
+
+  context "when client was not yet closed (`nil`)" do
+    it "is not closed" do
+      expect(client.closed?).to eq(false)
+    end
+
+    context "and attempt to close" do
+      it "calls the `destroy` binding" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+
+        client.close
+      end
+
+      it "indicates to the polling thread that it is closing" do
+        expect(thread).to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "joins the polling thread" do
+        expect(thread).to receive(:join)
+
+        client.close
+      end
+
+      it "closes and unassign the native client" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  context "when client was already closed" do
+    before { client.close }
+
+    it "is closed" do
+      expect(client.closed?).to eq(true)
+    end
+
+    context "and attempt to close again" do
+      it "does not call the `destroy` binding" do
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+
+        client.close
+      end
+
+      it "does not indicate to the polling thread that it is closing" do
+        expect(thread).not_to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "does not join the polling thread" do
+        expect(thread).not_to receive(:join)
+
+        client.close
+      end
+
+      it "does not close and unassign the native client again" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  it "provide a finalizer Proc that closes the `native` client" do
+    expect(client.closed?).to eq(false)
+
+    client.finalizer.call("some-ignored-object-id")
+
+    expect(client.closed?).to eq(true)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0.beta.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-23 00:00:00.000000000 Z
+date: 2022-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -177,6 +177,7 @@ files:
 - lib/rdkafka/error.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/producer.rb
+- lib/rdkafka/producer/client.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
@@ -196,6 +197,7 @@ files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -215,9 +217,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.4
 signing_key:
@@ -241,6 +243,7 @@ test_files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb