karafka-rdkafka 0.15.0.beta2 → 0.15.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35b0da819d785e6087c6dd43c762d815568a1bc69ab3b834d613f00ed7748e83
-  data.tar.gz: 4175fc335ebbbabfda1dd348c0c5c0f0f2a2cd4bcd848fd0adf5706acf550796
+  metadata.gz: f5e7e7fd2a934c69dc9fb5e6a7c12c46ccbafbe62306919926872cb160894a54
+  data.tar.gz: e0d1f6f2ec2c09e3f8c7631a8ab6490811bb76f583a0d9dc73f15826e396b7b0
 SHA512:
-  metadata.gz: e0d65e75c55f217ceb232e1d0dd6dee5789e8a704d0ec5895d4c87450500d0d480ad9db18a532a67ffc5b970b8dd1e97c8a6be6494e3985a678e811a84036b3c
-  data.tar.gz: 8e11cd4865f058af95499a68d5d11c51218b1318f67136e036612e86dc06241279804ed341000940b6bd965adc6711a21f187c532f1381efe51fd18cca0b4205
+  metadata.gz: 3ef0a6ea40dfea7b168152717f46c8ddfbf461fd6841ec262b8f15df2e0ff1b68d38b06664b9b59ad37b1f79839e2f0c57077e56d7557c9443710b5fb3d82507
+  data.tar.gz: 5ed98fb28a6e2755937deb05c36845b495df9ac1573e9974df395a101b6cd7cfabcd494cd21ddc4fbd97f326198aeb26bc83f7160fbf30f788c564a842fb6365

data/CHANGELOG.md

@@ -5,6 +5,7 @@
 - **[Feature]** Support incremental config describe + alter API (mensfeld)
 - [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
 - [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
+- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
 - [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
 
 ## 0.14.10 (2024-02-08)

data/lib/rdkafka/admin.rb

@@ -12,6 +12,19 @@ module Rdkafka
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
+    # @return [String] admin name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     def finalizer
       ->(_) { close }
     end

data/lib/rdkafka/config.rb

@@ -195,11 +195,13 @@ module Rdkafka
 
     # Creates a consumer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Consumer] The created consumer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def consumer
+    def consumer(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
 
@@ -214,25 +216,26 @@ module Rdkafka
       # Redirect the main queue to the consumer queue
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
 
-      yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
-
       # Return consumer with Kafka client
       Rdkafka::Consumer.new(
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: false,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end
 
     # Create a producer with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer
+    def producer(native_kafka_auto_start: true)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -243,13 +246,13 @@ module Rdkafka
       partitioner_name = self[:partitioner] || self["partitioner"]
 
       kafka = native_kafka(config, :rd_kafka_producer)
-      yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
 
       Rdkafka::Producer.new(
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         ),
         partitioner_name
       ).tap do |producer|
@@ -259,23 +262,25 @@ module Rdkafka
 
     # Creates an admin instance with this configuration.
     #
+    # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
+    #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin
+    def admin(native_kafka_auto_start: true)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
 
       kafka = native_kafka(config, :rd_kafka_producer)
-      yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
 
       Rdkafka::Admin.new(
         Rdkafka::NativeKafka.new(
           kafka,
           run_polling_thread: true,
-          opaque: opaque
+          opaque: opaque,
+          auto_start: native_kafka_auto_start
         )
       )
     end

data/lib/rdkafka/consumer.rb

@@ -20,6 +20,12 @@ module Rdkafka
       @native_kafka = native_kafka
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] consumer name
     def name
       @name ||= @native_kafka.with_inner do |inner|

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -28,30 +28,42 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
-      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
-      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+      @run_polling_thread = run_polling_thread
 
-      if run_polling_thread
-        # Start thread to poll client for delivery callbacks,
-        # not used in consumer.
-        @polling_thread = Thread.new do
-          loop do
-            @poll_mutex.synchronize do
-              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
-            end
+      start if auto_start
 
-            # Exit thread if closing and the poll queue is empty
-            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-              break
+      @closing = false
+    end
+
+    def start
+      synchronize do
+        @return if @started
+
+        @started = true
+
+        # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+        Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
+
+        if @run_polling_thread
+          # Start thread to poll client for delivery callbacks,
+          # not used in consumer.
+          @polling_thread = Thread.new do
+            loop do
+              @poll_mutex.synchronize do
+                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+              end
+
+              # Exit thread if closing and the poll queue is empty
+              if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
+                break
+              end
             end
           end
-        end
 
-        @polling_thread.abort_on_exception = true
-        @polling_thread[:closing] = false
+          @polling_thread.abort_on_exception = true
+          @polling_thread[:closing] = false
+        end
       end
-
-      @closing = false
     end
 
     def with_inner

data/lib/rdkafka/producer.rb

@@ -54,6 +54,12 @@ module Rdkafka
       end
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] producer name
     def name
       @name ||= @native_kafka.with_inner do |inner|

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.0.beta2"
+  VERSION = "0.15.0.beta3"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/admin_spec.rb

@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
   let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
   let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
 
+  describe 'admin without auto-start' do
+    let(:admin) { config.admin(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      admin.start
+      admin.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      admin.close
+    end
+  end
+
   describe "#create_topic" do
     describe "called with invalid input" do
       describe "with an invalid topic name" do

data/spec/rdkafka/consumer_spec.rb

@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
     it { expect(consumer.name).to include('rdkafka#consumer-') }
   end
 
+  describe 'consumer without auto-start' do
+    let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      consumer.start
+      consumer.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      consumer.close
+    end
+  end
+
   describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty

data/spec/rdkafka/producer_spec.rb

@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
     consumer.close
   end
 
+  describe 'producer without auto-start' do
+    let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      producer.start
+      producer.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      producer.close
+    end
+  end
+
   describe '#name' do
     it { expect(producer.name).to include('rdkafka#producer-') }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.15.0.beta2
+  version: 0.15.0.beta3
 platform: ruby
 authors:
 - Thijs Cadier