karafka-rdkafka 0.15.0.beta2 → 0.15.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 35b0da819d785e6087c6dd43c762d815568a1bc69ab3b834d613f00ed7748e83
4
- data.tar.gz: 4175fc335ebbbabfda1dd348c0c5c0f0f2a2cd4bcd848fd0adf5706acf550796
3
+ metadata.gz: f5e7e7fd2a934c69dc9fb5e6a7c12c46ccbafbe62306919926872cb160894a54
4
+ data.tar.gz: e0d1f6f2ec2c09e3f8c7631a8ab6490811bb76f583a0d9dc73f15826e396b7b0
5
5
  SHA512:
6
- metadata.gz: e0d65e75c55f217ceb232e1d0dd6dee5789e8a704d0ec5895d4c87450500d0d480ad9db18a532a67ffc5b970b8dd1e97c8a6be6494e3985a678e811a84036b3c
7
- data.tar.gz: 8e11cd4865f058af95499a68d5d11c51218b1318f67136e036612e86dc06241279804ed341000940b6bd965adc6711a21f187c532f1381efe51fd18cca0b4205
6
+ metadata.gz: 3ef0a6ea40dfea7b168152717f46c8ddfbf461fd6841ec262b8f15df2e0ff1b68d38b06664b9b59ad37b1f79839e2f0c57077e56d7557c9443710b5fb3d82507
7
+ data.tar.gz: 5ed98fb28a6e2755937deb05c36845b495df9ac1573e9974df395a101b6cd7cfabcd494cd21ddc4fbd97f326198aeb26bc83f7160fbf30f788c564a842fb6365
checksums.yaml.gz.sig CHANGED
Binary file
data/CHANGELOG.md CHANGED
@@ -5,6 +5,7 @@
5
5
  - **[Feature]** Support incremental config describe + alter API (mensfeld)
6
6
  - [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
7
7
  - [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
8
+ - [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
8
9
  - [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
9
10
 
10
11
  ## 0.14.10 (2024-02-08)
data/lib/rdkafka/admin.rb CHANGED
@@ -12,6 +12,19 @@ module Rdkafka
12
12
  ObjectSpace.define_finalizer(self, native_kafka.finalizer)
13
13
  end
14
14
 
15
+ # Starts the native Kafka polling thread and kicks off the init polling
16
+ # @note Not needed to run unless explicit start was disabled
17
+ def start
18
+ @native_kafka.start
19
+ end
20
+
21
+ # @return [String] admin name
22
+ def name
23
+ @name ||= @native_kafka.with_inner do |inner|
24
+ ::Rdkafka::Bindings.rd_kafka_name(inner)
25
+ end
26
+ end
27
+
15
28
  def finalizer
16
29
  ->(_) { close }
17
30
  end
@@ -195,11 +195,13 @@ module Rdkafka
195
195
 
196
196
  # Creates a consumer with this configuration.
197
197
  #
198
+ # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
199
+ # automatically. Defaults to true. Set to false only when doing complex initialization.
198
200
  # @return [Consumer] The created consumer
199
201
  #
200
202
  # @raise [ConfigError] When the configuration contains invalid options
201
203
  # @raise [ClientCreationError] When the native client cannot be created
202
- def consumer
204
+ def consumer(native_kafka_auto_start: true)
203
205
  opaque = Opaque.new
204
206
  config = native_config(opaque)
205
207
 
@@ -214,25 +216,26 @@ module Rdkafka
214
216
  # Redirect the main queue to the consumer queue
215
217
  Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
216
218
 
217
- yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
218
-
219
219
  # Return consumer with Kafka client
220
220
  Rdkafka::Consumer.new(
221
221
  Rdkafka::NativeKafka.new(
222
222
  kafka,
223
223
  run_polling_thread: false,
224
- opaque: opaque
224
+ opaque: opaque,
225
+ auto_start: native_kafka_auto_start
225
226
  )
226
227
  )
227
228
  end
228
229
 
229
230
  # Create a producer with this configuration.
230
231
  #
232
+ # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
233
+ # automatically. Defaults to true. Set to false only when doing complex initialization.
231
234
  # @return [Producer] The created producer
232
235
  #
233
236
  # @raise [ConfigError] When the configuration contains invalid options
234
237
  # @raise [ClientCreationError] When the native client cannot be created
235
- def producer
238
+ def producer(native_kafka_auto_start: true)
236
239
  # Create opaque
237
240
  opaque = Opaque.new
238
241
  # Create Kafka config
@@ -243,13 +246,13 @@ module Rdkafka
243
246
  partitioner_name = self[:partitioner] || self["partitioner"]
244
247
 
245
248
  kafka = native_kafka(config, :rd_kafka_producer)
246
- yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
247
249
 
248
250
  Rdkafka::Producer.new(
249
251
  Rdkafka::NativeKafka.new(
250
252
  kafka,
251
253
  run_polling_thread: true,
252
- opaque: opaque
254
+ opaque: opaque,
255
+ auto_start: native_kafka_auto_start
253
256
  ),
254
257
  partitioner_name
255
258
  ).tap do |producer|
@@ -259,23 +262,25 @@ module Rdkafka
259
262
 
260
263
  # Creates an admin instance with this configuration.
261
264
  #
265
+ # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
266
+ # automatically. Defaults to true. Set to false only when doing complex initialization.
262
267
  # @return [Admin] The created admin instance
263
268
  #
264
269
  # @raise [ConfigError] When the configuration contains invalid options
265
270
  # @raise [ClientCreationError] When the native client cannot be created
266
- def admin
271
+ def admin(native_kafka_auto_start: true)
267
272
  opaque = Opaque.new
268
273
  config = native_config(opaque)
269
274
  Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
270
275
 
271
276
  kafka = native_kafka(config, :rd_kafka_producer)
272
- yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
273
277
 
274
278
  Rdkafka::Admin.new(
275
279
  Rdkafka::NativeKafka.new(
276
280
  kafka,
277
281
  run_polling_thread: true,
278
- opaque: opaque
282
+ opaque: opaque,
283
+ auto_start: native_kafka_auto_start
279
284
  )
280
285
  )
281
286
  end
@@ -20,6 +20,12 @@ module Rdkafka
20
20
  @native_kafka = native_kafka
21
21
  end
22
22
 
23
+ # Starts the native Kafka polling thread and kicks off the init polling
24
+ # @note Not needed to run unless explicit start was disabled
25
+ def start
26
+ @native_kafka.start
27
+ end
28
+
23
29
  # @return [String] consumer name
24
30
  def name
25
31
  @name ||= @native_kafka.with_inner do |inner|
@@ -4,7 +4,7 @@ module Rdkafka
4
4
  # @private
5
5
  # A wrapper around a native kafka that polls and cleanly exits
6
6
  class NativeKafka
7
- def initialize(inner, run_polling_thread:, opaque:)
7
+ def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
8
8
  @inner = inner
9
9
  @opaque = opaque
10
10
  # Lock around external access
@@ -28,30 +28,42 @@ module Rdkafka
28
28
  # counter for operations in progress using inner
29
29
  @operations_in_progress = 0
30
30
 
31
- # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
32
- Rdkafka::Bindings.rd_kafka_poll(inner, 0)
31
+ @run_polling_thread = run_polling_thread
33
32
 
34
- if run_polling_thread
35
- # Start thread to poll client for delivery callbacks,
36
- # not used in consumer.
37
- @polling_thread = Thread.new do
38
- loop do
39
- @poll_mutex.synchronize do
40
- Rdkafka::Bindings.rd_kafka_poll(inner, 100)
41
- end
33
+ start if auto_start
42
34
 
43
- # Exit thread if closing and the poll queue is empty
44
- if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
45
- break
35
+ @closing = false
36
+ end
37
+
38
+ def start
39
+ synchronize do
40
+ @return if @started
41
+
42
+ @started = true
43
+
44
+ # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
45
+ Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
46
+
47
+ if @run_polling_thread
48
+ # Start thread to poll client for delivery callbacks,
49
+ # not used in consumer.
50
+ @polling_thread = Thread.new do
51
+ loop do
52
+ @poll_mutex.synchronize do
53
+ Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
54
+ end
55
+
56
+ # Exit thread if closing and the poll queue is empty
57
+ if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
58
+ break
59
+ end
46
60
  end
47
61
  end
48
- end
49
62
 
50
- @polling_thread.abort_on_exception = true
51
- @polling_thread[:closing] = false
63
+ @polling_thread.abort_on_exception = true
64
+ @polling_thread[:closing] = false
65
+ end
52
66
  end
53
-
54
- @closing = false
55
67
  end
56
68
 
57
69
  def with_inner
@@ -54,6 +54,12 @@ module Rdkafka
54
54
  end
55
55
  end
56
56
 
57
+ # Starts the native Kafka polling thread and kicks off the init polling
58
+ # @note Not needed to run unless explicit start was disabled
59
+ def start
60
+ @native_kafka.start
61
+ end
62
+
57
63
  # @return [String] producer name
58
64
  def name
59
65
  @name ||= @native_kafka.with_inner do |inner|
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
- VERSION = "0.15.0.beta2"
4
+ VERSION = "0.15.0.beta3"
5
5
  LIBRDKAFKA_VERSION = "2.3.0"
6
6
  LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
7
7
  end
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
31
31
  let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
32
32
  let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
33
33
 
34
+ describe 'admin without auto-start' do
35
+ let(:admin) { config.admin(native_kafka_auto_start: false) }
36
+
37
+ it 'expect to be able to start it later and close' do
38
+ admin.start
39
+ admin.close
40
+ end
41
+
42
+ it 'expect to be able to close it without starting' do
43
+ admin.close
44
+ end
45
+ end
46
+
34
47
  describe "#create_topic" do
35
48
  describe "called with invalid input" do
36
49
  describe "with an invalid topic name" do
@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
14
14
  it { expect(consumer.name).to include('rdkafka#consumer-') }
15
15
  end
16
16
 
17
+ describe 'consumer without auto-start' do
18
+ let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
19
+
20
+ it 'expect to be able to start it later and close' do
21
+ consumer.start
22
+ consumer.close
23
+ end
24
+
25
+ it 'expect to be able to close it without starting' do
26
+ consumer.close
27
+ end
28
+ end
29
+
17
30
  describe "#subscribe, #unsubscribe and #subscription" do
18
31
  it "should subscribe, unsubscribe and return the subscription" do
19
32
  expect(consumer.subscription).to be_empty
@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
14
14
  consumer.close
15
15
  end
16
16
 
17
+ describe 'producer without auto-start' do
18
+ let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
19
+
20
+ it 'expect to be able to start it later and close' do
21
+ producer.start
22
+ producer.close
23
+ end
24
+
25
+ it 'expect to be able to close it without starting' do
26
+ producer.close
27
+ end
28
+ end
29
+
17
30
  describe '#name' do
18
31
  it { expect(producer.name).to include('rdkafka#producer-') }
19
32
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka-rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.0.beta2
4
+ version: 0.15.0.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thijs Cadier
metadata.gz.sig CHANGED
Binary file