karafka-rdkafka 0.15.0.beta2 → 0.15.0.beta3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1 -0
- data/lib/rdkafka/admin.rb +13 -0
- data/lib/rdkafka/config.rb +15 -10
- data/lib/rdkafka/consumer.rb +6 -0
- data/lib/rdkafka/native_kafka.rb +31 -19
- data/lib/rdkafka/producer.rb +6 -0
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/admin_spec.rb +13 -0
- data/spec/rdkafka/consumer_spec.rb +13 -0
- data/spec/rdkafka/producer_spec.rb +13 -0
- data.tar.gz.sig +0 -0
- metadata +1 -1
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f5e7e7fd2a934c69dc9fb5e6a7c12c46ccbafbe62306919926872cb160894a54
|
4
|
+
data.tar.gz: e0d1f6f2ec2c09e3f8c7631a8ab6490811bb76f583a0d9dc73f15826e396b7b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ef0a6ea40dfea7b168152717f46c8ddfbf461fd6841ec262b8f15df2e0ff1b68d38b06664b9b59ad37b1f79839e2f0c57077e56d7557c9443710b5fb3d82507
|
7
|
+
data.tar.gz: 5ed98fb28a6e2755937deb05c36845b495df9ac1573e9974df395a101b6cd7cfabcd494cd21ddc4fbd97f326198aeb26bc83f7160fbf30f788c564a842fb6365
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,7 @@
|
|
5
5
|
- **[Feature]** Support incremental config describe + alter API (mensfeld)
|
6
6
|
- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
|
7
7
|
- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
|
8
|
+
- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
|
8
9
|
- [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
|
9
10
|
|
10
11
|
## 0.14.10 (2024-02-08)
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -12,6 +12,19 @@ module Rdkafka
|
|
12
12
|
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
13
13
|
end
|
14
14
|
|
15
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
16
|
+
# @note Not needed to run unless explicit start was disabled
|
17
|
+
def start
|
18
|
+
@native_kafka.start
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] admin name
|
22
|
+
def name
|
23
|
+
@name ||= @native_kafka.with_inner do |inner|
|
24
|
+
::Rdkafka::Bindings.rd_kafka_name(inner)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
15
28
|
def finalizer
|
16
29
|
->(_) { close }
|
17
30
|
end
|
data/lib/rdkafka/config.rb
CHANGED
@@ -195,11 +195,13 @@ module Rdkafka
|
|
195
195
|
|
196
196
|
# Creates a consumer with this configuration.
|
197
197
|
#
|
198
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
199
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
198
200
|
# @return [Consumer] The created consumer
|
199
201
|
#
|
200
202
|
# @raise [ConfigError] When the configuration contains invalid options
|
201
203
|
# @raise [ClientCreationError] When the native client cannot be created
|
202
|
-
def consumer
|
204
|
+
def consumer(native_kafka_auto_start: true)
|
203
205
|
opaque = Opaque.new
|
204
206
|
config = native_config(opaque)
|
205
207
|
|
@@ -214,25 +216,26 @@ module Rdkafka
|
|
214
216
|
# Redirect the main queue to the consumer queue
|
215
217
|
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
|
216
218
|
|
217
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
218
|
-
|
219
219
|
# Return consumer with Kafka client
|
220
220
|
Rdkafka::Consumer.new(
|
221
221
|
Rdkafka::NativeKafka.new(
|
222
222
|
kafka,
|
223
223
|
run_polling_thread: false,
|
224
|
-
opaque: opaque
|
224
|
+
opaque: opaque,
|
225
|
+
auto_start: native_kafka_auto_start
|
225
226
|
)
|
226
227
|
)
|
227
228
|
end
|
228
229
|
|
229
230
|
# Create a producer with this configuration.
|
230
231
|
#
|
232
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
233
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
231
234
|
# @return [Producer] The created producer
|
232
235
|
#
|
233
236
|
# @raise [ConfigError] When the configuration contains invalid options
|
234
237
|
# @raise [ClientCreationError] When the native client cannot be created
|
235
|
-
def producer
|
238
|
+
def producer(native_kafka_auto_start: true)
|
236
239
|
# Create opaque
|
237
240
|
opaque = Opaque.new
|
238
241
|
# Create Kafka config
|
@@ -243,13 +246,13 @@ module Rdkafka
|
|
243
246
|
partitioner_name = self[:partitioner] || self["partitioner"]
|
244
247
|
|
245
248
|
kafka = native_kafka(config, :rd_kafka_producer)
|
246
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
247
249
|
|
248
250
|
Rdkafka::Producer.new(
|
249
251
|
Rdkafka::NativeKafka.new(
|
250
252
|
kafka,
|
251
253
|
run_polling_thread: true,
|
252
|
-
opaque: opaque
|
254
|
+
opaque: opaque,
|
255
|
+
auto_start: native_kafka_auto_start
|
253
256
|
),
|
254
257
|
partitioner_name
|
255
258
|
).tap do |producer|
|
@@ -259,23 +262,25 @@ module Rdkafka
|
|
259
262
|
|
260
263
|
# Creates an admin instance with this configuration.
|
261
264
|
#
|
265
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
266
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
262
267
|
# @return [Admin] The created admin instance
|
263
268
|
#
|
264
269
|
# @raise [ConfigError] When the configuration contains invalid options
|
265
270
|
# @raise [ClientCreationError] When the native client cannot be created
|
266
|
-
def admin
|
271
|
+
def admin(native_kafka_auto_start: true)
|
267
272
|
opaque = Opaque.new
|
268
273
|
config = native_config(opaque)
|
269
274
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
270
275
|
|
271
276
|
kafka = native_kafka(config, :rd_kafka_producer)
|
272
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
273
277
|
|
274
278
|
Rdkafka::Admin.new(
|
275
279
|
Rdkafka::NativeKafka.new(
|
276
280
|
kafka,
|
277
281
|
run_polling_thread: true,
|
278
|
-
opaque: opaque
|
282
|
+
opaque: opaque,
|
283
|
+
auto_start: native_kafka_auto_start
|
279
284
|
)
|
280
285
|
)
|
281
286
|
end
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -20,6 +20,12 @@ module Rdkafka
|
|
20
20
|
@native_kafka = native_kafka
|
21
21
|
end
|
22
22
|
|
23
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
24
|
+
# @note Not needed to run unless explicit start was disabled
|
25
|
+
def start
|
26
|
+
@native_kafka.start
|
27
|
+
end
|
28
|
+
|
23
29
|
# @return [String] consumer name
|
24
30
|
def name
|
25
31
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/native_kafka.rb
CHANGED
@@ -4,7 +4,7 @@ module Rdkafka
|
|
4
4
|
# @private
|
5
5
|
# A wrapper around a native kafka that polls and cleanly exits
|
6
6
|
class NativeKafka
|
7
|
-
def initialize(inner, run_polling_thread:, opaque:)
|
7
|
+
def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
|
8
8
|
@inner = inner
|
9
9
|
@opaque = opaque
|
10
10
|
# Lock around external access
|
@@ -28,30 +28,42 @@ module Rdkafka
|
|
28
28
|
# counter for operations in progress using inner
|
29
29
|
@operations_in_progress = 0
|
30
30
|
|
31
|
-
|
32
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 0)
|
31
|
+
@run_polling_thread = run_polling_thread
|
33
32
|
|
34
|
-
if
|
35
|
-
# Start thread to poll client for delivery callbacks,
|
36
|
-
# not used in consumer.
|
37
|
-
@polling_thread = Thread.new do
|
38
|
-
loop do
|
39
|
-
@poll_mutex.synchronize do
|
40
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
41
|
-
end
|
33
|
+
start if auto_start
|
42
34
|
|
43
|
-
|
44
|
-
|
45
|
-
|
35
|
+
@closing = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
synchronize do
|
40
|
+
@return if @started
|
41
|
+
|
42
|
+
@started = true
|
43
|
+
|
44
|
+
# Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
|
45
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
|
46
|
+
|
47
|
+
if @run_polling_thread
|
48
|
+
# Start thread to poll client for delivery callbacks,
|
49
|
+
# not used in consumer.
|
50
|
+
@polling_thread = Thread.new do
|
51
|
+
loop do
|
52
|
+
@poll_mutex.synchronize do
|
53
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Exit thread if closing and the poll queue is empty
|
57
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
|
58
|
+
break
|
59
|
+
end
|
46
60
|
end
|
47
61
|
end
|
48
|
-
end
|
49
62
|
|
50
|
-
|
51
|
-
|
63
|
+
@polling_thread.abort_on_exception = true
|
64
|
+
@polling_thread[:closing] = false
|
65
|
+
end
|
52
66
|
end
|
53
|
-
|
54
|
-
@closing = false
|
55
67
|
end
|
56
68
|
|
57
69
|
def with_inner
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -54,6 +54,12 @@ module Rdkafka
|
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
57
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
58
|
+
# @note Not needed to run unless explicit start was disabled
|
59
|
+
def start
|
60
|
+
@native_kafka.start
|
61
|
+
end
|
62
|
+
|
57
63
|
# @return [String] producer name
|
58
64
|
def name
|
59
65
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/version.rb
CHANGED
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe 'admin without auto-start' do
|
35
|
+
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
|
+
|
37
|
+
it 'expect to be able to start it later and close' do
|
38
|
+
admin.start
|
39
|
+
admin.close
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'expect to be able to close it without starting' do
|
43
|
+
admin.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
34
47
|
describe "#create_topic" do
|
35
48
|
describe "called with invalid input" do
|
36
49
|
describe "with an invalid topic name" do
|
@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
|
|
14
14
|
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'consumer without auto-start' do
|
18
|
+
let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
consumer.start
|
22
|
+
consumer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
consumer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe "#subscribe, #unsubscribe and #subscription" do
|
18
31
|
it "should subscribe, unsubscribe and return the subscription" do
|
19
32
|
expect(consumer.subscription).to be_empty
|
@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
|
|
14
14
|
consumer.close
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'producer without auto-start' do
|
18
|
+
let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
producer.start
|
22
|
+
producer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
producer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe '#name' do
|
18
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
32
|
end
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
metadata.gz.sig
CHANGED
Binary file
|