karafka-rdkafka 0.15.0.beta2 → 0.15.0.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1 -0
- data/lib/rdkafka/admin.rb +13 -0
- data/lib/rdkafka/config.rb +15 -10
- data/lib/rdkafka/consumer.rb +6 -0
- data/lib/rdkafka/native_kafka.rb +31 -19
- data/lib/rdkafka/producer.rb +6 -0
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/admin_spec.rb +13 -0
- data/spec/rdkafka/consumer_spec.rb +13 -0
- data/spec/rdkafka/producer_spec.rb +13 -0
- data.tar.gz.sig +0 -0
- metadata +1 -1
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f5e7e7fd2a934c69dc9fb5e6a7c12c46ccbafbe62306919926872cb160894a54
|
4
|
+
data.tar.gz: e0d1f6f2ec2c09e3f8c7631a8ab6490811bb76f583a0d9dc73f15826e396b7b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ef0a6ea40dfea7b168152717f46c8ddfbf461fd6841ec262b8f15df2e0ff1b68d38b06664b9b59ad37b1f79839e2f0c57077e56d7557c9443710b5fb3d82507
|
7
|
+
data.tar.gz: 5ed98fb28a6e2755937deb05c36845b495df9ac1573e9974df395a101b6cd7cfabcd494cd21ddc4fbd97f326198aeb26bc83f7160fbf30f788c564a842fb6365
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,7 @@
|
|
5
5
|
- **[Feature]** Support incremental config describe + alter API (mensfeld)
|
6
6
|
- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
|
7
7
|
- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
|
8
|
+
- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
|
8
9
|
- [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
|
9
10
|
|
10
11
|
## 0.14.10 (2024-02-08)
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -12,6 +12,19 @@ module Rdkafka
|
|
12
12
|
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
13
13
|
end
|
14
14
|
|
15
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
16
|
+
# @note Not needed to run unless explicit start was disabled
|
17
|
+
def start
|
18
|
+
@native_kafka.start
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] admin name
|
22
|
+
def name
|
23
|
+
@name ||= @native_kafka.with_inner do |inner|
|
24
|
+
::Rdkafka::Bindings.rd_kafka_name(inner)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
15
28
|
def finalizer
|
16
29
|
->(_) { close }
|
17
30
|
end
|
data/lib/rdkafka/config.rb
CHANGED
@@ -195,11 +195,13 @@ module Rdkafka
|
|
195
195
|
|
196
196
|
# Creates a consumer with this configuration.
|
197
197
|
#
|
198
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
199
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
198
200
|
# @return [Consumer] The created consumer
|
199
201
|
#
|
200
202
|
# @raise [ConfigError] When the configuration contains invalid options
|
201
203
|
# @raise [ClientCreationError] When the native client cannot be created
|
202
|
-
def consumer
|
204
|
+
def consumer(native_kafka_auto_start: true)
|
203
205
|
opaque = Opaque.new
|
204
206
|
config = native_config(opaque)
|
205
207
|
|
@@ -214,25 +216,26 @@ module Rdkafka
|
|
214
216
|
# Redirect the main queue to the consumer queue
|
215
217
|
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
|
216
218
|
|
217
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
218
|
-
|
219
219
|
# Return consumer with Kafka client
|
220
220
|
Rdkafka::Consumer.new(
|
221
221
|
Rdkafka::NativeKafka.new(
|
222
222
|
kafka,
|
223
223
|
run_polling_thread: false,
|
224
|
-
opaque: opaque
|
224
|
+
opaque: opaque,
|
225
|
+
auto_start: native_kafka_auto_start
|
225
226
|
)
|
226
227
|
)
|
227
228
|
end
|
228
229
|
|
229
230
|
# Create a producer with this configuration.
|
230
231
|
#
|
232
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
233
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
231
234
|
# @return [Producer] The created producer
|
232
235
|
#
|
233
236
|
# @raise [ConfigError] When the configuration contains invalid options
|
234
237
|
# @raise [ClientCreationError] When the native client cannot be created
|
235
|
-
def producer
|
238
|
+
def producer(native_kafka_auto_start: true)
|
236
239
|
# Create opaque
|
237
240
|
opaque = Opaque.new
|
238
241
|
# Create Kafka config
|
@@ -243,13 +246,13 @@ module Rdkafka
|
|
243
246
|
partitioner_name = self[:partitioner] || self["partitioner"]
|
244
247
|
|
245
248
|
kafka = native_kafka(config, :rd_kafka_producer)
|
246
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
247
249
|
|
248
250
|
Rdkafka::Producer.new(
|
249
251
|
Rdkafka::NativeKafka.new(
|
250
252
|
kafka,
|
251
253
|
run_polling_thread: true,
|
252
|
-
opaque: opaque
|
254
|
+
opaque: opaque,
|
255
|
+
auto_start: native_kafka_auto_start
|
253
256
|
),
|
254
257
|
partitioner_name
|
255
258
|
).tap do |producer|
|
@@ -259,23 +262,25 @@ module Rdkafka
|
|
259
262
|
|
260
263
|
# Creates an admin instance with this configuration.
|
261
264
|
#
|
265
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
266
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
262
267
|
# @return [Admin] The created admin instance
|
263
268
|
#
|
264
269
|
# @raise [ConfigError] When the configuration contains invalid options
|
265
270
|
# @raise [ClientCreationError] When the native client cannot be created
|
266
|
-
def admin
|
271
|
+
def admin(native_kafka_auto_start: true)
|
267
272
|
opaque = Opaque.new
|
268
273
|
config = native_config(opaque)
|
269
274
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
270
275
|
|
271
276
|
kafka = native_kafka(config, :rd_kafka_producer)
|
272
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
273
277
|
|
274
278
|
Rdkafka::Admin.new(
|
275
279
|
Rdkafka::NativeKafka.new(
|
276
280
|
kafka,
|
277
281
|
run_polling_thread: true,
|
278
|
-
opaque: opaque
|
282
|
+
opaque: opaque,
|
283
|
+
auto_start: native_kafka_auto_start
|
279
284
|
)
|
280
285
|
)
|
281
286
|
end
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -20,6 +20,12 @@ module Rdkafka
|
|
20
20
|
@native_kafka = native_kafka
|
21
21
|
end
|
22
22
|
|
23
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
24
|
+
# @note Not needed to run unless explicit start was disabled
|
25
|
+
def start
|
26
|
+
@native_kafka.start
|
27
|
+
end
|
28
|
+
|
23
29
|
# @return [String] consumer name
|
24
30
|
def name
|
25
31
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/native_kafka.rb
CHANGED
@@ -4,7 +4,7 @@ module Rdkafka
|
|
4
4
|
# @private
|
5
5
|
# A wrapper around a native kafka that polls and cleanly exits
|
6
6
|
class NativeKafka
|
7
|
-
def initialize(inner, run_polling_thread:, opaque:)
|
7
|
+
def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
|
8
8
|
@inner = inner
|
9
9
|
@opaque = opaque
|
10
10
|
# Lock around external access
|
@@ -28,30 +28,42 @@ module Rdkafka
|
|
28
28
|
# counter for operations in progress using inner
|
29
29
|
@operations_in_progress = 0
|
30
30
|
|
31
|
-
|
32
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 0)
|
31
|
+
@run_polling_thread = run_polling_thread
|
33
32
|
|
34
|
-
if
|
35
|
-
# Start thread to poll client for delivery callbacks,
|
36
|
-
# not used in consumer.
|
37
|
-
@polling_thread = Thread.new do
|
38
|
-
loop do
|
39
|
-
@poll_mutex.synchronize do
|
40
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
41
|
-
end
|
33
|
+
start if auto_start
|
42
34
|
|
43
|
-
|
44
|
-
|
45
|
-
|
35
|
+
@closing = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
synchronize do
|
40
|
+
@return if @started
|
41
|
+
|
42
|
+
@started = true
|
43
|
+
|
44
|
+
# Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
|
45
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
|
46
|
+
|
47
|
+
if @run_polling_thread
|
48
|
+
# Start thread to poll client for delivery callbacks,
|
49
|
+
# not used in consumer.
|
50
|
+
@polling_thread = Thread.new do
|
51
|
+
loop do
|
52
|
+
@poll_mutex.synchronize do
|
53
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Exit thread if closing and the poll queue is empty
|
57
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
|
58
|
+
break
|
59
|
+
end
|
46
60
|
end
|
47
61
|
end
|
48
|
-
end
|
49
62
|
|
50
|
-
|
51
|
-
|
63
|
+
@polling_thread.abort_on_exception = true
|
64
|
+
@polling_thread[:closing] = false
|
65
|
+
end
|
52
66
|
end
|
53
|
-
|
54
|
-
@closing = false
|
55
67
|
end
|
56
68
|
|
57
69
|
def with_inner
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -54,6 +54,12 @@ module Rdkafka
|
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
57
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
58
|
+
# @note Not needed to run unless explicit start was disabled
|
59
|
+
def start
|
60
|
+
@native_kafka.start
|
61
|
+
end
|
62
|
+
|
57
63
|
# @return [String] producer name
|
58
64
|
def name
|
59
65
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/version.rb
CHANGED
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe 'admin without auto-start' do
|
35
|
+
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
|
+
|
37
|
+
it 'expect to be able to start it later and close' do
|
38
|
+
admin.start
|
39
|
+
admin.close
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'expect to be able to close it without starting' do
|
43
|
+
admin.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
34
47
|
describe "#create_topic" do
|
35
48
|
describe "called with invalid input" do
|
36
49
|
describe "with an invalid topic name" do
|
@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
|
|
14
14
|
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'consumer without auto-start' do
|
18
|
+
let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
consumer.start
|
22
|
+
consumer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
consumer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe "#subscribe, #unsubscribe and #subscription" do
|
18
31
|
it "should subscribe, unsubscribe and return the subscription" do
|
19
32
|
expect(consumer.subscription).to be_empty
|
@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
|
|
14
14
|
consumer.close
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'producer without auto-start' do
|
18
|
+
let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
producer.start
|
22
|
+
producer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
producer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe '#name' do
|
18
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
32
|
end
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
metadata.gz.sig
CHANGED
Binary file
|