karafka-rdkafka 0.15.0.beta2 → 0.15.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +2 -0
- data/lib/rdkafka/admin.rb +13 -0
- data/lib/rdkafka/config.rb +15 -10
- data/lib/rdkafka/consumer.rb +6 -0
- data/lib/rdkafka/native_kafka.rb +32 -19
- data/lib/rdkafka/producer.rb +6 -0
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/admin_spec.rb +13 -0
- data/spec/rdkafka/consumer_spec.rb +13 -0
- data/spec/rdkafka/native_kafka_spec.rb +8 -1
- data/spec/rdkafka/producer_spec.rb +13 -0
- data.tar.gz.sig +0 -0
- metadata +2 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 91ca920dc47a311632eb36adedc660a29d0cb72e6bcb2959451836a2fb99dd73
|
4
|
+
data.tar.gz: d4879f60d701be3c5b24738567d502eb63ca732f546e3fc54bc423892bf1521c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54a2b2d134bb46ecb888b699e5c3efb6240e5b7f7d659d17fed60280e88c3c748066b1d8912c81762e1275f3384318baadf238ae7f068839ed3ba0e3be085b34
|
7
|
+
data.tar.gz: b27d1257a263f5fc54eff2568891c68d012a6a2b4d48168379965dd9e30e4c3c37d71867f925b9324e32c8d1dea438f6b7760109662137d82806f1d08a0f006b
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
@@ -37,9 +37,9 @@ jobs:
|
|
37
37
|
- name: Install package dependencies
|
38
38
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
39
39
|
|
40
|
-
- name: Start Kafka with docker
|
40
|
+
- name: Start Kafka with docker compose
|
41
41
|
run: |
|
42
|
-
docker
|
42
|
+
docker compose up -d || (sleep 5 && docker compose up -d)
|
43
43
|
|
44
44
|
- name: Set up Ruby
|
45
45
|
uses: ruby/setup-ruby@v1
|
data/CHANGELOG.md
CHANGED
@@ -3,8 +3,10 @@
|
|
3
3
|
## 0.15.0 (Unreleased)
|
4
4
|
- **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
|
5
5
|
- **[Feature]** Support incremental config describe + alter API (mensfeld)
|
6
|
+
- [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
|
6
7
|
- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
|
7
8
|
- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
|
9
|
+
- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
|
8
10
|
- [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
|
9
11
|
|
10
12
|
## 0.14.10 (2024-02-08)
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -12,6 +12,19 @@ module Rdkafka
|
|
12
12
|
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
13
13
|
end
|
14
14
|
|
15
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
16
|
+
# @note Not needed to run unless explicit start was disabled
|
17
|
+
def start
|
18
|
+
@native_kafka.start
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] admin name
|
22
|
+
def name
|
23
|
+
@name ||= @native_kafka.with_inner do |inner|
|
24
|
+
::Rdkafka::Bindings.rd_kafka_name(inner)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
15
28
|
def finalizer
|
16
29
|
->(_) { close }
|
17
30
|
end
|
data/lib/rdkafka/config.rb
CHANGED
@@ -195,11 +195,13 @@ module Rdkafka
|
|
195
195
|
|
196
196
|
# Creates a consumer with this configuration.
|
197
197
|
#
|
198
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
199
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
198
200
|
# @return [Consumer] The created consumer
|
199
201
|
#
|
200
202
|
# @raise [ConfigError] When the configuration contains invalid options
|
201
203
|
# @raise [ClientCreationError] When the native client cannot be created
|
202
|
-
def consumer
|
204
|
+
def consumer(native_kafka_auto_start: true)
|
203
205
|
opaque = Opaque.new
|
204
206
|
config = native_config(opaque)
|
205
207
|
|
@@ -214,25 +216,26 @@ module Rdkafka
|
|
214
216
|
# Redirect the main queue to the consumer queue
|
215
217
|
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
|
216
218
|
|
217
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
218
|
-
|
219
219
|
# Return consumer with Kafka client
|
220
220
|
Rdkafka::Consumer.new(
|
221
221
|
Rdkafka::NativeKafka.new(
|
222
222
|
kafka,
|
223
223
|
run_polling_thread: false,
|
224
|
-
opaque: opaque
|
224
|
+
opaque: opaque,
|
225
|
+
auto_start: native_kafka_auto_start
|
225
226
|
)
|
226
227
|
)
|
227
228
|
end
|
228
229
|
|
229
230
|
# Create a producer with this configuration.
|
230
231
|
#
|
232
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
233
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
231
234
|
# @return [Producer] The created producer
|
232
235
|
#
|
233
236
|
# @raise [ConfigError] When the configuration contains invalid options
|
234
237
|
# @raise [ClientCreationError] When the native client cannot be created
|
235
|
-
def producer
|
238
|
+
def producer(native_kafka_auto_start: true)
|
236
239
|
# Create opaque
|
237
240
|
opaque = Opaque.new
|
238
241
|
# Create Kafka config
|
@@ -243,13 +246,13 @@ module Rdkafka
|
|
243
246
|
partitioner_name = self[:partitioner] || self["partitioner"]
|
244
247
|
|
245
248
|
kafka = native_kafka(config, :rd_kafka_producer)
|
246
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
247
249
|
|
248
250
|
Rdkafka::Producer.new(
|
249
251
|
Rdkafka::NativeKafka.new(
|
250
252
|
kafka,
|
251
253
|
run_polling_thread: true,
|
252
|
-
opaque: opaque
|
254
|
+
opaque: opaque,
|
255
|
+
auto_start: native_kafka_auto_start
|
253
256
|
),
|
254
257
|
partitioner_name
|
255
258
|
).tap do |producer|
|
@@ -259,23 +262,25 @@ module Rdkafka
|
|
259
262
|
|
260
263
|
# Creates an admin instance with this configuration.
|
261
264
|
#
|
265
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
266
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
262
267
|
# @return [Admin] The created admin instance
|
263
268
|
#
|
264
269
|
# @raise [ConfigError] When the configuration contains invalid options
|
265
270
|
# @raise [ClientCreationError] When the native client cannot be created
|
266
|
-
def admin
|
271
|
+
def admin(native_kafka_auto_start: true)
|
267
272
|
opaque = Opaque.new
|
268
273
|
config = native_config(opaque)
|
269
274
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
270
275
|
|
271
276
|
kafka = native_kafka(config, :rd_kafka_producer)
|
272
|
-
yield(kafka, Rdkafka::Bindings.rd_kafka_name(kafka)) if block_given?
|
273
277
|
|
274
278
|
Rdkafka::Admin.new(
|
275
279
|
Rdkafka::NativeKafka.new(
|
276
280
|
kafka,
|
277
281
|
run_polling_thread: true,
|
278
|
-
opaque: opaque
|
282
|
+
opaque: opaque,
|
283
|
+
auto_start: native_kafka_auto_start
|
279
284
|
)
|
280
285
|
)
|
281
286
|
end
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -20,6 +20,12 @@ module Rdkafka
|
|
20
20
|
@native_kafka = native_kafka
|
21
21
|
end
|
22
22
|
|
23
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
24
|
+
# @note Not needed to run unless explicit start was disabled
|
25
|
+
def start
|
26
|
+
@native_kafka.start
|
27
|
+
end
|
28
|
+
|
23
29
|
# @return [String] consumer name
|
24
30
|
def name
|
25
31
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/native_kafka.rb
CHANGED
@@ -4,7 +4,7 @@ module Rdkafka
|
|
4
4
|
# @private
|
5
5
|
# A wrapper around a native kafka that polls and cleanly exits
|
6
6
|
class NativeKafka
|
7
|
-
def initialize(inner, run_polling_thread:, opaque:)
|
7
|
+
def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
|
8
8
|
@inner = inner
|
9
9
|
@opaque = opaque
|
10
10
|
# Lock around external access
|
@@ -28,30 +28,43 @@ module Rdkafka
|
|
28
28
|
# counter for operations in progress using inner
|
29
29
|
@operations_in_progress = 0
|
30
30
|
|
31
|
-
|
32
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 0)
|
31
|
+
@run_polling_thread = run_polling_thread
|
33
32
|
|
34
|
-
if
|
35
|
-
# Start thread to poll client for delivery callbacks,
|
36
|
-
# not used in consumer.
|
37
|
-
@polling_thread = Thread.new do
|
38
|
-
loop do
|
39
|
-
@poll_mutex.synchronize do
|
40
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
41
|
-
end
|
33
|
+
start if auto_start
|
42
34
|
|
43
|
-
|
44
|
-
|
45
|
-
|
35
|
+
@closing = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
synchronize do
|
40
|
+
@return if @started
|
41
|
+
|
42
|
+
@started = true
|
43
|
+
|
44
|
+
# Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
|
45
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
|
46
|
+
|
47
|
+
if @run_polling_thread
|
48
|
+
# Start thread to poll client for delivery callbacks,
|
49
|
+
# not used in consumer.
|
50
|
+
@polling_thread = Thread.new do
|
51
|
+
loop do
|
52
|
+
@poll_mutex.synchronize do
|
53
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Exit thread if closing and the poll queue is empty
|
57
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
|
58
|
+
break
|
59
|
+
end
|
46
60
|
end
|
47
61
|
end
|
48
|
-
end
|
49
62
|
|
50
|
-
|
51
|
-
|
63
|
+
@polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
|
64
|
+
@polling_thread.abort_on_exception = true
|
65
|
+
@polling_thread[:closing] = false
|
66
|
+
end
|
52
67
|
end
|
53
|
-
|
54
|
-
@closing = false
|
55
68
|
end
|
56
69
|
|
57
70
|
def with_inner
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -54,6 +54,12 @@ module Rdkafka
|
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
57
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
58
|
+
# @note Not needed to run unless explicit start was disabled
|
59
|
+
def start
|
60
|
+
@native_kafka.start
|
61
|
+
end
|
62
|
+
|
57
63
|
# @return [String] producer name
|
58
64
|
def name
|
59
65
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/version.rb
CHANGED
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe 'admin without auto-start' do
|
35
|
+
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
|
+
|
37
|
+
it 'expect to be able to start it later and close' do
|
38
|
+
admin.start
|
39
|
+
admin.close
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'expect to be able to close it without starting' do
|
43
|
+
admin.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
34
47
|
describe "#create_topic" do
|
35
48
|
describe "called with invalid input" do
|
36
49
|
describe "with an invalid topic name" do
|
@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
|
|
14
14
|
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'consumer without auto-start' do
|
18
|
+
let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
consumer.start
|
22
|
+
consumer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
consumer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe "#subscribe, #unsubscribe and #subscription" do
|
18
31
|
it "should subscribe, unsubscribe and return the subscription" do
|
19
32
|
expect(consumer.subscription).to be_empty
|
@@ -10,8 +10,9 @@ describe Rdkafka::NativeKafka do
|
|
10
10
|
subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
|
11
11
|
|
12
12
|
before do
|
13
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_name).and_return('producer-1')
|
13
14
|
allow(Thread).to receive(:new).and_return(thread)
|
14
|
-
|
15
|
+
allow(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
|
15
16
|
allow(thread).to receive(:[]=).with(:closing, anything)
|
16
17
|
allow(thread).to receive(:join)
|
17
18
|
allow(thread).to receive(:abort_on_exception=).with(anything)
|
@@ -20,6 +21,12 @@ describe Rdkafka::NativeKafka do
|
|
20
21
|
after { client.close }
|
21
22
|
|
22
23
|
context "defaults" do
|
24
|
+
it "sets the thread name" do
|
25
|
+
expect(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
|
26
|
+
|
27
|
+
client
|
28
|
+
end
|
29
|
+
|
23
30
|
it "sets the thread to abort on exception" do
|
24
31
|
expect(thread).to receive(:abort_on_exception=).with(true)
|
25
32
|
|
@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
|
|
14
14
|
consumer.close
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'producer without auto-start' do
|
18
|
+
let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
producer.start
|
22
|
+
producer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
producer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe '#name' do
|
18
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
32
|
end
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.15.0.
|
4
|
+
version: 0.15.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-04-10 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
metadata.gz.sig
CHANGED
Binary file
|