rdkafka 0.12.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +58 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +141 -93
- data/Gemfile +2 -0
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +64 -29
- data/Rakefile +2 -0
- data/certs/cert_chain.pem +26 -0
- data/docker-compose.yml +18 -15
- data/ext/README.md +1 -1
- data/ext/Rakefile +3 -1
- data/lib/rdkafka/abstract_handle.rb +41 -25
- data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
- data/lib/rdkafka/admin/create_acl_report.rb +24 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
- data/lib/rdkafka/admin.rb +494 -35
- data/lib/rdkafka/bindings.rb +175 -40
- data/lib/rdkafka/callbacks.rb +194 -1
- data/lib/rdkafka/config.rb +62 -25
- data/lib/rdkafka/consumer/headers.rb +24 -9
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +13 -8
- data/lib/rdkafka/consumer.rb +219 -102
- data/lib/rdkafka/error.rb +15 -0
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +25 -2
- data/lib/rdkafka/native_kafka.rb +120 -0
- data/lib/rdkafka/producer/delivery_handle.rb +5 -2
- data/lib/rdkafka/producer/delivery_report.rb +9 -2
- data/lib/rdkafka/producer.rb +117 -17
- data/lib/rdkafka/version.rb +5 -3
- data/lib/rdkafka.rb +24 -2
- data/rdkafka.gemspec +19 -3
- data/renovate.json +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
- data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/create_topic_report_spec.rb +1 -1
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +1 -1
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
- data/spec/rdkafka/admin_spec.rb +209 -5
- data/spec/rdkafka/bindings_spec.rb +2 -1
- data/spec/rdkafka/callbacks_spec.rb +1 -1
- data/spec/rdkafka/config_spec.rb +24 -3
- data/spec/rdkafka/consumer/headers_spec.rb +60 -0
- data/spec/rdkafka/consumer/message_spec.rb +1 -1
- data/spec/rdkafka/consumer/partition_spec.rb +1 -1
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +20 -1
- data/spec/rdkafka/consumer_spec.rb +332 -61
- data/spec/rdkafka/error_spec.rb +1 -1
- data/spec/rdkafka/metadata_spec.rb +4 -3
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -35
- data/spec/rdkafka/producer/delivery_handle_spec.rb +4 -1
- data/spec/rdkafka/producer/delivery_report_spec.rb +7 -3
- data/spec/rdkafka/producer_spec.rb +208 -20
- data/spec/spec_helper.rb +20 -2
- data.tar.gz.sig +3 -0
- metadata +79 -16
- metadata.gz.sig +3 -0
- data/.semaphore/semaphore.yml +0 -23
- data/bin/console +0 -11
- data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,17 +1,15 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
|
-
describe Rdkafka::
|
3
|
+
describe Rdkafka::NativeKafka do
|
4
4
|
let(:config) { rdkafka_producer_config }
|
5
5
|
let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
|
6
6
|
let(:closing) { false }
|
7
7
|
let(:thread) { double(Thread) }
|
8
|
+
let(:opaque) { Rdkafka::Opaque.new }
|
8
9
|
|
9
|
-
subject(:client) { described_class.new(native) }
|
10
|
+
subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
|
10
11
|
|
11
12
|
before do
|
12
|
-
allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
|
13
|
-
allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
|
14
|
-
allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
|
15
13
|
allow(Thread).to receive(:new).and_return(thread)
|
16
14
|
|
17
15
|
allow(thread).to receive(:[]=).with(:closing, anything)
|
@@ -19,6 +17,8 @@ describe Rdkafka::Producer::Client do
|
|
19
17
|
allow(thread).to receive(:abort_on_exception=).with(anything)
|
20
18
|
end
|
21
19
|
|
20
|
+
after { client.close }
|
21
|
+
|
22
22
|
context "defaults" do
|
23
23
|
it "sets the thread to abort on exception" do
|
24
24
|
expect(thread).to receive(:abort_on_exception=).with(true)
|
@@ -39,32 +39,12 @@ describe Rdkafka::Producer::Client do
|
|
39
39
|
|
40
40
|
client
|
41
41
|
end
|
42
|
-
|
43
|
-
it "polls the native with default 250ms timeout" do
|
44
|
-
polling_loop_expects do
|
45
|
-
expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
it "check the out queue of native client" do
|
50
|
-
polling_loop_expects do
|
51
|
-
expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def polling_loop_expects(&block)
|
57
|
-
Thread.current[:closing] = true # this forces the loop break with line #12
|
58
|
-
|
59
|
-
allow(Thread).to receive(:new).and_yield do |_|
|
60
|
-
block.call
|
61
|
-
end.and_return(thread)
|
62
|
-
|
63
|
-
client
|
64
42
|
end
|
65
43
|
|
66
|
-
it "exposes
|
67
|
-
|
44
|
+
it "exposes the inner client" do
|
45
|
+
client.with_inner do |inner|
|
46
|
+
expect(inner).to eq(native)
|
47
|
+
end
|
68
48
|
end
|
69
49
|
|
70
50
|
context "when client was not yet closed (`nil`)" do
|
@@ -74,7 +54,7 @@ describe Rdkafka::Producer::Client do
|
|
74
54
|
|
75
55
|
context "and attempt to close" do
|
76
56
|
it "calls the `destroy` binding" do
|
77
|
-
expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
|
57
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
|
78
58
|
|
79
59
|
client.close
|
80
60
|
end
|
@@ -94,7 +74,6 @@ describe Rdkafka::Producer::Client do
|
|
94
74
|
it "closes and unassign the native client" do
|
95
75
|
client.close
|
96
76
|
|
97
|
-
expect(client.native).to eq(nil)
|
98
77
|
expect(client.closed?).to eq(true)
|
99
78
|
end
|
100
79
|
end
|
@@ -109,7 +88,7 @@ describe Rdkafka::Producer::Client do
|
|
109
88
|
|
110
89
|
context "and attempt to close again" do
|
111
90
|
it "does not call the `destroy` binding" do
|
112
|
-
expect(Rdkafka::Bindings).not_to receive(:
|
91
|
+
expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
|
113
92
|
|
114
93
|
client.close
|
115
94
|
end
|
@@ -129,13 +108,12 @@ describe Rdkafka::Producer::Client do
|
|
129
108
|
it "does not close and unassign the native client again" do
|
130
109
|
client.close
|
131
110
|
|
132
|
-
expect(client.native).to eq(nil)
|
133
111
|
expect(client.closed?).to eq(true)
|
134
112
|
end
|
135
113
|
end
|
136
114
|
end
|
137
115
|
|
138
|
-
it "
|
116
|
+
it "provides a finalizer that closes the native kafka client" do
|
139
117
|
expect(client.closed?).to eq(false)
|
140
118
|
|
141
119
|
client.finalizer.call("some-ignored-object-id")
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
describe Rdkafka::Producer::DeliveryHandle do
|
4
4
|
let(:response) { 0 }
|
@@ -9,6 +9,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
9
9
|
handle[:response] = response
|
10
10
|
handle[:partition] = 2
|
11
11
|
handle[:offset] = 100
|
12
|
+
handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
|
12
13
|
end
|
13
14
|
end
|
14
15
|
|
@@ -29,6 +30,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
29
30
|
|
30
31
|
expect(report.partition).to eq(2)
|
31
32
|
expect(report.offset).to eq(100)
|
33
|
+
expect(report.topic_name).to eq("produce_test_topic")
|
32
34
|
end
|
33
35
|
|
34
36
|
it "should wait without a timeout" do
|
@@ -36,6 +38,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
36
38
|
|
37
39
|
expect(report.partition).to eq(2)
|
38
40
|
expect(report.offset).to eq(100)
|
41
|
+
expect(report.topic_name).to eq("produce_test_topic")
|
39
42
|
end
|
40
43
|
end
|
41
44
|
end
|
@@ -1,7 +1,7 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
describe Rdkafka::Producer::DeliveryReport do
|
4
|
-
subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "
|
4
|
+
subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
|
5
5
|
|
6
6
|
it "should get the partition" do
|
7
7
|
expect(subject.partition).to eq 2
|
@@ -11,7 +11,11 @@ describe Rdkafka::Producer::DeliveryReport do
|
|
11
11
|
expect(subject.offset).to eq 100
|
12
12
|
end
|
13
13
|
|
14
|
+
it "should get the topic_name" do
|
15
|
+
expect(subject.topic_name).to eq "topic"
|
16
|
+
end
|
17
|
+
|
14
18
|
it "should get the error" do
|
15
|
-
expect(subject.error).to eq
|
19
|
+
expect(subject.error).to eq -1
|
16
20
|
end
|
17
21
|
end
|
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "zlib"
|
3
4
|
|
4
5
|
describe Rdkafka::Producer do
|
@@ -7,11 +8,16 @@ describe Rdkafka::Producer do
|
|
7
8
|
|
8
9
|
after do
|
9
10
|
# Registry should always end up being empty
|
10
|
-
|
11
|
+
registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
|
12
|
+
expect(registry).to be_empty, registry.inspect
|
11
13
|
producer.close
|
12
14
|
consumer.close
|
13
15
|
end
|
14
16
|
|
17
|
+
describe '#name' do
|
18
|
+
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
|
+
end
|
20
|
+
|
15
21
|
context "delivery callback" do
|
16
22
|
context "with a proc/lambda" do
|
17
23
|
it "should set the callback" do
|
@@ -30,6 +36,7 @@ describe Rdkafka::Producer do
|
|
30
36
|
expect(report).not_to be_nil
|
31
37
|
expect(report.partition).to eq 1
|
32
38
|
expect(report.offset).to be >= 0
|
39
|
+
expect(report.topic_name).to eq "produce_test_topic"
|
33
40
|
@callback_called = true
|
34
41
|
end
|
35
42
|
|
@@ -113,6 +120,7 @@ describe Rdkafka::Producer do
|
|
113
120
|
expect(called_report.first).not_to be_nil
|
114
121
|
expect(called_report.first.partition).to eq 1
|
115
122
|
expect(called_report.first.offset).to be >= 0
|
123
|
+
expect(called_report.first.topic_name).to eq "produce_test_topic"
|
116
124
|
end
|
117
125
|
|
118
126
|
it "should provide handle" do
|
@@ -180,10 +188,11 @@ describe Rdkafka::Producer do
|
|
180
188
|
expect(report.partition).to eq 1
|
181
189
|
expect(report.offset).to be >= 0
|
182
190
|
|
183
|
-
#
|
191
|
+
# Flush and close producer
|
192
|
+
producer.flush
|
184
193
|
producer.close
|
185
194
|
|
186
|
-
# Consume message and verify
|
195
|
+
# Consume message and verify its content
|
187
196
|
message = wait_for_message(
|
188
197
|
topic: "produce_test_topic",
|
189
198
|
delivery_report: report,
|
@@ -207,7 +216,7 @@ describe Rdkafka::Producer do
|
|
207
216
|
)
|
208
217
|
report = handle.wait(max_wait_timeout: 5)
|
209
218
|
|
210
|
-
# Consume message and verify
|
219
|
+
# Consume message and verify its content
|
211
220
|
message = wait_for_message(
|
212
221
|
topic: "produce_test_topic",
|
213
222
|
delivery_report: report,
|
@@ -251,6 +260,28 @@ describe Rdkafka::Producer do
|
|
251
260
|
expect(messages[2].key).to eq key
|
252
261
|
end
|
253
262
|
|
263
|
+
it "should produce a message with empty string without crashing" do
|
264
|
+
messages = [{key: 'a', partition_key: ''}]
|
265
|
+
|
266
|
+
messages = messages.map do |m|
|
267
|
+
handle = producer.produce(
|
268
|
+
topic: "partitioner_test_topic",
|
269
|
+
payload: "payload partition",
|
270
|
+
key: m[:key],
|
271
|
+
partition_key: m[:partition_key]
|
272
|
+
)
|
273
|
+
report = handle.wait(max_wait_timeout: 5)
|
274
|
+
|
275
|
+
wait_for_message(
|
276
|
+
topic: "partitioner_test_topic",
|
277
|
+
delivery_report: report,
|
278
|
+
)
|
279
|
+
end
|
280
|
+
|
281
|
+
expect(messages[0].partition).to eq 0
|
282
|
+
expect(messages[0].key).to eq 'a'
|
283
|
+
end
|
284
|
+
|
254
285
|
it "should produce a message with utf-8 encoding" do
|
255
286
|
handle = producer.produce(
|
256
287
|
topic: "produce_test_topic",
|
@@ -259,7 +290,7 @@ describe Rdkafka::Producer do
|
|
259
290
|
)
|
260
291
|
report = handle.wait(max_wait_timeout: 5)
|
261
292
|
|
262
|
-
# Consume message and verify
|
293
|
+
# Consume message and verify its content
|
263
294
|
message = wait_for_message(
|
264
295
|
topic: "produce_test_topic",
|
265
296
|
delivery_report: report,
|
@@ -292,7 +323,7 @@ describe Rdkafka::Producer do
|
|
292
323
|
)
|
293
324
|
report = handle.wait(max_wait_timeout: 5)
|
294
325
|
|
295
|
-
# Consume message and verify
|
326
|
+
# Consume message and verify its content
|
296
327
|
message = wait_for_message(
|
297
328
|
topic: "produce_test_topic",
|
298
329
|
delivery_report: report,
|
@@ -313,7 +344,7 @@ describe Rdkafka::Producer do
|
|
313
344
|
)
|
314
345
|
report = handle.wait(max_wait_timeout: 5)
|
315
346
|
|
316
|
-
# Consume message and verify
|
347
|
+
# Consume message and verify its content
|
317
348
|
message = wait_for_message(
|
318
349
|
topic: "produce_test_topic",
|
319
350
|
delivery_report: report,
|
@@ -333,7 +364,7 @@ describe Rdkafka::Producer do
|
|
333
364
|
)
|
334
365
|
report = handle.wait(max_wait_timeout: 5)
|
335
366
|
|
336
|
-
# Consume message and verify
|
367
|
+
# Consume message and verify its content
|
337
368
|
message = wait_for_message(
|
338
369
|
topic: "produce_test_topic",
|
339
370
|
delivery_report: report,
|
@@ -351,7 +382,7 @@ describe Rdkafka::Producer do
|
|
351
382
|
)
|
352
383
|
report = handle.wait(max_wait_timeout: 5)
|
353
384
|
|
354
|
-
# Consume message and verify
|
385
|
+
# Consume message and verify its content
|
355
386
|
message = wait_for_message(
|
356
387
|
topic: "produce_test_topic",
|
357
388
|
delivery_report: report,
|
@@ -371,7 +402,7 @@ describe Rdkafka::Producer do
|
|
371
402
|
)
|
372
403
|
report = handle.wait(max_wait_timeout: 5)
|
373
404
|
|
374
|
-
# Consume message and verify
|
405
|
+
# Consume message and verify its content
|
375
406
|
message = wait_for_message(
|
376
407
|
topic: "produce_test_topic",
|
377
408
|
delivery_report: report,
|
@@ -380,9 +411,9 @@ describe Rdkafka::Producer do
|
|
380
411
|
|
381
412
|
expect(message.payload).to eq "payload headers"
|
382
413
|
expect(message.key).to eq "key headers"
|
383
|
-
expect(message.headers[
|
384
|
-
expect(message.headers[
|
385
|
-
expect(message.headers[
|
414
|
+
expect(message.headers["foo"]).to eq "bar"
|
415
|
+
expect(message.headers["baz"]).to eq "foobar"
|
416
|
+
expect(message.headers["foobar"]).to be_nil
|
386
417
|
end
|
387
418
|
|
388
419
|
it "should produce a message with empty headers" do
|
@@ -394,7 +425,7 @@ describe Rdkafka::Producer do
|
|
394
425
|
)
|
395
426
|
report = handle.wait(max_wait_timeout: 5)
|
396
427
|
|
397
|
-
# Consume message and verify
|
428
|
+
# Consume message and verify its content
|
398
429
|
message = wait_for_message(
|
399
430
|
topic: "produce_test_topic",
|
400
431
|
delivery_report: report,
|
@@ -432,10 +463,10 @@ describe Rdkafka::Producer do
|
|
432
463
|
# wait for and check the message in the main process.
|
433
464
|
reader, writer = IO.pipe
|
434
465
|
|
435
|
-
fork do
|
466
|
+
pid = fork do
|
436
467
|
reader.close
|
437
468
|
|
438
|
-
#
|
469
|
+
# Avoid sharing the client between processes.
|
439
470
|
producer = rdkafka_producer_config.producer
|
440
471
|
|
441
472
|
handle = producer.produce(
|
@@ -448,24 +479,28 @@ describe Rdkafka::Producer do
|
|
448
479
|
|
449
480
|
report_json = JSON.generate(
|
450
481
|
"partition" => report.partition,
|
451
|
-
"offset" => report.offset
|
482
|
+
"offset" => report.offset,
|
483
|
+
"topic_name" => report.topic_name
|
452
484
|
)
|
453
485
|
|
454
486
|
writer.write(report_json)
|
455
487
|
writer.close
|
488
|
+
producer.flush
|
456
489
|
producer.close
|
457
490
|
end
|
491
|
+
Process.wait(pid)
|
458
492
|
|
459
493
|
writer.close
|
460
494
|
report_hash = JSON.parse(reader.read)
|
461
495
|
report = Rdkafka::Producer::DeliveryReport.new(
|
462
496
|
report_hash["partition"],
|
463
|
-
report_hash["offset"]
|
497
|
+
report_hash["offset"],
|
498
|
+
report_hash["topic_name"]
|
464
499
|
)
|
465
500
|
|
466
501
|
reader.close
|
467
502
|
|
468
|
-
# Consume message and verify
|
503
|
+
# Consume message and verify its content
|
469
504
|
message = wait_for_message(
|
470
505
|
topic: "produce_test_topic",
|
471
506
|
delivery_report: report,
|
@@ -522,4 +557,157 @@ describe Rdkafka::Producer do
|
|
522
557
|
end
|
523
558
|
end
|
524
559
|
end
|
560
|
+
|
561
|
+
describe '#partition_count' do
|
562
|
+
it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
|
563
|
+
|
564
|
+
context 'when the partition count value is already cached' do
|
565
|
+
before do
|
566
|
+
producer.partition_count('consume_test_topic')
|
567
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
568
|
+
end
|
569
|
+
|
570
|
+
it 'expect not to query it again' do
|
571
|
+
producer.partition_count('consume_test_topic')
|
572
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
context 'when the partition count value was cached but time expired' do
|
577
|
+
before do
|
578
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
|
579
|
+
producer.partition_count('consume_test_topic')
|
580
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
581
|
+
end
|
582
|
+
|
583
|
+
it 'expect not to query it again' do
|
584
|
+
producer.partition_count('consume_test_topic')
|
585
|
+
expect(::Rdkafka::Metadata).to have_received(:new)
|
586
|
+
end
|
587
|
+
end
|
588
|
+
|
589
|
+
context 'when the partition count value was cached and time did not expire' do
|
590
|
+
before do
|
591
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 29.001)
|
592
|
+
producer.partition_count('consume_test_topic')
|
593
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
594
|
+
end
|
595
|
+
|
596
|
+
it 'expect not to query it again' do
|
597
|
+
producer.partition_count('consume_test_topic')
|
598
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
599
|
+
end
|
600
|
+
end
|
601
|
+
end
|
602
|
+
|
603
|
+
describe '#flush' do
|
604
|
+
it "should return flush when it can flush all outstanding messages or when no messages" do
|
605
|
+
producer.produce(
|
606
|
+
topic: "produce_test_topic",
|
607
|
+
payload: "payload headers",
|
608
|
+
key: "key headers",
|
609
|
+
headers: {}
|
610
|
+
)
|
611
|
+
|
612
|
+
expect(producer.flush(5_000)).to eq(true)
|
613
|
+
end
|
614
|
+
|
615
|
+
context 'when it cannot flush due to a timeout' do
|
616
|
+
let(:producer) do
|
617
|
+
rdkafka_producer_config(
|
618
|
+
"bootstrap.servers": "localhost:9093",
|
619
|
+
"message.timeout.ms": 2_000
|
620
|
+
).producer
|
621
|
+
end
|
622
|
+
|
623
|
+
after do
|
624
|
+
# Allow rdkafka to evict message preventing memory-leak
|
625
|
+
sleep(2)
|
626
|
+
end
|
627
|
+
|
628
|
+
it "should return false on flush when cannot deliver and beyond timeout" do
|
629
|
+
producer.produce(
|
630
|
+
topic: "produce_test_topic",
|
631
|
+
payload: "payload headers",
|
632
|
+
key: "key headers",
|
633
|
+
headers: {}
|
634
|
+
)
|
635
|
+
|
636
|
+
expect(producer.flush(1_000)).to eq(false)
|
637
|
+
end
|
638
|
+
end
|
639
|
+
|
640
|
+
context 'when there is a different error' do
|
641
|
+
before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
|
642
|
+
|
643
|
+
it 'should raise it' do
|
644
|
+
expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
|
645
|
+
end
|
646
|
+
end
|
647
|
+
end
|
648
|
+
|
649
|
+
describe '#purge' do
|
650
|
+
context 'when no outgoing messages' do
|
651
|
+
it { expect(producer.purge).to eq(true) }
|
652
|
+
end
|
653
|
+
|
654
|
+
context 'when librdkafka purge returns an error' do
|
655
|
+
before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
|
656
|
+
|
657
|
+
it 'expect to raise an error' do
|
658
|
+
expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
|
659
|
+
end
|
660
|
+
end
|
661
|
+
|
662
|
+
context 'when there are outgoing things in the queue' do
|
663
|
+
let(:producer) do
|
664
|
+
rdkafka_producer_config(
|
665
|
+
"bootstrap.servers": "localhost:9093",
|
666
|
+
"message.timeout.ms": 2_000
|
667
|
+
).producer
|
668
|
+
end
|
669
|
+
|
670
|
+
it "should should purge and move forward" do
|
671
|
+
producer.produce(
|
672
|
+
topic: "produce_test_topic",
|
673
|
+
payload: "payload headers"
|
674
|
+
)
|
675
|
+
|
676
|
+
expect(producer.purge).to eq(true)
|
677
|
+
expect(producer.flush(1_000)).to eq(true)
|
678
|
+
end
|
679
|
+
|
680
|
+
it "should materialize the delivery handles" do
|
681
|
+
handle = producer.produce(
|
682
|
+
topic: "produce_test_topic",
|
683
|
+
payload: "payload headers"
|
684
|
+
)
|
685
|
+
|
686
|
+
expect(producer.purge).to eq(true)
|
687
|
+
|
688
|
+
expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
|
689
|
+
end
|
690
|
+
|
691
|
+
context "when using delivery_callback" do
|
692
|
+
let(:delivery_reports) { [] }
|
693
|
+
|
694
|
+
let(:delivery_callback) do
|
695
|
+
->(delivery_report) { delivery_reports << delivery_report }
|
696
|
+
end
|
697
|
+
|
698
|
+
before { producer.delivery_callback = delivery_callback }
|
699
|
+
|
700
|
+
it "should run the callback" do
|
701
|
+
handle = producer.produce(
|
702
|
+
topic: "produce_test_topic",
|
703
|
+
payload: "payload headers"
|
704
|
+
)
|
705
|
+
|
706
|
+
expect(producer.purge).to eq(true)
|
707
|
+
# queue purge
|
708
|
+
expect(delivery_reports[0].error).to eq(-152)
|
709
|
+
end
|
710
|
+
end
|
711
|
+
end
|
712
|
+
end
|
525
713
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
unless ENV["CI"] == "true"
|
2
4
|
require "simplecov"
|
3
5
|
SimpleCov.start do
|
@@ -9,6 +11,7 @@ require "pry"
|
|
9
11
|
require "rspec"
|
10
12
|
require "rdkafka"
|
11
13
|
require "timeout"
|
14
|
+
require "securerandom"
|
12
15
|
|
13
16
|
def rdkafka_base_config
|
14
17
|
{
|
@@ -33,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
|
|
33
36
|
# Add consumer specific fields to it
|
34
37
|
config[:"auto.offset.reset"] = "earliest"
|
35
38
|
config[:"enable.partition.eof"] = false
|
36
|
-
config[:"group.id"] = "ruby-test-#{
|
39
|
+
config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
|
37
40
|
# Enable debug mode if required
|
38
41
|
if ENV["DEBUG_CONSUMER"]
|
39
42
|
config[:debug] = "cgrp,topic,fetch"
|
@@ -71,7 +74,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
|
|
71
74
|
end
|
72
75
|
|
73
76
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
|
74
|
-
new_consumer =
|
77
|
+
new_consumer = consumer.nil?
|
75
78
|
consumer ||= rdkafka_consumer_config.consumer
|
76
79
|
consumer.subscribe(topic)
|
77
80
|
timeout = Time.now.to_i + timeout_in_seconds
|
@@ -104,6 +107,20 @@ def wait_for_unassignment(consumer)
|
|
104
107
|
end
|
105
108
|
end
|
106
109
|
|
110
|
+
def notify_listener(listener, &block)
|
111
|
+
# 1. subscribe and poll
|
112
|
+
consumer.subscribe("consume_test_topic")
|
113
|
+
wait_for_assignment(consumer)
|
114
|
+
consumer.poll(100)
|
115
|
+
|
116
|
+
block.call if block
|
117
|
+
|
118
|
+
# 2. unsubscribe
|
119
|
+
consumer.unsubscribe
|
120
|
+
wait_for_unassignment(consumer)
|
121
|
+
consumer.close
|
122
|
+
end
|
123
|
+
|
107
124
|
RSpec.configure do |config|
|
108
125
|
config.filter_run focus: true
|
109
126
|
config.run_all_when_everything_filtered = true
|
@@ -118,6 +135,7 @@ RSpec.configure do |config|
|
|
118
135
|
rake_test_topic: 3,
|
119
136
|
watermarks_test_topic: 3,
|
120
137
|
partitioner_test_topic: 25,
|
138
|
+
example_topic: 1
|
121
139
|
}.each do |topic, partitions|
|
122
140
|
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
123
141
|
begin
|
data.tar.gz.sig
ADDED
@@ -0,0 +1,3 @@
|
|
1
|
+
I��:z�mY0��+��!vr��V��,g��,>���56�r��5���*Z�魇Q]##�)����y9��e\�w����^�F2ֲ�u�(7��u�A��(;]@���5��_��I�vF���&8���_�}2ܐ��.��F�=�f��V٭fI$<�1�M�8����C�o'2]�ŵg�6H��o_��B��|A�5F�Mx�J�N��k�)�Y��gC�r��"ib���sy>����Q��ڋCW9ڮoz�
|
2
|
+
�k���q�q�������s�M&Y�4&�����+�]1�,Tf(DE
|
3
|
+
9��U���"Ķ>�_g;��
|