rdkafka 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +57 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +155 -111
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +60 -39
- data/certs/cert_chain.pem +26 -0
- data/docker-compose.yml +18 -15
- data/ext/README.md +1 -1
- data/ext/Rakefile +43 -26
- data/lib/rdkafka/abstract_handle.rb +40 -26
- data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
- data/lib/rdkafka/admin/create_acl_report.rb +24 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
- data/lib/rdkafka/admin.rb +449 -7
- data/lib/rdkafka/bindings.rb +133 -7
- data/lib/rdkafka/callbacks.rb +196 -1
- data/lib/rdkafka/config.rb +53 -19
- data/lib/rdkafka/consumer/headers.rb +2 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +11 -8
- data/lib/rdkafka/consumer.rb +164 -74
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +22 -1
- data/lib/rdkafka/native_kafka.rb +6 -1
- data/lib/rdkafka/producer/delivery_handle.rb +12 -1
- data/lib/rdkafka/producer/delivery_report.rb +16 -3
- data/lib/rdkafka/producer.rb +121 -13
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +21 -1
- data/rdkafka.gemspec +19 -5
- data/renovate.json +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
- data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
- data/spec/rdkafka/admin_spec.rb +205 -2
- data/spec/rdkafka/bindings_spec.rb +0 -1
- data/spec/rdkafka/callbacks_spec.rb +0 -2
- data/spec/rdkafka/config_spec.rb +8 -2
- data/spec/rdkafka/consumer/headers_spec.rb +0 -2
- data/spec/rdkafka/consumer/message_spec.rb +0 -2
- data/spec/rdkafka/consumer/partition_spec.rb +0 -2
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +19 -2
- data/spec/rdkafka/consumer_spec.rb +232 -39
- data/spec/rdkafka/error_spec.rb +0 -2
- data/spec/rdkafka/metadata_spec.rb +2 -3
- data/spec/rdkafka/native_kafka_spec.rb +2 -3
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
- data/spec/rdkafka/producer/delivery_report_spec.rb +4 -2
- data/spec/rdkafka/producer_spec.rb +183 -3
- data/spec/spec_helper.rb +3 -1
- data.tar.gz.sig +0 -0
- metadata +78 -14
- metadata.gz.sig +0 -0
- data/.semaphore/semaphore.yml +0 -27
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
3
|
require "ostruct"
|
5
4
|
require 'securerandom'
|
6
5
|
|
@@ -11,6 +10,10 @@ describe Rdkafka::Consumer do
|
|
11
10
|
after { consumer.close }
|
12
11
|
after { producer.close }
|
13
12
|
|
13
|
+
describe '#name' do
|
14
|
+
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
|
+
end
|
16
|
+
|
14
17
|
describe "#subscribe, #unsubscribe and #subscription" do
|
15
18
|
it "should subscribe, unsubscribe and return the subscription" do
|
16
19
|
expect(consumer.subscription).to be_empty
|
@@ -51,6 +54,30 @@ describe Rdkafka::Consumer do
|
|
51
54
|
consumer.subscription
|
52
55
|
}.to raise_error(Rdkafka::RdkafkaError)
|
53
56
|
end
|
57
|
+
|
58
|
+
context "when using consumer without the poll set" do
|
59
|
+
let(:consumer) do
|
60
|
+
config = rdkafka_consumer_config
|
61
|
+
config.consumer_poll_set = false
|
62
|
+
config.consumer
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should subscribe, unsubscribe and return the subscription" do
|
66
|
+
expect(consumer.subscription).to be_empty
|
67
|
+
|
68
|
+
consumer.subscribe("consume_test_topic")
|
69
|
+
|
70
|
+
expect(consumer.subscription).not_to be_empty
|
71
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
72
|
+
list.add_topic("consume_test_topic")
|
73
|
+
end
|
74
|
+
expect(consumer.subscription).to eq expected_subscription
|
75
|
+
|
76
|
+
consumer.unsubscribe
|
77
|
+
|
78
|
+
expect(consumer.subscription).to be_empty
|
79
|
+
end
|
80
|
+
end
|
54
81
|
end
|
55
82
|
|
56
83
|
describe "#pause and #resume" do
|
@@ -270,6 +297,28 @@ describe Rdkafka::Consumer do
|
|
270
297
|
end
|
271
298
|
end
|
272
299
|
|
300
|
+
describe '#assignment_lost?' do
|
301
|
+
it "should not return true as we do have an assignment" do
|
302
|
+
consumer.subscribe("consume_test_topic")
|
303
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
304
|
+
list.add_topic("consume_test_topic")
|
305
|
+
end
|
306
|
+
|
307
|
+
expect(consumer.assignment_lost?).to eq false
|
308
|
+
consumer.unsubscribe
|
309
|
+
end
|
310
|
+
|
311
|
+
it "should not return true after voluntary unsubscribing" do
|
312
|
+
consumer.subscribe("consume_test_topic")
|
313
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
314
|
+
list.add_topic("consume_test_topic")
|
315
|
+
end
|
316
|
+
|
317
|
+
consumer.unsubscribe
|
318
|
+
expect(consumer.assignment_lost?).to eq false
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
273
322
|
describe "#close" do
|
274
323
|
it "should close a consumer" do
|
275
324
|
consumer.subscribe("consume_test_topic")
|
@@ -311,8 +360,9 @@ describe Rdkafka::Consumer do
|
|
311
360
|
end
|
312
361
|
end
|
313
362
|
|
314
|
-
|
315
|
-
|
363
|
+
|
364
|
+
describe "#position, #commit, #committed and #store_offset" do
|
365
|
+
# Make sure there are messages to work with
|
316
366
|
let!(:report) do
|
317
367
|
producer.produce(
|
318
368
|
topic: "consume_test_topic",
|
@@ -330,29 +380,33 @@ describe Rdkafka::Consumer do
|
|
330
380
|
)
|
331
381
|
end
|
332
382
|
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
383
|
+
describe "#position" do
|
384
|
+
it "should only accept a topic partition list in position if not nil" do
|
385
|
+
expect {
|
386
|
+
consumer.position("list")
|
387
|
+
}.to raise_error TypeError
|
388
|
+
end
|
337
389
|
end
|
338
390
|
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
391
|
+
describe "#committed" do
|
392
|
+
it "should only accept a topic partition list in commit if not nil" do
|
393
|
+
expect {
|
394
|
+
consumer.commit("list")
|
395
|
+
}.to raise_error TypeError
|
396
|
+
end
|
344
397
|
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
398
|
+
it "should commit in sync mode" do
|
399
|
+
expect {
|
400
|
+
consumer.commit(nil, true)
|
401
|
+
}.not_to raise_error
|
402
|
+
end
|
349
403
|
end
|
350
404
|
|
351
405
|
context "with a committed consumer" do
|
352
406
|
before :all do
|
353
407
|
# Make sure there are some messages.
|
354
408
|
handles = []
|
355
|
-
producer =
|
409
|
+
producer = rdkafka_config.producer
|
356
410
|
10.times do
|
357
411
|
(0..2).each do |i|
|
358
412
|
handles << producer.produce(
|
@@ -396,34 +450,38 @@ describe Rdkafka::Consumer do
|
|
396
450
|
}.to raise_error(Rdkafka::RdkafkaError)
|
397
451
|
end
|
398
452
|
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
453
|
+
describe "#committed" do
|
454
|
+
it "should fetch the committed offsets for the current assignment" do
|
455
|
+
partitions = consumer.committed.to_h["consume_test_topic"]
|
456
|
+
expect(partitions).not_to be_nil
|
457
|
+
expect(partitions[0].offset).to eq 1
|
458
|
+
end
|
404
459
|
|
405
|
-
|
406
|
-
|
407
|
-
|
460
|
+
it "should fetch the committed offsets for a specified topic partition list" do
|
461
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
462
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
463
|
+
end
|
464
|
+
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
465
|
+
expect(partitions).not_to be_nil
|
466
|
+
expect(partitions[0].offset).to eq 1
|
467
|
+
expect(partitions[1].offset).to eq 1
|
468
|
+
expect(partitions[2].offset).to eq 1
|
408
469
|
end
|
409
|
-
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
410
|
-
expect(partitions).not_to be_nil
|
411
|
-
expect(partitions[0].offset).to eq 1
|
412
|
-
expect(partitions[1].offset).to eq 1
|
413
|
-
expect(partitions[2].offset).to eq 1
|
414
|
-
end
|
415
470
|
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
471
|
+
it "should raise an error when getting committed fails" do
|
472
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
|
473
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
474
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
475
|
+
end
|
476
|
+
expect {
|
477
|
+
consumer.committed(list)
|
478
|
+
}.to raise_error Rdkafka::RdkafkaError
|
420
479
|
end
|
421
|
-
expect {
|
422
|
-
consumer.committed(list)
|
423
|
-
}.to raise_error Rdkafka::RdkafkaError
|
424
480
|
end
|
425
481
|
|
426
482
|
describe "#store_offset" do
|
483
|
+
let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
|
484
|
+
|
427
485
|
before do
|
428
486
|
config = {}
|
429
487
|
config[:'enable.auto.offset.store'] = false
|
@@ -441,6 +499,8 @@ describe Rdkafka::Consumer do
|
|
441
499
|
@new_consumer.store_offset(message)
|
442
500
|
@new_consumer.commit
|
443
501
|
|
502
|
+
# TODO use position here, should be at offset
|
503
|
+
|
444
504
|
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
445
505
|
list.add_topic("consume_test_topic", [0, 1, 2])
|
446
506
|
end
|
@@ -455,6 +515,43 @@ describe Rdkafka::Consumer do
|
|
455
515
|
@new_consumer.store_offset(message)
|
456
516
|
}.to raise_error Rdkafka::RdkafkaError
|
457
517
|
end
|
518
|
+
|
519
|
+
describe "#position" do
|
520
|
+
it "should fetch the positions for the current assignment" do
|
521
|
+
consumer.store_offset(message)
|
522
|
+
|
523
|
+
partitions = consumer.position.to_h["consume_test_topic"]
|
524
|
+
expect(partitions).not_to be_nil
|
525
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
526
|
+
end
|
527
|
+
|
528
|
+
it "should fetch the positions for a specified assignment" do
|
529
|
+
consumer.store_offset(message)
|
530
|
+
|
531
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
532
|
+
list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
|
533
|
+
end
|
534
|
+
partitions = consumer.position(list).to_h["consume_test_topic"]
|
535
|
+
expect(partitions).not_to be_nil
|
536
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
537
|
+
end
|
538
|
+
|
539
|
+
it "should raise an error when getting the position fails" do
|
540
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
|
541
|
+
|
542
|
+
expect {
|
543
|
+
consumer.position
|
544
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
545
|
+
end
|
546
|
+
end
|
547
|
+
|
548
|
+
context "when trying to use with enable.auto.offset.store set to true" do
|
549
|
+
let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': true).consumer }
|
550
|
+
|
551
|
+
it "expect to raise invalid configuration error" do
|
552
|
+
expect { consumer.store_offset(message) }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
|
553
|
+
end
|
554
|
+
end
|
458
555
|
end
|
459
556
|
end
|
460
557
|
end
|
@@ -950,6 +1047,102 @@ describe Rdkafka::Consumer do
|
|
950
1047
|
end
|
951
1048
|
end
|
952
1049
|
|
1050
|
+
describe "#offsets_for_times" do
|
1051
|
+
it "should raise when not TopicPartitionList" do
|
1052
|
+
expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
|
1053
|
+
end
|
1054
|
+
|
1055
|
+
it "should raise an error when offsets_for_times fails" do
|
1056
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
1057
|
+
|
1058
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
|
1059
|
+
|
1060
|
+
expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
|
1061
|
+
end
|
1062
|
+
|
1063
|
+
context "when subscribed" do
|
1064
|
+
let(:timeout) { 1000 }
|
1065
|
+
|
1066
|
+
before do
|
1067
|
+
consumer.subscribe("consume_test_topic")
|
1068
|
+
|
1069
|
+
# 1. partitions are assigned
|
1070
|
+
wait_for_assignment(consumer)
|
1071
|
+
expect(consumer.assignment).not_to be_empty
|
1072
|
+
|
1073
|
+
# 2. eat unrelated messages
|
1074
|
+
while(consumer.poll(timeout)) do; end
|
1075
|
+
end
|
1076
|
+
|
1077
|
+
after { consumer.unsubscribe }
|
1078
|
+
|
1079
|
+
def send_one_message(val)
|
1080
|
+
producer.produce(
|
1081
|
+
topic: "consume_test_topic",
|
1082
|
+
payload: "payload #{val}",
|
1083
|
+
key: "key 0",
|
1084
|
+
partition: 0
|
1085
|
+
).wait
|
1086
|
+
end
|
1087
|
+
|
1088
|
+
it "returns a TopicParticionList with updated offsets" do
|
1089
|
+
send_one_message("a")
|
1090
|
+
send_one_message("b")
|
1091
|
+
send_one_message("c")
|
1092
|
+
|
1093
|
+
consumer.poll(timeout)
|
1094
|
+
message = consumer.poll(timeout)
|
1095
|
+
consumer.poll(timeout)
|
1096
|
+
|
1097
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
1098
|
+
list.add_topic_and_partitions_with_offsets(
|
1099
|
+
"consume_test_topic",
|
1100
|
+
[
|
1101
|
+
[0, message.timestamp]
|
1102
|
+
]
|
1103
|
+
)
|
1104
|
+
end
|
1105
|
+
|
1106
|
+
tpl_response = consumer.offsets_for_times(tpl)
|
1107
|
+
|
1108
|
+
expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
|
1109
|
+
end
|
1110
|
+
end
|
1111
|
+
end
|
1112
|
+
|
1113
|
+
# Only relevant in case of a consumer with separate queues
|
1114
|
+
describe '#events_poll' do
|
1115
|
+
let(:stats) { [] }
|
1116
|
+
|
1117
|
+
before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
|
1118
|
+
|
1119
|
+
after { Rdkafka::Config.statistics_callback = nil }
|
1120
|
+
|
1121
|
+
let(:consumer) do
|
1122
|
+
config = rdkafka_consumer_config('statistics.interval.ms': 100)
|
1123
|
+
config.consumer_poll_set = false
|
1124
|
+
config.consumer
|
1125
|
+
end
|
1126
|
+
|
1127
|
+
it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
|
1128
|
+
consumer.subscribe("consume_test_topic")
|
1129
|
+
consumer.poll(1_000)
|
1130
|
+
expect(stats).to be_empty
|
1131
|
+
consumer.events_poll(-1)
|
1132
|
+
expect(stats).not_to be_empty
|
1133
|
+
end
|
1134
|
+
end
|
1135
|
+
|
1136
|
+
describe '#consumer_group_metadata_pointer' do
|
1137
|
+
let(:pointer) { consumer.consumer_group_metadata_pointer }
|
1138
|
+
|
1139
|
+
after { Rdkafka::Bindings.rd_kafka_consumer_group_metadata_destroy(pointer) }
|
1140
|
+
|
1141
|
+
it 'expect to return a pointer' do
|
1142
|
+
expect(pointer).to be_a(FFI::Pointer)
|
1143
|
+
end
|
1144
|
+
end
|
1145
|
+
|
953
1146
|
describe "a rebalance listener" do
|
954
1147
|
let(:consumer) do
|
955
1148
|
config = rdkafka_consumer_config
|
@@ -1024,7 +1217,7 @@ describe Rdkafka::Consumer do
|
|
1024
1217
|
:assign => [ nil ],
|
1025
1218
|
:assignment => nil,
|
1026
1219
|
:committed => [],
|
1027
|
-
:query_watermark_offsets => [ nil, nil ]
|
1220
|
+
:query_watermark_offsets => [ nil, nil ]
|
1028
1221
|
}.each do |method, args|
|
1029
1222
|
it "raises an exception if #{method} is called" do
|
1030
1223
|
expect {
|
data/spec/rdkafka/error_spec.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
3
|
require "securerandom"
|
5
4
|
|
6
5
|
describe Rdkafka::Metadata do
|
@@ -31,7 +30,7 @@ describe Rdkafka::Metadata do
|
|
31
30
|
it "#brokers returns our single broker" do
|
32
31
|
expect(subject.brokers.length).to eq(1)
|
33
32
|
expect(subject.brokers[0][:broker_id]).to eq(1)
|
34
|
-
expect(subject.brokers[0][:broker_name]).to eq("
|
33
|
+
expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
|
35
34
|
expect(subject.brokers[0][:broker_port]).to eq(9092)
|
36
35
|
end
|
37
36
|
|
@@ -54,7 +53,7 @@ describe Rdkafka::Metadata do
|
|
54
53
|
it "#brokers returns our single broker" do
|
55
54
|
expect(subject.brokers.length).to eq(1)
|
56
55
|
expect(subject.brokers[0][:broker_id]).to eq(1)
|
57
|
-
expect(subject.brokers[0][:broker_name]).to eq("
|
56
|
+
expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
|
58
57
|
expect(subject.brokers[0][:broker_port]).to eq(9092)
|
59
58
|
end
|
60
59
|
|
@@ -1,14 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
|
-
|
5
3
|
describe Rdkafka::NativeKafka do
|
6
4
|
let(:config) { rdkafka_producer_config }
|
7
5
|
let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
|
8
6
|
let(:closing) { false }
|
9
7
|
let(:thread) { double(Thread) }
|
8
|
+
let(:opaque) { Rdkafka::Opaque.new }
|
10
9
|
|
11
|
-
subject(:client) { described_class.new(native, run_polling_thread: true) }
|
10
|
+
subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
|
12
11
|
|
13
12
|
before do
|
14
13
|
allow(Thread).to receive(:new).and_return(thread)
|
@@ -1,7 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
|
-
|
5
3
|
describe Rdkafka::Producer::DeliveryReport do
|
6
4
|
subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
|
7
5
|
|
@@ -17,6 +15,10 @@ describe Rdkafka::Producer::DeliveryReport do
|
|
17
15
|
expect(subject.topic_name).to eq "topic"
|
18
16
|
end
|
19
17
|
|
18
|
+
it "should get the same topic name under topic alias" do
|
19
|
+
expect(subject.topic).to eq "topic"
|
20
|
+
end
|
21
|
+
|
20
22
|
it "should get the error" do
|
21
23
|
expect(subject.error).to eq -1
|
22
24
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
3
|
require "zlib"
|
5
4
|
|
6
5
|
describe Rdkafka::Producer do
|
@@ -15,6 +14,10 @@ describe Rdkafka::Producer do
|
|
15
14
|
consumer.close
|
16
15
|
end
|
17
16
|
|
17
|
+
describe '#name' do
|
18
|
+
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
|
+
end
|
20
|
+
|
18
21
|
context "delivery callback" do
|
19
22
|
context "with a proc/lambda" do
|
20
23
|
it "should set the callback" do
|
@@ -31,6 +34,7 @@ describe Rdkafka::Producer do
|
|
31
34
|
|
32
35
|
producer.delivery_callback = lambda do |report|
|
33
36
|
expect(report).not_to be_nil
|
37
|
+
expect(report.label).to eq "label"
|
34
38
|
expect(report.partition).to eq 1
|
35
39
|
expect(report.offset).to be >= 0
|
36
40
|
expect(report.topic_name).to eq "produce_test_topic"
|
@@ -41,9 +45,12 @@ describe Rdkafka::Producer do
|
|
41
45
|
handle = producer.produce(
|
42
46
|
topic: "produce_test_topic",
|
43
47
|
payload: "payload",
|
44
|
-
key: "key"
|
48
|
+
key: "key",
|
49
|
+
label: "label"
|
45
50
|
)
|
46
51
|
|
52
|
+
expect(handle.label).to eq "label"
|
53
|
+
|
47
54
|
# Wait for it to be delivered
|
48
55
|
handle.wait(max_wait_timeout: 15)
|
49
56
|
|
@@ -172,11 +179,13 @@ describe Rdkafka::Producer do
|
|
172
179
|
handle = producer.produce(
|
173
180
|
topic: "produce_test_topic",
|
174
181
|
payload: "payload",
|
175
|
-
key: "key"
|
182
|
+
key: "key",
|
183
|
+
label: "label"
|
176
184
|
)
|
177
185
|
|
178
186
|
# Should be pending at first
|
179
187
|
expect(handle.pending?).to be true
|
188
|
+
expect(handle.label).to eq "label"
|
180
189
|
|
181
190
|
# Check delivery handle and report
|
182
191
|
report = handle.wait(max_wait_timeout: 5)
|
@@ -184,6 +193,7 @@ describe Rdkafka::Producer do
|
|
184
193
|
expect(report).not_to be_nil
|
185
194
|
expect(report.partition).to eq 1
|
186
195
|
expect(report.offset).to be >= 0
|
196
|
+
expect(report.label).to eq "label"
|
187
197
|
|
188
198
|
# Flush and close producer
|
189
199
|
producer.flush
|
@@ -554,4 +564,174 @@ describe Rdkafka::Producer do
|
|
554
564
|
end
|
555
565
|
end
|
556
566
|
end
|
567
|
+
|
568
|
+
context "when not being able to deliver the message" do
|
569
|
+
let(:producer) do
|
570
|
+
rdkafka_producer_config(
|
571
|
+
"bootstrap.servers": "localhost:9093",
|
572
|
+
"message.timeout.ms": 100
|
573
|
+
).producer
|
574
|
+
end
|
575
|
+
|
576
|
+
it "should contain the error in the response when not deliverable" do
|
577
|
+
handler = producer.produce(topic: 'produce_test_topic', payload: nil, label: 'na')
|
578
|
+
# Wait for the async callbacks and delivery registry to update
|
579
|
+
sleep(2)
|
580
|
+
expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
|
581
|
+
expect(handler.create_result.label).to eq('na')
|
582
|
+
end
|
583
|
+
end
|
584
|
+
|
585
|
+
describe '#partition_count' do
|
586
|
+
it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
|
587
|
+
|
588
|
+
context 'when the partition count value is already cached' do
|
589
|
+
before do
|
590
|
+
producer.partition_count('consume_test_topic')
|
591
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
592
|
+
end
|
593
|
+
|
594
|
+
it 'expect not to query it again' do
|
595
|
+
producer.partition_count('consume_test_topic')
|
596
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
597
|
+
end
|
598
|
+
end
|
599
|
+
|
600
|
+
context 'when the partition count value was cached but time expired' do
|
601
|
+
before do
|
602
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
|
603
|
+
producer.partition_count('consume_test_topic')
|
604
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
605
|
+
end
|
606
|
+
|
607
|
+
it 'expect not to query it again' do
|
608
|
+
producer.partition_count('consume_test_topic')
|
609
|
+
expect(::Rdkafka::Metadata).to have_received(:new)
|
610
|
+
end
|
611
|
+
end
|
612
|
+
|
613
|
+
context 'when the partition count value was cached and time did not expire' do
|
614
|
+
before do
|
615
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 29.001)
|
616
|
+
producer.partition_count('consume_test_topic')
|
617
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
618
|
+
end
|
619
|
+
|
620
|
+
it 'expect not to query it again' do
|
621
|
+
producer.partition_count('consume_test_topic')
|
622
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
623
|
+
end
|
624
|
+
end
|
625
|
+
end
|
626
|
+
|
627
|
+
describe '#flush' do
|
628
|
+
it "should return flush when it can flush all outstanding messages or when no messages" do
|
629
|
+
producer.produce(
|
630
|
+
topic: "produce_test_topic",
|
631
|
+
payload: "payload headers",
|
632
|
+
key: "key headers",
|
633
|
+
headers: {}
|
634
|
+
)
|
635
|
+
|
636
|
+
expect(producer.flush(5_000)).to eq(true)
|
637
|
+
end
|
638
|
+
|
639
|
+
context 'when it cannot flush due to a timeout' do
|
640
|
+
let(:producer) do
|
641
|
+
rdkafka_producer_config(
|
642
|
+
"bootstrap.servers": "localhost:9093",
|
643
|
+
"message.timeout.ms": 2_000
|
644
|
+
).producer
|
645
|
+
end
|
646
|
+
|
647
|
+
after do
|
648
|
+
# Allow rdkafka to evict message preventing memory-leak
|
649
|
+
sleep(2)
|
650
|
+
end
|
651
|
+
|
652
|
+
it "should return false on flush when cannot deliver and beyond timeout" do
|
653
|
+
producer.produce(
|
654
|
+
topic: "produce_test_topic",
|
655
|
+
payload: "payload headers",
|
656
|
+
key: "key headers",
|
657
|
+
headers: {}
|
658
|
+
)
|
659
|
+
|
660
|
+
expect(producer.flush(1_000)).to eq(false)
|
661
|
+
end
|
662
|
+
end
|
663
|
+
|
664
|
+
context 'when there is a different error' do
|
665
|
+
before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
|
666
|
+
|
667
|
+
it 'should raise it' do
|
668
|
+
expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
|
669
|
+
end
|
670
|
+
end
|
671
|
+
end
|
672
|
+
|
673
|
+
describe '#purge' do
|
674
|
+
context 'when no outgoing messages' do
|
675
|
+
it { expect(producer.purge).to eq(true) }
|
676
|
+
end
|
677
|
+
|
678
|
+
context 'when librdkafka purge returns an error' do
|
679
|
+
before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
|
680
|
+
|
681
|
+
it 'expect to raise an error' do
|
682
|
+
expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
|
683
|
+
end
|
684
|
+
end
|
685
|
+
|
686
|
+
context 'when there are outgoing things in the queue' do
|
687
|
+
let(:producer) do
|
688
|
+
rdkafka_producer_config(
|
689
|
+
"bootstrap.servers": "localhost:9093",
|
690
|
+
"message.timeout.ms": 2_000
|
691
|
+
).producer
|
692
|
+
end
|
693
|
+
|
694
|
+
it "should should purge and move forward" do
|
695
|
+
producer.produce(
|
696
|
+
topic: "produce_test_topic",
|
697
|
+
payload: "payload headers"
|
698
|
+
)
|
699
|
+
|
700
|
+
expect(producer.purge).to eq(true)
|
701
|
+
expect(producer.flush(1_000)).to eq(true)
|
702
|
+
end
|
703
|
+
|
704
|
+
it "should materialize the delivery handles" do
|
705
|
+
handle = producer.produce(
|
706
|
+
topic: "produce_test_topic",
|
707
|
+
payload: "payload headers"
|
708
|
+
)
|
709
|
+
|
710
|
+
expect(producer.purge).to eq(true)
|
711
|
+
|
712
|
+
expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
|
713
|
+
end
|
714
|
+
|
715
|
+
context "when using delivery_callback" do
|
716
|
+
let(:delivery_reports) { [] }
|
717
|
+
|
718
|
+
let(:delivery_callback) do
|
719
|
+
->(delivery_report) { delivery_reports << delivery_report }
|
720
|
+
end
|
721
|
+
|
722
|
+
before { producer.delivery_callback = delivery_callback }
|
723
|
+
|
724
|
+
it "should run the callback" do
|
725
|
+
handle = producer.produce(
|
726
|
+
topic: "produce_test_topic",
|
727
|
+
payload: "payload headers"
|
728
|
+
)
|
729
|
+
|
730
|
+
expect(producer.purge).to eq(true)
|
731
|
+
# queue purge
|
732
|
+
expect(delivery_reports[0].error).to eq(-152)
|
733
|
+
end
|
734
|
+
end
|
735
|
+
end
|
736
|
+
end
|
557
737
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -11,6 +11,7 @@ require "pry"
|
|
11
11
|
require "rspec"
|
12
12
|
require "rdkafka"
|
13
13
|
require "timeout"
|
14
|
+
require "securerandom"
|
14
15
|
|
15
16
|
def rdkafka_base_config
|
16
17
|
{
|
@@ -35,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
|
|
35
36
|
# Add consumer specific fields to it
|
36
37
|
config[:"auto.offset.reset"] = "earliest"
|
37
38
|
config[:"enable.partition.eof"] = false
|
38
|
-
config[:"group.id"] = "ruby-test-#{
|
39
|
+
config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
|
39
40
|
# Enable debug mode if required
|
40
41
|
if ENV["DEBUG_CONSUMER"]
|
41
42
|
config[:debug] = "cgrp,topic,fetch"
|
@@ -134,6 +135,7 @@ RSpec.configure do |config|
|
|
134
135
|
rake_test_topic: 3,
|
135
136
|
watermarks_test_topic: 3,
|
136
137
|
partitioner_test_topic: 25,
|
138
|
+
example_topic: 1
|
137
139
|
}.each do |topic, partitions|
|
138
140
|
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
139
141
|
begin
|