rdkafka 0.12.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +58 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +141 -93
- data/Gemfile +2 -0
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +64 -29
- data/Rakefile +2 -0
- data/certs/cert_chain.pem +26 -0
- data/docker-compose.yml +18 -15
- data/ext/README.md +1 -1
- data/ext/Rakefile +3 -1
- data/lib/rdkafka/abstract_handle.rb +41 -25
- data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
- data/lib/rdkafka/admin/create_acl_report.rb +24 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
- data/lib/rdkafka/admin.rb +494 -35
- data/lib/rdkafka/bindings.rb +175 -40
- data/lib/rdkafka/callbacks.rb +194 -1
- data/lib/rdkafka/config.rb +62 -25
- data/lib/rdkafka/consumer/headers.rb +24 -9
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +13 -8
- data/lib/rdkafka/consumer.rb +219 -102
- data/lib/rdkafka/error.rb +15 -0
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +25 -2
- data/lib/rdkafka/native_kafka.rb +120 -0
- data/lib/rdkafka/producer/delivery_handle.rb +5 -2
- data/lib/rdkafka/producer/delivery_report.rb +9 -2
- data/lib/rdkafka/producer.rb +117 -17
- data/lib/rdkafka/version.rb +5 -3
- data/lib/rdkafka.rb +24 -2
- data/rdkafka.gemspec +19 -3
- data/renovate.json +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
- data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/create_topic_report_spec.rb +1 -1
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +1 -1
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +1 -1
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
- data/spec/rdkafka/admin_spec.rb +209 -5
- data/spec/rdkafka/bindings_spec.rb +2 -1
- data/spec/rdkafka/callbacks_spec.rb +1 -1
- data/spec/rdkafka/config_spec.rb +24 -3
- data/spec/rdkafka/consumer/headers_spec.rb +60 -0
- data/spec/rdkafka/consumer/message_spec.rb +1 -1
- data/spec/rdkafka/consumer/partition_spec.rb +1 -1
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +20 -1
- data/spec/rdkafka/consumer_spec.rb +332 -61
- data/spec/rdkafka/error_spec.rb +1 -1
- data/spec/rdkafka/metadata_spec.rb +4 -3
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -35
- data/spec/rdkafka/producer/delivery_handle_spec.rb +4 -1
- data/spec/rdkafka/producer/delivery_report_spec.rb +7 -3
- data/spec/rdkafka/producer_spec.rb +208 -20
- data/spec/spec_helper.rb +20 -2
- data.tar.gz.sig +3 -0
- metadata +79 -16
- metadata.gz.sig +3 -0
- data/.semaphore/semaphore.yml +0 -23
- data/bin/console +0 -11
- data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "ostruct"
|
3
4
|
require 'securerandom'
|
4
5
|
|
@@ -9,6 +10,10 @@ describe Rdkafka::Consumer do
|
|
9
10
|
after { consumer.close }
|
10
11
|
after { producer.close }
|
11
12
|
|
13
|
+
describe '#name' do
|
14
|
+
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
|
+
end
|
16
|
+
|
12
17
|
describe "#subscribe, #unsubscribe and #subscription" do
|
13
18
|
it "should subscribe, unsubscribe and return the subscription" do
|
14
19
|
expect(consumer.subscription).to be_empty
|
@@ -49,11 +54,35 @@ describe Rdkafka::Consumer do
|
|
49
54
|
consumer.subscription
|
50
55
|
}.to raise_error(Rdkafka::RdkafkaError)
|
51
56
|
end
|
57
|
+
|
58
|
+
context "when using consumer without the poll set" do
|
59
|
+
let(:consumer) do
|
60
|
+
config = rdkafka_consumer_config
|
61
|
+
config.consumer_poll_set = false
|
62
|
+
config.consumer
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should subscribe, unsubscribe and return the subscription" do
|
66
|
+
expect(consumer.subscription).to be_empty
|
67
|
+
|
68
|
+
consumer.subscribe("consume_test_topic")
|
69
|
+
|
70
|
+
expect(consumer.subscription).not_to be_empty
|
71
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
72
|
+
list.add_topic("consume_test_topic")
|
73
|
+
end
|
74
|
+
expect(consumer.subscription).to eq expected_subscription
|
75
|
+
|
76
|
+
consumer.unsubscribe
|
77
|
+
|
78
|
+
expect(consumer.subscription).to be_empty
|
79
|
+
end
|
80
|
+
end
|
52
81
|
end
|
53
82
|
|
54
83
|
describe "#pause and #resume" do
|
55
84
|
context "subscription" do
|
56
|
-
let(:timeout) {
|
85
|
+
let(:timeout) { 2000 }
|
57
86
|
|
58
87
|
before { consumer.subscribe("consume_test_topic") }
|
59
88
|
after { consumer.unsubscribe }
|
@@ -268,6 +297,28 @@ describe Rdkafka::Consumer do
|
|
268
297
|
end
|
269
298
|
end
|
270
299
|
|
300
|
+
describe '#assignment_lost?' do
|
301
|
+
it "should not return true as we do have an assignment" do
|
302
|
+
consumer.subscribe("consume_test_topic")
|
303
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
304
|
+
list.add_topic("consume_test_topic")
|
305
|
+
end
|
306
|
+
|
307
|
+
expect(consumer.assignment_lost?).to eq false
|
308
|
+
consumer.unsubscribe
|
309
|
+
end
|
310
|
+
|
311
|
+
it "should not return true after voluntary unsubscribing" do
|
312
|
+
consumer.subscribe("consume_test_topic")
|
313
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
314
|
+
list.add_topic("consume_test_topic")
|
315
|
+
end
|
316
|
+
|
317
|
+
consumer.unsubscribe
|
318
|
+
expect(consumer.assignment_lost?).to eq false
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
271
322
|
describe "#close" do
|
272
323
|
it "should close a consumer" do
|
273
324
|
consumer.subscribe("consume_test_topic")
|
@@ -284,10 +335,34 @@ describe Rdkafka::Consumer do
|
|
284
335
|
consumer.poll(100)
|
285
336
|
}.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
|
286
337
|
end
|
338
|
+
|
339
|
+
context 'when there are outgoing operations in other threads' do
|
340
|
+
it 'should wait and not crash' do
|
341
|
+
times = []
|
342
|
+
|
343
|
+
# Run a long running poll
|
344
|
+
thread = Thread.new do
|
345
|
+
times << Time.now
|
346
|
+
consumer.subscribe("empty_test_topic")
|
347
|
+
times << Time.now
|
348
|
+
consumer.poll(1_000)
|
349
|
+
times << Time.now
|
350
|
+
end
|
351
|
+
|
352
|
+
# Make sure it starts before we close
|
353
|
+
sleep(0.1)
|
354
|
+
consumer.close
|
355
|
+
close_time = Time.now
|
356
|
+
thread.join
|
357
|
+
|
358
|
+
times.each { |op_time| expect(op_time).to be < close_time }
|
359
|
+
end
|
360
|
+
end
|
287
361
|
end
|
288
362
|
|
289
|
-
|
290
|
-
|
363
|
+
|
364
|
+
describe "#position, #commit, #committed and #store_offset" do
|
365
|
+
# Make sure there are messages to work with
|
291
366
|
let!(:report) do
|
292
367
|
producer.produce(
|
293
368
|
topic: "consume_test_topic",
|
@@ -305,29 +380,33 @@ describe Rdkafka::Consumer do
|
|
305
380
|
)
|
306
381
|
end
|
307
382
|
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
383
|
+
describe "#position" do
|
384
|
+
it "should only accept a topic partition list in position if not nil" do
|
385
|
+
expect {
|
386
|
+
consumer.position("list")
|
387
|
+
}.to raise_error TypeError
|
388
|
+
end
|
312
389
|
end
|
313
390
|
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
391
|
+
describe "#committed" do
|
392
|
+
it "should only accept a topic partition list in commit if not nil" do
|
393
|
+
expect {
|
394
|
+
consumer.commit("list")
|
395
|
+
}.to raise_error TypeError
|
396
|
+
end
|
319
397
|
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
398
|
+
it "should commit in sync mode" do
|
399
|
+
expect {
|
400
|
+
consumer.commit(nil, true)
|
401
|
+
}.not_to raise_error
|
402
|
+
end
|
324
403
|
end
|
325
404
|
|
326
405
|
context "with a committed consumer" do
|
327
406
|
before :all do
|
328
407
|
# Make sure there are some messages.
|
329
408
|
handles = []
|
330
|
-
producer =
|
409
|
+
producer = rdkafka_config.producer
|
331
410
|
10.times do
|
332
411
|
(0..2).each do |i|
|
333
412
|
handles << producer.produce(
|
@@ -371,31 +450,33 @@ describe Rdkafka::Consumer do
|
|
371
450
|
}.to raise_error(Rdkafka::RdkafkaError)
|
372
451
|
end
|
373
452
|
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
453
|
+
describe "#committed" do
|
454
|
+
it "should fetch the committed offsets for the current assignment" do
|
455
|
+
partitions = consumer.committed.to_h["consume_test_topic"]
|
456
|
+
expect(partitions).not_to be_nil
|
457
|
+
expect(partitions[0].offset).to eq 1
|
458
|
+
end
|
379
459
|
|
380
|
-
|
381
|
-
|
382
|
-
|
460
|
+
it "should fetch the committed offsets for a specified topic partition list" do
|
461
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
462
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
463
|
+
end
|
464
|
+
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
465
|
+
expect(partitions).not_to be_nil
|
466
|
+
expect(partitions[0].offset).to eq 1
|
467
|
+
expect(partitions[1].offset).to eq 1
|
468
|
+
expect(partitions[2].offset).to eq 1
|
383
469
|
end
|
384
|
-
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
385
|
-
expect(partitions).not_to be_nil
|
386
|
-
expect(partitions[0].offset).to eq 1
|
387
|
-
expect(partitions[1].offset).to eq 1
|
388
|
-
expect(partitions[2].offset).to eq 1
|
389
|
-
end
|
390
470
|
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
471
|
+
it "should raise an error when getting committed fails" do
|
472
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
|
473
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
474
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
475
|
+
end
|
476
|
+
expect {
|
477
|
+
consumer.committed(list)
|
478
|
+
}.to raise_error Rdkafka::RdkafkaError
|
395
479
|
end
|
396
|
-
expect {
|
397
|
-
consumer.committed(list)
|
398
|
-
}.to raise_error Rdkafka::RdkafkaError
|
399
480
|
end
|
400
481
|
|
401
482
|
describe "#store_offset" do
|
@@ -416,6 +497,8 @@ describe Rdkafka::Consumer do
|
|
416
497
|
@new_consumer.store_offset(message)
|
417
498
|
@new_consumer.commit
|
418
499
|
|
500
|
+
# TODO use position here, should be at offset
|
501
|
+
|
419
502
|
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
420
503
|
list.add_topic("consume_test_topic", [0, 1, 2])
|
421
504
|
end
|
@@ -430,6 +513,35 @@ describe Rdkafka::Consumer do
|
|
430
513
|
@new_consumer.store_offset(message)
|
431
514
|
}.to raise_error Rdkafka::RdkafkaError
|
432
515
|
end
|
516
|
+
|
517
|
+
describe "#position" do
|
518
|
+
it "should fetch the positions for the current assignment" do
|
519
|
+
consumer.store_offset(message)
|
520
|
+
|
521
|
+
partitions = consumer.position.to_h["consume_test_topic"]
|
522
|
+
expect(partitions).not_to be_nil
|
523
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
524
|
+
end
|
525
|
+
|
526
|
+
it "should fetch the positions for a specified assignment" do
|
527
|
+
consumer.store_offset(message)
|
528
|
+
|
529
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
530
|
+
list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
|
531
|
+
end
|
532
|
+
partitions = consumer.position(list).to_h["consume_test_topic"]
|
533
|
+
expect(partitions).not_to be_nil
|
534
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
535
|
+
end
|
536
|
+
|
537
|
+
it "should raise an error when getting the position fails" do
|
538
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
|
539
|
+
|
540
|
+
expect {
|
541
|
+
consumer.position
|
542
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
543
|
+
end
|
544
|
+
end
|
433
545
|
end
|
434
546
|
end
|
435
547
|
end
|
@@ -593,7 +705,7 @@ describe Rdkafka::Consumer do
|
|
593
705
|
end
|
594
706
|
|
595
707
|
describe "#poll with headers" do
|
596
|
-
it "should return message with headers" do
|
708
|
+
it "should return message with headers using string keys (when produced with symbol keys)" do
|
597
709
|
report = producer.produce(
|
598
710
|
topic: "consume_test_topic",
|
599
711
|
key: "key headers",
|
@@ -603,7 +715,20 @@ describe Rdkafka::Consumer do
|
|
603
715
|
message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
|
604
716
|
expect(message).to be
|
605
717
|
expect(message.key).to eq('key headers')
|
606
|
-
expect(message.headers).to include(foo
|
718
|
+
expect(message.headers).to include('foo' => 'bar')
|
719
|
+
end
|
720
|
+
|
721
|
+
it "should return message with headers using string keys (when produced with string keys)" do
|
722
|
+
report = producer.produce(
|
723
|
+
topic: "consume_test_topic",
|
724
|
+
key: "key headers",
|
725
|
+
headers: { 'foo' => 'bar' }
|
726
|
+
).wait
|
727
|
+
|
728
|
+
message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
|
729
|
+
expect(message).to be
|
730
|
+
expect(message.key).to eq('key headers')
|
731
|
+
expect(message.headers).to include('foo' => 'bar')
|
607
732
|
end
|
608
733
|
|
609
734
|
it "should return message with no headers" do
|
@@ -698,7 +823,7 @@ describe Rdkafka::Consumer do
|
|
698
823
|
n.times do |i|
|
699
824
|
handles << producer.produce(
|
700
825
|
topic: topic_name,
|
701
|
-
payload: Time.new.to_f.to_s,
|
826
|
+
payload: i % 10 == 0 ? nil : Time.new.to_f.to_s,
|
702
827
|
key: i.to_s,
|
703
828
|
partition: 0
|
704
829
|
)
|
@@ -723,7 +848,8 @@ describe Rdkafka::Consumer do
|
|
723
848
|
#
|
724
849
|
# This is, in effect, an integration test and the subsequent specs are
|
725
850
|
# unit tests.
|
726
|
-
|
851
|
+
admin = rdkafka_config.admin
|
852
|
+
create_topic_handle = admin.create_topic(topic_name, 1, 1)
|
727
853
|
create_topic_handle.wait(max_wait_timeout: 15.0)
|
728
854
|
consumer.subscribe(topic_name)
|
729
855
|
produce_n 42
|
@@ -736,6 +862,7 @@ describe Rdkafka::Consumer do
|
|
736
862
|
expect(all_yields.flatten.size).to eq 42
|
737
863
|
expect(all_yields.size).to be > 4
|
738
864
|
expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
|
865
|
+
admin.close
|
739
866
|
end
|
740
867
|
|
741
868
|
it "should batch poll results and yield arrays of messages" do
|
@@ -778,13 +905,15 @@ describe Rdkafka::Consumer do
|
|
778
905
|
end
|
779
906
|
|
780
907
|
it "should yield [] if nothing is received before the timeout" do
|
781
|
-
|
908
|
+
admin = rdkafka_config.admin
|
909
|
+
create_topic_handle = admin.create_topic(topic_name, 1, 1)
|
782
910
|
create_topic_handle.wait(max_wait_timeout: 15.0)
|
783
911
|
consumer.subscribe(topic_name)
|
784
912
|
consumer.each_batch do |batch|
|
785
913
|
expect(batch).to eq([])
|
786
914
|
break
|
787
915
|
end
|
916
|
+
admin.close
|
788
917
|
end
|
789
918
|
|
790
919
|
it "should yield batchs of max_items in size if messages are already fetched" do
|
@@ -861,6 +990,7 @@ describe Rdkafka::Consumer do
|
|
861
990
|
expect(batches_yielded.first.size).to eq 2
|
862
991
|
expect(exceptions_yielded.flatten.size).to eq 1
|
863
992
|
expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
|
993
|
+
consumer.close
|
864
994
|
end
|
865
995
|
end
|
866
996
|
|
@@ -902,10 +1032,97 @@ describe Rdkafka::Consumer do
|
|
902
1032
|
expect(each_batch_iterations).to eq 0
|
903
1033
|
expect(batches_yielded.size).to eq 0
|
904
1034
|
expect(exceptions_yielded.size).to eq 0
|
1035
|
+
consumer.close
|
905
1036
|
end
|
906
1037
|
end
|
907
1038
|
end
|
908
1039
|
|
1040
|
+
describe "#offsets_for_times" do
|
1041
|
+
it "should raise when not TopicPartitionList" do
|
1042
|
+
expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
|
1043
|
+
end
|
1044
|
+
|
1045
|
+
it "should raise an error when offsets_for_times fails" do
|
1046
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
1047
|
+
|
1048
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
|
1049
|
+
|
1050
|
+
expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
|
1051
|
+
end
|
1052
|
+
|
1053
|
+
context "when subscribed" do
|
1054
|
+
let(:timeout) { 1000 }
|
1055
|
+
|
1056
|
+
before do
|
1057
|
+
consumer.subscribe("consume_test_topic")
|
1058
|
+
|
1059
|
+
# 1. partitions are assigned
|
1060
|
+
wait_for_assignment(consumer)
|
1061
|
+
expect(consumer.assignment).not_to be_empty
|
1062
|
+
|
1063
|
+
# 2. eat unrelated messages
|
1064
|
+
while(consumer.poll(timeout)) do; end
|
1065
|
+
end
|
1066
|
+
|
1067
|
+
after { consumer.unsubscribe }
|
1068
|
+
|
1069
|
+
def send_one_message(val)
|
1070
|
+
producer.produce(
|
1071
|
+
topic: "consume_test_topic",
|
1072
|
+
payload: "payload #{val}",
|
1073
|
+
key: "key 0",
|
1074
|
+
partition: 0
|
1075
|
+
).wait
|
1076
|
+
end
|
1077
|
+
|
1078
|
+
it "returns a TopicParticionList with updated offsets" do
|
1079
|
+
send_one_message("a")
|
1080
|
+
send_one_message("b")
|
1081
|
+
send_one_message("c")
|
1082
|
+
|
1083
|
+
consumer.poll(timeout)
|
1084
|
+
message = consumer.poll(timeout)
|
1085
|
+
consumer.poll(timeout)
|
1086
|
+
|
1087
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
1088
|
+
list.add_topic_and_partitions_with_offsets(
|
1089
|
+
"consume_test_topic",
|
1090
|
+
[
|
1091
|
+
[0, message.timestamp]
|
1092
|
+
]
|
1093
|
+
)
|
1094
|
+
end
|
1095
|
+
|
1096
|
+
tpl_response = consumer.offsets_for_times(tpl)
|
1097
|
+
|
1098
|
+
expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
|
1099
|
+
end
|
1100
|
+
end
|
1101
|
+
end
|
1102
|
+
|
1103
|
+
# Only relevant in case of a consumer with separate queues
|
1104
|
+
describe '#events_poll' do
|
1105
|
+
let(:stats) { [] }
|
1106
|
+
|
1107
|
+
before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
|
1108
|
+
|
1109
|
+
after { Rdkafka::Config.statistics_callback = nil }
|
1110
|
+
|
1111
|
+
let(:consumer) do
|
1112
|
+
config = rdkafka_consumer_config('statistics.interval.ms': 100)
|
1113
|
+
config.consumer_poll_set = false
|
1114
|
+
config.consumer
|
1115
|
+
end
|
1116
|
+
|
1117
|
+
it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
|
1118
|
+
consumer.subscribe("consume_test_topic")
|
1119
|
+
consumer.poll(1_000)
|
1120
|
+
expect(stats).to be_empty
|
1121
|
+
consumer.events_poll(-1)
|
1122
|
+
expect(stats).not_to be_empty
|
1123
|
+
end
|
1124
|
+
end
|
1125
|
+
|
909
1126
|
describe "a rebalance listener" do
|
910
1127
|
let(:consumer) do
|
911
1128
|
config = rdkafka_consumer_config
|
@@ -916,11 +1133,11 @@ describe Rdkafka::Consumer do
|
|
916
1133
|
context "with a working listener" do
|
917
1134
|
let(:listener) do
|
918
1135
|
Struct.new(:queue) do
|
919
|
-
def on_partitions_assigned(
|
1136
|
+
def on_partitions_assigned(list)
|
920
1137
|
collect(:assign, list)
|
921
1138
|
end
|
922
1139
|
|
923
|
-
def on_partitions_revoked(
|
1140
|
+
def on_partitions_revoked(list)
|
924
1141
|
collect(:revoke, list)
|
925
1142
|
end
|
926
1143
|
|
@@ -944,12 +1161,12 @@ describe Rdkafka::Consumer do
|
|
944
1161
|
context "with a broken listener" do
|
945
1162
|
let(:listener) do
|
946
1163
|
Struct.new(:queue) do
|
947
|
-
def on_partitions_assigned(
|
1164
|
+
def on_partitions_assigned(list)
|
948
1165
|
queue << :assigned
|
949
1166
|
raise 'boom'
|
950
1167
|
end
|
951
1168
|
|
952
|
-
def on_partitions_revoked(
|
1169
|
+
def on_partitions_revoked(list)
|
953
1170
|
queue << :revoked
|
954
1171
|
raise 'boom'
|
955
1172
|
end
|
@@ -962,18 +1179,6 @@ describe Rdkafka::Consumer do
|
|
962
1179
|
expect(listener.queue).to eq([:assigned, :revoked])
|
963
1180
|
end
|
964
1181
|
end
|
965
|
-
|
966
|
-
def notify_listener(listener)
|
967
|
-
# 1. subscribe and poll
|
968
|
-
consumer.subscribe("consume_test_topic")
|
969
|
-
wait_for_assignment(consumer)
|
970
|
-
consumer.poll(100)
|
971
|
-
|
972
|
-
# 2. unsubscribe
|
973
|
-
consumer.unsubscribe
|
974
|
-
wait_for_unassignment(consumer)
|
975
|
-
consumer.close
|
976
|
-
end
|
977
1182
|
end
|
978
1183
|
|
979
1184
|
context "methods that should not be called after a consumer has been closed" do
|
@@ -992,7 +1197,7 @@ describe Rdkafka::Consumer do
|
|
992
1197
|
:assign => [ nil ],
|
993
1198
|
:assignment => nil,
|
994
1199
|
:committed => [],
|
995
|
-
:query_watermark_offsets => [ nil, nil ]
|
1200
|
+
:query_watermark_offsets => [ nil, nil ]
|
996
1201
|
}.each do |method, args|
|
997
1202
|
it "raises an exception if #{method} is called" do
|
998
1203
|
expect {
|
@@ -1005,4 +1210,70 @@ describe Rdkafka::Consumer do
|
|
1005
1210
|
end
|
1006
1211
|
end
|
1007
1212
|
end
|
1213
|
+
|
1214
|
+
it "provides a finalizer that closes the native kafka client" do
|
1215
|
+
expect(consumer.closed?).to eq(false)
|
1216
|
+
|
1217
|
+
consumer.finalizer.call("some-ignored-object-id")
|
1218
|
+
|
1219
|
+
expect(consumer.closed?).to eq(true)
|
1220
|
+
end
|
1221
|
+
|
1222
|
+
context "when the rebalance protocol is cooperative" do
|
1223
|
+
let(:consumer) do
|
1224
|
+
config = rdkafka_consumer_config(
|
1225
|
+
{
|
1226
|
+
:"partition.assignment.strategy" => "cooperative-sticky",
|
1227
|
+
:"debug" => "consumer",
|
1228
|
+
}
|
1229
|
+
)
|
1230
|
+
config.consumer_rebalance_listener = listener
|
1231
|
+
config.consumer
|
1232
|
+
end
|
1233
|
+
|
1234
|
+
let(:listener) do
|
1235
|
+
Struct.new(:queue) do
|
1236
|
+
def on_partitions_assigned(list)
|
1237
|
+
collect(:assign, list)
|
1238
|
+
end
|
1239
|
+
|
1240
|
+
def on_partitions_revoked(list)
|
1241
|
+
collect(:revoke, list)
|
1242
|
+
end
|
1243
|
+
|
1244
|
+
def collect(name, list)
|
1245
|
+
partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
|
1246
|
+
queue << ([name] + partitions)
|
1247
|
+
end
|
1248
|
+
end.new([])
|
1249
|
+
end
|
1250
|
+
|
1251
|
+
it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
|
1252
|
+
notify_listener(listener) do
|
1253
|
+
handles = []
|
1254
|
+
10.times do
|
1255
|
+
handles << producer.produce(
|
1256
|
+
topic: "consume_test_topic",
|
1257
|
+
payload: "payload 1",
|
1258
|
+
key: "key 1",
|
1259
|
+
partition: 0
|
1260
|
+
)
|
1261
|
+
end
|
1262
|
+
handles.each(&:wait)
|
1263
|
+
|
1264
|
+
consumer.subscribe("consume_test_topic")
|
1265
|
+
# Check the first 10 messages. Then close the consumer, which
|
1266
|
+
# should break the each loop.
|
1267
|
+
consumer.each_with_index do |message, i|
|
1268
|
+
expect(message).to be_a Rdkafka::Consumer::Message
|
1269
|
+
break if i == 10
|
1270
|
+
end
|
1271
|
+
end
|
1272
|
+
|
1273
|
+
expect(listener.queue).to eq([
|
1274
|
+
[:assign, "consume_test_topic", 0, 1, 2],
|
1275
|
+
[:revoke, "consume_test_topic", 0, 1, 2]
|
1276
|
+
])
|
1277
|
+
end
|
1278
|
+
end
|
1008
1279
|
end
|
data/spec/rdkafka/error_spec.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "securerandom"
|
3
4
|
|
4
5
|
describe Rdkafka::Metadata do
|
@@ -29,7 +30,7 @@ describe Rdkafka::Metadata do
|
|
29
30
|
it "#brokers returns our single broker" do
|
30
31
|
expect(subject.brokers.length).to eq(1)
|
31
32
|
expect(subject.brokers[0][:broker_id]).to eq(1)
|
32
|
-
expect(subject.brokers[0][:broker_name]).to eq("
|
33
|
+
expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
|
33
34
|
expect(subject.brokers[0][:broker_port]).to eq(9092)
|
34
35
|
end
|
35
36
|
|
@@ -52,7 +53,7 @@ describe Rdkafka::Metadata do
|
|
52
53
|
it "#brokers returns our single broker" do
|
53
54
|
expect(subject.brokers.length).to eq(1)
|
54
55
|
expect(subject.brokers[0][:broker_id]).to eq(1)
|
55
|
-
expect(subject.brokers[0][:broker_name]).to eq("
|
56
|
+
expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
|
56
57
|
expect(subject.brokers[0][:broker_port]).to eq(9092)
|
57
58
|
end
|
58
59
|
|