rdkafka 0.22.0.beta1-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
  5. data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
  6. data/.github/workflows/ci_macos_arm64.yml +306 -0
  7. data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
  8. data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
  9. data/.github/workflows/push_macos_arm64.yml +54 -0
  10. data/.github/workflows/push_ruby.yml +37 -0
  11. data/.github/workflows/verify-action-pins.yml +16 -0
  12. data/.gitignore +14 -0
  13. data/.rspec +2 -0
  14. data/.ruby-gemset +1 -0
  15. data/.ruby-version +1 -0
  16. data/.yardopts +2 -0
  17. data/CHANGELOG.md +247 -0
  18. data/Gemfile +5 -0
  19. data/MIT-LICENSE +22 -0
  20. data/README.md +178 -0
  21. data/Rakefile +96 -0
  22. data/docker-compose.yml +25 -0
  23. data/ext/README.md +19 -0
  24. data/ext/Rakefile +131 -0
  25. data/ext/build_common.sh +361 -0
  26. data/ext/build_linux_x86_64_gnu.sh +306 -0
  27. data/ext/build_linux_x86_64_musl.sh +763 -0
  28. data/ext/build_macos_arm64.sh +550 -0
  29. data/ext/librdkafka.so +0 -0
  30. data/lib/rdkafka/abstract_handle.rb +116 -0
  31. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  32. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  33. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  34. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  35. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  36. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  37. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  38. data/lib/rdkafka/admin/create_topic_handle.rb +29 -0
  39. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  40. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  41. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  42. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  43. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  44. data/lib/rdkafka/admin/delete_topic_handle.rb +29 -0
  45. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  46. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  47. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  48. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  49. data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
  50. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  51. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
  52. data/lib/rdkafka/admin.rb +833 -0
  53. data/lib/rdkafka/bindings.rb +566 -0
  54. data/lib/rdkafka/callbacks.rb +415 -0
  55. data/lib/rdkafka/config.rb +398 -0
  56. data/lib/rdkafka/consumer/headers.rb +79 -0
  57. data/lib/rdkafka/consumer/message.rb +86 -0
  58. data/lib/rdkafka/consumer/partition.rb +51 -0
  59. data/lib/rdkafka/consumer/topic_partition_list.rb +169 -0
  60. data/lib/rdkafka/consumer.rb +653 -0
  61. data/lib/rdkafka/error.rb +101 -0
  62. data/lib/rdkafka/helpers/oauth.rb +58 -0
  63. data/lib/rdkafka/helpers/time.rb +14 -0
  64. data/lib/rdkafka/metadata.rb +115 -0
  65. data/lib/rdkafka/native_kafka.rb +139 -0
  66. data/lib/rdkafka/producer/delivery_handle.rb +40 -0
  67. data/lib/rdkafka/producer/delivery_report.rb +46 -0
  68. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  69. data/lib/rdkafka/producer.rb +430 -0
  70. data/lib/rdkafka/version.rb +7 -0
  71. data/lib/rdkafka.rb +54 -0
  72. data/rdkafka.gemspec +65 -0
  73. data/renovate.json +92 -0
  74. data/spec/rdkafka/abstract_handle_spec.rb +117 -0
  75. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  76. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  77. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  78. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  79. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  80. data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  81. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  82. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  83. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  84. data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  85. data/spec/rdkafka/admin_spec.rb +770 -0
  86. data/spec/rdkafka/bindings_spec.rb +223 -0
  87. data/spec/rdkafka/callbacks_spec.rb +20 -0
  88. data/spec/rdkafka/config_spec.rb +258 -0
  89. data/spec/rdkafka/consumer/headers_spec.rb +73 -0
  90. data/spec/rdkafka/consumer/message_spec.rb +139 -0
  91. data/spec/rdkafka/consumer/partition_spec.rb +57 -0
  92. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  93. data/spec/rdkafka/consumer_spec.rb +1274 -0
  94. data/spec/rdkafka/error_spec.rb +89 -0
  95. data/spec/rdkafka/metadata_spec.rb +79 -0
  96. data/spec/rdkafka/native_kafka_spec.rb +130 -0
  97. data/spec/rdkafka/producer/delivery_handle_spec.rb +45 -0
  98. data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
  99. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  100. data/spec/rdkafka/producer_spec.rb +1052 -0
  101. data/spec/spec_helper.rb +195 -0
  102. metadata +276 -0
@@ -0,0 +1,1274 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ostruct"
4
+ require 'securerandom'
5
+
6
+ describe Rdkafka::Consumer do
7
+ let(:consumer) { rdkafka_consumer_config.consumer }
8
+ let(:producer) { rdkafka_producer_config.producer }
9
+
10
+ after { consumer.close }
11
+ after { producer.close }
12
+
13
+ describe '#name' do
14
+ it { expect(consumer.name).to include('rdkafka#consumer-') }
15
+ end
16
+
17
+ describe 'consumer without auto-start' do
18
+ let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
19
+
20
+ it 'expect to be able to start it later and close' do
21
+ consumer.start
22
+ consumer.close
23
+ end
24
+
25
+ it 'expect to be able to close it without starting' do
26
+ consumer.close
27
+ end
28
+ end
29
+
30
+ describe "#subscribe, #unsubscribe and #subscription" do
31
+ it "should subscribe, unsubscribe and return the subscription" do
32
+ expect(consumer.subscription).to be_empty
33
+
34
+ consumer.subscribe("consume_test_topic")
35
+
36
+ expect(consumer.subscription).not_to be_empty
37
+ expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
38
+ list.add_topic("consume_test_topic")
39
+ end
40
+ expect(consumer.subscription).to eq expected_subscription
41
+
42
+ consumer.unsubscribe
43
+
44
+ expect(consumer.subscription).to be_empty
45
+ end
46
+
47
+ it "should raise an error when subscribing fails" do
48
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_subscribe).and_return(20)
49
+
50
+ expect {
51
+ consumer.subscribe("consume_test_topic")
52
+ }.to raise_error(Rdkafka::RdkafkaError)
53
+ end
54
+
55
+ it "should raise an error when unsubscribing fails" do
56
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_unsubscribe).and_return(20)
57
+
58
+ expect {
59
+ consumer.unsubscribe
60
+ }.to raise_error(Rdkafka::RdkafkaError)
61
+ end
62
+
63
+ it "should raise an error when fetching the subscription fails" do
64
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_subscription).and_return(20)
65
+
66
+ expect {
67
+ consumer.subscription
68
+ }.to raise_error(Rdkafka::RdkafkaError)
69
+ end
70
+
71
+ context "when using consumer without the poll set" do
72
+ let(:consumer) do
73
+ config = rdkafka_consumer_config
74
+ config.consumer_poll_set = false
75
+ config.consumer
76
+ end
77
+
78
+ it "should subscribe, unsubscribe and return the subscription" do
79
+ expect(consumer.subscription).to be_empty
80
+
81
+ consumer.subscribe("consume_test_topic")
82
+
83
+ expect(consumer.subscription).not_to be_empty
84
+ expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
85
+ list.add_topic("consume_test_topic")
86
+ end
87
+ expect(consumer.subscription).to eq expected_subscription
88
+
89
+ consumer.unsubscribe
90
+
91
+ expect(consumer.subscription).to be_empty
92
+ end
93
+ end
94
+ end
95
+
96
+ describe "#pause and #resume" do
97
+ context "subscription" do
98
+ let(:timeout) { 2000 }
99
+
100
+ before { consumer.subscribe("consume_test_topic") }
101
+ after { consumer.unsubscribe }
102
+
103
+ it "should pause and then resume" do
104
+ # 1. partitions are assigned
105
+ wait_for_assignment(consumer)
106
+ expect(consumer.assignment).not_to be_empty
107
+
108
+ # 2. send a first message
109
+ send_one_message
110
+
111
+ # 3. ensure that message is successfully consumed
112
+ records = consumer.poll(timeout)
113
+ expect(records).not_to be_nil
114
+ consumer.commit
115
+
116
+ # 4. send a second message
117
+ send_one_message
118
+
119
+ # 5. pause the subscription
120
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
121
+ tpl.add_topic("consume_test_topic", (0..2))
122
+ consumer.pause(tpl)
123
+
124
+ # 6. ensure that messages are not available
125
+ records = consumer.poll(timeout)
126
+ expect(records).to be_nil
127
+
128
+ # 7. resume the subscription
129
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
130
+ tpl.add_topic("consume_test_topic", (0..2))
131
+ consumer.resume(tpl)
132
+
133
+ # 8. ensure that message is successfully consumed
134
+ records = consumer.poll(timeout)
135
+ expect(records).not_to be_nil
136
+ end
137
+ end
138
+
139
+ it "should raise when not TopicPartitionList" do
140
+ expect { consumer.pause(true) }.to raise_error(TypeError)
141
+ expect { consumer.resume(true) }.to raise_error(TypeError)
142
+ end
143
+
144
+ it "should raise an error when pausing fails" do
145
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap { |tpl| tpl.add_topic('topic', (0..1)) }
146
+
147
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_pause_partitions).and_return(20)
148
+ expect {
149
+ consumer.pause(list)
150
+ }.to raise_error do |err|
151
+ expect(err).to be_instance_of(Rdkafka::RdkafkaTopicPartitionListError)
152
+ expect(err.topic_partition_list).to be
153
+ end
154
+ end
155
+
156
+ it "should raise an error when resume fails" do
157
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_resume_partitions).and_return(20)
158
+ expect {
159
+ consumer.resume(Rdkafka::Consumer::TopicPartitionList.new)
160
+ }.to raise_error Rdkafka::RdkafkaError
161
+ end
162
+
163
+ def send_one_message
164
+ producer.produce(
165
+ topic: "consume_test_topic",
166
+ payload: "payload 1",
167
+ key: "key 1"
168
+ ).wait
169
+ end
170
+ end
171
+
172
+ describe "#seek" do
173
+ let(:topic) { "it-#{SecureRandom.uuid}" }
174
+
175
+ before do
176
+ admin = rdkafka_producer_config.admin
177
+ admin.create_topic(topic, 1, 1).wait
178
+ admin.close
179
+ end
180
+
181
+ it "should raise an error when seeking fails" do
182
+ fake_msg = OpenStruct.new(topic: topic, partition: 0, offset: 0)
183
+
184
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
185
+ expect {
186
+ consumer.seek(fake_msg)
187
+ }.to raise_error Rdkafka::RdkafkaError
188
+ end
189
+
190
+ context "subscription" do
191
+ let(:timeout) { 1000 }
192
+ # Some specs here test the manual offset commit hence we want to ensure, that we have some
193
+ # offsets in-memory that we can manually commit
194
+ let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
195
+
196
+ before do
197
+ consumer.subscribe(topic)
198
+
199
+ # 1. partitions are assigned
200
+ wait_for_assignment(consumer)
201
+ expect(consumer.assignment).not_to be_empty
202
+
203
+ # 2. eat unrelated messages
204
+ while(consumer.poll(timeout)) do; end
205
+ end
206
+ after { consumer.unsubscribe }
207
+
208
+ def send_one_message(val)
209
+ producer.produce(
210
+ topic: topic,
211
+ payload: "payload #{val}",
212
+ key: "key 1",
213
+ partition: 0
214
+ ).wait
215
+ end
216
+
217
+ it "works when a partition is paused" do
218
+ # 3. get reference message
219
+ send_one_message(:a)
220
+ message1 = consumer.poll(timeout)
221
+ expect(message1&.payload).to eq "payload a"
222
+
223
+ # 4. pause the subscription
224
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
225
+ tpl.add_topic(topic, 1)
226
+ consumer.pause(tpl)
227
+
228
+ # 5. seek to previous message
229
+ consumer.seek(message1)
230
+
231
+ # 6. resume the subscription
232
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
233
+ tpl.add_topic(topic, 1)
234
+ consumer.resume(tpl)
235
+
236
+ # 7. ensure same message is read again
237
+ message2 = consumer.poll(timeout)
238
+
239
+ # This is needed because `enable.auto.offset.store` is true but when running in CI that
240
+ # is overloaded, offset store lags
241
+ sleep(1)
242
+
243
+ consumer.commit
244
+ expect(message1.offset).to eq message2.offset
245
+ expect(message1.payload).to eq message2.payload
246
+ end
247
+
248
+ it "allows skipping messages" do
249
+ # 3. send messages
250
+ send_one_message(:a)
251
+ send_one_message(:b)
252
+ send_one_message(:c)
253
+
254
+ # 4. get reference message
255
+ message = consumer.poll(timeout)
256
+ expect(message&.payload).to eq "payload a"
257
+
258
+ # 5. seek over one message
259
+ fake_msg = message.dup
260
+ fake_msg.instance_variable_set(:@offset, fake_msg.offset + 2)
261
+ consumer.seek(fake_msg)
262
+
263
+ # 6. ensure that only one message is available
264
+ records = consumer.poll(timeout)
265
+ expect(records&.payload).to eq "payload c"
266
+ records = consumer.poll(timeout)
267
+ expect(records).to be_nil
268
+ end
269
+ end
270
+ end
271
+
272
+ describe "#seek_by" do
273
+ let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
274
+ let(:topic) { "it-#{SecureRandom.uuid}" }
275
+ let(:partition) { 0 }
276
+ let(:offset) { 0 }
277
+
278
+ before do
279
+ admin = rdkafka_producer_config.admin
280
+ admin.create_topic(topic, 1, 1).wait
281
+ admin.close
282
+ end
283
+
284
+ it "should raise an error when seeking fails" do
285
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
286
+ expect {
287
+ consumer.seek_by(topic, partition, offset)
288
+ }.to raise_error Rdkafka::RdkafkaError
289
+ end
290
+
291
+ context "subscription" do
292
+ let(:timeout) { 1000 }
293
+
294
+ before do
295
+ consumer.subscribe(topic)
296
+
297
+ # 1. partitions are assigned
298
+ wait_for_assignment(consumer)
299
+ expect(consumer.assignment).not_to be_empty
300
+
301
+ # 2. eat unrelated messages
302
+ while(consumer.poll(timeout)) do; end
303
+ end
304
+ after { consumer.unsubscribe }
305
+
306
+ def send_one_message(val)
307
+ producer.produce(
308
+ topic: topic,
309
+ payload: "payload #{val}",
310
+ key: "key 1",
311
+ partition: 0
312
+ ).wait
313
+ end
314
+
315
+ it "works when a partition is paused" do
316
+ # 3. get reference message
317
+ send_one_message(:a)
318
+ message1 = consumer.poll(timeout)
319
+ expect(message1&.payload).to eq "payload a"
320
+
321
+ # 4. pause the subscription
322
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
323
+ tpl.add_topic(topic, 1)
324
+ consumer.pause(tpl)
325
+
326
+ # 5. seek by the previous message fields
327
+ consumer.seek_by(message1.topic, message1.partition, message1.offset)
328
+
329
+ # 6. resume the subscription
330
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
331
+ tpl.add_topic(topic, 1)
332
+ consumer.resume(tpl)
333
+
334
+ # 7. ensure same message is read again
335
+ message2 = consumer.poll(timeout)
336
+
337
+ # This is needed because `enable.auto.offset.store` is true but when running in CI that
338
+ # is overloaded, offset store lags
339
+ sleep(2)
340
+
341
+ consumer.commit
342
+ expect(message1.offset).to eq message2.offset
343
+ expect(message1.payload).to eq message2.payload
344
+ end
345
+
346
+ it "allows skipping messages" do
347
+ # 3. send messages
348
+ send_one_message(:a)
349
+ send_one_message(:b)
350
+ send_one_message(:c)
351
+
352
+ # 4. get reference message
353
+ message = consumer.poll(timeout)
354
+ expect(message&.payload).to eq "payload a"
355
+
356
+ # 5. seek over one message
357
+ consumer.seek_by(message.topic, message.partition, message.offset + 2)
358
+
359
+ # 6. ensure that only one message is available
360
+ records = consumer.poll(timeout)
361
+ expect(records&.payload).to eq "payload c"
362
+ records = consumer.poll(timeout)
363
+ expect(records).to be_nil
364
+ end
365
+ end
366
+ end
367
+
368
+ describe "#assign and #assignment" do
369
+ it "should return an empty assignment if nothing is assigned" do
370
+ expect(consumer.assignment).to be_empty
371
+ end
372
+
373
+ it "should only accept a topic partition list in assign" do
374
+ expect {
375
+ consumer.assign("list")
376
+ }.to raise_error TypeError
377
+ end
378
+
379
+ it "should raise an error when assigning fails" do
380
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_assign).and_return(20)
381
+ expect {
382
+ consumer.assign(Rdkafka::Consumer::TopicPartitionList.new)
383
+ }.to raise_error Rdkafka::RdkafkaError
384
+ end
385
+
386
+ it "should assign specific topic/partitions and return that assignment" do
387
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
388
+ tpl.add_topic("consume_test_topic", (0..2))
389
+ consumer.assign(tpl)
390
+
391
+ assignment = consumer.assignment
392
+ expect(assignment).not_to be_empty
393
+ expect(assignment.to_h["consume_test_topic"].length).to eq 3
394
+ end
395
+
396
+ it "should return the assignment when subscribed" do
397
+ # Make sure there's a message
398
+ producer.produce(
399
+ topic: "consume_test_topic",
400
+ payload: "payload 1",
401
+ key: "key 1",
402
+ partition: 0
403
+ ).wait
404
+
405
+ # Subscribe and poll until partitions are assigned
406
+ consumer.subscribe("consume_test_topic")
407
+ 100.times do
408
+ consumer.poll(100)
409
+ break unless consumer.assignment.empty?
410
+ end
411
+
412
+ assignment = consumer.assignment
413
+ expect(assignment).not_to be_empty
414
+ expect(assignment.to_h["consume_test_topic"].length).to eq 3
415
+ end
416
+
417
+ it "should raise an error when getting assignment fails" do
418
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_assignment).and_return(20)
419
+ expect {
420
+ consumer.assignment
421
+ }.to raise_error Rdkafka::RdkafkaError
422
+ end
423
+ end
424
+
425
+ describe '#assignment_lost?' do
426
+ it "should not return true as we do have an assignment" do
427
+ consumer.subscribe("consume_test_topic")
428
+ expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
429
+ list.add_topic("consume_test_topic")
430
+ end
431
+
432
+ expect(consumer.assignment_lost?).to eq false
433
+ consumer.unsubscribe
434
+ end
435
+
436
+ it "should not return true after voluntary unsubscribing" do
437
+ consumer.subscribe("consume_test_topic")
438
+ expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
439
+ list.add_topic("consume_test_topic")
440
+ end
441
+
442
+ consumer.unsubscribe
443
+ expect(consumer.assignment_lost?).to eq false
444
+ end
445
+ end
446
+
447
+ describe "#close" do
448
+ it "should close a consumer" do
449
+ consumer.subscribe("consume_test_topic")
450
+ 100.times do |i|
451
+ producer.produce(
452
+ topic: "consume_test_topic",
453
+ payload: "payload #{i}",
454
+ key: "key #{i}",
455
+ partition: 0
456
+ ).wait
457
+ end
458
+ consumer.close
459
+ expect {
460
+ consumer.poll(100)
461
+ }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
462
+ end
463
+
464
+ context 'when there are outgoing operations in other threads' do
465
+ it 'should wait and not crash' do
466
+ times = []
467
+
468
+ # Run a long running poll
469
+ thread = Thread.new do
470
+ times << Time.now
471
+ consumer.subscribe("empty_test_topic")
472
+ times << Time.now
473
+ consumer.poll(1_000)
474
+ times << Time.now
475
+ end
476
+
477
+ # Make sure it starts before we close
478
+ sleep(0.1)
479
+ consumer.close
480
+ close_time = Time.now
481
+ thread.join
482
+
483
+ times.each { |op_time| expect(op_time).to be < close_time }
484
+ end
485
+ end
486
+ end
487
+
488
+
489
+ describe "#position, #commit, #committed and #store_offset" do
490
+ # Make sure there are messages to work with
491
+ let!(:report) do
492
+ producer.produce(
493
+ topic: "consume_test_topic",
494
+ payload: "payload 1",
495
+ key: "key 1",
496
+ partition: 0
497
+ ).wait
498
+ end
499
+
500
+ let(:message) do
501
+ wait_for_message(
502
+ topic: "consume_test_topic",
503
+ delivery_report: report,
504
+ consumer: consumer
505
+ )
506
+ end
507
+
508
+ describe "#position" do
509
+ it "should only accept a topic partition list in position if not nil" do
510
+ expect {
511
+ consumer.position("list")
512
+ }.to raise_error TypeError
513
+ end
514
+ end
515
+
516
+ describe "#committed" do
517
+ it "should only accept a topic partition list in commit if not nil" do
518
+ expect {
519
+ consumer.commit("list")
520
+ }.to raise_error TypeError
521
+ end
522
+
523
+ it "should commit in sync mode" do
524
+ expect {
525
+ consumer.commit(nil, true)
526
+ }.not_to raise_error
527
+ end
528
+ end
529
+
530
+ context "with a committed consumer" do
531
+ before :all do
532
+ # Make sure there are some messages.
533
+ handles = []
534
+ producer = rdkafka_config.producer
535
+ 10.times do
536
+ (0..2).each do |i|
537
+ handles << producer.produce(
538
+ topic: "consume_test_topic",
539
+ payload: "payload 1",
540
+ key: "key 1",
541
+ partition: i
542
+ )
543
+ end
544
+ end
545
+ handles.each(&:wait)
546
+ producer.close
547
+ end
548
+
549
+ before do
550
+ consumer.subscribe("consume_test_topic")
551
+ wait_for_assignment(consumer)
552
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
553
+ list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => 1, 1 => 1, 2 => 1)
554
+ end
555
+ consumer.commit(list)
556
+ end
557
+
558
+ it "should commit a specific topic partion list" do
559
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
560
+ list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => 1, 1 => 2, 2 => 3)
561
+ end
562
+ consumer.commit(list)
563
+
564
+ partitions = consumer.committed(list).to_h["consume_test_topic"]
565
+ expect(partitions[0].offset).to eq 1
566
+ expect(partitions[1].offset).to eq 2
567
+ expect(partitions[2].offset).to eq 3
568
+ end
569
+
570
+ it "should raise an error when committing fails" do
571
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
572
+
573
+ expect {
574
+ consumer.commit
575
+ }.to raise_error(Rdkafka::RdkafkaError)
576
+ end
577
+
578
+ describe "#committed" do
579
+ it "should fetch the committed offsets for the current assignment" do
580
+ partitions = consumer.committed.to_h["consume_test_topic"]
581
+ expect(partitions).not_to be_nil
582
+ expect(partitions[0].offset).to eq 1
583
+ end
584
+
585
+ it "should fetch the committed offsets for a specified topic partition list" do
586
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
587
+ list.add_topic("consume_test_topic", [0, 1, 2])
588
+ end
589
+ partitions = consumer.committed(list).to_h["consume_test_topic"]
590
+ expect(partitions).not_to be_nil
591
+ expect(partitions[0].offset).to eq 1
592
+ expect(partitions[1].offset).to eq 1
593
+ expect(partitions[2].offset).to eq 1
594
+ end
595
+
596
+ it "should raise an error when getting committed fails" do
597
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
598
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
599
+ list.add_topic("consume_test_topic", [0, 1, 2])
600
+ end
601
+ expect {
602
+ consumer.committed(list)
603
+ }.to raise_error Rdkafka::RdkafkaError
604
+ end
605
+ end
606
+
607
+ describe "#store_offset" do
608
+ let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
609
+ let(:metadata) { SecureRandom.uuid }
610
+ let(:group_id) { SecureRandom.uuid }
611
+ let(:base_config) do
612
+ {
613
+ 'group.id': group_id,
614
+ 'enable.auto.offset.store': false,
615
+ 'enable.auto.commit': false
616
+ }
617
+ end
618
+
619
+ before do
620
+ @new_consumer = rdkafka_consumer_config(base_config).consumer
621
+ @new_consumer.subscribe("consume_test_topic")
622
+ wait_for_assignment(@new_consumer)
623
+ end
624
+
625
+ after do
626
+ @new_consumer.close
627
+ end
628
+
629
+ it "should store the offset for a message" do
630
+ @new_consumer.store_offset(message)
631
+ @new_consumer.commit
632
+
633
+ # TODO use position here, should be at offset
634
+
635
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
636
+ list.add_topic("consume_test_topic", [0, 1, 2])
637
+ end
638
+ partitions = @new_consumer.committed(list).to_h["consume_test_topic"]
639
+ expect(partitions).not_to be_nil
640
+ expect(partitions[message.partition].offset).to eq(message.offset + 1)
641
+ end
642
+
643
+ it "should raise an error with invalid input" do
644
+ allow(message).to receive(:partition).and_return(9999)
645
+ expect {
646
+ @new_consumer.store_offset(message)
647
+ }.to raise_error Rdkafka::RdkafkaError
648
+ end
649
+
650
+ describe "#position" do
651
+ it "should fetch the positions for the current assignment" do
652
+ consumer.store_offset(message)
653
+
654
+ partitions = consumer.position.to_h["consume_test_topic"]
655
+ expect(partitions).not_to be_nil
656
+ expect(partitions[0].offset).to eq message.offset + 1
657
+ end
658
+
659
+ it "should fetch the positions for a specified assignment" do
660
+ consumer.store_offset(message)
661
+
662
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
663
+ list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
664
+ end
665
+ partitions = consumer.position(list).to_h["consume_test_topic"]
666
+ expect(partitions).not_to be_nil
667
+ expect(partitions[0].offset).to eq message.offset + 1
668
+ end
669
+
670
+ it "should raise an error when getting the position fails" do
671
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
672
+
673
+ expect {
674
+ consumer.position
675
+ }.to raise_error(Rdkafka::RdkafkaError)
676
+ end
677
+ end
678
+
679
+ context "when trying to use with enable.auto.offset.store set to true" do
680
+ let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': true).consumer }
681
+
682
+ it "expect to raise invalid configuration error" do
683
+ expect { consumer.store_offset(message) }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
684
+ end
685
+ end
686
+ end
687
+ end
688
+ end
689
+
690
+ describe "#query_watermark_offsets" do
691
+ it "should return the watermark offsets" do
692
+ # Make sure there's a message
693
+ producer.produce(
694
+ topic: "watermarks_test_topic",
695
+ payload: "payload 1",
696
+ key: "key 1",
697
+ partition: 0
698
+ ).wait
699
+
700
+ low, high = consumer.query_watermark_offsets("watermarks_test_topic", 0, 5000)
701
+ expect(low).to eq 0
702
+ expect(high).to be > 0
703
+ end
704
+
705
+ it "should raise an error when querying offsets fails" do
706
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_query_watermark_offsets).and_return(20)
707
+ expect {
708
+ consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
709
+ }.to raise_error Rdkafka::RdkafkaError
710
+ end
711
+ end
712
+
713
+ describe "#lag" do
714
+ let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
715
+
716
+ it "should calculate the consumer lag" do
717
+ # Make sure there's a message in every partition and
718
+ # wait for the message to make sure everything is committed.
719
+ (0..2).each do |i|
720
+ producer.produce(
721
+ topic: "consume_test_topic",
722
+ key: "key lag #{i}",
723
+ partition: i
724
+ ).wait
725
+ end
726
+
727
+ # Consume to the end
728
+ consumer.subscribe("consume_test_topic")
729
+ eof_count = 0
730
+ loop do
731
+ begin
732
+ consumer.poll(100)
733
+ rescue Rdkafka::RdkafkaError => error
734
+ if error.is_partition_eof?
735
+ eof_count += 1
736
+ end
737
+ break if eof_count == 3
738
+ end
739
+ end
740
+
741
+ # Commit
742
+ consumer.commit
743
+
744
+ # Create list to fetch lag for. TODO creating the list will not be necessary
745
+ # after committed uses the subscription.
746
+ list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
747
+ l.add_topic("consume_test_topic", (0..2))
748
+ end)
749
+
750
+ # Lag should be 0 now
751
+ lag = consumer.lag(list)
752
+ expected_lag = {
753
+ "consume_test_topic" => {
754
+ 0 => 0,
755
+ 1 => 0,
756
+ 2 => 0
757
+ }
758
+ }
759
+ expect(lag).to eq(expected_lag)
760
+
761
+ # Produce message on every topic again
762
+ (0..2).each do |i|
763
+ producer.produce(
764
+ topic: "consume_test_topic",
765
+ key: "key lag #{i}",
766
+ partition: i
767
+ ).wait
768
+ end
769
+
770
+ # Lag should be 1 now
771
+ lag = consumer.lag(list)
772
+ expected_lag = {
773
+ "consume_test_topic" => {
774
+ 0 => 1,
775
+ 1 => 1,
776
+ 2 => 1
777
+ }
778
+ }
779
+ expect(lag).to eq(expected_lag)
780
+ end
781
+
782
+ it "returns nil if there are no messages on the topic" do
783
+ list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
784
+ l.add_topic("consume_test_topic", (0..2))
785
+ end)
786
+
787
+ lag = consumer.lag(list)
788
+ expected_lag = {
789
+ "consume_test_topic" => {}
790
+ }
791
+ expect(lag).to eq(expected_lag)
792
+ end
793
+ end
794
+
795
+ describe "#cluster_id" do
796
+ it 'should return the current ClusterId' do
797
+ consumer.subscribe("consume_test_topic")
798
+ wait_for_assignment(consumer)
799
+ expect(consumer.cluster_id).not_to be_empty
800
+ end
801
+ end
802
+
803
+ describe "#member_id" do
804
+ it 'should return the current MemberId' do
805
+ consumer.subscribe("consume_test_topic")
806
+ wait_for_assignment(consumer)
807
+ expect(consumer.member_id).to start_with('rdkafka-')
808
+ end
809
+ end
810
+
811
+ describe "#poll" do
812
+ it "should return nil if there is no subscription" do
813
+ expect(consumer.poll(1000)).to be_nil
814
+ end
815
+
816
+ it "should return nil if there are no messages" do
817
+ consumer.subscribe("empty_test_topic")
818
+ expect(consumer.poll(1000)).to be_nil
819
+ end
820
+
821
+ it "should return a message if there is one" do
822
+ topic = "it-#{SecureRandom.uuid}"
823
+
824
+ producer.produce(
825
+ topic: topic,
826
+ payload: "payload 1",
827
+ key: "key 1"
828
+ ).wait
829
+ consumer.subscribe(topic)
830
+ message = consumer.each {|m| break m}
831
+
832
+ expect(message).to be_a Rdkafka::Consumer::Message
833
+ expect(message.payload).to eq('payload 1')
834
+ expect(message.key).to eq('key 1')
835
+ end
836
+
837
+ it "should raise an error when polling fails" do
838
+ message = Rdkafka::Bindings::Message.new.tap do |message|
839
+ message[:err] = 20
840
+ end
841
+ message_pointer = message.to_ptr
842
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_consumer_poll).and_return(message_pointer)
843
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_message_destroy).with(message_pointer)
844
+ expect {
845
+ consumer.poll(100)
846
+ }.to raise_error Rdkafka::RdkafkaError
847
+ end
848
+ end
849
+
850
+ describe "#poll with headers" do
851
+ it "should return message with headers using string keys (when produced with symbol keys)" do
852
+ report = producer.produce(
853
+ topic: "consume_test_topic",
854
+ key: "key headers",
855
+ headers: { foo: 'bar' }
856
+ ).wait
857
+
858
+ message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
859
+ expect(message).to be
860
+ expect(message.key).to eq('key headers')
861
+ expect(message.headers).to include('foo' => 'bar')
862
+ end
863
+
864
+ it "should return message with headers using string keys (when produced with string keys)" do
865
+ report = producer.produce(
866
+ topic: "consume_test_topic",
867
+ key: "key headers",
868
+ headers: { 'foo' => 'bar' }
869
+ ).wait
870
+
871
+ message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
872
+ expect(message).to be
873
+ expect(message.key).to eq('key headers')
874
+ expect(message.headers).to include('foo' => 'bar')
875
+ end
876
+
877
+ it "should return message with no headers" do
878
+ report = producer.produce(
879
+ topic: "consume_test_topic",
880
+ key: "key no headers",
881
+ headers: nil
882
+ ).wait
883
+
884
+ message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
885
+ expect(message).to be
886
+ expect(message.key).to eq('key no headers')
887
+ expect(message.headers).to be_empty
888
+ end
889
+
890
+ it "should raise an error when message headers aren't readable" do
891
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) { 1 }
892
+
893
+ report = producer.produce(
894
+ topic: "consume_test_topic",
895
+ key: "key err headers",
896
+ headers: nil
897
+ ).wait
898
+
899
+ expect {
900
+ wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
901
+ }.to raise_error do |err|
902
+ expect(err).to be_instance_of(Rdkafka::RdkafkaError)
903
+ expect(err.message).to start_with("Error reading message headers")
904
+ end
905
+ end
906
+
907
+ it "should raise an error when the first message header aren't readable" do
908
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_header_get_all).with(any_args) { 1 }
909
+
910
+ report = producer.produce(
911
+ topic: "consume_test_topic",
912
+ key: "key err headers",
913
+ headers: { foo: 'bar' }
914
+ ).wait
915
+
916
+ expect {
917
+ wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
918
+ }.to raise_error do |err|
919
+ expect(err).to be_instance_of(Rdkafka::RdkafkaError)
920
+ expect(err.message).to start_with("Error reading a message header at index 0")
921
+ end
922
+ end
923
+ end
924
+
925
+ describe "#each" do
926
+ it "should yield messages" do
927
+ handles = []
928
+ 10.times do
929
+ handles << producer.produce(
930
+ topic: "consume_test_topic",
931
+ payload: "payload 1",
932
+ key: "key 1",
933
+ partition: 0
934
+ )
935
+ end
936
+ handles.each(&:wait)
937
+
938
+ consumer.subscribe("consume_test_topic")
939
+ # Check the first 10 messages. Then close the consumer, which
940
+ # should break the each loop.
941
+ consumer.each_with_index do |message, i|
942
+ expect(message).to be_a Rdkafka::Consumer::Message
943
+ break if i == 10
944
+ end
945
+ consumer.close
946
+ end
947
+ end
948
+
949
+ describe "#each_batch" do
950
+ it 'expect to raise an error' do
951
+ expect do
952
+ consumer.each_batch {}
953
+ end.to raise_error(NotImplementedError)
954
+ end
955
+ end
956
+
957
+ describe "#offsets_for_times" do
958
+ it "should raise when not TopicPartitionList" do
959
+ expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
960
+ end
961
+
962
+ it "should raise an error when offsets_for_times fails" do
963
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
964
+
965
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
966
+
967
+ expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
968
+ end
969
+
970
+ context "when subscribed" do
971
+ let(:timeout) { 1000 }
972
+
973
+ before do
974
+ consumer.subscribe("consume_test_topic")
975
+
976
+ # 1. partitions are assigned
977
+ wait_for_assignment(consumer)
978
+ expect(consumer.assignment).not_to be_empty
979
+
980
+ # 2. eat unrelated messages
981
+ while(consumer.poll(timeout)) do; end
982
+ end
983
+
984
+ after { consumer.unsubscribe }
985
+
986
+ def send_one_message(val)
987
+ producer.produce(
988
+ topic: "consume_test_topic",
989
+ payload: "payload #{val}",
990
+ key: "key 0",
991
+ partition: 0
992
+ ).wait
993
+ end
994
+
995
+ it "returns a TopicParticionList with updated offsets" do
996
+ send_one_message("a")
997
+ send_one_message("b")
998
+ send_one_message("c")
999
+
1000
+ consumer.poll(timeout)
1001
+ message = consumer.poll(timeout)
1002
+ consumer.poll(timeout)
1003
+
1004
+ tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
1005
+ list.add_topic_and_partitions_with_offsets(
1006
+ "consume_test_topic",
1007
+ [
1008
+ [0, message.timestamp]
1009
+ ]
1010
+ )
1011
+ end
1012
+
1013
+ tpl_response = consumer.offsets_for_times(tpl)
1014
+
1015
+ expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
1016
+ end
1017
+ end
1018
+ end
1019
+
1020
+ # Only relevant in case of a consumer with separate queues
1021
+ describe '#events_poll' do
1022
+ let(:stats) { [] }
1023
+
1024
+ before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
1025
+
1026
+ after { Rdkafka::Config.statistics_callback = nil }
1027
+
1028
+ let(:consumer) do
1029
+ config = rdkafka_consumer_config('statistics.interval.ms': 500)
1030
+ config.consumer_poll_set = false
1031
+ config.consumer
1032
+ end
1033
+
1034
+ it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
1035
+ consumer.subscribe("consume_test_topic")
1036
+ consumer.poll(1_000)
1037
+ expect(stats).to be_empty
1038
+ consumer.events_poll(-1)
1039
+ expect(stats).not_to be_empty
1040
+ end
1041
+ end
1042
+
1043
+ describe '#consumer_group_metadata_pointer' do
1044
+ let(:pointer) { consumer.consumer_group_metadata_pointer }
1045
+
1046
+ after { Rdkafka::Bindings.rd_kafka_consumer_group_metadata_destroy(pointer) }
1047
+
1048
+ it 'expect to return a pointer' do
1049
+ expect(pointer).to be_a(FFI::Pointer)
1050
+ end
1051
+ end
1052
+
1053
+ describe "a rebalance listener" do
1054
+ let(:consumer) do
1055
+ config = rdkafka_consumer_config
1056
+ config.consumer_rebalance_listener = listener
1057
+ config.consumer
1058
+ end
1059
+
1060
+ context "with a working listener" do
1061
+ let(:listener) do
1062
+ Struct.new(:queue) do
1063
+ def on_partitions_assigned(list)
1064
+ collect(:assign, list)
1065
+ end
1066
+
1067
+ def on_partitions_revoked(list)
1068
+ collect(:revoke, list)
1069
+ end
1070
+
1071
+ def collect(name, list)
1072
+ partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
1073
+ queue << ([name] + partitions)
1074
+ end
1075
+ end.new([])
1076
+ end
1077
+
1078
+ it "should get notifications" do
1079
+ notify_listener(listener)
1080
+
1081
+ expect(listener.queue).to eq([
1082
+ [:assign, "consume_test_topic", 0, 1, 2],
1083
+ [:revoke, "consume_test_topic", 0, 1, 2]
1084
+ ])
1085
+ end
1086
+ end
1087
+
1088
+ context "with a broken listener" do
1089
+ let(:listener) do
1090
+ Struct.new(:queue) do
1091
+ def on_partitions_assigned(list)
1092
+ queue << :assigned
1093
+ raise 'boom'
1094
+ end
1095
+
1096
+ def on_partitions_revoked(list)
1097
+ queue << :revoked
1098
+ raise 'boom'
1099
+ end
1100
+ end.new([])
1101
+ end
1102
+
1103
+ it 'should handle callback exceptions' do
1104
+ notify_listener(listener)
1105
+
1106
+ expect(listener.queue).to eq([:assigned, :revoked])
1107
+ end
1108
+ end
1109
+ end
1110
+
1111
+ context "methods that should not be called after a consumer has been closed" do
1112
+ before do
1113
+ consumer.close
1114
+ end
1115
+
1116
+ # Affected methods and a non-invalid set of parameters for the method
1117
+ {
1118
+ :subscribe => [ nil ],
1119
+ :unsubscribe => nil,
1120
+ :pause => [ nil ],
1121
+ :resume => [ nil ],
1122
+ :subscription => nil,
1123
+ :assign => [ nil ],
1124
+ :assignment => nil,
1125
+ :committed => [],
1126
+ :query_watermark_offsets => [ nil, nil ],
1127
+ :assignment_lost? => []
1128
+ }.each do |method, args|
1129
+ it "raises an exception if #{method} is called" do
1130
+ expect {
1131
+ if args.nil?
1132
+ consumer.public_send(method)
1133
+ else
1134
+ consumer.public_send(method, *args)
1135
+ end
1136
+ }.to raise_exception(Rdkafka::ClosedConsumerError, /#{method.to_s}/)
1137
+ end
1138
+ end
1139
+ end
1140
+
1141
+ it "provides a finalizer that closes the native kafka client" do
1142
+ expect(consumer.closed?).to eq(false)
1143
+
1144
+ consumer.finalizer.call("some-ignored-object-id")
1145
+
1146
+ expect(consumer.closed?).to eq(true)
1147
+ end
1148
+
1149
+ context "when the rebalance protocol is cooperative" do
1150
+ let(:consumer) do
1151
+ config = rdkafka_consumer_config(
1152
+ {
1153
+ :"partition.assignment.strategy" => "cooperative-sticky",
1154
+ :"debug" => "consumer",
1155
+ }
1156
+ )
1157
+ config.consumer_rebalance_listener = listener
1158
+ config.consumer
1159
+ end
1160
+
1161
+ let(:listener) do
1162
+ Struct.new(:queue) do
1163
+ def on_partitions_assigned(list)
1164
+ collect(:assign, list)
1165
+ end
1166
+
1167
+ def on_partitions_revoked(list)
1168
+ collect(:revoke, list)
1169
+ end
1170
+
1171
+ def collect(name, list)
1172
+ partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
1173
+ queue << ([name] + partitions)
1174
+ end
1175
+ end.new([])
1176
+ end
1177
+
1178
+ it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
1179
+ notify_listener(listener) do
1180
+ handles = []
1181
+ 10.times do
1182
+ handles << producer.produce(
1183
+ topic: "consume_test_topic",
1184
+ payload: "payload 1",
1185
+ key: "key 1",
1186
+ partition: 0
1187
+ )
1188
+ end
1189
+ handles.each(&:wait)
1190
+
1191
+ consumer.subscribe("consume_test_topic")
1192
+ # Check the first 10 messages. Then close the consumer, which
1193
+ # should break the each loop.
1194
+ consumer.each_with_index do |message, i|
1195
+ expect(message).to be_a Rdkafka::Consumer::Message
1196
+ break if i == 10
1197
+ end
1198
+ end
1199
+
1200
+ expect(listener.queue).to eq([
1201
+ [:assign, "consume_test_topic", 0, 1, 2],
1202
+ [:revoke, "consume_test_topic", 0, 1, 2]
1203
+ ])
1204
+ end
1205
+ end
1206
+
1207
+ describe '#oauthbearer_set_token' do
1208
+ context 'when sasl not configured' do
1209
+ it 'should return RD_KAFKA_RESP_ERR__STATE' do
1210
+ response = consumer.oauthbearer_set_token(
1211
+ token: "foo",
1212
+ lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
1213
+ principal_name: "kafka-cluster"
1214
+ )
1215
+ expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
1216
+ end
1217
+ end
1218
+
1219
+ context 'when sasl configured' do
1220
+ before do
1221
+ $consumer_sasl = rdkafka_producer_config(
1222
+ "security.protocol": "sasl_ssl",
1223
+ "sasl.mechanisms": 'OAUTHBEARER'
1224
+ ).consumer
1225
+ end
1226
+
1227
+ after do
1228
+ $consumer_sasl.close
1229
+ end
1230
+
1231
+ it 'should succeed' do
1232
+
1233
+ response = $consumer_sasl.oauthbearer_set_token(
1234
+ token: "foo",
1235
+ lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
1236
+ principal_name: "kafka-cluster"
1237
+ )
1238
+ expect(response).to eq(0)
1239
+ end
1240
+ end
1241
+ end
1242
+
1243
+ describe "when reaching eof on a topic and eof reporting enabled" do
1244
+ let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
1245
+
1246
+ it "should return proper details" do
1247
+ (0..2).each do |i|
1248
+ producer.produce(
1249
+ topic: "consume_test_topic",
1250
+ key: "key lag #{i}",
1251
+ partition: i
1252
+ ).wait
1253
+ end
1254
+
1255
+ # Consume to the end
1256
+ consumer.subscribe("consume_test_topic")
1257
+ eof_count = 0
1258
+ eof_error = nil
1259
+
1260
+ loop do
1261
+ begin
1262
+ consumer.poll(100)
1263
+ rescue Rdkafka::RdkafkaError => error
1264
+ if error.is_partition_eof?
1265
+ eof_error = error
1266
+ end
1267
+ break if eof_error
1268
+ end
1269
+ end
1270
+
1271
+ expect(eof_error.code).to eq(:partition_eof)
1272
+ end
1273
+ end
1274
+ end