rdkafka 0.12.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +58 -0
  5. data/.gitignore +4 -0
  6. data/.rspec +1 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +141 -93
  10. data/Gemfile +2 -0
  11. data/{LICENSE → MIT-LICENSE} +2 -1
  12. data/README.md +64 -29
  13. data/Rakefile +2 -0
  14. data/certs/cert_chain.pem +26 -0
  15. data/docker-compose.yml +18 -15
  16. data/ext/README.md +1 -1
  17. data/ext/Rakefile +3 -1
  18. data/lib/rdkafka/abstract_handle.rb +41 -25
  19. data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
  20. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  21. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  22. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  23. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  24. data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
  25. data/lib/rdkafka/admin/create_topic_report.rb +2 -0
  26. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  27. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  28. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  29. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  30. data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
  31. data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
  32. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  33. data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
  34. data/lib/rdkafka/admin.rb +494 -35
  35. data/lib/rdkafka/bindings.rb +175 -40
  36. data/lib/rdkafka/callbacks.rb +194 -1
  37. data/lib/rdkafka/config.rb +62 -25
  38. data/lib/rdkafka/consumer/headers.rb +24 -9
  39. data/lib/rdkafka/consumer/message.rb +3 -1
  40. data/lib/rdkafka/consumer/partition.rb +2 -0
  41. data/lib/rdkafka/consumer/topic_partition_list.rb +13 -8
  42. data/lib/rdkafka/consumer.rb +219 -102
  43. data/lib/rdkafka/error.rb +15 -0
  44. data/lib/rdkafka/helpers/time.rb +14 -0
  45. data/lib/rdkafka/metadata.rb +25 -2
  46. data/lib/rdkafka/native_kafka.rb +120 -0
  47. data/lib/rdkafka/producer/delivery_handle.rb +5 -2
  48. data/lib/rdkafka/producer/delivery_report.rb +9 -2
  49. data/lib/rdkafka/producer.rb +117 -17
  50. data/lib/rdkafka/version.rb +5 -3
  51. data/lib/rdkafka.rb +24 -2
  52. data/rdkafka.gemspec +19 -3
  53. data/renovate.json +6 -0
  54. data/spec/rdkafka/abstract_handle_spec.rb +1 -1
  55. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  56. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  57. data/spec/rdkafka/admin/create_topic_handle_spec.rb +1 -1
  58. data/spec/rdkafka/admin/create_topic_report_spec.rb +1 -1
  59. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  60. data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
  61. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +1 -1
  62. data/spec/rdkafka/admin/delete_topic_report_spec.rb +1 -1
  63. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  64. data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
  65. data/spec/rdkafka/admin_spec.rb +209 -5
  66. data/spec/rdkafka/bindings_spec.rb +2 -1
  67. data/spec/rdkafka/callbacks_spec.rb +1 -1
  68. data/spec/rdkafka/config_spec.rb +24 -3
  69. data/spec/rdkafka/consumer/headers_spec.rb +60 -0
  70. data/spec/rdkafka/consumer/message_spec.rb +1 -1
  71. data/spec/rdkafka/consumer/partition_spec.rb +1 -1
  72. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +20 -1
  73. data/spec/rdkafka/consumer_spec.rb +332 -61
  74. data/spec/rdkafka/error_spec.rb +1 -1
  75. data/spec/rdkafka/metadata_spec.rb +4 -3
  76. data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -35
  77. data/spec/rdkafka/producer/delivery_handle_spec.rb +4 -1
  78. data/spec/rdkafka/producer/delivery_report_spec.rb +7 -3
  79. data/spec/rdkafka/producer_spec.rb +208 -20
  80. data/spec/spec_helper.rb +20 -2
  81. data.tar.gz.sig +3 -0
  82. metadata +79 -16
  83. metadata.gz.sig +3 -0
  84. data/.semaphore/semaphore.yml +0 -23
  85. data/bin/console +0 -11
  86. data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,17 +1,15 @@
1
- require "spec_helper"
1
+ # frozen_string_literal: true
2
2
 
3
- describe Rdkafka::Producer::Client do
3
+ describe Rdkafka::NativeKafka do
4
4
  let(:config) { rdkafka_producer_config }
5
5
  let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
6
6
  let(:closing) { false }
7
7
  let(:thread) { double(Thread) }
8
+ let(:opaque) { Rdkafka::Opaque.new }
8
9
 
9
- subject(:client) { described_class.new(native) }
10
+ subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
10
11
 
11
12
  before do
12
- allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
13
- allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
14
- allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
15
13
  allow(Thread).to receive(:new).and_return(thread)
16
14
 
17
15
  allow(thread).to receive(:[]=).with(:closing, anything)
@@ -19,6 +17,8 @@ describe Rdkafka::Producer::Client do
19
17
  allow(thread).to receive(:abort_on_exception=).with(anything)
20
18
  end
21
19
 
20
+ after { client.close }
21
+
22
22
  context "defaults" do
23
23
  it "sets the thread to abort on exception" do
24
24
  expect(thread).to receive(:abort_on_exception=).with(true)
@@ -39,32 +39,12 @@ describe Rdkafka::Producer::Client do
39
39
 
40
40
  client
41
41
  end
42
-
43
- it "polls the native with default 250ms timeout" do
44
- polling_loop_expects do
45
- expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
46
- end
47
- end
48
-
49
- it "check the out queue of native client" do
50
- polling_loop_expects do
51
- expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
52
- end
53
- end
54
- end
55
-
56
- def polling_loop_expects(&block)
57
- Thread.current[:closing] = true # this forces the loop break with line #12
58
-
59
- allow(Thread).to receive(:new).and_yield do |_|
60
- block.call
61
- end.and_return(thread)
62
-
63
- client
64
42
  end
65
43
 
66
- it "exposes `native` client" do
67
- expect(client.native).to eq(native)
44
+ it "exposes the inner client" do
45
+ client.with_inner do |inner|
46
+ expect(inner).to eq(native)
47
+ end
68
48
  end
69
49
 
70
50
  context "when client was not yet closed (`nil`)" do
@@ -74,7 +54,7 @@ describe Rdkafka::Producer::Client do
74
54
 
75
55
  context "and attempt to close" do
76
56
  it "calls the `destroy` binding" do
77
- expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
57
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
78
58
 
79
59
  client.close
80
60
  end
@@ -94,7 +74,6 @@ describe Rdkafka::Producer::Client do
94
74
  it "closes and unassign the native client" do
95
75
  client.close
96
76
 
97
- expect(client.native).to eq(nil)
98
77
  expect(client.closed?).to eq(true)
99
78
  end
100
79
  end
@@ -109,7 +88,7 @@ describe Rdkafka::Producer::Client do
109
88
 
110
89
  context "and attempt to close again" do
111
90
  it "does not call the `destroy` binding" do
112
- expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
91
+ expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
113
92
 
114
93
  client.close
115
94
  end
@@ -129,13 +108,12 @@ describe Rdkafka::Producer::Client do
129
108
  it "does not close and unassign the native client again" do
130
109
  client.close
131
110
 
132
- expect(client.native).to eq(nil)
133
111
  expect(client.closed?).to eq(true)
134
112
  end
135
113
  end
136
114
  end
137
115
 
138
- it "provide a finalizer Proc that closes the `native` client" do
116
+ it "provides a finalizer that closes the native kafka client" do
139
117
  expect(client.closed?).to eq(false)
140
118
 
141
119
  client.finalizer.call("some-ignored-object-id")
@@ -1,4 +1,4 @@
1
- require "spec_helper"
1
+ # frozen_string_literal: true
2
2
 
3
3
  describe Rdkafka::Producer::DeliveryHandle do
4
4
  let(:response) { 0 }
@@ -9,6 +9,7 @@ describe Rdkafka::Producer::DeliveryHandle do
9
9
  handle[:response] = response
10
10
  handle[:partition] = 2
11
11
  handle[:offset] = 100
12
+ handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
12
13
  end
13
14
  end
14
15
 
@@ -29,6 +30,7 @@ describe Rdkafka::Producer::DeliveryHandle do
29
30
 
30
31
  expect(report.partition).to eq(2)
31
32
  expect(report.offset).to eq(100)
33
+ expect(report.topic_name).to eq("produce_test_topic")
32
34
  end
33
35
 
34
36
  it "should wait without a timeout" do
@@ -36,6 +38,7 @@ describe Rdkafka::Producer::DeliveryHandle do
36
38
 
37
39
  expect(report.partition).to eq(2)
38
40
  expect(report.offset).to eq(100)
41
+ expect(report.topic_name).to eq("produce_test_topic")
39
42
  end
40
43
  end
41
44
  end
@@ -1,7 +1,7 @@
1
- require "spec_helper"
1
+ # frozen_string_literal: true
2
2
 
3
3
  describe Rdkafka::Producer::DeliveryReport do
4
- subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
4
+ subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
5
5
 
6
6
  it "should get the partition" do
7
7
  expect(subject.partition).to eq 2
@@ -11,7 +11,11 @@ describe Rdkafka::Producer::DeliveryReport do
11
11
  expect(subject.offset).to eq 100
12
12
  end
13
13
 
14
+ it "should get the topic_name" do
15
+ expect(subject.topic_name).to eq "topic"
16
+ end
17
+
14
18
  it "should get the error" do
15
- expect(subject.error).to eq "error"
19
+ expect(subject.error).to eq -1
16
20
  end
17
21
  end
@@ -1,4 +1,5 @@
1
- require "spec_helper"
1
+ # frozen_string_literal: true
2
+
2
3
  require "zlib"
3
4
 
4
5
  describe Rdkafka::Producer do
@@ -7,11 +8,16 @@ describe Rdkafka::Producer do
7
8
 
8
9
  after do
9
10
  # Registry should always end up being empty
10
- expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
11
+ registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
12
+ expect(registry).to be_empty, registry.inspect
11
13
  producer.close
12
14
  consumer.close
13
15
  end
14
16
 
17
+ describe '#name' do
18
+ it { expect(producer.name).to include('rdkafka#producer-') }
19
+ end
20
+
15
21
  context "delivery callback" do
16
22
  context "with a proc/lambda" do
17
23
  it "should set the callback" do
@@ -30,6 +36,7 @@ describe Rdkafka::Producer do
30
36
  expect(report).not_to be_nil
31
37
  expect(report.partition).to eq 1
32
38
  expect(report.offset).to be >= 0
39
+ expect(report.topic_name).to eq "produce_test_topic"
33
40
  @callback_called = true
34
41
  end
35
42
 
@@ -113,6 +120,7 @@ describe Rdkafka::Producer do
113
120
  expect(called_report.first).not_to be_nil
114
121
  expect(called_report.first.partition).to eq 1
115
122
  expect(called_report.first.offset).to be >= 0
123
+ expect(called_report.first.topic_name).to eq "produce_test_topic"
116
124
  end
117
125
 
118
126
  it "should provide handle" do
@@ -180,10 +188,11 @@ describe Rdkafka::Producer do
180
188
  expect(report.partition).to eq 1
181
189
  expect(report.offset).to be >= 0
182
190
 
183
- # Close producer
191
+ # Flush and close producer
192
+ producer.flush
184
193
  producer.close
185
194
 
186
- # Consume message and verify it's content
195
+ # Consume message and verify its content
187
196
  message = wait_for_message(
188
197
  topic: "produce_test_topic",
189
198
  delivery_report: report,
@@ -207,7 +216,7 @@ describe Rdkafka::Producer do
207
216
  )
208
217
  report = handle.wait(max_wait_timeout: 5)
209
218
 
210
- # Consume message and verify it's content
219
+ # Consume message and verify its content
211
220
  message = wait_for_message(
212
221
  topic: "produce_test_topic",
213
222
  delivery_report: report,
@@ -251,6 +260,28 @@ describe Rdkafka::Producer do
251
260
  expect(messages[2].key).to eq key
252
261
  end
253
262
 
263
+ it "should produce a message with empty string without crashing" do
264
+ messages = [{key: 'a', partition_key: ''}]
265
+
266
+ messages = messages.map do |m|
267
+ handle = producer.produce(
268
+ topic: "partitioner_test_topic",
269
+ payload: "payload partition",
270
+ key: m[:key],
271
+ partition_key: m[:partition_key]
272
+ )
273
+ report = handle.wait(max_wait_timeout: 5)
274
+
275
+ wait_for_message(
276
+ topic: "partitioner_test_topic",
277
+ delivery_report: report,
278
+ )
279
+ end
280
+
281
+ expect(messages[0].partition).to eq 0
282
+ expect(messages[0].key).to eq 'a'
283
+ end
284
+
254
285
  it "should produce a message with utf-8 encoding" do
255
286
  handle = producer.produce(
256
287
  topic: "produce_test_topic",
@@ -259,7 +290,7 @@ describe Rdkafka::Producer do
259
290
  )
260
291
  report = handle.wait(max_wait_timeout: 5)
261
292
 
262
- # Consume message and verify it's content
293
+ # Consume message and verify its content
263
294
  message = wait_for_message(
264
295
  topic: "produce_test_topic",
265
296
  delivery_report: report,
@@ -292,7 +323,7 @@ describe Rdkafka::Producer do
292
323
  )
293
324
  report = handle.wait(max_wait_timeout: 5)
294
325
 
295
- # Consume message and verify it's content
326
+ # Consume message and verify its content
296
327
  message = wait_for_message(
297
328
  topic: "produce_test_topic",
298
329
  delivery_report: report,
@@ -313,7 +344,7 @@ describe Rdkafka::Producer do
313
344
  )
314
345
  report = handle.wait(max_wait_timeout: 5)
315
346
 
316
- # Consume message and verify it's content
347
+ # Consume message and verify its content
317
348
  message = wait_for_message(
318
349
  topic: "produce_test_topic",
319
350
  delivery_report: report,
@@ -333,7 +364,7 @@ describe Rdkafka::Producer do
333
364
  )
334
365
  report = handle.wait(max_wait_timeout: 5)
335
366
 
336
- # Consume message and verify it's content
367
+ # Consume message and verify its content
337
368
  message = wait_for_message(
338
369
  topic: "produce_test_topic",
339
370
  delivery_report: report,
@@ -351,7 +382,7 @@ describe Rdkafka::Producer do
351
382
  )
352
383
  report = handle.wait(max_wait_timeout: 5)
353
384
 
354
- # Consume message and verify it's content
385
+ # Consume message and verify its content
355
386
  message = wait_for_message(
356
387
  topic: "produce_test_topic",
357
388
  delivery_report: report,
@@ -371,7 +402,7 @@ describe Rdkafka::Producer do
371
402
  )
372
403
  report = handle.wait(max_wait_timeout: 5)
373
404
 
374
- # Consume message and verify it's content
405
+ # Consume message and verify its content
375
406
  message = wait_for_message(
376
407
  topic: "produce_test_topic",
377
408
  delivery_report: report,
@@ -380,9 +411,9 @@ describe Rdkafka::Producer do
380
411
 
381
412
  expect(message.payload).to eq "payload headers"
382
413
  expect(message.key).to eq "key headers"
383
- expect(message.headers[:foo]).to eq "bar"
384
- expect(message.headers[:baz]).to eq "foobar"
385
- expect(message.headers[:foobar]).to be_nil
414
+ expect(message.headers["foo"]).to eq "bar"
415
+ expect(message.headers["baz"]).to eq "foobar"
416
+ expect(message.headers["foobar"]).to be_nil
386
417
  end
387
418
 
388
419
  it "should produce a message with empty headers" do
@@ -394,7 +425,7 @@ describe Rdkafka::Producer do
394
425
  )
395
426
  report = handle.wait(max_wait_timeout: 5)
396
427
 
397
- # Consume message and verify it's content
428
+ # Consume message and verify its content
398
429
  message = wait_for_message(
399
430
  topic: "produce_test_topic",
400
431
  delivery_report: report,
@@ -432,10 +463,10 @@ describe Rdkafka::Producer do
432
463
  # wait for and check the message in the main process.
433
464
  reader, writer = IO.pipe
434
465
 
435
- fork do
466
+ pid = fork do
436
467
  reader.close
437
468
 
438
- # Avoids sharing the socket between processes.
469
+ # Avoid sharing the client between processes.
439
470
  producer = rdkafka_producer_config.producer
440
471
 
441
472
  handle = producer.produce(
@@ -448,24 +479,28 @@ describe Rdkafka::Producer do
448
479
 
449
480
  report_json = JSON.generate(
450
481
  "partition" => report.partition,
451
- "offset" => report.offset
482
+ "offset" => report.offset,
483
+ "topic_name" => report.topic_name
452
484
  )
453
485
 
454
486
  writer.write(report_json)
455
487
  writer.close
488
+ producer.flush
456
489
  producer.close
457
490
  end
491
+ Process.wait(pid)
458
492
 
459
493
  writer.close
460
494
  report_hash = JSON.parse(reader.read)
461
495
  report = Rdkafka::Producer::DeliveryReport.new(
462
496
  report_hash["partition"],
463
- report_hash["offset"]
497
+ report_hash["offset"],
498
+ report_hash["topic_name"]
464
499
  )
465
500
 
466
501
  reader.close
467
502
 
468
- # Consume message and verify it's content
503
+ # Consume message and verify its content
469
504
  message = wait_for_message(
470
505
  topic: "produce_test_topic",
471
506
  delivery_report: report,
@@ -522,4 +557,157 @@ describe Rdkafka::Producer do
522
557
  end
523
558
  end
524
559
  end
560
+
561
+ describe '#partition_count' do
562
+ it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
563
+
564
+ context 'when the partition count value is already cached' do
565
+ before do
566
+ producer.partition_count('consume_test_topic')
567
+ allow(::Rdkafka::Metadata).to receive(:new).and_call_original
568
+ end
569
+
570
+ it 'expect not to query it again' do
571
+ producer.partition_count('consume_test_topic')
572
+ expect(::Rdkafka::Metadata).not_to have_received(:new)
573
+ end
574
+ end
575
+
576
+ context 'when the partition count value was cached but time expired' do
577
+ before do
578
+ allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
579
+ producer.partition_count('consume_test_topic')
580
+ allow(::Rdkafka::Metadata).to receive(:new).and_call_original
581
+ end
582
+
583
+ it 'expect not to query it again' do
584
+ producer.partition_count('consume_test_topic')
585
+ expect(::Rdkafka::Metadata).to have_received(:new)
586
+ end
587
+ end
588
+
589
+ context 'when the partition count value was cached and time did not expire' do
590
+ before do
591
+ allow(::Process).to receive(:clock_gettime).and_return(0, 29.001)
592
+ producer.partition_count('consume_test_topic')
593
+ allow(::Rdkafka::Metadata).to receive(:new).and_call_original
594
+ end
595
+
596
+ it 'expect not to query it again' do
597
+ producer.partition_count('consume_test_topic')
598
+ expect(::Rdkafka::Metadata).not_to have_received(:new)
599
+ end
600
+ end
601
+ end
602
+
603
+ describe '#flush' do
604
+ it "should return flush when it can flush all outstanding messages or when no messages" do
605
+ producer.produce(
606
+ topic: "produce_test_topic",
607
+ payload: "payload headers",
608
+ key: "key headers",
609
+ headers: {}
610
+ )
611
+
612
+ expect(producer.flush(5_000)).to eq(true)
613
+ end
614
+
615
+ context 'when it cannot flush due to a timeout' do
616
+ let(:producer) do
617
+ rdkafka_producer_config(
618
+ "bootstrap.servers": "localhost:9093",
619
+ "message.timeout.ms": 2_000
620
+ ).producer
621
+ end
622
+
623
+ after do
624
+ # Allow rdkafka to evict message preventing memory-leak
625
+ sleep(2)
626
+ end
627
+
628
+ it "should return false on flush when cannot deliver and beyond timeout" do
629
+ producer.produce(
630
+ topic: "produce_test_topic",
631
+ payload: "payload headers",
632
+ key: "key headers",
633
+ headers: {}
634
+ )
635
+
636
+ expect(producer.flush(1_000)).to eq(false)
637
+ end
638
+ end
639
+
640
+ context 'when there is a different error' do
641
+ before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
642
+
643
+ it 'should raise it' do
644
+ expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
645
+ end
646
+ end
647
+ end
648
+
649
+ describe '#purge' do
650
+ context 'when no outgoing messages' do
651
+ it { expect(producer.purge).to eq(true) }
652
+ end
653
+
654
+ context 'when librdkafka purge returns an error' do
655
+ before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
656
+
657
+ it 'expect to raise an error' do
658
+ expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
659
+ end
660
+ end
661
+
662
+ context 'when there are outgoing things in the queue' do
663
+ let(:producer) do
664
+ rdkafka_producer_config(
665
+ "bootstrap.servers": "localhost:9093",
666
+ "message.timeout.ms": 2_000
667
+ ).producer
668
+ end
669
+
670
+ it "should should purge and move forward" do
671
+ producer.produce(
672
+ topic: "produce_test_topic",
673
+ payload: "payload headers"
674
+ )
675
+
676
+ expect(producer.purge).to eq(true)
677
+ expect(producer.flush(1_000)).to eq(true)
678
+ end
679
+
680
+ it "should materialize the delivery handles" do
681
+ handle = producer.produce(
682
+ topic: "produce_test_topic",
683
+ payload: "payload headers"
684
+ )
685
+
686
+ expect(producer.purge).to eq(true)
687
+
688
+ expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
689
+ end
690
+
691
+ context "when using delivery_callback" do
692
+ let(:delivery_reports) { [] }
693
+
694
+ let(:delivery_callback) do
695
+ ->(delivery_report) { delivery_reports << delivery_report }
696
+ end
697
+
698
+ before { producer.delivery_callback = delivery_callback }
699
+
700
+ it "should run the callback" do
701
+ handle = producer.produce(
702
+ topic: "produce_test_topic",
703
+ payload: "payload headers"
704
+ )
705
+
706
+ expect(producer.purge).to eq(true)
707
+ # queue purge
708
+ expect(delivery_reports[0].error).to eq(-152)
709
+ end
710
+ end
711
+ end
712
+ end
525
713
  end
data/spec/spec_helper.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  unless ENV["CI"] == "true"
2
4
  require "simplecov"
3
5
  SimpleCov.start do
@@ -9,6 +11,7 @@ require "pry"
9
11
  require "rspec"
10
12
  require "rdkafka"
11
13
  require "timeout"
14
+ require "securerandom"
12
15
 
13
16
  def rdkafka_base_config
14
17
  {
@@ -33,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
33
36
  # Add consumer specific fields to it
34
37
  config[:"auto.offset.reset"] = "earliest"
35
38
  config[:"enable.partition.eof"] = false
36
- config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
39
+ config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
37
40
  # Enable debug mode if required
38
41
  if ENV["DEBUG_CONSUMER"]
39
42
  config[:debug] = "cgrp,topic,fetch"
@@ -71,7 +74,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
71
74
  end
72
75
 
73
76
  def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
74
- new_consumer = !!consumer
77
+ new_consumer = consumer.nil?
75
78
  consumer ||= rdkafka_consumer_config.consumer
76
79
  consumer.subscribe(topic)
77
80
  timeout = Time.now.to_i + timeout_in_seconds
@@ -104,6 +107,20 @@ def wait_for_unassignment(consumer)
104
107
  end
105
108
  end
106
109
 
110
+ def notify_listener(listener, &block)
111
+ # 1. subscribe and poll
112
+ consumer.subscribe("consume_test_topic")
113
+ wait_for_assignment(consumer)
114
+ consumer.poll(100)
115
+
116
+ block.call if block
117
+
118
+ # 2. unsubscribe
119
+ consumer.unsubscribe
120
+ wait_for_unassignment(consumer)
121
+ consumer.close
122
+ end
123
+
107
124
  RSpec.configure do |config|
108
125
  config.filter_run focus: true
109
126
  config.run_all_when_everything_filtered = true
@@ -118,6 +135,7 @@ RSpec.configure do |config|
118
135
  rake_test_topic: 3,
119
136
  watermarks_test_topic: 3,
120
137
  partitioner_test_topic: 25,
138
+ example_topic: 1
121
139
  }.each do |topic, partitions|
122
140
  create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
123
141
  begin
data.tar.gz.sig ADDED
@@ -0,0 +1,3 @@
1
+ I��:z�mY0� �+��!vr��V��,g��,>���56�r��5���*Z�魇Q]##�)����y9��e\�w����^�F2ֲ�u�(7��u�A��(;]@���5��_��I�vF΀� ��&8���_�}2ܐ��.��F�=�f��V٭fI$<�1�M�8����C�o'2]�ŵg �6H��o_��B��|A�5F�Mx�J�N��k�)�Y��gC�r��"ib���sy>����Q��ڋCW9ڮo–z�
2
+ �k���q�q�������s�M&Y�4& �����+�]1�,Tf(DE
3
+ 9��U���"Ķ>�_g;��