karafka-rdkafka 0.13.7 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +40 -31
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +11 -11
- data/ext/README.md +1 -1
- data/ext/Rakefile +1 -1
- data/lib/rdkafka/abstract_handle.rb +37 -24
- data/lib/rdkafka/admin.rb +6 -7
- data/lib/rdkafka/bindings.rb +1 -4
- data/lib/rdkafka/config.rb +30 -15
- data/lib/rdkafka/consumer/headers.rb +2 -4
- data/lib/rdkafka/consumer.rb +83 -53
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/producer.rb +8 -15
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +10 -1
- data/spec/rdkafka/abstract_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin_spec.rb +0 -1
- data/spec/rdkafka/bindings_spec.rb +0 -1
- data/spec/rdkafka/callbacks_spec.rb +0 -2
- data/spec/rdkafka/config_spec.rb +8 -2
- data/spec/rdkafka/consumer/headers_spec.rb +0 -2
- data/spec/rdkafka/consumer/message_spec.rb +0 -2
- data/spec/rdkafka/consumer/partition_spec.rb +0 -2
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -2
- data/spec/rdkafka/consumer_spec.rb +122 -38
- data/spec/rdkafka/error_spec.rb +0 -2
- data/spec/rdkafka/metadata_spec.rb +0 -1
- data/spec/rdkafka/native_kafka_spec.rb +0 -2
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -2
- data/spec/rdkafka/producer_spec.rb +0 -1
- data/spec/spec_helper.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +7 -4
- metadata.gz.sig +0 -0
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
3
|
require "ostruct"
|
5
4
|
require 'securerandom'
|
6
5
|
|
@@ -55,6 +54,30 @@ describe Rdkafka::Consumer do
|
|
55
54
|
consumer.subscription
|
56
55
|
}.to raise_error(Rdkafka::RdkafkaError)
|
57
56
|
end
|
57
|
+
|
58
|
+
context "when using consumer without the poll set" do
|
59
|
+
let(:consumer) do
|
60
|
+
config = rdkafka_consumer_config
|
61
|
+
config.consumer_poll_set = false
|
62
|
+
config.consumer
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should subscribe, unsubscribe and return the subscription" do
|
66
|
+
expect(consumer.subscription).to be_empty
|
67
|
+
|
68
|
+
consumer.subscribe("consume_test_topic")
|
69
|
+
|
70
|
+
expect(consumer.subscription).not_to be_empty
|
71
|
+
expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
72
|
+
list.add_topic("consume_test_topic")
|
73
|
+
end
|
74
|
+
expect(consumer.subscription).to eq expected_subscription
|
75
|
+
|
76
|
+
consumer.unsubscribe
|
77
|
+
|
78
|
+
expect(consumer.subscription).to be_empty
|
79
|
+
end
|
80
|
+
end
|
58
81
|
end
|
59
82
|
|
60
83
|
describe "#pause and #resume" do
|
@@ -337,8 +360,9 @@ describe Rdkafka::Consumer do
|
|
337
360
|
end
|
338
361
|
end
|
339
362
|
|
340
|
-
|
341
|
-
|
363
|
+
|
364
|
+
describe "#position, #commit, #committed and #store_offset" do
|
365
|
+
# Make sure there are messages to work with
|
342
366
|
let!(:report) do
|
343
367
|
producer.produce(
|
344
368
|
topic: "consume_test_topic",
|
@@ -356,29 +380,33 @@ describe Rdkafka::Consumer do
|
|
356
380
|
)
|
357
381
|
end
|
358
382
|
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
383
|
+
describe "#position" do
|
384
|
+
it "should only accept a topic partition list in position if not nil" do
|
385
|
+
expect {
|
386
|
+
consumer.position("list")
|
387
|
+
}.to raise_error TypeError
|
388
|
+
end
|
363
389
|
end
|
364
390
|
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
391
|
+
describe "#committed" do
|
392
|
+
it "should only accept a topic partition list in commit if not nil" do
|
393
|
+
expect {
|
394
|
+
consumer.commit("list")
|
395
|
+
}.to raise_error TypeError
|
396
|
+
end
|
370
397
|
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
398
|
+
it "should commit in sync mode" do
|
399
|
+
expect {
|
400
|
+
consumer.commit(nil, true)
|
401
|
+
}.not_to raise_error
|
402
|
+
end
|
375
403
|
end
|
376
404
|
|
377
405
|
context "with a committed consumer" do
|
378
406
|
before :all do
|
379
407
|
# Make sure there are some messages.
|
380
408
|
handles = []
|
381
|
-
producer =
|
409
|
+
producer = rdkafka_config.producer
|
382
410
|
10.times do
|
383
411
|
(0..2).each do |i|
|
384
412
|
handles << producer.produce(
|
@@ -422,31 +450,33 @@ describe Rdkafka::Consumer do
|
|
422
450
|
}.to raise_error(Rdkafka::RdkafkaError)
|
423
451
|
end
|
424
452
|
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
453
|
+
describe "#committed" do
|
454
|
+
it "should fetch the committed offsets for the current assignment" do
|
455
|
+
partitions = consumer.committed.to_h["consume_test_topic"]
|
456
|
+
expect(partitions).not_to be_nil
|
457
|
+
expect(partitions[0].offset).to eq 1
|
458
|
+
end
|
430
459
|
|
431
|
-
|
432
|
-
|
433
|
-
|
460
|
+
it "should fetch the committed offsets for a specified topic partition list" do
|
461
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
462
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
463
|
+
end
|
464
|
+
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
465
|
+
expect(partitions).not_to be_nil
|
466
|
+
expect(partitions[0].offset).to eq 1
|
467
|
+
expect(partitions[1].offset).to eq 1
|
468
|
+
expect(partitions[2].offset).to eq 1
|
434
469
|
end
|
435
|
-
partitions = consumer.committed(list).to_h["consume_test_topic"]
|
436
|
-
expect(partitions).not_to be_nil
|
437
|
-
expect(partitions[0].offset).to eq 1
|
438
|
-
expect(partitions[1].offset).to eq 1
|
439
|
-
expect(partitions[2].offset).to eq 1
|
440
|
-
end
|
441
470
|
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
471
|
+
it "should raise an error when getting committed fails" do
|
472
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
|
473
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
474
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
475
|
+
end
|
476
|
+
expect {
|
477
|
+
consumer.committed(list)
|
478
|
+
}.to raise_error Rdkafka::RdkafkaError
|
446
479
|
end
|
447
|
-
expect {
|
448
|
-
consumer.committed(list)
|
449
|
-
}.to raise_error Rdkafka::RdkafkaError
|
450
480
|
end
|
451
481
|
|
452
482
|
describe "#store_offset" do
|
@@ -467,6 +497,8 @@ describe Rdkafka::Consumer do
|
|
467
497
|
@new_consumer.store_offset(message)
|
468
498
|
@new_consumer.commit
|
469
499
|
|
500
|
+
# TODO use position here, should be at offset
|
501
|
+
|
470
502
|
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
471
503
|
list.add_topic("consume_test_topic", [0, 1, 2])
|
472
504
|
end
|
@@ -481,6 +513,35 @@ describe Rdkafka::Consumer do
|
|
481
513
|
@new_consumer.store_offset(message)
|
482
514
|
}.to raise_error Rdkafka::RdkafkaError
|
483
515
|
end
|
516
|
+
|
517
|
+
describe "#position" do
|
518
|
+
it "should fetch the positions for the current assignment" do
|
519
|
+
consumer.store_offset(message)
|
520
|
+
|
521
|
+
partitions = consumer.position.to_h["consume_test_topic"]
|
522
|
+
expect(partitions).not_to be_nil
|
523
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
524
|
+
end
|
525
|
+
|
526
|
+
it "should fetch the positions for a specified assignment" do
|
527
|
+
consumer.store_offset(message)
|
528
|
+
|
529
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
530
|
+
list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
|
531
|
+
end
|
532
|
+
partitions = consumer.position(list).to_h["consume_test_topic"]
|
533
|
+
expect(partitions).not_to be_nil
|
534
|
+
expect(partitions[0].offset).to eq message.offset + 1
|
535
|
+
end
|
536
|
+
|
537
|
+
it "should raise an error when getting the position fails" do
|
538
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
|
539
|
+
|
540
|
+
expect {
|
541
|
+
consumer.position
|
542
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
543
|
+
end
|
544
|
+
end
|
484
545
|
end
|
485
546
|
end
|
486
547
|
end
|
@@ -1039,6 +1100,29 @@ describe Rdkafka::Consumer do
|
|
1039
1100
|
end
|
1040
1101
|
end
|
1041
1102
|
|
1103
|
+
# Only relevant in case of a consumer with separate queues
|
1104
|
+
describe '#events_poll' do
|
1105
|
+
let(:stats) { [] }
|
1106
|
+
|
1107
|
+
before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
|
1108
|
+
|
1109
|
+
after { Rdkafka::Config.statistics_callback = nil }
|
1110
|
+
|
1111
|
+
let(:consumer) do
|
1112
|
+
config = rdkafka_consumer_config('statistics.interval.ms': 100)
|
1113
|
+
config.consumer_poll_set = false
|
1114
|
+
config.consumer
|
1115
|
+
end
|
1116
|
+
|
1117
|
+
it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
|
1118
|
+
consumer.subscribe("consume_test_topic")
|
1119
|
+
consumer.poll(1_000)
|
1120
|
+
expect(stats).to be_empty
|
1121
|
+
consumer.events_poll(-1)
|
1122
|
+
expect(stats).not_to be_empty
|
1123
|
+
end
|
1124
|
+
end
|
1125
|
+
|
1042
1126
|
describe "a rebalance listener" do
|
1043
1127
|
let(:consumer) do
|
1044
1128
|
config = rdkafka_consumer_config
|
data/spec/rdkafka/error_spec.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
@@ -36,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
|
|
36
36
|
# Add consumer specific fields to it
|
37
37
|
config[:"auto.offset.reset"] = "earliest"
|
38
38
|
config[:"enable.partition.eof"] = false
|
39
|
-
config[:"group.id"] = "ruby-test-#{
|
39
|
+
config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
|
40
40
|
# Enable debug mode if required
|
41
41
|
if ENV["DEBUG_CONSUMER"]
|
42
42
|
config[:debug] = "cgrp,topic,fetch"
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.13.
|
4
|
+
version: 0.13.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-
|
38
|
+
date: 2023-11-07 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: ffi
|
@@ -174,11 +174,13 @@ files:
|
|
174
174
|
- ".github/workflows/ci.yml"
|
175
175
|
- ".gitignore"
|
176
176
|
- ".rspec"
|
177
|
+
- ".ruby-gemset"
|
178
|
+
- ".ruby-version"
|
177
179
|
- ".yardopts"
|
178
180
|
- CHANGELOG.md
|
179
181
|
- Gemfile
|
180
182
|
- Guardfile
|
181
|
-
- LICENSE
|
183
|
+
- MIT-LICENSE
|
182
184
|
- README.md
|
183
185
|
- Rakefile
|
184
186
|
- certs/cert_chain.pem
|
@@ -204,6 +206,7 @@ files:
|
|
204
206
|
- lib/rdkafka/consumer/partition.rb
|
205
207
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
206
208
|
- lib/rdkafka/error.rb
|
209
|
+
- lib/rdkafka/helpers/time.rb
|
207
210
|
- lib/rdkafka/metadata.rb
|
208
211
|
- lib/rdkafka/native_kafka.rb
|
209
212
|
- lib/rdkafka/producer.rb
|
@@ -258,7 +261,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
258
261
|
- !ruby/object:Gem::Version
|
259
262
|
version: '0'
|
260
263
|
requirements: []
|
261
|
-
rubygems_version: 3.
|
264
|
+
rubygems_version: 3.4.19
|
262
265
|
signing_key:
|
263
266
|
specification_version: 4
|
264
267
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
metadata.gz.sig
CHANGED
Binary file
|