rdkafka 0.13.0 → 0.14.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +1 -0
- data/.github/workflows/ci.yml +58 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +39 -21
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +19 -20
- data/certs/cert_chain.pem +26 -0
- data/docker-compose.yml +16 -15
- data/ext/README.md +1 -1
- data/ext/Rakefile +1 -1
- data/lib/rdkafka/abstract_handle.rb +37 -24
- data/lib/rdkafka/admin.rb +6 -7
- data/lib/rdkafka/bindings.rb +8 -5
- data/lib/rdkafka/config.rb +30 -17
- data/lib/rdkafka/consumer/headers.rb +2 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +3 -1
- data/lib/rdkafka/consumer.rb +92 -53
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +22 -1
- data/lib/rdkafka/native_kafka.rb +6 -1
- data/lib/rdkafka/producer.rb +85 -7
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +10 -1
- data/rdkafka.gemspec +17 -3
- data/renovate.json +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin_spec.rb +1 -2
- data/spec/rdkafka/bindings_spec.rb +0 -1
- data/spec/rdkafka/callbacks_spec.rb +0 -2
- data/spec/rdkafka/config_spec.rb +0 -2
- data/spec/rdkafka/consumer/headers_spec.rb +0 -2
- data/spec/rdkafka/consumer/message_spec.rb +0 -2
- data/spec/rdkafka/consumer/partition_spec.rb +0 -2
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +19 -2
- data/spec/rdkafka/consumer_spec.rb +143 -39
- data/spec/rdkafka/error_spec.rb +0 -2
- data/spec/rdkafka/metadata_spec.rb +2 -3
- data/spec/rdkafka/native_kafka_spec.rb +2 -3
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -2
- data/spec/rdkafka/producer_spec.rb +157 -1
- data.tar.gz.sig +0 -0
- metadata +54 -15
- metadata.gz.sig +0 -0
- data/.semaphore/semaphore.yml +0 -27
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "spec_helper"
|
4
3
|
require "zlib"
|
5
4
|
|
6
5
|
describe Rdkafka::Producer do
|
@@ -15,6 +14,10 @@ describe Rdkafka::Producer do
|
|
15
14
|
consumer.close
|
16
15
|
end
|
17
16
|
|
17
|
+
describe '#name' do
|
18
|
+
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
|
+
end
|
20
|
+
|
18
21
|
context "delivery callback" do
|
19
22
|
context "with a proc/lambda" do
|
20
23
|
it "should set the callback" do
|
@@ -554,4 +557,157 @@ describe Rdkafka::Producer do
|
|
554
557
|
end
|
555
558
|
end
|
556
559
|
end
|
560
|
+
|
561
|
+
describe '#partition_count' do
|
562
|
+
it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
|
563
|
+
|
564
|
+
context 'when the partition count value is already cached' do
|
565
|
+
before do
|
566
|
+
producer.partition_count('consume_test_topic')
|
567
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
568
|
+
end
|
569
|
+
|
570
|
+
it 'expect not to query it again' do
|
571
|
+
producer.partition_count('consume_test_topic')
|
572
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
context 'when the partition count value was cached but time expired' do
|
577
|
+
before do
|
578
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 30.02)
|
579
|
+
producer.partition_count('consume_test_topic')
|
580
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
581
|
+
end
|
582
|
+
|
583
|
+
it 'expect not to query it again' do
|
584
|
+
producer.partition_count('consume_test_topic')
|
585
|
+
expect(::Rdkafka::Metadata).to have_received(:new)
|
586
|
+
end
|
587
|
+
end
|
588
|
+
|
589
|
+
context 'when the partition count value was cached and time did not expire' do
|
590
|
+
before do
|
591
|
+
allow(::Process).to receive(:clock_gettime).and_return(0, 29.001)
|
592
|
+
producer.partition_count('consume_test_topic')
|
593
|
+
allow(::Rdkafka::Metadata).to receive(:new).and_call_original
|
594
|
+
end
|
595
|
+
|
596
|
+
it 'expect not to query it again' do
|
597
|
+
producer.partition_count('consume_test_topic')
|
598
|
+
expect(::Rdkafka::Metadata).not_to have_received(:new)
|
599
|
+
end
|
600
|
+
end
|
601
|
+
end
|
602
|
+
|
603
|
+
describe '#flush' do
|
604
|
+
it "should return flush when it can flush all outstanding messages or when no messages" do
|
605
|
+
producer.produce(
|
606
|
+
topic: "produce_test_topic",
|
607
|
+
payload: "payload headers",
|
608
|
+
key: "key headers",
|
609
|
+
headers: {}
|
610
|
+
)
|
611
|
+
|
612
|
+
expect(producer.flush(5_000)).to eq(true)
|
613
|
+
end
|
614
|
+
|
615
|
+
context 'when it cannot flush due to a timeout' do
|
616
|
+
let(:producer) do
|
617
|
+
rdkafka_producer_config(
|
618
|
+
"bootstrap.servers": "localhost:9093",
|
619
|
+
"message.timeout.ms": 2_000
|
620
|
+
).producer
|
621
|
+
end
|
622
|
+
|
623
|
+
after do
|
624
|
+
# Allow rdkafka to evict message preventing memory-leak
|
625
|
+
sleep(2)
|
626
|
+
end
|
627
|
+
|
628
|
+
it "should return false on flush when cannot deliver and beyond timeout" do
|
629
|
+
producer.produce(
|
630
|
+
topic: "produce_test_topic",
|
631
|
+
payload: "payload headers",
|
632
|
+
key: "key headers",
|
633
|
+
headers: {}
|
634
|
+
)
|
635
|
+
|
636
|
+
expect(producer.flush(1_000)).to eq(false)
|
637
|
+
end
|
638
|
+
end
|
639
|
+
|
640
|
+
context 'when there is a different error' do
|
641
|
+
before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
|
642
|
+
|
643
|
+
it 'should raise it' do
|
644
|
+
expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
|
645
|
+
end
|
646
|
+
end
|
647
|
+
end
|
648
|
+
|
649
|
+
describe '#purge' do
|
650
|
+
context 'when no outgoing messages' do
|
651
|
+
it { expect(producer.purge).to eq(true) }
|
652
|
+
end
|
653
|
+
|
654
|
+
context 'when librdkafka purge returns an error' do
|
655
|
+
before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
|
656
|
+
|
657
|
+
it 'expect to raise an error' do
|
658
|
+
expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
|
659
|
+
end
|
660
|
+
end
|
661
|
+
|
662
|
+
context 'when there are outgoing things in the queue' do
|
663
|
+
let(:producer) do
|
664
|
+
rdkafka_producer_config(
|
665
|
+
"bootstrap.servers": "localhost:9093",
|
666
|
+
"message.timeout.ms": 2_000
|
667
|
+
).producer
|
668
|
+
end
|
669
|
+
|
670
|
+
it "should should purge and move forward" do
|
671
|
+
producer.produce(
|
672
|
+
topic: "produce_test_topic",
|
673
|
+
payload: "payload headers"
|
674
|
+
)
|
675
|
+
|
676
|
+
expect(producer.purge).to eq(true)
|
677
|
+
expect(producer.flush(1_000)).to eq(true)
|
678
|
+
end
|
679
|
+
|
680
|
+
it "should materialize the delivery handles" do
|
681
|
+
handle = producer.produce(
|
682
|
+
topic: "produce_test_topic",
|
683
|
+
payload: "payload headers"
|
684
|
+
)
|
685
|
+
|
686
|
+
expect(producer.purge).to eq(true)
|
687
|
+
|
688
|
+
expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
|
689
|
+
end
|
690
|
+
|
691
|
+
context "when using delivery_callback" do
|
692
|
+
let(:delivery_reports) { [] }
|
693
|
+
|
694
|
+
let(:delivery_callback) do
|
695
|
+
->(delivery_report) { delivery_reports << delivery_report }
|
696
|
+
end
|
697
|
+
|
698
|
+
before { producer.delivery_callback = delivery_callback }
|
699
|
+
|
700
|
+
it "should run the callback" do
|
701
|
+
handle = producer.produce(
|
702
|
+
topic: "produce_test_topic",
|
703
|
+
payload: "payload headers"
|
704
|
+
)
|
705
|
+
|
706
|
+
expect(producer.purge).to eq(true)
|
707
|
+
# queue purge
|
708
|
+
expect(delivery_reports[0].error).to eq(-152)
|
709
|
+
end
|
710
|
+
end
|
711
|
+
end
|
712
|
+
end
|
557
713
|
end
|
data.tar.gz.sig
ADDED
Binary file
|
metadata
CHANGED
@@ -1,14 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
|
-
cert_chain:
|
11
|
-
|
10
|
+
cert_chain:
|
11
|
+
- |
|
12
|
+
-----BEGIN CERTIFICATE-----
|
13
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
14
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
15
|
+
MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
|
16
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
17
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
|
18
|
+
8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
|
19
|
+
sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
|
20
|
+
wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
|
21
|
+
b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
|
22
|
+
ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
|
23
|
+
5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
|
24
|
+
YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
|
25
|
+
kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
|
26
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
|
27
|
+
1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
28
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
|
29
|
+
bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
|
30
|
+
3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
|
31
|
+
kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
|
32
|
+
IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
|
33
|
+
fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
|
34
|
+
qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
|
35
|
+
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
|
+
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
|
+
-----END CERTIFICATE-----
|
38
|
+
date: 2023-11-17 00:00:00.000000000 Z
|
12
39
|
dependencies:
|
13
40
|
- !ruby/object:Gem::Dependency
|
14
41
|
name: ffi
|
@@ -138,22 +165,25 @@ dependencies:
|
|
138
165
|
version: '0'
|
139
166
|
description: Modern Kafka client library for Ruby based on librdkafka
|
140
167
|
email:
|
141
|
-
-
|
168
|
+
- contact@karafka.io
|
142
169
|
executables: []
|
143
170
|
extensions:
|
144
171
|
- ext/Rakefile
|
145
172
|
extra_rdoc_files: []
|
146
173
|
files:
|
174
|
+
- ".github/workflows/ci.yml"
|
147
175
|
- ".gitignore"
|
148
176
|
- ".rspec"
|
149
|
-
- ".
|
177
|
+
- ".ruby-gemset"
|
178
|
+
- ".ruby-version"
|
150
179
|
- ".yardopts"
|
151
180
|
- CHANGELOG.md
|
152
181
|
- Gemfile
|
153
182
|
- Guardfile
|
154
|
-
- LICENSE
|
183
|
+
- MIT-LICENSE
|
155
184
|
- README.md
|
156
185
|
- Rakefile
|
186
|
+
- certs/cert_chain.pem
|
157
187
|
- docker-compose.yml
|
158
188
|
- ext/README.md
|
159
189
|
- ext/Rakefile
|
@@ -173,6 +203,7 @@ files:
|
|
173
203
|
- lib/rdkafka/consumer/partition.rb
|
174
204
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
175
205
|
- lib/rdkafka/error.rb
|
206
|
+
- lib/rdkafka/helpers/time.rb
|
176
207
|
- lib/rdkafka/metadata.rb
|
177
208
|
- lib/rdkafka/native_kafka.rb
|
178
209
|
- lib/rdkafka/producer.rb
|
@@ -180,6 +211,7 @@ files:
|
|
180
211
|
- lib/rdkafka/producer/delivery_report.rb
|
181
212
|
- lib/rdkafka/version.rb
|
182
213
|
- rdkafka.gemspec
|
214
|
+
- renovate.json
|
183
215
|
- spec/rdkafka/abstract_handle_spec.rb
|
184
216
|
- spec/rdkafka/admin/create_topic_handle_spec.rb
|
185
217
|
- spec/rdkafka/admin/create_topic_report_spec.rb
|
@@ -201,11 +233,18 @@ files:
|
|
201
233
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
202
234
|
- spec/rdkafka/producer_spec.rb
|
203
235
|
- spec/spec_helper.rb
|
204
|
-
homepage:
|
236
|
+
homepage:
|
205
237
|
licenses:
|
206
238
|
- MIT
|
207
|
-
metadata:
|
208
|
-
|
239
|
+
metadata:
|
240
|
+
funding_uri: https://karafka.io/#become-pro
|
241
|
+
homepage_uri: https://karafka.io
|
242
|
+
changelog_uri: https://github.com/karafka/rdkafka-ruby/blob/main/CHANGELOG.md
|
243
|
+
bug_tracker_uri: https://github.com/karafka/rdkafka-ruby/issues
|
244
|
+
source_code_uri: https://github.com/karafka/rdkafka-ruby
|
245
|
+
documentation_uri: https://github.com/karafka/rdkafka-ruby/blob/main/README.md
|
246
|
+
rubygems_mfa_required: 'true'
|
247
|
+
post_install_message:
|
209
248
|
rdoc_options: []
|
210
249
|
require_paths:
|
211
250
|
- lib
|
@@ -213,15 +252,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
213
252
|
requirements:
|
214
253
|
- - ">="
|
215
254
|
- !ruby/object:Gem::Version
|
216
|
-
version: '2.
|
255
|
+
version: '2.7'
|
217
256
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
218
257
|
requirements:
|
219
|
-
- - "
|
258
|
+
- - ">"
|
220
259
|
- !ruby/object:Gem::Version
|
221
|
-
version:
|
260
|
+
version: 1.3.1
|
222
261
|
requirements: []
|
223
|
-
rubygems_version: 3.4.
|
224
|
-
signing_key:
|
262
|
+
rubygems_version: 3.4.19
|
263
|
+
signing_key:
|
225
264
|
specification_version: 4
|
226
265
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
227
266
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
metadata.gz.sig
ADDED
Binary file
|
data/.semaphore/semaphore.yml
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
version: v1.0
|
2
|
-
name: Rdkafka Ruby
|
3
|
-
|
4
|
-
agent:
|
5
|
-
machine:
|
6
|
-
type: e1-standard-4
|
7
|
-
os_image: ubuntu1804
|
8
|
-
|
9
|
-
blocks:
|
10
|
-
- name: Run specs
|
11
|
-
task:
|
12
|
-
prologue:
|
13
|
-
commands:
|
14
|
-
- sudo apt install -y valgrind
|
15
|
-
jobs:
|
16
|
-
- name: bundle exec rspec
|
17
|
-
matrix:
|
18
|
-
- env_var: RUBY_VERSION
|
19
|
-
values: [ "2.6.10", "2.7.6", "3.0.4", "3.1.2"]
|
20
|
-
commands:
|
21
|
-
- sem-version ruby $RUBY_VERSION
|
22
|
-
- checkout
|
23
|
-
- docker-compose up -d --no-recreate
|
24
|
-
- bundle install --path vendor/bundle
|
25
|
-
- cd ext && bundle exec rake && cd ..
|
26
|
-
- ulimit -c unlimited
|
27
|
-
- valgrind -v bundle exec rspec
|