rdkafka 0.15.1 → 0.16.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -4
- data/.gitignore +2 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +11 -1
- data/README.md +19 -9
- data/docker-compose.yml +1 -1
- data/ext/Rakefile +8 -0
- data/lib/rdkafka/abstract_handle.rb +44 -20
- data/lib/rdkafka/admin/create_topic_report.rb +1 -1
- data/lib/rdkafka/admin/delete_groups_report.rb +1 -1
- data/lib/rdkafka/admin/delete_topic_report.rb +1 -1
- data/lib/rdkafka/admin.rb +15 -0
- data/lib/rdkafka/bindings.rb +35 -3
- data/lib/rdkafka/callbacks.rb +18 -10
- data/lib/rdkafka/config.rb +69 -15
- data/lib/rdkafka/consumer.rb +7 -0
- data/lib/rdkafka/helpers/oauth.rb +58 -0
- data/lib/rdkafka/native_kafka.rb +32 -19
- data/lib/rdkafka/producer.rb +7 -0
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +1 -0
- data/spec/rdkafka/abstract_handle_spec.rb +34 -21
- data/spec/rdkafka/admin_spec.rb +53 -0
- data/spec/rdkafka/bindings_spec.rb +97 -0
- data/spec/rdkafka/config_spec.rb +53 -0
- data/spec/rdkafka/consumer_spec.rb +54 -0
- data/spec/rdkafka/native_kafka_spec.rb +8 -1
- data/spec/rdkafka/producer_spec.rb +43 -0
- data/spec/spec_helper.rb +16 -1
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
|
|
14
14
|
it { expect(consumer.name).to include('rdkafka#consumer-') }
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'consumer without auto-start' do
|
18
|
+
let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
consumer.start
|
22
|
+
consumer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
consumer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe "#subscribe, #unsubscribe and #subscription" do
|
18
31
|
it "should subscribe, unsubscribe and return the subscription" do
|
19
32
|
expect(consumer.subscription).to be_empty
|
@@ -211,6 +224,11 @@ describe Rdkafka::Consumer do
|
|
211
224
|
|
212
225
|
# 7. ensure same message is read again
|
213
226
|
message2 = consumer.poll(timeout)
|
227
|
+
|
228
|
+
# This is needed because `enable.auto.offset.store` is true but when running in CI that
|
229
|
+
# is overloaded, offset store lags
|
230
|
+
sleep(2)
|
231
|
+
|
214
232
|
consumer.commit
|
215
233
|
expect(message1.offset).to eq message2.offset
|
216
234
|
expect(message1.payload).to eq message2.payload
|
@@ -1296,4 +1314,40 @@ describe Rdkafka::Consumer do
|
|
1296
1314
|
])
|
1297
1315
|
end
|
1298
1316
|
end
|
1317
|
+
|
1318
|
+
describe '#oauthbearer_set_token' do
|
1319
|
+
context 'when sasl not configured' do
|
1320
|
+
it 'should return RD_KAFKA_RESP_ERR__STATE' do
|
1321
|
+
response = consumer.oauthbearer_set_token(
|
1322
|
+
token: "foo",
|
1323
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
1324
|
+
principal_name: "kafka-cluster"
|
1325
|
+
)
|
1326
|
+
expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
|
1327
|
+
end
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
context 'when sasl configured' do
|
1331
|
+
before do
|
1332
|
+
$consumer_sasl = rdkafka_producer_config(
|
1333
|
+
"security.protocol": "sasl_ssl",
|
1334
|
+
"sasl.mechanisms": 'OAUTHBEARER'
|
1335
|
+
).consumer
|
1336
|
+
end
|
1337
|
+
|
1338
|
+
after do
|
1339
|
+
$consumer_sasl.close
|
1340
|
+
end
|
1341
|
+
|
1342
|
+
it 'should succeed' do
|
1343
|
+
|
1344
|
+
response = $consumer_sasl.oauthbearer_set_token(
|
1345
|
+
token: "foo",
|
1346
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
1347
|
+
principal_name: "kafka-cluster"
|
1348
|
+
)
|
1349
|
+
expect(response).to eq(0)
|
1350
|
+
end
|
1351
|
+
end
|
1352
|
+
end
|
1299
1353
|
end
|
@@ -10,8 +10,9 @@ describe Rdkafka::NativeKafka do
|
|
10
10
|
subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
|
11
11
|
|
12
12
|
before do
|
13
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_name).and_return('producer-1')
|
13
14
|
allow(Thread).to receive(:new).and_return(thread)
|
14
|
-
|
15
|
+
allow(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
|
15
16
|
allow(thread).to receive(:[]=).with(:closing, anything)
|
16
17
|
allow(thread).to receive(:join)
|
17
18
|
allow(thread).to receive(:abort_on_exception=).with(anything)
|
@@ -20,6 +21,12 @@ describe Rdkafka::NativeKafka do
|
|
20
21
|
after { client.close }
|
21
22
|
|
22
23
|
context "defaults" do
|
24
|
+
it "sets the thread name" do
|
25
|
+
expect(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
|
26
|
+
|
27
|
+
client
|
28
|
+
end
|
29
|
+
|
23
30
|
it "sets the thread to abort on exception" do
|
24
31
|
expect(thread).to receive(:abort_on_exception=).with(true)
|
25
32
|
|
@@ -14,6 +14,19 @@ describe Rdkafka::Producer do
|
|
14
14
|
consumer.close
|
15
15
|
end
|
16
16
|
|
17
|
+
describe 'producer without auto-start' do
|
18
|
+
let(:producer) { rdkafka_producer_config.producer(native_kafka_auto_start: false) }
|
19
|
+
|
20
|
+
it 'expect to be able to start it later and close' do
|
21
|
+
producer.start
|
22
|
+
producer.close
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'expect to be able to close it without starting' do
|
26
|
+
producer.close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
17
30
|
describe '#name' do
|
18
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
19
32
|
end
|
@@ -734,4 +747,34 @@ describe Rdkafka::Producer do
|
|
734
747
|
end
|
735
748
|
end
|
736
749
|
end
|
750
|
+
|
751
|
+
describe '#oauthbearer_set_token' do
|
752
|
+
context 'when sasl not configured' do
|
753
|
+
it 'should return RD_KAFKA_RESP_ERR__STATE' do
|
754
|
+
response = producer.oauthbearer_set_token(
|
755
|
+
token: "foo",
|
756
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
757
|
+
principal_name: "kafka-cluster"
|
758
|
+
)
|
759
|
+
expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
|
760
|
+
end
|
761
|
+
end
|
762
|
+
|
763
|
+
context 'when sasl configured' do
|
764
|
+
it 'should succeed' do
|
765
|
+
producer_sasl = rdkafka_producer_config(
|
766
|
+
{
|
767
|
+
"security.protocol": "sasl_ssl",
|
768
|
+
"sasl.mechanisms": 'OAUTHBEARER'
|
769
|
+
}
|
770
|
+
).producer
|
771
|
+
response = producer_sasl.oauthbearer_set_token(
|
772
|
+
token: "foo",
|
773
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
774
|
+
principal_name: "kafka-cluster"
|
775
|
+
)
|
776
|
+
expect(response).to eq(0)
|
777
|
+
end
|
778
|
+
end
|
779
|
+
end
|
737
780
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -139,7 +139,7 @@ RSpec.configure do |config|
|
|
139
139
|
}.each do |topic, partitions|
|
140
140
|
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
141
141
|
begin
|
142
|
-
create_topic_handle.wait(max_wait_timeout:
|
142
|
+
create_topic_handle.wait(max_wait_timeout: 1.0)
|
143
143
|
rescue Rdkafka::RdkafkaError => ex
|
144
144
|
raise unless ex.message.match?(/topic_already_exists/)
|
145
145
|
end
|
@@ -155,3 +155,18 @@ RSpec.configure do |config|
|
|
155
155
|
end
|
156
156
|
end
|
157
157
|
end
|
158
|
+
|
159
|
+
class RdKafkaTestConsumer
|
160
|
+
def self.with
|
161
|
+
consumer = Rdkafka::Bindings.rd_kafka_new(
|
162
|
+
:rd_kafka_consumer,
|
163
|
+
nil,
|
164
|
+
nil,
|
165
|
+
0
|
166
|
+
)
|
167
|
+
yield consumer
|
168
|
+
ensure
|
169
|
+
Rdkafka::Bindings.rd_kafka_consumer_close(consumer)
|
170
|
+
Rdkafka::Bindings.rd_kafka_destroy(consumer)
|
171
|
+
end
|
172
|
+
end
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-04-29 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
@@ -216,6 +216,7 @@ files:
|
|
216
216
|
- lib/rdkafka/consumer/partition.rb
|
217
217
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
218
218
|
- lib/rdkafka/error.rb
|
219
|
+
- lib/rdkafka/helpers/oauth.rb
|
219
220
|
- lib/rdkafka/helpers/time.rb
|
220
221
|
- lib/rdkafka/metadata.rb
|
221
222
|
- lib/rdkafka/native_kafka.rb
|
@@ -278,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
278
279
|
- !ruby/object:Gem::Version
|
279
280
|
version: '0'
|
280
281
|
requirements: []
|
281
|
-
rubygems_version: 3.5.
|
282
|
+
rubygems_version: 3.5.9
|
282
283
|
signing_key:
|
283
284
|
specification_version: 4
|
284
285
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
metadata.gz.sig
CHANGED
Binary file
|