rdkafka 0.15.1 → 0.16.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -4
- data/.gitignore +2 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +11 -1
- data/README.md +19 -9
- data/docker-compose.yml +1 -1
- data/ext/Rakefile +8 -0
- data/lib/rdkafka/abstract_handle.rb +44 -20
- data/lib/rdkafka/admin/create_topic_report.rb +1 -1
- data/lib/rdkafka/admin/delete_groups_report.rb +1 -1
- data/lib/rdkafka/admin/delete_topic_report.rb +1 -1
- data/lib/rdkafka/admin.rb +15 -0
- data/lib/rdkafka/bindings.rb +35 -3
- data/lib/rdkafka/callbacks.rb +18 -10
- data/lib/rdkafka/config.rb +69 -15
- data/lib/rdkafka/consumer.rb +7 -0
- data/lib/rdkafka/helpers/oauth.rb +58 -0
- data/lib/rdkafka/native_kafka.rb +32 -19
- data/lib/rdkafka/producer.rb +7 -0
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +1 -0
- data/spec/rdkafka/abstract_handle_spec.rb +34 -21
- data/spec/rdkafka/admin_spec.rb +53 -0
- data/spec/rdkafka/bindings_spec.rb +97 -0
- data/spec/rdkafka/config_spec.rb +53 -0
- data/spec/rdkafka/consumer_spec.rb +54 -0
- data/spec/rdkafka/native_kafka_spec.rb +8 -1
- data/spec/rdkafka/producer_spec.rb +43 -0
- data/spec/spec_helper.rb +16 -1
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
data/lib/rdkafka/config.rb
CHANGED
@@ -15,13 +15,13 @@ module Rdkafka
|
|
15
15
|
@@opaques = ObjectSpace::WeakMap.new
|
16
16
|
# @private
|
17
17
|
@@log_queue = Queue.new
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
18
|
+
# We memoize thread on the first log flush
|
19
|
+
# This allows us also to restart logger thread on forks
|
20
|
+
@@log_thread = nil
|
21
|
+
# @private
|
22
|
+
@@log_mutex = Mutex.new
|
23
|
+
# @private
|
24
|
+
@@oauthbearer_token_refresh_callback = nil
|
25
25
|
|
26
26
|
# Returns the current logger, by default this is a logger to stdout.
|
27
27
|
#
|
@@ -30,6 +30,24 @@ module Rdkafka
|
|
30
30
|
@@logger
|
31
31
|
end
|
32
32
|
|
33
|
+
# Makes sure that there is a thread for consuming logs
|
34
|
+
# We do not spawn thread immediately and we need to check if it operates to support forking
|
35
|
+
def self.ensure_log_thread
|
36
|
+
return if @@log_thread && @@log_thread.alive?
|
37
|
+
|
38
|
+
@@log_mutex.synchronize do
|
39
|
+
# Restart if dead (fork, crash)
|
40
|
+
@@log_thread = nil if @@log_thread && !@@log_thread.alive?
|
41
|
+
|
42
|
+
@@log_thread ||= Thread.start do
|
43
|
+
loop do
|
44
|
+
severity, msg = @@log_queue.pop
|
45
|
+
@@logger.add(severity, msg)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
33
51
|
# Returns a queue whose contents will be passed to the configured logger. Each entry
|
34
52
|
# should follow the format [Logger::Severity, String]. The benefit over calling the
|
35
53
|
# logger directly is that this is safe to use from trap contexts.
|
@@ -87,6 +105,24 @@ module Rdkafka
|
|
87
105
|
@@error_callback
|
88
106
|
end
|
89
107
|
|
108
|
+
# Sets the SASL/OAUTHBEARER token refresh callback.
|
109
|
+
# This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
|
110
|
+
#
|
111
|
+
# @param callback [Proc, #call] The callback
|
112
|
+
#
|
113
|
+
# @return [nil]
|
114
|
+
def self.oauthbearer_token_refresh_callback=(callback)
|
115
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
|
116
|
+
@@oauthbearer_token_refresh_callback = callback
|
117
|
+
end
|
118
|
+
|
119
|
+
# Returns the current oauthbearer_token_refresh_callback callback, by default this is nil.
|
120
|
+
#
|
121
|
+
# @return [Proc, nil]
|
122
|
+
def self.oauthbearer_token_refresh_callback
|
123
|
+
@@oauthbearer_token_refresh_callback
|
124
|
+
end
|
125
|
+
|
90
126
|
# @private
|
91
127
|
def self.opaques
|
92
128
|
@@opaques
|
@@ -159,11 +195,13 @@ module Rdkafka
|
|
159
195
|
|
160
196
|
# Creates a consumer with this configuration.
|
161
197
|
#
|
198
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
199
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
162
200
|
# @return [Consumer] The created consumer
|
163
201
|
#
|
164
202
|
# @raise [ConfigError] When the configuration contains invalid options
|
165
203
|
# @raise [ClientCreationError] When the native client cannot be created
|
166
|
-
def consumer
|
204
|
+
def consumer(native_kafka_auto_start: true)
|
167
205
|
opaque = Opaque.new
|
168
206
|
config = native_config(opaque)
|
169
207
|
|
@@ -183,18 +221,21 @@ module Rdkafka
|
|
183
221
|
Rdkafka::NativeKafka.new(
|
184
222
|
kafka,
|
185
223
|
run_polling_thread: false,
|
186
|
-
opaque: opaque
|
224
|
+
opaque: opaque,
|
225
|
+
auto_start: native_kafka_auto_start
|
187
226
|
)
|
188
227
|
)
|
189
228
|
end
|
190
229
|
|
191
230
|
# Create a producer with this configuration.
|
192
231
|
#
|
232
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
233
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
193
234
|
# @return [Producer] The created producer
|
194
235
|
#
|
195
236
|
# @raise [ConfigError] When the configuration contains invalid options
|
196
237
|
# @raise [ClientCreationError] When the native client cannot be created
|
197
|
-
def producer
|
238
|
+
def producer(native_kafka_auto_start: true)
|
198
239
|
# Create opaque
|
199
240
|
opaque = Opaque.new
|
200
241
|
# Create Kafka config
|
@@ -203,11 +244,15 @@ module Rdkafka
|
|
203
244
|
Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
|
204
245
|
# Return producer with Kafka client
|
205
246
|
partitioner_name = self[:partitioner] || self["partitioner"]
|
247
|
+
|
248
|
+
kafka = native_kafka(config, :rd_kafka_producer)
|
249
|
+
|
206
250
|
Rdkafka::Producer.new(
|
207
251
|
Rdkafka::NativeKafka.new(
|
208
|
-
|
252
|
+
kafka,
|
209
253
|
run_polling_thread: true,
|
210
|
-
opaque: opaque
|
254
|
+
opaque: opaque,
|
255
|
+
auto_start: native_kafka_auto_start
|
211
256
|
),
|
212
257
|
partitioner_name
|
213
258
|
).tap do |producer|
|
@@ -217,19 +262,25 @@ module Rdkafka
|
|
217
262
|
|
218
263
|
# Creates an admin instance with this configuration.
|
219
264
|
#
|
265
|
+
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
266
|
+
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
220
267
|
# @return [Admin] The created admin instance
|
221
268
|
#
|
222
269
|
# @raise [ConfigError] When the configuration contains invalid options
|
223
270
|
# @raise [ClientCreationError] When the native client cannot be created
|
224
|
-
def admin
|
271
|
+
def admin(native_kafka_auto_start: true)
|
225
272
|
opaque = Opaque.new
|
226
273
|
config = native_config(opaque)
|
227
274
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
275
|
+
|
276
|
+
kafka = native_kafka(config, :rd_kafka_producer)
|
277
|
+
|
228
278
|
Rdkafka::Admin.new(
|
229
279
|
Rdkafka::NativeKafka.new(
|
230
|
-
|
280
|
+
kafka,
|
231
281
|
run_polling_thread: true,
|
232
|
-
opaque: opaque
|
282
|
+
opaque: opaque,
|
283
|
+
auto_start: native_kafka_auto_start
|
233
284
|
)
|
234
285
|
)
|
235
286
|
end
|
@@ -283,6 +334,9 @@ module Rdkafka
|
|
283
334
|
|
284
335
|
# Set error callback
|
285
336
|
Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
|
337
|
+
|
338
|
+
# Set oauth callback
|
339
|
+
Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
|
286
340
|
end
|
287
341
|
end
|
288
342
|
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -13,12 +13,19 @@ module Rdkafka
|
|
13
13
|
class Consumer
|
14
14
|
include Enumerable
|
15
15
|
include Helpers::Time
|
16
|
+
include Helpers::OAuth
|
16
17
|
|
17
18
|
# @private
|
18
19
|
def initialize(native_kafka)
|
19
20
|
@native_kafka = native_kafka
|
20
21
|
end
|
21
22
|
|
23
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
24
|
+
# @note Not needed to run unless explicit start was disabled
|
25
|
+
def start
|
26
|
+
@native_kafka.start
|
27
|
+
end
|
28
|
+
|
22
29
|
# @return [String] consumer name
|
23
30
|
def name
|
24
31
|
@name ||= @native_kafka.with_inner do |inner|
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Rdkafka
|
2
|
+
module Helpers
|
3
|
+
|
4
|
+
module OAuth
|
5
|
+
|
6
|
+
# Set the OAuthBearer token
|
7
|
+
#
|
8
|
+
# @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
|
9
|
+
# @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
|
10
|
+
# @param principal_name [String] the mandatory Kafka principal name associated with the token.
|
11
|
+
# @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
|
12
|
+
# @return [Integer] 0 on success
|
13
|
+
def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
|
14
|
+
error_buffer = FFI::MemoryPointer.from_string(" " * 256)
|
15
|
+
|
16
|
+
response = @native_kafka.with_inner do |inner|
|
17
|
+
Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
|
18
|
+
inner, token, lifetime_ms, principal_name,
|
19
|
+
flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
|
20
|
+
)
|
21
|
+
end
|
22
|
+
|
23
|
+
return response if response.zero?
|
24
|
+
|
25
|
+
oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
|
26
|
+
|
27
|
+
response
|
28
|
+
end
|
29
|
+
|
30
|
+
# Marks failed oauth token acquire in librdkafka
|
31
|
+
#
|
32
|
+
# @param reason [String] human readable error reason for failing to acquire token
|
33
|
+
def oauthbearer_set_token_failure(reason)
|
34
|
+
@native_kafka.with_inner do |inner|
|
35
|
+
Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
|
36
|
+
inner,
|
37
|
+
reason
|
38
|
+
)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
# Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
|
45
|
+
def flatten_extensions(extensions)
|
46
|
+
return nil unless extensions
|
47
|
+
"\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
|
48
|
+
end
|
49
|
+
|
50
|
+
# extension_size is the number of keys + values which should be a non-negative even number
|
51
|
+
# https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
|
52
|
+
def extension_size(extensions)
|
53
|
+
return 0 unless extensions
|
54
|
+
extensions.size * 2
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/rdkafka/native_kafka.rb
CHANGED
@@ -4,7 +4,7 @@ module Rdkafka
|
|
4
4
|
# @private
|
5
5
|
# A wrapper around a native kafka that polls and cleanly exits
|
6
6
|
class NativeKafka
|
7
|
-
def initialize(inner, run_polling_thread:, opaque:)
|
7
|
+
def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
|
8
8
|
@inner = inner
|
9
9
|
@opaque = opaque
|
10
10
|
# Lock around external access
|
@@ -28,30 +28,43 @@ module Rdkafka
|
|
28
28
|
# counter for operations in progress using inner
|
29
29
|
@operations_in_progress = 0
|
30
30
|
|
31
|
-
|
32
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 0)
|
31
|
+
@run_polling_thread = run_polling_thread
|
33
32
|
|
34
|
-
if
|
35
|
-
# Start thread to poll client for delivery callbacks,
|
36
|
-
# not used in consumer.
|
37
|
-
@polling_thread = Thread.new do
|
38
|
-
loop do
|
39
|
-
@poll_mutex.synchronize do
|
40
|
-
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
41
|
-
end
|
33
|
+
start if auto_start
|
42
34
|
|
43
|
-
|
44
|
-
|
45
|
-
|
35
|
+
@closing = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def start
|
39
|
+
synchronize do
|
40
|
+
return if @started
|
41
|
+
|
42
|
+
@started = true
|
43
|
+
|
44
|
+
# Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
|
45
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
|
46
|
+
|
47
|
+
if @run_polling_thread
|
48
|
+
# Start thread to poll client for delivery callbacks,
|
49
|
+
# not used in consumer.
|
50
|
+
@polling_thread = Thread.new do
|
51
|
+
loop do
|
52
|
+
@poll_mutex.synchronize do
|
53
|
+
Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Exit thread if closing and the poll queue is empty
|
57
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
|
58
|
+
break
|
59
|
+
end
|
46
60
|
end
|
47
61
|
end
|
48
|
-
end
|
49
62
|
|
50
|
-
|
51
|
-
|
63
|
+
@polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
|
64
|
+
@polling_thread.abort_on_exception = true
|
65
|
+
@polling_thread[:closing] = false
|
66
|
+
end
|
52
67
|
end
|
53
|
-
|
54
|
-
@closing = false
|
55
68
|
end
|
56
69
|
|
57
70
|
def with_inner
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -4,6 +4,7 @@ module Rdkafka
|
|
4
4
|
# A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
|
5
5
|
class Producer
|
6
6
|
include Helpers::Time
|
7
|
+
include Helpers::OAuth
|
7
8
|
|
8
9
|
# Cache partitions count for 30 seconds
|
9
10
|
PARTITIONS_COUNT_TTL = 30
|
@@ -53,6 +54,12 @@ module Rdkafka
|
|
53
54
|
end
|
54
55
|
end
|
55
56
|
|
57
|
+
# Starts the native Kafka polling thread and kicks off the init polling
|
58
|
+
# @note Not needed to run unless explicit start was disabled
|
59
|
+
def start
|
60
|
+
@native_kafka.start
|
61
|
+
end
|
62
|
+
|
56
63
|
# @return [String] producer name
|
57
64
|
def name
|
58
65
|
@name ||= @native_kafka.with_inner do |inner|
|
data/lib/rdkafka/version.rb
CHANGED
data/lib/rdkafka.rb
CHANGED
@@ -76,37 +76,50 @@ describe Rdkafka::AbstractHandle do
|
|
76
76
|
end
|
77
77
|
|
78
78
|
describe "#wait" do
|
79
|
-
|
79
|
+
context 'when pending_handle true' do
|
80
|
+
let(:pending_handle) { true }
|
80
81
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
82
|
+
it "should wait until the timeout and then raise an error" do
|
83
|
+
expect(Kernel).not_to receive(:warn)
|
84
|
+
expect {
|
85
|
+
subject.wait(max_wait_timeout: 0.1)
|
86
|
+
}.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
|
87
|
+
end
|
85
88
|
end
|
86
89
|
|
87
|
-
context
|
90
|
+
context 'when pending_handle false' do
|
88
91
|
let(:pending_handle) { false }
|
89
|
-
let(:result) { 1 }
|
90
92
|
|
91
|
-
it
|
92
|
-
|
93
|
-
|
93
|
+
it 'should show a deprecation warning when wait_timeout is set' do
|
94
|
+
expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
|
95
|
+
subject.wait(wait_timeout: 0.1)
|
94
96
|
end
|
95
97
|
|
96
|
-
|
97
|
-
|
98
|
-
|
98
|
+
context "without error" do
|
99
|
+
let(:result) { 1 }
|
100
|
+
|
101
|
+
it "should return a result" do
|
102
|
+
expect(Kernel).not_to receive(:warn)
|
103
|
+
wait_result = subject.wait
|
104
|
+
expect(wait_result).to eq(result)
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should wait without a timeout" do
|
108
|
+
expect(Kernel).not_to receive(:warn)
|
109
|
+
wait_result = subject.wait(max_wait_timeout: nil)
|
110
|
+
expect(wait_result).to eq(result)
|
111
|
+
end
|
99
112
|
end
|
100
|
-
end
|
101
113
|
|
102
|
-
|
103
|
-
|
104
|
-
let(:response) { 20 }
|
114
|
+
context "with error" do
|
115
|
+
let(:response) { 20 }
|
105
116
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
117
|
+
it "should raise an rdkafka error" do
|
118
|
+
expect(Kernel).not_to receive(:warn)
|
119
|
+
expect {
|
120
|
+
subject.wait
|
121
|
+
}.to raise_error Rdkafka::RdkafkaError
|
122
|
+
end
|
110
123
|
end
|
111
124
|
end
|
112
125
|
end
|
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe 'admin without auto-start' do
|
35
|
+
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
|
+
|
37
|
+
it 'expect to be able to start it later and close' do
|
38
|
+
admin.start
|
39
|
+
admin.close
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'expect to be able to close it without starting' do
|
43
|
+
admin.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
34
47
|
describe "#create_topic" do
|
35
48
|
describe "called with invalid input" do
|
36
49
|
describe "with an invalid topic name" do
|
@@ -275,6 +288,9 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
275
288
|
expect(create_acl_report.rdkafka_response).to eq(0)
|
276
289
|
expect(create_acl_report.rdkafka_response_string).to eq("")
|
277
290
|
|
291
|
+
# Since we create and immediately check, this is slow on loaded CIs, hence we wait
|
292
|
+
sleep(2)
|
293
|
+
|
278
294
|
#describe_acl
|
279
295
|
describe_acl_handle = admin.describe_acl(resource_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_ANY, resource_name: nil, resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY, principal: nil, host: nil, operation: Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_ANY, permission_type: Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ANY)
|
280
296
|
describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
|
@@ -404,4 +420,41 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
404
420
|
end
|
405
421
|
end
|
406
422
|
end
|
423
|
+
|
424
|
+
describe '#oauthbearer_set_token' do
|
425
|
+
context 'when sasl not configured' do
|
426
|
+
it 'should return RD_KAFKA_RESP_ERR__STATE' do
|
427
|
+
response = admin.oauthbearer_set_token(
|
428
|
+
token: "foo",
|
429
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
430
|
+
principal_name: "kafka-cluster"
|
431
|
+
)
|
432
|
+
expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
|
433
|
+
end
|
434
|
+
end
|
435
|
+
|
436
|
+
context 'when sasl configured' do
|
437
|
+
before do
|
438
|
+
config_sasl = rdkafka_config(
|
439
|
+
"security.protocol": "sasl_ssl",
|
440
|
+
"sasl.mechanisms": 'OAUTHBEARER'
|
441
|
+
)
|
442
|
+
$admin_sasl = config_sasl.admin
|
443
|
+
end
|
444
|
+
|
445
|
+
after do
|
446
|
+
$admin_sasl.close
|
447
|
+
end
|
448
|
+
|
449
|
+
it 'should succeed' do
|
450
|
+
|
451
|
+
response = $admin_sasl.oauthbearer_set_token(
|
452
|
+
token: "foo",
|
453
|
+
lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
|
454
|
+
principal_name: "kafka-cluster"
|
455
|
+
)
|
456
|
+
expect(response).to eq(0)
|
457
|
+
end
|
458
|
+
end
|
459
|
+
end
|
407
460
|
end
|
@@ -36,6 +36,16 @@ describe Rdkafka::Bindings do
|
|
36
36
|
expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
|
37
37
|
end
|
38
38
|
|
39
|
+
it "should log fatal messages" do
|
40
|
+
Rdkafka::Bindings::LogCallback.call(nil, 1, nil, "log line")
|
41
|
+
expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should log fatal messages" do
|
45
|
+
Rdkafka::Bindings::LogCallback.call(nil, 2, nil, "log line")
|
46
|
+
expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
|
47
|
+
end
|
48
|
+
|
39
49
|
it "should log error messages" do
|
40
50
|
Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
|
41
51
|
expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
|
@@ -51,6 +61,11 @@ describe Rdkafka::Bindings do
|
|
51
61
|
expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
|
52
62
|
end
|
53
63
|
|
64
|
+
it "should log info messages" do
|
65
|
+
Rdkafka::Bindings::LogCallback.call(nil, 6, nil, "log line")
|
66
|
+
expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
|
67
|
+
end
|
68
|
+
|
54
69
|
it "should log debug messages" do
|
55
70
|
Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
|
56
71
|
expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
|
@@ -132,4 +147,86 @@ describe Rdkafka::Bindings do
|
|
132
147
|
end
|
133
148
|
end
|
134
149
|
end
|
150
|
+
|
151
|
+
describe "oauthbearer set token" do
|
152
|
+
|
153
|
+
context "without args" do
|
154
|
+
it "should raise argument error" do
|
155
|
+
expect {
|
156
|
+
Rdkafka::Bindings.rd_kafka_oauthbearer_set_token
|
157
|
+
}.to raise_error(ArgumentError)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
context "with args" do
|
162
|
+
before do
|
163
|
+
DEFAULT_TOKEN_EXPIRY_SECONDS = 900
|
164
|
+
$token_value = "token"
|
165
|
+
$md_lifetime_ms = Time.now.to_i*1000 + DEFAULT_TOKEN_EXPIRY_SECONDS * 1000
|
166
|
+
$md_principal_name = "kafka-cluster"
|
167
|
+
$extensions = nil
|
168
|
+
$extension_size = 0
|
169
|
+
$error_buffer = FFI::MemoryPointer.from_string(" " * 256)
|
170
|
+
end
|
171
|
+
|
172
|
+
it "should set token or capture failure" do
|
173
|
+
RdKafkaTestConsumer.with do |consumer_ptr|
|
174
|
+
response = Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(consumer_ptr, $token_value, $md_lifetime_ms, $md_principal_name, $extensions, $extension_size, $error_buffer, 256)
|
175
|
+
expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
|
176
|
+
expect($error_buffer.read_string).to eq("SASL/OAUTHBEARER is not the configured authentication mechanism")
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
describe "oauthbearer set token failure" do
|
183
|
+
|
184
|
+
context "without args" do
|
185
|
+
|
186
|
+
it "should fail" do
|
187
|
+
expect {
|
188
|
+
Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure
|
189
|
+
}.to raise_error(ArgumentError)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
context "with args" do
|
194
|
+
it "should succeed" do
|
195
|
+
expect {
|
196
|
+
errstr = "error"
|
197
|
+
RdKafkaTestConsumer.with do |consumer_ptr|
|
198
|
+
Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(consumer_ptr, errstr)
|
199
|
+
end
|
200
|
+
}.to_not raise_error
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
describe "oauthbearer callback" do
|
206
|
+
|
207
|
+
context "without an oauthbearer callback" do
|
208
|
+
it "should do nothing" do
|
209
|
+
expect {
|
210
|
+
Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(nil, "", nil)
|
211
|
+
}.not_to raise_error
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
context "with an oauthbearer callback" do
|
216
|
+
before do
|
217
|
+
Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
|
218
|
+
$received_config = config
|
219
|
+
$received_client_name = client_name
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
it "should call the oauth bearer callback and receive config and client name" do
|
224
|
+
RdKafkaTestConsumer.with do |consumer_ptr|
|
225
|
+
Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(consumer_ptr, "{}", nil)
|
226
|
+
expect($received_config).to eq("{}")
|
227
|
+
expect($received_client_name).to match(/consumer/)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
135
232
|
end
|
data/spec/rdkafka/config_spec.rb
CHANGED
@@ -22,6 +22,7 @@ describe Rdkafka::Config do
|
|
22
22
|
it "supports logging queue" do
|
23
23
|
log = StringIO.new
|
24
24
|
Rdkafka::Config.logger = Logger.new(log)
|
25
|
+
Rdkafka::Config.ensure_log_thread
|
25
26
|
|
26
27
|
Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
|
27
28
|
20.times do
|
@@ -31,6 +32,25 @@ describe Rdkafka::Config do
|
|
31
32
|
|
32
33
|
expect(log.string).to include "FATAL -- : I love testing"
|
33
34
|
end
|
35
|
+
|
36
|
+
it "expect to start new logger thread after fork and work" do
|
37
|
+
reader, writer = IO.pipe
|
38
|
+
|
39
|
+
pid = fork do
|
40
|
+
$stdout.reopen(writer)
|
41
|
+
Rdkafka::Config.logger = Logger.new($stdout)
|
42
|
+
reader.close
|
43
|
+
producer = rdkafka_producer_config(debug: 'all').producer
|
44
|
+
producer.close
|
45
|
+
writer.close
|
46
|
+
sleep(1)
|
47
|
+
end
|
48
|
+
|
49
|
+
writer.close
|
50
|
+
Process.wait(pid)
|
51
|
+
output = reader.read
|
52
|
+
expect(output.split("\n").size).to be >= 20
|
53
|
+
end
|
34
54
|
end
|
35
55
|
|
36
56
|
context "statistics callback" do
|
@@ -95,6 +115,39 @@ describe Rdkafka::Config do
|
|
95
115
|
end
|
96
116
|
end
|
97
117
|
|
118
|
+
context "oauthbearer calllback" do
|
119
|
+
context "with a proc/lambda" do
|
120
|
+
it "should set the callback" do
|
121
|
+
expect {
|
122
|
+
Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
|
123
|
+
puts config
|
124
|
+
puts client_name
|
125
|
+
end
|
126
|
+
}.not_to raise_error
|
127
|
+
expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
context "with a callable object" do
|
132
|
+
it "should set the callback" do
|
133
|
+
callback = Class.new do
|
134
|
+
def call(config, client_name); end
|
135
|
+
end
|
136
|
+
|
137
|
+
expect {
|
138
|
+
Rdkafka::Config.oauthbearer_token_refresh_callback = callback.new
|
139
|
+
}.not_to raise_error
|
140
|
+
expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
it "should not accept a callback that's not callable" do
|
145
|
+
expect {
|
146
|
+
Rdkafka::Config.oauthbearer_token_refresh_callback = 'not a callback'
|
147
|
+
}.to raise_error(TypeError)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
98
151
|
context "configuration" do
|
99
152
|
it "should store configuration" do
|
100
153
|
config = Rdkafka::Config.new
|