karafka-rdkafka 0.14.11 → 0.15.0.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +4 -2
- data/README.md +19 -9
- data/docker-compose.yml +1 -1
- data/ext/Rakefile +1 -11
- data/lib/rdkafka/abstract_handle.rb +44 -20
- data/lib/rdkafka/admin/config_binding_result.rb +30 -0
- data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
- data/lib/rdkafka/admin/create_topic_report.rb +1 -1
- data/lib/rdkafka/admin/delete_groups_report.rb +1 -1
- data/lib/rdkafka/admin/delete_topic_report.rb +1 -1
- data/lib/rdkafka/admin/describe_acl_report.rb +1 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +48 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +48 -0
- data/lib/rdkafka/admin.rb +159 -0
- data/lib/rdkafka/bindings.rb +42 -0
- data/lib/rdkafka/callbacks.rb +103 -19
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +34 -21
- data/spec/rdkafka/admin_spec.rb +275 -3
- data/spec/rdkafka/consumer_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +8 -3
- metadata.gz.sig +0 -0
- data/dist/librdkafka_2.3.0.tar.gz +0 -0
data/lib/rdkafka/bindings.rb
CHANGED
@@ -97,6 +97,48 @@ module Rdkafka
|
|
97
97
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
98
98
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
99
99
|
|
100
|
+
# Configs management
|
101
|
+
#
|
102
|
+
# Structs for management of configurations
|
103
|
+
# Each configuration is attached to a resource and one resource can have many configuration
|
104
|
+
# details. Each resource will also have separate errors results if obtaining configuration
|
105
|
+
# was not possible for any reason
|
106
|
+
class ConfigResource < FFI::Struct
|
107
|
+
layout :type, :int,
|
108
|
+
:name, :string
|
109
|
+
end
|
110
|
+
|
111
|
+
attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
112
|
+
attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
|
113
|
+
attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
|
114
|
+
attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
|
115
|
+
attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
|
116
|
+
attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
|
117
|
+
attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
|
118
|
+
attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
|
119
|
+
attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
|
120
|
+
attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
|
121
|
+
attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
|
122
|
+
attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
|
123
|
+
attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
|
124
|
+
attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
|
125
|
+
attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
|
126
|
+
attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
127
|
+
attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
|
128
|
+
attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
|
129
|
+
attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
|
130
|
+
|
131
|
+
RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
|
132
|
+
RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
|
133
|
+
|
134
|
+
RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
|
135
|
+
RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
|
136
|
+
|
137
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
|
138
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
|
139
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
140
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
141
|
+
|
100
142
|
# Errors
|
101
143
|
|
102
144
|
attach_function :rd_kafka_err2name, [:int], :string
|
data/lib/rdkafka/callbacks.rb
CHANGED
@@ -113,6 +113,42 @@ module Rdkafka
|
|
113
113
|
end
|
114
114
|
end
|
115
115
|
|
116
|
+
class DescribeConfigsResult
|
117
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
118
|
+
|
119
|
+
def initialize(event_ptr)
|
120
|
+
@results=[]
|
121
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
122
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
123
|
+
|
124
|
+
if @result_error == 0
|
125
|
+
configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
|
126
|
+
# Get the number of matching acls
|
127
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
128
|
+
@results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
|
129
|
+
@results_count = pointer_to_size_t.read_int
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class IncrementalAlterConfigsResult
|
135
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
136
|
+
|
137
|
+
def initialize(event_ptr)
|
138
|
+
@results=[]
|
139
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
140
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
141
|
+
|
142
|
+
if @result_error == 0
|
143
|
+
incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
|
144
|
+
# Get the number of matching acls
|
145
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
146
|
+
@results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
|
147
|
+
@results_count = pointer_to_size_t.read_int
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
116
152
|
# FFI Function used for Create Topic and Delete Topic callbacks
|
117
153
|
BackgroundEventCallbackFunction = FFI::Function.new(
|
118
154
|
:void, [:pointer, :pointer, :pointer]
|
@@ -123,20 +159,24 @@ module Rdkafka
|
|
123
159
|
# @private
|
124
160
|
class BackgroundEventCallback
|
125
161
|
def self.call(_, event_ptr, _)
|
126
|
-
|
127
|
-
|
162
|
+
case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
163
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
|
128
164
|
process_create_topic(event_ptr)
|
129
|
-
|
165
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
|
166
|
+
process_describe_configs(event_ptr)
|
167
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
|
168
|
+
process_incremental_alter_configs(event_ptr)
|
169
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
|
130
170
|
process_delete_topic(event_ptr)
|
131
|
-
|
171
|
+
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
|
132
172
|
process_create_partitions(event_ptr)
|
133
|
-
|
173
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
|
134
174
|
process_create_acl(event_ptr)
|
135
|
-
|
175
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
|
136
176
|
process_delete_acl(event_ptr)
|
137
|
-
|
177
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
|
138
178
|
process_describe_acl(event_ptr)
|
139
|
-
|
179
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
|
140
180
|
process_delete_groups(event_ptr)
|
141
181
|
end
|
142
182
|
end
|
@@ -156,7 +196,44 @@ module Rdkafka
|
|
156
196
|
create_topic_handle[:response] = create_topic_results[0].result_error
|
157
197
|
create_topic_handle[:error_string] = create_topic_results[0].error_string
|
158
198
|
create_topic_handle[:result_name] = create_topic_results[0].result_name
|
159
|
-
|
199
|
+
|
200
|
+
create_topic_handle.unlock
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def self.process_describe_configs(event_ptr)
|
205
|
+
describe_configs = DescribeConfigsResult.new(event_ptr)
|
206
|
+
describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
207
|
+
|
208
|
+
if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
|
209
|
+
describe_configs_handle[:response] = describe_configs.result_error
|
210
|
+
describe_configs_handle[:response_string] = describe_configs.error_string
|
211
|
+
describe_configs_handle[:pending] = false
|
212
|
+
|
213
|
+
if describe_configs.result_error == 0
|
214
|
+
describe_configs_handle[:config_entries] = describe_configs.results
|
215
|
+
describe_configs_handle[:entry_count] = describe_configs.results_count
|
216
|
+
end
|
217
|
+
|
218
|
+
describe_configs_handle.unlock
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def self.process_incremental_alter_configs(event_ptr)
|
223
|
+
incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
|
224
|
+
incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
225
|
+
|
226
|
+
if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
|
227
|
+
incremental_alter_handle[:response] = incremental_alter.result_error
|
228
|
+
incremental_alter_handle[:response_string] = incremental_alter.error_string
|
229
|
+
incremental_alter_handle[:pending] = false
|
230
|
+
|
231
|
+
if incremental_alter.result_error == 0
|
232
|
+
incremental_alter_handle[:config_entries] = incremental_alter.results
|
233
|
+
incremental_alter_handle[:entry_count] = incremental_alter.results_count
|
234
|
+
end
|
235
|
+
|
236
|
+
incremental_alter_handle.unlock
|
160
237
|
end
|
161
238
|
end
|
162
239
|
|
@@ -173,7 +250,8 @@ module Rdkafka
|
|
173
250
|
delete_group_handle[:response] = delete_group_results[0].result_error
|
174
251
|
delete_group_handle[:error_string] = delete_group_results[0].error_string
|
175
252
|
delete_group_handle[:result_name] = delete_group_results[0].result_name
|
176
|
-
|
253
|
+
|
254
|
+
delete_group_handle.unlock
|
177
255
|
end
|
178
256
|
end
|
179
257
|
|
@@ -190,7 +268,8 @@ module Rdkafka
|
|
190
268
|
delete_topic_handle[:response] = delete_topic_results[0].result_error
|
191
269
|
delete_topic_handle[:error_string] = delete_topic_results[0].error_string
|
192
270
|
delete_topic_handle[:result_name] = delete_topic_results[0].result_name
|
193
|
-
|
271
|
+
|
272
|
+
delete_topic_handle.unlock
|
194
273
|
end
|
195
274
|
end
|
196
275
|
|
@@ -207,7 +286,8 @@ module Rdkafka
|
|
207
286
|
create_partitions_handle[:response] = create_partitions_results[0].result_error
|
208
287
|
create_partitions_handle[:error_string] = create_partitions_results[0].error_string
|
209
288
|
create_partitions_handle[:result_name] = create_partitions_results[0].result_name
|
210
|
-
|
289
|
+
|
290
|
+
create_partitions_handle.unlock
|
211
291
|
end
|
212
292
|
end
|
213
293
|
|
@@ -223,7 +303,8 @@ module Rdkafka
|
|
223
303
|
if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
|
224
304
|
create_acl_handle[:response] = create_acl_results[0].result_error
|
225
305
|
create_acl_handle[:response_string] = create_acl_results[0].error_string
|
226
|
-
|
306
|
+
|
307
|
+
create_acl_handle.unlock
|
227
308
|
end
|
228
309
|
end
|
229
310
|
|
@@ -239,11 +320,13 @@ module Rdkafka
|
|
239
320
|
if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
|
240
321
|
delete_acl_handle[:response] = delete_acl_results[0].result_error
|
241
322
|
delete_acl_handle[:response_string] = delete_acl_results[0].error_string
|
242
|
-
|
323
|
+
|
243
324
|
if delete_acl_results[0].result_error == 0
|
244
325
|
delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
|
245
326
|
delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
|
246
327
|
end
|
328
|
+
|
329
|
+
delete_acl_handle.unlock
|
247
330
|
end
|
248
331
|
end
|
249
332
|
|
@@ -254,17 +337,18 @@ module Rdkafka
|
|
254
337
|
if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
|
255
338
|
describe_acl_handle[:response] = describe_acl.result_error
|
256
339
|
describe_acl_handle[:response_string] = describe_acl.error_string
|
257
|
-
|
340
|
+
|
258
341
|
if describe_acl.result_error == 0
|
259
|
-
describe_acl_handle[:acls]
|
342
|
+
describe_acl_handle[:acls] = describe_acl.matching_acls
|
260
343
|
describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
|
261
344
|
end
|
345
|
+
|
346
|
+
describe_acl_handle.unlock
|
262
347
|
end
|
263
348
|
end
|
264
349
|
end
|
265
350
|
|
266
351
|
# FFI Function used for Message Delivery callbacks
|
267
|
-
|
268
352
|
DeliveryCallbackFunction = FFI::Function.new(
|
269
353
|
:void, [:pointer, :pointer, :pointer]
|
270
354
|
) do |client_ptr, message_ptr, opaque_ptr|
|
@@ -284,7 +368,6 @@ module Rdkafka
|
|
284
368
|
delivery_handle[:partition] = message[:partition]
|
285
369
|
delivery_handle[:offset] = message[:offset]
|
286
370
|
delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
|
287
|
-
delivery_handle[:pending] = false
|
288
371
|
|
289
372
|
# Call delivery callback on opaque
|
290
373
|
if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
|
@@ -299,9 +382,10 @@ module Rdkafka
|
|
299
382
|
delivery_handle
|
300
383
|
)
|
301
384
|
end
|
385
|
+
|
386
|
+
delivery_handle.unlock
|
302
387
|
end
|
303
388
|
end
|
304
389
|
end
|
305
|
-
|
306
390
|
end
|
307
391
|
end
|
data/lib/rdkafka/version.rb
CHANGED
data/lib/rdkafka.rb
CHANGED
@@ -23,7 +23,13 @@ require "rdkafka/admin/delete_acl_handle"
|
|
23
23
|
require "rdkafka/admin/delete_acl_report"
|
24
24
|
require "rdkafka/admin/describe_acl_handle"
|
25
25
|
require "rdkafka/admin/describe_acl_report"
|
26
|
+
require "rdkafka/admin/describe_configs_handle"
|
27
|
+
require "rdkafka/admin/describe_configs_report"
|
28
|
+
require "rdkafka/admin/incremental_alter_configs_handle"
|
29
|
+
require "rdkafka/admin/incremental_alter_configs_report"
|
26
30
|
require "rdkafka/admin/acl_binding_result"
|
31
|
+
require "rdkafka/admin/config_binding_result"
|
32
|
+
require "rdkafka/admin/config_resource_binding_result"
|
27
33
|
require "rdkafka/bindings"
|
28
34
|
require "rdkafka/callbacks"
|
29
35
|
require "rdkafka/config"
|
@@ -76,37 +76,50 @@ describe Rdkafka::AbstractHandle do
|
|
76
76
|
end
|
77
77
|
|
78
78
|
describe "#wait" do
|
79
|
-
|
79
|
+
context 'when pending_handle true' do
|
80
|
+
let(:pending_handle) { true }
|
80
81
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
82
|
+
it "should wait until the timeout and then raise an error" do
|
83
|
+
expect(Kernel).not_to receive(:warn)
|
84
|
+
expect {
|
85
|
+
subject.wait(max_wait_timeout: 0.1)
|
86
|
+
}.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
|
87
|
+
end
|
85
88
|
end
|
86
89
|
|
87
|
-
context
|
90
|
+
context 'when pending_handle false' do
|
88
91
|
let(:pending_handle) { false }
|
89
|
-
let(:result) { 1 }
|
90
92
|
|
91
|
-
it
|
92
|
-
|
93
|
-
|
93
|
+
it 'should show a deprecation warning when wait_timeout is set' do
|
94
|
+
expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
|
95
|
+
subject.wait(wait_timeout: 0.1)
|
94
96
|
end
|
95
97
|
|
96
|
-
|
97
|
-
|
98
|
-
|
98
|
+
context "without error" do
|
99
|
+
let(:result) { 1 }
|
100
|
+
|
101
|
+
it "should return a result" do
|
102
|
+
expect(Kernel).not_to receive(:warn)
|
103
|
+
wait_result = subject.wait
|
104
|
+
expect(wait_result).to eq(result)
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should wait without a timeout" do
|
108
|
+
expect(Kernel).not_to receive(:warn)
|
109
|
+
wait_result = subject.wait(max_wait_timeout: nil)
|
110
|
+
expect(wait_result).to eq(result)
|
111
|
+
end
|
99
112
|
end
|
100
|
-
end
|
101
113
|
|
102
|
-
|
103
|
-
|
104
|
-
let(:response) { 20 }
|
114
|
+
context "with error" do
|
115
|
+
let(:response) { 20 }
|
105
116
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
117
|
+
it "should raise an rdkafka error" do
|
118
|
+
expect(Kernel).not_to receive(:warn)
|
119
|
+
expect {
|
120
|
+
subject.wait
|
121
|
+
}.to raise_error Rdkafka::RdkafkaError
|
122
|
+
end
|
110
123
|
end
|
111
124
|
end
|
112
125
|
end
|
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -16,12 +16,12 @@ describe Rdkafka::Admin do
|
|
16
16
|
admin.close
|
17
17
|
end
|
18
18
|
|
19
|
-
let(:topic_name) { "test-topic-#{
|
19
|
+
let(:topic_name) { "test-topic-#{SecureRandom.uuid}" }
|
20
20
|
let(:topic_partition_count) { 3 }
|
21
21
|
let(:topic_replication_factor) { 1 }
|
22
22
|
let(:topic_config) { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
|
23
23
|
let(:invalid_topic_config) { {"cleeeeenup.policee" => "campact"} }
|
24
|
-
let(:group_name) { "test-group-#{
|
24
|
+
let(:group_name) { "test-group-#{SecureRandom.uuid}" }
|
25
25
|
|
26
26
|
let(:resource_name) {"acl-test-topic"}
|
27
27
|
let(:resource_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
|
@@ -129,6 +129,275 @@ describe Rdkafka::Admin do
|
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
|
+
describe "describe_configs" do
|
133
|
+
subject(:resources_results) { admin.describe_configs(resources).wait.resources }
|
134
|
+
|
135
|
+
before do
|
136
|
+
admin.create_topic(topic_name, 2, 1).wait
|
137
|
+
sleep(1)
|
138
|
+
end
|
139
|
+
|
140
|
+
context 'when describing config of an existing topic' do
|
141
|
+
let(:resources) { [{ resource_type: 2, resource_name: topic_name }] }
|
142
|
+
|
143
|
+
it do
|
144
|
+
expect(resources_results.size).to eq(1)
|
145
|
+
expect(resources_results.first.type).to eq(2)
|
146
|
+
expect(resources_results.first.name).to eq(topic_name)
|
147
|
+
expect(resources_results.first.configs.size).to be > 25
|
148
|
+
expect(resources_results.first.configs.first.name).to eq('compression.type')
|
149
|
+
expect(resources_results.first.configs.first.value).to eq('producer')
|
150
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
context 'when describing config of a non-existing topic' do
|
155
|
+
let(:resources) { [{ resource_type: 2, resource_name: SecureRandom.uuid }] }
|
156
|
+
|
157
|
+
it 'expect to raise error' do
|
158
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
context 'when describing both existing and non-existing topics' do
|
163
|
+
let(:resources) do
|
164
|
+
[
|
165
|
+
{ resource_type: 2, resource_name: topic_name },
|
166
|
+
{ resource_type: 2, resource_name: SecureRandom.uuid }
|
167
|
+
]
|
168
|
+
end
|
169
|
+
|
170
|
+
it 'expect to raise error' do
|
171
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
context 'when describing multiple existing topics' do
|
176
|
+
let(:resources) do
|
177
|
+
[
|
178
|
+
{ resource_type: 2, resource_name: 'example_topic' },
|
179
|
+
{ resource_type: 2, resource_name: topic_name }
|
180
|
+
]
|
181
|
+
end
|
182
|
+
|
183
|
+
it do
|
184
|
+
expect(resources_results.size).to eq(2)
|
185
|
+
expect(resources_results.first.type).to eq(2)
|
186
|
+
expect(resources_results.first.name).to eq('example_topic')
|
187
|
+
expect(resources_results.last.type).to eq(2)
|
188
|
+
expect(resources_results.last.name).to eq(topic_name)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
context 'when trying to describe invalid resource type' do
|
193
|
+
let(:resources) { [{ resource_type: 0, resource_name: SecureRandom.uuid }] }
|
194
|
+
|
195
|
+
it 'expect to raise error' do
|
196
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_request/)
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
context 'when trying to describe invalid broker' do
|
201
|
+
let(:resources) { [{ resource_type: 4, resource_name: 'non-existing' }] }
|
202
|
+
|
203
|
+
it 'expect to raise error' do
|
204
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
context 'when trying to describe valid broker' do
|
209
|
+
let(:resources) { [{ resource_type: 4, resource_name: '1' }] }
|
210
|
+
|
211
|
+
it do
|
212
|
+
expect(resources_results.size).to eq(1)
|
213
|
+
expect(resources_results.first.type).to eq(4)
|
214
|
+
expect(resources_results.first.name).to eq('1')
|
215
|
+
expect(resources_results.first.configs.size).to be > 230
|
216
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
217
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
218
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
context 'when describing valid broker with topics in one request' do
|
223
|
+
let(:resources) do
|
224
|
+
[
|
225
|
+
{ resource_type: 4, resource_name: '1' },
|
226
|
+
{ resource_type: 2, resource_name: topic_name }
|
227
|
+
]
|
228
|
+
end
|
229
|
+
|
230
|
+
it do
|
231
|
+
expect(resources_results.size).to eq(2)
|
232
|
+
expect(resources_results.first.type).to eq(4)
|
233
|
+
expect(resources_results.first.name).to eq('1')
|
234
|
+
expect(resources_results.first.configs.size).to be > 230
|
235
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
236
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
237
|
+
expect(resources_results.last.type).to eq(2)
|
238
|
+
expect(resources_results.last.name).to eq(topic_name)
|
239
|
+
expect(resources_results.last.configs.size).to be > 25
|
240
|
+
expect(resources_results.last.configs.first.name).to eq('compression.type')
|
241
|
+
expect(resources_results.last.configs.first.value).to eq('producer')
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
describe "incremental_alter_configs" do
|
247
|
+
subject(:resources_results) { admin.incremental_alter_configs(resources_with_configs).wait.resources }
|
248
|
+
|
249
|
+
before do
|
250
|
+
admin.create_topic(topic_name, 2, 1).wait
|
251
|
+
sleep(1)
|
252
|
+
end
|
253
|
+
|
254
|
+
context 'when altering one topic with one valid config via set' do
|
255
|
+
let(:target_retention) { (86400002 + rand(10_000)).to_s }
|
256
|
+
let(:resources_with_configs) do
|
257
|
+
[
|
258
|
+
{
|
259
|
+
resource_type: 2,
|
260
|
+
resource_name: topic_name,
|
261
|
+
configs: [
|
262
|
+
{
|
263
|
+
name: 'delete.retention.ms',
|
264
|
+
value: target_retention,
|
265
|
+
op_type: 0
|
266
|
+
}
|
267
|
+
]
|
268
|
+
}
|
269
|
+
]
|
270
|
+
end
|
271
|
+
|
272
|
+
it do
|
273
|
+
expect(resources_results.size).to eq(1)
|
274
|
+
expect(resources_results.first.type).to eq(2)
|
275
|
+
expect(resources_results.first.name).to eq(topic_name)
|
276
|
+
|
277
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
278
|
+
config.name == 'delete.retention.ms'
|
279
|
+
end
|
280
|
+
|
281
|
+
expect(ret_config.value).to eq(target_retention)
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
context 'when altering one topic with one valid config via delete' do
|
286
|
+
let(:target_retention) { (8640002 + rand(10_000)).to_s }
|
287
|
+
let(:resources_with_configs) do
|
288
|
+
[
|
289
|
+
{
|
290
|
+
resource_type: 2,
|
291
|
+
resource_name: topic_name,
|
292
|
+
configs: [
|
293
|
+
{
|
294
|
+
name: 'delete.retention.ms',
|
295
|
+
value: target_retention,
|
296
|
+
op_type: 1
|
297
|
+
}
|
298
|
+
]
|
299
|
+
}
|
300
|
+
]
|
301
|
+
end
|
302
|
+
|
303
|
+
it do
|
304
|
+
expect(resources_results.size).to eq(1)
|
305
|
+
expect(resources_results.first.type).to eq(2)
|
306
|
+
expect(resources_results.first.name).to eq(topic_name)
|
307
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
308
|
+
config.name == 'delete.retention.ms'
|
309
|
+
end
|
310
|
+
|
311
|
+
expect(ret_config.value).to eq('86400000')
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
context 'when altering one topic with one valid config via append' do
|
316
|
+
let(:target_policy) { 'compact' }
|
317
|
+
let(:resources_with_configs) do
|
318
|
+
[
|
319
|
+
{
|
320
|
+
resource_type: 2,
|
321
|
+
resource_name: topic_name,
|
322
|
+
configs: [
|
323
|
+
{
|
324
|
+
name: 'cleanup.policy',
|
325
|
+
value: target_policy,
|
326
|
+
op_type: 2
|
327
|
+
}
|
328
|
+
]
|
329
|
+
}
|
330
|
+
]
|
331
|
+
end
|
332
|
+
|
333
|
+
it do
|
334
|
+
expect(resources_results.size).to eq(1)
|
335
|
+
expect(resources_results.first.type).to eq(2)
|
336
|
+
expect(resources_results.first.name).to eq(topic_name)
|
337
|
+
|
338
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
339
|
+
config.name == 'cleanup.policy'
|
340
|
+
end
|
341
|
+
|
342
|
+
expect(ret_config.value).to eq("delete,#{target_policy}")
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
context 'when altering one topic with one valid config via subtrack' do
|
347
|
+
let(:target_policy) { 'delete' }
|
348
|
+
let(:resources_with_configs) do
|
349
|
+
[
|
350
|
+
{
|
351
|
+
resource_type: 2,
|
352
|
+
resource_name: topic_name,
|
353
|
+
configs: [
|
354
|
+
{
|
355
|
+
name: 'cleanup.policy',
|
356
|
+
value: target_policy,
|
357
|
+
op_type: 3
|
358
|
+
}
|
359
|
+
]
|
360
|
+
}
|
361
|
+
]
|
362
|
+
end
|
363
|
+
|
364
|
+
it do
|
365
|
+
expect(resources_results.size).to eq(1)
|
366
|
+
expect(resources_results.first.type).to eq(2)
|
367
|
+
expect(resources_results.first.name).to eq(topic_name)
|
368
|
+
|
369
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
370
|
+
config.name == 'cleanup.policy'
|
371
|
+
end
|
372
|
+
|
373
|
+
expect(ret_config.value).to eq('')
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
377
|
+
context 'when altering one topic with invalid config' do
|
378
|
+
let(:target_retention) { '-10' }
|
379
|
+
let(:resources_with_configs) do
|
380
|
+
[
|
381
|
+
{
|
382
|
+
resource_type: 2,
|
383
|
+
resource_name: topic_name,
|
384
|
+
configs: [
|
385
|
+
{
|
386
|
+
name: 'delete.retention.ms',
|
387
|
+
value: target_retention,
|
388
|
+
op_type: 0
|
389
|
+
}
|
390
|
+
]
|
391
|
+
}
|
392
|
+
]
|
393
|
+
end
|
394
|
+
|
395
|
+
it 'expect to raise error' do
|
396
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_config/)
|
397
|
+
end
|
398
|
+
end
|
399
|
+
end
|
400
|
+
|
132
401
|
describe "#delete_topic" do
|
133
402
|
describe "called with invalid input" do
|
134
403
|
# https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
|
@@ -396,7 +665,10 @@ describe Rdkafka::Admin do
|
|
396
665
|
end
|
397
666
|
|
398
667
|
context 'when topic has less then desired number of partitions' do
|
399
|
-
before
|
668
|
+
before do
|
669
|
+
admin.create_topic(topic_name, 1, 1).wait
|
670
|
+
sleep(1)
|
671
|
+
end
|
400
672
|
|
401
673
|
it 'expect to change number of partitions' do
|
402
674
|
admin.create_partitions(topic_name, 10).wait
|
@@ -214,7 +214,7 @@ describe Rdkafka::Consumer do
|
|
214
214
|
|
215
215
|
# This is needed because `enable.auto.offset.store` is true but when running in CI that
|
216
216
|
# is overloaded, offset store lags
|
217
|
-
sleep(
|
217
|
+
sleep(2)
|
218
218
|
|
219
219
|
consumer.commit
|
220
220
|
expect(message1.offset).to eq message2.offset
|
data/spec/spec_helper.rb
CHANGED
@@ -139,7 +139,7 @@ RSpec.configure do |config|
|
|
139
139
|
}.each do |topic, partitions|
|
140
140
|
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
141
141
|
begin
|
142
|
-
create_topic_handle.wait(max_wait_timeout:
|
142
|
+
create_topic_handle.wait(max_wait_timeout: 1.0)
|
143
143
|
rescue Rdkafka::RdkafkaError => ex
|
144
144
|
raise unless ex.message.match?(/topic_already_exists/)
|
145
145
|
end
|
data.tar.gz.sig
CHANGED
Binary file
|