rdkafka 0.24.2-aarch64-linux-gnu → 0.25.1-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/Gemfile +8 -0
- data/Gemfile.lint +14 -0
- data/Gemfile.lint.lock +123 -0
- data/README.md +2 -1
- data/Rakefile +21 -21
- data/docker-compose-ssl.yml +1 -1
- data/docker-compose.yml +1 -1
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/abstract_handle.rb +23 -5
- data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
- data/lib/rdkafka/admin/create_acl_report.rb +3 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
- data/lib/rdkafka/admin.rb +194 -132
- data/lib/rdkafka/bindings.rb +155 -107
- data/lib/rdkafka/callbacks.rb +81 -21
- data/lib/rdkafka/config.rb +36 -24
- data/lib/rdkafka/consumer/headers.rb +3 -2
- data/lib/rdkafka/consumer/message.rb +12 -11
- data/lib/rdkafka/consumer/partition.rb +8 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +18 -18
- data/lib/rdkafka/consumer.rb +247 -42
- data/lib/rdkafka/defaults.rb +106 -0
- data/lib/rdkafka/error.rb +28 -13
- data/lib/rdkafka/helpers/oauth.rb +11 -6
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +45 -21
- data/lib/rdkafka/native_kafka.rb +89 -4
- data/lib/rdkafka/producer/delivery_handle.rb +5 -5
- data/lib/rdkafka/producer/delivery_report.rb +8 -4
- data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
- data/lib/rdkafka/producer.rb +165 -79
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +1 -0
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +39 -47
- data/renovate.json +22 -8
- metadata +7 -86
data/lib/rdkafka/callbacks.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
+
# Callback handlers for librdkafka events
|
|
5
|
+
# @private
|
|
4
6
|
module Callbacks
|
|
5
7
|
# Extracts attributes of a rd_kafka_topic_result_t
|
|
6
8
|
#
|
|
@@ -8,12 +10,16 @@ module Rdkafka
|
|
|
8
10
|
class TopicResult
|
|
9
11
|
attr_reader :result_error, :error_string, :result_name
|
|
10
12
|
|
|
13
|
+
# @param topic_result_pointer [FFI::Pointer] pointer to the topic result struct
|
|
11
14
|
def initialize(topic_result_pointer)
|
|
12
15
|
@result_error = Rdkafka::Bindings.rd_kafka_topic_result_error(topic_result_pointer)
|
|
13
16
|
@error_string = Rdkafka::Bindings.rd_kafka_topic_result_error_string(topic_result_pointer)
|
|
14
17
|
@result_name = Rdkafka::Bindings.rd_kafka_topic_result_name(topic_result_pointer)
|
|
15
18
|
end
|
|
16
19
|
|
|
20
|
+
# @param count [Integer] number of results
|
|
21
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
22
|
+
# @return [Array<TopicResult>] array of topic results
|
|
17
23
|
def self.create_topic_results_from_array(count, array_pointer)
|
|
18
24
|
(1..count).map do |index|
|
|
19
25
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -22,13 +28,18 @@ module Rdkafka
|
|
|
22
28
|
end
|
|
23
29
|
end
|
|
24
30
|
|
|
31
|
+
# Extracts attributes of rd_kafka_group_result_t
|
|
32
|
+
#
|
|
33
|
+
# @private
|
|
25
34
|
class GroupResult
|
|
26
35
|
attr_reader :result_error, :error_string, :result_name
|
|
36
|
+
|
|
37
|
+
# @param group_result_pointer [FFI::Pointer] pointer to the group result struct
|
|
27
38
|
def initialize(group_result_pointer)
|
|
28
39
|
native_error = Rdkafka::Bindings.rd_kafka_group_result_error(group_result_pointer)
|
|
29
40
|
|
|
30
41
|
if native_error.null?
|
|
31
|
-
@result_error =
|
|
42
|
+
@result_error = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
32
43
|
@error_string = FFI::Pointer::NULL
|
|
33
44
|
else
|
|
34
45
|
@result_error = native_error[:code]
|
|
@@ -37,6 +48,10 @@ module Rdkafka
|
|
|
37
48
|
|
|
38
49
|
@result_name = Rdkafka::Bindings.rd_kafka_group_result_name(group_result_pointer)
|
|
39
50
|
end
|
|
51
|
+
|
|
52
|
+
# @param count [Integer] number of results
|
|
53
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
54
|
+
# @return [Array<GroupResult>] array of group results
|
|
40
55
|
def self.create_group_results_from_array(count, array_pointer)
|
|
41
56
|
(1..count).map do |index|
|
|
42
57
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -51,12 +66,16 @@ module Rdkafka
|
|
|
51
66
|
class CreateAclResult
|
|
52
67
|
attr_reader :result_error, :error_string
|
|
53
68
|
|
|
69
|
+
# @param acl_result_pointer [FFI::Pointer] pointer to the ACL result struct
|
|
54
70
|
def initialize(acl_result_pointer)
|
|
55
71
|
rd_kafka_error_pointer = Bindings.rd_kafka_acl_result_error(acl_result_pointer)
|
|
56
72
|
@result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
|
|
57
73
|
@error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
|
|
58
74
|
end
|
|
59
75
|
|
|
76
|
+
# @param count [Integer] number of results
|
|
77
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
78
|
+
# @return [Array<CreateAclResult>] array of ACL results
|
|
60
79
|
def self.create_acl_results_from_array(count, array_pointer)
|
|
61
80
|
(1..count).map do |index|
|
|
62
81
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -71,19 +90,23 @@ module Rdkafka
|
|
|
71
90
|
class DeleteAclResult
|
|
72
91
|
attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
|
|
73
92
|
|
|
93
|
+
# @param acl_result_pointer [FFI::Pointer] pointer to the delete ACL result response struct
|
|
74
94
|
def initialize(acl_result_pointer)
|
|
75
|
-
@matching_acls=[]
|
|
95
|
+
@matching_acls = []
|
|
76
96
|
rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
|
|
77
97
|
@result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
|
|
78
98
|
@error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
|
|
79
|
-
if @result_error ==
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
100
|
+
# Get the number of matching acls
|
|
101
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
102
|
+
@matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
|
|
103
|
+
@matching_acls_count = pointer_to_size_t.read_int
|
|
84
104
|
end
|
|
85
105
|
end
|
|
86
106
|
|
|
107
|
+
# @param count [Integer] number of results
|
|
108
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
109
|
+
# @return [Array<DeleteAclResult>] array of delete ACL results
|
|
87
110
|
def self.delete_acl_results_from_array(count, array_pointer)
|
|
88
111
|
(1..count).map do |index|
|
|
89
112
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -98,11 +121,12 @@ module Rdkafka
|
|
|
98
121
|
class DescribeAclResult
|
|
99
122
|
attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
|
|
100
123
|
|
|
124
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
101
125
|
def initialize(event_ptr)
|
|
102
|
-
@matching_acls=[]
|
|
126
|
+
@matching_acls = []
|
|
103
127
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
104
128
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
105
|
-
if @result_error ==
|
|
129
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
106
130
|
acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
|
|
107
131
|
# Get the number of matching acls
|
|
108
132
|
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
@@ -112,15 +136,19 @@ module Rdkafka
|
|
|
112
136
|
end
|
|
113
137
|
end
|
|
114
138
|
|
|
139
|
+
# Extracts attributes of rd_kafka_DescribeConfigs_result_t
|
|
140
|
+
#
|
|
141
|
+
# @private
|
|
115
142
|
class DescribeConfigsResult
|
|
116
143
|
attr_reader :result_error, :error_string, :results, :results_count
|
|
117
144
|
|
|
145
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
118
146
|
def initialize(event_ptr)
|
|
119
|
-
@results=[]
|
|
147
|
+
@results = []
|
|
120
148
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
121
149
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
122
150
|
|
|
123
|
-
if @result_error ==
|
|
151
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
124
152
|
configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
|
|
125
153
|
# Get the number of matching acls
|
|
126
154
|
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
@@ -130,15 +158,19 @@ module Rdkafka
|
|
|
130
158
|
end
|
|
131
159
|
end
|
|
132
160
|
|
|
161
|
+
# Extracts attributes of rd_kafka_IncrementalAlterConfigs_result_t
|
|
162
|
+
#
|
|
163
|
+
# @private
|
|
133
164
|
class IncrementalAlterConfigsResult
|
|
134
165
|
attr_reader :result_error, :error_string, :results, :results_count
|
|
135
166
|
|
|
167
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
136
168
|
def initialize(event_ptr)
|
|
137
|
-
@results=[]
|
|
169
|
+
@results = []
|
|
138
170
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
139
171
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
140
172
|
|
|
141
|
-
if @result_error ==
|
|
173
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
142
174
|
incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
|
|
143
175
|
# Get the number of matching acls
|
|
144
176
|
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
@@ -150,7 +182,11 @@ module Rdkafka
|
|
|
150
182
|
|
|
151
183
|
# @private
|
|
152
184
|
class BackgroundEventCallback
|
|
153
|
-
|
|
185
|
+
# Handles background events from librdkafka
|
|
186
|
+
# @param _client_ptr [FFI::Pointer] unused client pointer
|
|
187
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
188
|
+
# @param _opaque_ptr [FFI::Pointer] unused opaque pointer
|
|
189
|
+
def self.call(_client_ptr, event_ptr, _opaque_ptr)
|
|
154
190
|
case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
|
155
191
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
|
|
156
192
|
process_create_topic(event_ptr)
|
|
@@ -175,6 +211,8 @@ module Rdkafka
|
|
|
175
211
|
|
|
176
212
|
private
|
|
177
213
|
|
|
214
|
+
# Processes create topic result event
|
|
215
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
178
216
|
def self.process_create_topic(event_ptr)
|
|
179
217
|
create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
|
|
180
218
|
|
|
@@ -193,6 +231,8 @@ module Rdkafka
|
|
|
193
231
|
end
|
|
194
232
|
end
|
|
195
233
|
|
|
234
|
+
# Processes describe configs result event
|
|
235
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
196
236
|
def self.process_describe_configs(event_ptr)
|
|
197
237
|
describe_configs = DescribeConfigsResult.new(event_ptr)
|
|
198
238
|
describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
@@ -202,7 +242,7 @@ module Rdkafka
|
|
|
202
242
|
describe_configs_handle[:response_string] = describe_configs.error_string
|
|
203
243
|
describe_configs_handle[:pending] = false
|
|
204
244
|
|
|
205
|
-
if describe_configs.result_error ==
|
|
245
|
+
if describe_configs.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
206
246
|
describe_configs_handle[:config_entries] = describe_configs.results
|
|
207
247
|
describe_configs_handle[:entry_count] = describe_configs.results_count
|
|
208
248
|
end
|
|
@@ -211,6 +251,8 @@ module Rdkafka
|
|
|
211
251
|
end
|
|
212
252
|
end
|
|
213
253
|
|
|
254
|
+
# Processes incremental alter configs result event
|
|
255
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
214
256
|
def self.process_incremental_alter_configs(event_ptr)
|
|
215
257
|
incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
|
|
216
258
|
incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
@@ -220,7 +262,7 @@ module Rdkafka
|
|
|
220
262
|
incremental_alter_handle[:response_string] = incremental_alter.error_string
|
|
221
263
|
incremental_alter_handle[:pending] = false
|
|
222
264
|
|
|
223
|
-
if incremental_alter.result_error ==
|
|
265
|
+
if incremental_alter.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
224
266
|
incremental_alter_handle[:config_entries] = incremental_alter.results
|
|
225
267
|
incremental_alter_handle[:entry_count] = incremental_alter.results_count
|
|
226
268
|
end
|
|
@@ -229,6 +271,8 @@ module Rdkafka
|
|
|
229
271
|
end
|
|
230
272
|
end
|
|
231
273
|
|
|
274
|
+
# Processes delete groups result event
|
|
275
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
232
276
|
def self.process_delete_groups(event_ptr)
|
|
233
277
|
delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
|
|
234
278
|
|
|
@@ -247,6 +291,8 @@ module Rdkafka
|
|
|
247
291
|
end
|
|
248
292
|
end
|
|
249
293
|
|
|
294
|
+
# Processes delete topic result event
|
|
295
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
250
296
|
def self.process_delete_topic(event_ptr)
|
|
251
297
|
delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
|
|
252
298
|
|
|
@@ -265,6 +311,8 @@ module Rdkafka
|
|
|
265
311
|
end
|
|
266
312
|
end
|
|
267
313
|
|
|
314
|
+
# Processes create partitions result event
|
|
315
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
268
316
|
def self.process_create_partitions(event_ptr)
|
|
269
317
|
create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
|
|
270
318
|
|
|
@@ -283,6 +331,8 @@ module Rdkafka
|
|
|
283
331
|
end
|
|
284
332
|
end
|
|
285
333
|
|
|
334
|
+
# Processes create ACL result event
|
|
335
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
286
336
|
def self.process_create_acl(event_ptr)
|
|
287
337
|
create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
|
|
288
338
|
|
|
@@ -300,6 +350,8 @@ module Rdkafka
|
|
|
300
350
|
end
|
|
301
351
|
end
|
|
302
352
|
|
|
353
|
+
# Processes delete ACL result event
|
|
354
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
303
355
|
def self.process_delete_acl(event_ptr)
|
|
304
356
|
delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
|
|
305
357
|
|
|
@@ -313,7 +365,7 @@ module Rdkafka
|
|
|
313
365
|
delete_acl_handle[:response] = delete_acl_results[0].result_error
|
|
314
366
|
delete_acl_handle[:response_string] = delete_acl_results[0].error_string
|
|
315
367
|
|
|
316
|
-
if delete_acl_results[0].result_error ==
|
|
368
|
+
if delete_acl_results[0].result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
317
369
|
delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
|
|
318
370
|
delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
|
|
319
371
|
end
|
|
@@ -322,6 +374,8 @@ module Rdkafka
|
|
|
322
374
|
end
|
|
323
375
|
end
|
|
324
376
|
|
|
377
|
+
# Processes describe ACL result event
|
|
378
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
325
379
|
def self.process_describe_acl(event_ptr)
|
|
326
380
|
describe_acl = DescribeAclResult.new(event_ptr)
|
|
327
381
|
describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
@@ -330,7 +384,7 @@ module Rdkafka
|
|
|
330
384
|
describe_acl_handle[:response] = describe_acl.result_error
|
|
331
385
|
describe_acl_handle[:response_string] = describe_acl.error_string
|
|
332
386
|
|
|
333
|
-
if describe_acl.result_error ==
|
|
387
|
+
if describe_acl.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
334
388
|
describe_acl_handle[:acls] = describe_acl.matching_acls
|
|
335
389
|
describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
|
|
336
390
|
end
|
|
@@ -342,7 +396,11 @@ module Rdkafka
|
|
|
342
396
|
|
|
343
397
|
# @private
|
|
344
398
|
class DeliveryCallback
|
|
345
|
-
|
|
399
|
+
# Handles message delivery callbacks
|
|
400
|
+
# @param _client_ptr [FFI::Pointer] unused client pointer
|
|
401
|
+
# @param message_ptr [FFI::Pointer] pointer to the delivered message
|
|
402
|
+
# @param opaque_ptr [FFI::Pointer] pointer to the opaque object for callback context
|
|
403
|
+
def self.call(_client_ptr, message_ptr, opaque_ptr)
|
|
346
404
|
message = Rdkafka::Bindings::Message.new(message_ptr)
|
|
347
405
|
delivery_handle_ptr_address = message[:_private].address
|
|
348
406
|
if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
|
|
@@ -373,7 +431,9 @@ module Rdkafka
|
|
|
373
431
|
end
|
|
374
432
|
end
|
|
375
433
|
|
|
434
|
+
# @private
|
|
376
435
|
@@mutex = Mutex.new
|
|
436
|
+
# @private
|
|
377
437
|
@@current_pid = nil
|
|
378
438
|
|
|
379
439
|
class << self
|
|
@@ -392,14 +452,14 @@ module Rdkafka
|
|
|
392
452
|
|
|
393
453
|
# FFI Function used for Create Topic and Delete Topic callbacks
|
|
394
454
|
background_event_callback_function = FFI::Function.new(
|
|
395
|
-
|
|
455
|
+
:void, [:pointer, :pointer, :pointer]
|
|
396
456
|
) do |client_ptr, event_ptr, opaque_ptr|
|
|
397
457
|
BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
|
|
398
458
|
end
|
|
399
459
|
|
|
400
460
|
# FFI Function used for Message Delivery callbacks
|
|
401
461
|
delivery_callback_function = FFI::Function.new(
|
|
402
|
-
|
|
462
|
+
:void, [:pointer, :pointer, :pointer]
|
|
403
463
|
) do |client_ptr, message_ptr, opaque_ptr|
|
|
404
464
|
DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
|
|
405
465
|
end
|
data/lib/rdkafka/config.rb
CHANGED
|
@@ -6,7 +6,7 @@ module Rdkafka
|
|
|
6
6
|
# configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
|
|
7
7
|
class Config
|
|
8
8
|
# @private
|
|
9
|
-
@@logger = Logger.new(
|
|
9
|
+
@@logger = Logger.new($stdout)
|
|
10
10
|
# @private
|
|
11
11
|
@@statistics_callback = nil
|
|
12
12
|
# @private
|
|
@@ -33,7 +33,7 @@ module Rdkafka
|
|
|
33
33
|
# Makes sure that there is a thread for consuming logs
|
|
34
34
|
# We do not spawn thread immediately and we need to check if it operates to support forking
|
|
35
35
|
def self.ensure_log_thread
|
|
36
|
-
return if @@log_thread
|
|
36
|
+
return if @@log_thread&.alive?
|
|
37
37
|
|
|
38
38
|
@@log_mutex.synchronize do
|
|
39
39
|
# Restart if dead (fork, crash)
|
|
@@ -71,11 +71,10 @@ module Rdkafka
|
|
|
71
71
|
# You can configure if and how often this happens using `statistics.interval.ms`.
|
|
72
72
|
# The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
|
|
73
73
|
#
|
|
74
|
-
# @param callback [Proc, #call]
|
|
75
|
-
#
|
|
74
|
+
# @param callback [Proc, #call, nil] callable object or nil to clear
|
|
76
75
|
# @return [nil]
|
|
77
76
|
def self.statistics_callback=(callback)
|
|
78
|
-
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback
|
|
77
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
|
|
79
78
|
@@statistics_callback = callback
|
|
80
79
|
end
|
|
81
80
|
|
|
@@ -90,8 +89,7 @@ module Rdkafka
|
|
|
90
89
|
# If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
|
|
91
90
|
# The callback is called with an instance of RdKafka::Error.
|
|
92
91
|
#
|
|
93
|
-
# @param callback [Proc, #call]
|
|
94
|
-
#
|
|
92
|
+
# @param callback [Proc, #call] callable object to handle errors
|
|
95
93
|
# @return [nil]
|
|
96
94
|
def self.error_callback=(callback)
|
|
97
95
|
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
|
|
@@ -108,11 +106,10 @@ module Rdkafka
|
|
|
108
106
|
# Sets the SASL/OAUTHBEARER token refresh callback.
|
|
109
107
|
# This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
|
|
110
108
|
#
|
|
111
|
-
# @param callback [Proc, #call]
|
|
112
|
-
#
|
|
109
|
+
# @param callback [Proc, #call, nil] callable object to handle token refresh or nil to clear
|
|
113
110
|
# @return [nil]
|
|
114
111
|
def self.oauthbearer_token_refresh_callback=(callback)
|
|
115
|
-
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback
|
|
112
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
|
|
116
113
|
@@oauthbearer_token_refresh_callback = callback
|
|
117
114
|
end
|
|
118
115
|
|
|
@@ -134,7 +131,7 @@ module Rdkafka
|
|
|
134
131
|
# Required config that cannot be overwritten.
|
|
135
132
|
REQUIRED_CONFIG = {
|
|
136
133
|
# Enable log queues so we get callbacks in our own Ruby threads
|
|
137
|
-
|
|
134
|
+
"log.queue": true
|
|
138
135
|
}.freeze
|
|
139
136
|
|
|
140
137
|
# Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
|
|
@@ -171,10 +168,8 @@ module Rdkafka
|
|
|
171
168
|
|
|
172
169
|
# Get notifications on partition assignment/revocation for the subscribed topics
|
|
173
170
|
#
|
|
174
|
-
# @
|
|
175
|
-
|
|
176
|
-
@consumer_rebalance_listener = listener
|
|
177
|
-
end
|
|
171
|
+
# @return [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
|
|
172
|
+
attr_writer :consumer_rebalance_listener
|
|
178
173
|
|
|
179
174
|
# Should we use a single queue for the underlying consumer and events.
|
|
180
175
|
#
|
|
@@ -187,10 +182,8 @@ module Rdkafka
|
|
|
187
182
|
# It is recommended to use the defaults and only set it to `false` in advance multi-threaded
|
|
188
183
|
# and complex cases where granular events handling control is needed.
|
|
189
184
|
#
|
|
190
|
-
# @
|
|
191
|
-
|
|
192
|
-
@consumer_poll_set = poll_set
|
|
193
|
-
end
|
|
185
|
+
# @return [Boolean]
|
|
186
|
+
attr_writer :consumer_poll_set
|
|
194
187
|
|
|
195
188
|
# Creates a consumer with this configuration.
|
|
196
189
|
#
|
|
@@ -231,11 +224,13 @@ module Rdkafka
|
|
|
231
224
|
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
|
232
225
|
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
|
233
226
|
# @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
|
|
227
|
+
# @param run_polling_thread [Boolean] should the background polling thread be started.
|
|
228
|
+
# Defaults to true. Set to false when using the FD API for fiber scheduler integration.
|
|
234
229
|
# @return [Producer] The created producer
|
|
235
230
|
#
|
|
236
231
|
# @raise [ConfigError] When the configuration contains invalid options
|
|
237
232
|
# @raise [ClientCreationError] When the native client cannot be created
|
|
238
|
-
def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms:
|
|
233
|
+
def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
|
|
239
234
|
# Create opaque
|
|
240
235
|
opaque = Opaque.new
|
|
241
236
|
# Create Kafka config
|
|
@@ -250,7 +245,7 @@ module Rdkafka
|
|
|
250
245
|
Rdkafka::Producer.new(
|
|
251
246
|
Rdkafka::NativeKafka.new(
|
|
252
247
|
kafka,
|
|
253
|
-
run_polling_thread:
|
|
248
|
+
run_polling_thread: run_polling_thread,
|
|
254
249
|
opaque: opaque,
|
|
255
250
|
auto_start: native_kafka_auto_start,
|
|
256
251
|
timeout_ms: native_kafka_poll_timeout_ms
|
|
@@ -266,11 +261,13 @@ module Rdkafka
|
|
|
266
261
|
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
|
267
262
|
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
|
268
263
|
# @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
|
|
264
|
+
# @param run_polling_thread [Boolean] should the background polling thread be started.
|
|
265
|
+
# Defaults to true. Set to false when using the FD API for fiber scheduler integration.
|
|
269
266
|
# @return [Admin] The created admin instance
|
|
270
267
|
#
|
|
271
268
|
# @raise [ConfigError] When the configuration contains invalid options
|
|
272
269
|
# @raise [ClientCreationError] When the native client cannot be created
|
|
273
|
-
def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms:
|
|
270
|
+
def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
|
|
274
271
|
opaque = Opaque.new
|
|
275
272
|
config = native_config(opaque)
|
|
276
273
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
|
@@ -280,7 +277,7 @@ module Rdkafka
|
|
|
280
277
|
Rdkafka::Admin.new(
|
|
281
278
|
Rdkafka::NativeKafka.new(
|
|
282
279
|
kafka,
|
|
283
|
-
run_polling_thread:
|
|
280
|
+
run_polling_thread: run_polling_thread,
|
|
284
281
|
opaque: opaque,
|
|
285
282
|
auto_start: native_kafka_auto_start,
|
|
286
283
|
timeout_ms: native_kafka_poll_timeout_ms
|
|
@@ -301,6 +298,9 @@ module Rdkafka
|
|
|
301
298
|
|
|
302
299
|
# This method is only intended to be used to create a client,
|
|
303
300
|
# using it in another way will leak memory.
|
|
301
|
+
#
|
|
302
|
+
# @param opaque [Object, nil] optional opaque pointer for callbacks
|
|
303
|
+
# @return [FFI::Pointer] native rdkafka configuration pointer
|
|
304
304
|
def native_config(opaque = nil)
|
|
305
305
|
Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
|
|
306
306
|
# Create config
|
|
@@ -345,6 +345,11 @@ module Rdkafka
|
|
|
345
345
|
end
|
|
346
346
|
end
|
|
347
347
|
|
|
348
|
+
# Creates a native Kafka handle
|
|
349
|
+
# @param config [FFI::Pointer] pointer to the native config
|
|
350
|
+
# @param type [Symbol] type of client (:rd_kafka_producer or :rd_kafka_consumer)
|
|
351
|
+
# @return [FFI::Pointer] pointer to the native Kafka handle
|
|
352
|
+
# @private
|
|
348
353
|
def native_kafka(config, type)
|
|
349
354
|
error_buffer = FFI::MemoryPointer.from_string(" " * 256)
|
|
350
355
|
handle = Rdkafka::Bindings.rd_kafka_new(
|
|
@@ -374,10 +379,15 @@ module Rdkafka
|
|
|
374
379
|
attr_accessor :producer
|
|
375
380
|
attr_accessor :consumer_rebalance_listener
|
|
376
381
|
|
|
382
|
+
# Invokes the delivery callback on the producer if one is set
|
|
383
|
+
# @param delivery_report [Rdkafka::Producer::DeliveryReport] the delivery report
|
|
384
|
+
# @param delivery_handle [Rdkafka::Producer::DeliveryHandle] the delivery handle
|
|
377
385
|
def call_delivery_callback(delivery_report, delivery_handle)
|
|
378
|
-
producer
|
|
386
|
+
producer&.call_delivery_callback(delivery_report, delivery_handle)
|
|
379
387
|
end
|
|
380
388
|
|
|
389
|
+
# Invokes the on_partitions_assigned callback on the rebalance listener if set
|
|
390
|
+
# @param list [Rdkafka::Consumer::TopicPartitionList] the assigned partitions
|
|
381
391
|
def call_on_partitions_assigned(list)
|
|
382
392
|
return unless consumer_rebalance_listener
|
|
383
393
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
|
|
@@ -385,6 +395,8 @@ module Rdkafka
|
|
|
385
395
|
consumer_rebalance_listener.on_partitions_assigned(list)
|
|
386
396
|
end
|
|
387
397
|
|
|
398
|
+
# Invokes the on_partitions_revoked callback on the rebalance listener if set
|
|
399
|
+
# @param list [Rdkafka::Consumer::TopicPartitionList] the revoked partitions
|
|
388
400
|
def call_on_partitions_revoked(list)
|
|
389
401
|
return unless consumer_rebalance_listener
|
|
390
402
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
|
|
@@ -4,6 +4,7 @@ module Rdkafka
|
|
|
4
4
|
class Consumer
|
|
5
5
|
# Interface to return headers for a consumer message
|
|
6
6
|
module Headers
|
|
7
|
+
# Empty frozen hash used when there are no headers
|
|
7
8
|
EMPTY_HEADERS = {}.freeze
|
|
8
9
|
|
|
9
10
|
# Reads a librdkafka native message's headers and returns them as a Ruby Hash
|
|
@@ -12,8 +13,8 @@ module Rdkafka
|
|
|
12
13
|
#
|
|
13
14
|
# @private
|
|
14
15
|
#
|
|
15
|
-
# @param [Rdkafka::Bindings::Message]
|
|
16
|
-
# @return [Hash
|
|
16
|
+
# @param native_message [Rdkafka::Bindings::Message] the native message to read headers from
|
|
17
|
+
# @return [Hash{String => String, Array<String>}] headers Hash for the native_message
|
|
17
18
|
# @raise [Rdkafka::RdkafkaError] when fail to read headers
|
|
18
19
|
def self.from_native(native_message)
|
|
19
20
|
headers_ptrptr = FFI::MemoryPointer.new(:pointer)
|
|
@@ -28,10 +28,11 @@ module Rdkafka
|
|
|
28
28
|
# @return [Time, nil]
|
|
29
29
|
attr_reader :timestamp
|
|
30
30
|
|
|
31
|
-
# @return [Hash
|
|
31
|
+
# @return [Hash{String => String}] message headers
|
|
32
32
|
attr_reader :headers
|
|
33
33
|
|
|
34
34
|
# @private
|
|
35
|
+
# @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
|
|
35
36
|
def initialize(native_message)
|
|
36
37
|
# Set topic
|
|
37
38
|
unless native_message[:rkt].null?
|
|
@@ -52,13 +53,11 @@ module Rdkafka
|
|
|
52
53
|
# Set timestamp
|
|
53
54
|
raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
|
|
54
55
|
@timestamp = if raw_timestamp && raw_timestamp > -1
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
nil
|
|
61
|
-
end
|
|
56
|
+
# Calculate seconds and microseconds
|
|
57
|
+
seconds = raw_timestamp / 1000
|
|
58
|
+
milliseconds = (raw_timestamp - seconds * 1000) * 1000
|
|
59
|
+
Time.at(seconds, milliseconds)
|
|
60
|
+
end
|
|
62
61
|
|
|
63
62
|
@headers = Headers.from_native(native_message)
|
|
64
63
|
end
|
|
@@ -71,6 +70,11 @@ module Rdkafka
|
|
|
71
70
|
"<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
|
|
72
71
|
end
|
|
73
72
|
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
# Truncates a string for display purposes
|
|
76
|
+
# @param string [String, nil] the string to truncate
|
|
77
|
+
# @return [String, nil] truncated string or nil
|
|
74
78
|
def truncate(string)
|
|
75
79
|
if string && string.length > 40
|
|
76
80
|
"#{string[0..39]}..."
|
|
@@ -78,9 +82,6 @@ module Rdkafka
|
|
|
78
82
|
string
|
|
79
83
|
end
|
|
80
84
|
end
|
|
81
|
-
|
|
82
|
-
private
|
|
83
|
-
|
|
84
85
|
end
|
|
85
86
|
end
|
|
86
87
|
end
|
|
@@ -17,7 +17,10 @@ module Rdkafka
|
|
|
17
17
|
attr_reader :err
|
|
18
18
|
|
|
19
19
|
# @private
|
|
20
|
-
|
|
20
|
+
# @param partition [Integer] partition number
|
|
21
|
+
# @param offset [Integer, nil] partition offset
|
|
22
|
+
# @param err [Integer] error code from librdkafka
|
|
23
|
+
def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR)
|
|
21
24
|
@partition = partition
|
|
22
25
|
@offset = offset
|
|
23
26
|
@err = err
|
|
@@ -28,7 +31,7 @@ module Rdkafka
|
|
|
28
31
|
def to_s
|
|
29
32
|
message = "<Partition #{partition}"
|
|
30
33
|
message += " offset=#{offset}" if offset
|
|
31
|
-
message += " err=#{err}" if err !=
|
|
34
|
+
message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
32
35
|
message += ">"
|
|
33
36
|
message
|
|
34
37
|
end
|
|
@@ -40,11 +43,12 @@ module Rdkafka
|
|
|
40
43
|
end
|
|
41
44
|
|
|
42
45
|
# Whether another partition is equal to this
|
|
46
|
+
# @param other [Object] object to compare with
|
|
43
47
|
# @return [Boolean]
|
|
44
48
|
def ==(other)
|
|
45
49
|
self.class == other.class &&
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
partition == other.partition &&
|
|
51
|
+
offset == other.offset
|
|
48
52
|
end
|
|
49
53
|
end
|
|
50
54
|
end
|