rdkafka 0.29.1-aarch64-linux-gnu → 0.30.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -11
- data/README.md +1 -0
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/admin/alter_consumer_group_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/alter_consumer_group_offsets_report.rb +50 -0
- data/lib/rdkafka/admin/delete_consumer_group_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/delete_consumer_group_offsets_report.rb +50 -0
- data/lib/rdkafka/admin.rb +263 -85
- data/lib/rdkafka/bindings.rb +46 -2
- data/lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb +65 -0
- data/lib/rdkafka/callbacks/delete_consumer_group_offsets_handler.rb +65 -0
- data/lib/rdkafka/callbacks.rb +14 -16
- data/lib/rdkafka/clients.rb +114 -0
- data/lib/rdkafka/config.rb +118 -39
- data/lib/rdkafka/consumer/allocator.rb +38 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +1 -2
- data/lib/rdkafka/consumer.rb +44 -40
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +10 -0
- data/package-lock.json +6 -6
- data/renovate.json +1 -1
- metadata +10 -2
data/lib/rdkafka/callbacks.rb
CHANGED
|
@@ -21,10 +21,9 @@ module Rdkafka
|
|
|
21
21
|
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
22
22
|
# @return [Array<TopicResult>] array of topic results
|
|
23
23
|
def self.create_topic_results_from_array(count, array_pointer)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
end
|
|
24
|
+
return [] if count.zero?
|
|
25
|
+
|
|
26
|
+
array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
|
|
28
27
|
end
|
|
29
28
|
end
|
|
30
29
|
|
|
@@ -53,10 +52,9 @@ module Rdkafka
|
|
|
53
52
|
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
54
53
|
# @return [Array<GroupResult>] array of group results
|
|
55
54
|
def self.create_group_results_from_array(count, array_pointer)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
end
|
|
55
|
+
return [] if count.zero?
|
|
56
|
+
|
|
57
|
+
array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
|
|
60
58
|
end
|
|
61
59
|
end
|
|
62
60
|
|
|
@@ -77,10 +75,9 @@ module Rdkafka
|
|
|
77
75
|
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
78
76
|
# @return [Array<CreateAclResult>] array of ACL results
|
|
79
77
|
def self.create_acl_results_from_array(count, array_pointer)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
end
|
|
78
|
+
return [] if count.zero?
|
|
79
|
+
|
|
80
|
+
array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
|
|
84
81
|
end
|
|
85
82
|
end
|
|
86
83
|
|
|
@@ -108,10 +105,9 @@ module Rdkafka
|
|
|
108
105
|
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
109
106
|
# @return [Array<DeleteAclResult>] array of delete ACL results
|
|
110
107
|
def self.delete_acl_results_from_array(count, array_pointer)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
end
|
|
108
|
+
return [] if count.zero?
|
|
109
|
+
|
|
110
|
+
array_pointer.read_array_of_pointer(count).map { |result_pointer| new(result_pointer) }
|
|
115
111
|
end
|
|
116
112
|
end
|
|
117
113
|
|
|
@@ -213,6 +209,8 @@ module Rdkafka
|
|
|
213
209
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
|
|
214
210
|
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
|
|
215
211
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
|
|
212
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT then AlterConsumerGroupOffsetsHandler
|
|
213
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT then DeleteConsumerGroupOffsetsHandler
|
|
216
214
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
|
|
217
215
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
|
|
218
216
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
# Tracks live public clients so they can be closed before Ruby begins forced finalization.
|
|
5
|
+
# @private
|
|
6
|
+
module Clients
|
|
7
|
+
# Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
|
|
8
|
+
# foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
|
|
9
|
+
# the moment the value is touched. Track weakly only where that is safe and fall back to a
|
|
10
|
+
# strong registry - pruned as clients close - on older Rubies. The strong registry holds an
|
|
11
|
+
# open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
|
|
12
|
+
WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
|
|
13
|
+
|
|
14
|
+
@pid = Process.pid
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
|
|
17
|
+
if WEAK_TRACKING
|
|
18
|
+
@clients = ObjectSpace::WeakMap.new
|
|
19
|
+
@tokens = [].freeze
|
|
20
|
+
else
|
|
21
|
+
@clients = {}.freeze
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Registers a successfully constructed client so it can be closed during process shutdown.
|
|
25
|
+
# @param client [Consumer, Producer, Admin] client to close during process shutdown
|
|
26
|
+
# @return [Consumer, Producer, Admin] the registered client
|
|
27
|
+
def self.register(client)
|
|
28
|
+
reset_after_fork
|
|
29
|
+
|
|
30
|
+
@mutex.synchronize do
|
|
31
|
+
if WEAK_TRACKING
|
|
32
|
+
live_clients
|
|
33
|
+
|
|
34
|
+
# The strongly-held token is the key and the client is the weak value, so registering a
|
|
35
|
+
# client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
|
|
36
|
+
# look the value up by token instead.
|
|
37
|
+
token = Object.new
|
|
38
|
+
@clients[token] = client
|
|
39
|
+
@tokens = (@tokens + [token]).freeze
|
|
40
|
+
else
|
|
41
|
+
# Strong references, so drop clients that have already been closed to bound growth.
|
|
42
|
+
live = @clients.reject { |_id, existing| existing.closed? }
|
|
43
|
+
live[client.object_id] = client
|
|
44
|
+
@clients = live.freeze
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
client
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Closes each live client registered in this process.
|
|
52
|
+
# @return [nil]
|
|
53
|
+
# @raise [StandardError] the first close error, after attempting every client
|
|
54
|
+
def self.close_all
|
|
55
|
+
return unless @pid == Process.pid
|
|
56
|
+
|
|
57
|
+
clients = @mutex.synchronize { current_clients }
|
|
58
|
+
first_error = nil
|
|
59
|
+
|
|
60
|
+
clients.each do |client|
|
|
61
|
+
client.close unless client.closed?
|
|
62
|
+
rescue => error
|
|
63
|
+
first_error ||= error
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
raise first_error if first_error
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Returns the currently registered live clients.
|
|
70
|
+
# @return [Array<Consumer, Producer, Admin>] live clients
|
|
71
|
+
# @private
|
|
72
|
+
def self.current_clients
|
|
73
|
+
WEAK_TRACKING ? live_clients : @clients.values
|
|
74
|
+
end
|
|
75
|
+
private_class_method :current_clients
|
|
76
|
+
|
|
77
|
+
# Returns the live clients and releases tokens whose clients have been collected.
|
|
78
|
+
# Weak-tracking Rubies only.
|
|
79
|
+
# @return [Array<Consumer, Producer, Admin>] live clients
|
|
80
|
+
# @private
|
|
81
|
+
def self.live_clients
|
|
82
|
+
clients = []
|
|
83
|
+
tokens = []
|
|
84
|
+
|
|
85
|
+
@tokens.each do |token|
|
|
86
|
+
client = @clients[token]
|
|
87
|
+
clients << client if client
|
|
88
|
+
tokens << token if client
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
@tokens = tokens.freeze
|
|
92
|
+
clients
|
|
93
|
+
end
|
|
94
|
+
private_class_method :live_clients
|
|
95
|
+
|
|
96
|
+
# Replaces inherited state before registering a client in a forked child.
|
|
97
|
+
# @return [nil]
|
|
98
|
+
# @private
|
|
99
|
+
def self.reset_after_fork
|
|
100
|
+
return if @pid == Process.pid
|
|
101
|
+
|
|
102
|
+
if WEAK_TRACKING
|
|
103
|
+
@clients = ObjectSpace::WeakMap.new
|
|
104
|
+
@tokens = [].freeze
|
|
105
|
+
else
|
|
106
|
+
@clients = {}.freeze
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
@pid = Process.pid
|
|
110
|
+
@mutex = Mutex.new
|
|
111
|
+
end
|
|
112
|
+
private_class_method :reset_after_fork
|
|
113
|
+
end
|
|
114
|
+
end
|
data/lib/rdkafka/config.rb
CHANGED
|
@@ -22,6 +22,12 @@ module Rdkafka
|
|
|
22
22
|
@@log_mutex = Mutex.new
|
|
23
23
|
# @!visibility private
|
|
24
24
|
@@oauthbearer_token_refresh_callback = nil
|
|
25
|
+
# @!visibility private
|
|
26
|
+
# Whether the partition key length passed to the partitioner is the byte length (`bytesize`)
|
|
27
|
+
# rather than the character length (`size`). `false` (character length) is the legacy default
|
|
28
|
+
# to avoid changing partition assignment for existing multibyte/binary keys; set to `true` to
|
|
29
|
+
# opt into the correct, other-client-consistent byte-length hashing.
|
|
30
|
+
@@partitioner_key_uses_bytesize = false
|
|
25
31
|
|
|
26
32
|
# Returns the current logger, by default this is a logger to stdout.
|
|
27
33
|
#
|
|
@@ -122,6 +128,34 @@ module Rdkafka
|
|
|
122
128
|
@@oauthbearer_token_refresh_callback
|
|
123
129
|
end
|
|
124
130
|
|
|
131
|
+
# Whether the partitioner hashes the partition key by its byte length instead of its character
|
|
132
|
+
# length. Defaults to `false` (character length) to preserve existing partition assignment for
|
|
133
|
+
# multibyte/binary keys. Passing the byte length is the correct, other-client-consistent
|
|
134
|
+
# behavior; enable it (and be aware it can move such keys to a different partition) with
|
|
135
|
+
# `Rdkafka::Config.partitioner_key_uses_bytesize = true`. This default is expected to change in
|
|
136
|
+
# a future major version.
|
|
137
|
+
#
|
|
138
|
+
# @return [Boolean]
|
|
139
|
+
def self.partitioner_key_uses_bytesize
|
|
140
|
+
@@partitioner_key_uses_bytesize
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Sets whether the partitioner hashes the partition key by its byte length rather than its
|
|
144
|
+
# character length. The key length method is resolved here once rather than on every produced
|
|
145
|
+
# message, and each call re-resolves it, so changing the setting at runtime takes effect
|
|
146
|
+
# immediately.
|
|
147
|
+
#
|
|
148
|
+
# @param value [Boolean] true to hash by byte length, false (default) for character length
|
|
149
|
+
# @return [void]
|
|
150
|
+
def self.partitioner_key_uses_bytesize=(value)
|
|
151
|
+
@@partitioner_key_uses_bytesize = value
|
|
152
|
+
|
|
153
|
+
Rdkafka::Bindings.singleton_class.alias_method(
|
|
154
|
+
:partition_key_length,
|
|
155
|
+
value ? :partition_key_bytesize : :partition_key_size
|
|
156
|
+
)
|
|
157
|
+
end
|
|
158
|
+
|
|
125
159
|
# @private
|
|
126
160
|
def self.opaques
|
|
127
161
|
@@opaques
|
|
@@ -209,21 +243,16 @@ module Rdkafka
|
|
|
209
243
|
Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
|
|
210
244
|
end
|
|
211
245
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
Rdkafka::
|
|
221
|
-
|
|
222
|
-
run_polling_thread: false,
|
|
223
|
-
opaque: opaque,
|
|
224
|
-
auto_start: native_kafka_auto_start
|
|
225
|
-
)
|
|
226
|
-
)
|
|
246
|
+
build_native_client(
|
|
247
|
+
config,
|
|
248
|
+
:rd_kafka_consumer,
|
|
249
|
+
opaque: opaque,
|
|
250
|
+
run_polling_thread: false,
|
|
251
|
+
auto_start: native_kafka_auto_start
|
|
252
|
+
) do |kafka, native_kafka|
|
|
253
|
+
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
|
|
254
|
+
Rdkafka::Consumer.new(native_kafka)
|
|
255
|
+
end
|
|
227
256
|
end
|
|
228
257
|
|
|
229
258
|
# Create a producer with this configuration.
|
|
@@ -247,19 +276,17 @@ module Rdkafka
|
|
|
247
276
|
# Return producer with Kafka client
|
|
248
277
|
partitioner_name = self[:partitioner] || self["partitioner"]
|
|
249
278
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
).tap do |producer|
|
|
262
|
-
opaque.producer = producer
|
|
279
|
+
build_native_client(
|
|
280
|
+
config,
|
|
281
|
+
:rd_kafka_producer,
|
|
282
|
+
opaque: opaque,
|
|
283
|
+
run_polling_thread: run_polling_thread,
|
|
284
|
+
auto_start: native_kafka_auto_start,
|
|
285
|
+
timeout_ms: native_kafka_poll_timeout_ms
|
|
286
|
+
) do |_kafka, native_kafka|
|
|
287
|
+
Rdkafka::Producer.new(native_kafka, partitioner_name).tap do |producer|
|
|
288
|
+
opaque.producer = producer
|
|
289
|
+
end
|
|
263
290
|
end
|
|
264
291
|
end
|
|
265
292
|
|
|
@@ -279,17 +306,16 @@ module Rdkafka
|
|
|
279
306
|
config = native_config(opaque)
|
|
280
307
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
|
281
308
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
)
|
|
309
|
+
build_native_client(
|
|
310
|
+
config,
|
|
311
|
+
:rd_kafka_producer,
|
|
312
|
+
opaque: opaque,
|
|
313
|
+
run_polling_thread: run_polling_thread,
|
|
314
|
+
auto_start: native_kafka_auto_start,
|
|
315
|
+
timeout_ms: native_kafka_poll_timeout_ms
|
|
316
|
+
) do |_kafka, native_kafka|
|
|
317
|
+
Rdkafka::Admin.new(native_kafka)
|
|
318
|
+
end
|
|
293
319
|
end
|
|
294
320
|
|
|
295
321
|
# Returns all configuration properties and their current values for this config.
|
|
@@ -343,6 +369,56 @@ module Rdkafka
|
|
|
343
369
|
|
|
344
370
|
private
|
|
345
371
|
|
|
372
|
+
# Builds a public client around a native Kafka handle, destroying the current owner if setup
|
|
373
|
+
# fails before ownership is transferred to the caller.
|
|
374
|
+
# @param config [FFI::Pointer] native rdkafka configuration
|
|
375
|
+
# @param type [Symbol] native client type
|
|
376
|
+
# @param opaque [Object] callback context retained for the native client
|
|
377
|
+
# @param run_polling_thread [Boolean] whether to run a Ruby polling thread
|
|
378
|
+
# @param auto_start [Boolean] whether to start the polling thread after wrapping the client
|
|
379
|
+
# @param timeout_ms [Integer] polling timeout in milliseconds
|
|
380
|
+
# @yieldparam kafka [FFI::Pointer] native Kafka handle
|
|
381
|
+
# @yieldparam native_kafka [NativeKafka] Ruby owner of the native handle
|
|
382
|
+
# @return [Consumer, Producer, Admin] the constructed public client
|
|
383
|
+
# @private
|
|
384
|
+
def build_native_client(
|
|
385
|
+
config,
|
|
386
|
+
type,
|
|
387
|
+
opaque:,
|
|
388
|
+
run_polling_thread:,
|
|
389
|
+
auto_start:,
|
|
390
|
+
timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS
|
|
391
|
+
)
|
|
392
|
+
kafka = native_kafka(config, type)
|
|
393
|
+
native_kafka = nil
|
|
394
|
+
client = nil
|
|
395
|
+
|
|
396
|
+
begin
|
|
397
|
+
native_kafka = Rdkafka::NativeKafka.new(
|
|
398
|
+
kafka,
|
|
399
|
+
run_polling_thread: run_polling_thread,
|
|
400
|
+
opaque: opaque,
|
|
401
|
+
auto_start: false,
|
|
402
|
+
timeout_ms: timeout_ms
|
|
403
|
+
)
|
|
404
|
+
client = yield(kafka, native_kafka)
|
|
405
|
+
native_kafka.start if auto_start
|
|
406
|
+
Rdkafka::Clients.register(client)
|
|
407
|
+
rescue Exception
|
|
408
|
+
owner = client || native_kafka
|
|
409
|
+
|
|
410
|
+
begin
|
|
411
|
+
owner ? owner.close : Rdkafka::Bindings.rd_kafka_destroy(kafka)
|
|
412
|
+
rescue Exception => err
|
|
413
|
+
# Never let a teardown failure replace the error that caused it: that error is what the
|
|
414
|
+
# caller needs to see, and it is re-raised below regardless.
|
|
415
|
+
Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
raise
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
|
|
346
422
|
# This method is only intended to be used to create a client,
|
|
347
423
|
# using it in another way will leak memory.
|
|
348
424
|
#
|
|
@@ -429,6 +505,9 @@ module Rdkafka
|
|
|
429
505
|
|
|
430
506
|
# Return handle which should be closed using rd_kafka_destroy after usage.
|
|
431
507
|
handle
|
|
508
|
+
rescue Exception
|
|
509
|
+
Rdkafka::Bindings.rd_kafka_destroy(handle) if handle && !handle.null?
|
|
510
|
+
raise
|
|
432
511
|
end
|
|
433
512
|
end
|
|
434
513
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
class Consumer
|
|
5
|
+
# Creates FFI pointers from the process-global allocator so that memory handed to librdkafka
|
|
6
|
+
# can be released by it with the matching `free`.
|
|
7
|
+
module Allocator
|
|
8
|
+
extend FFI::Library
|
|
9
|
+
|
|
10
|
+
ffi_lib FFI::CURRENT_PROCESS # use process-global malloc not FFI::Library::LIBC
|
|
11
|
+
attach_function :malloc, [:size_t], :pointer
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
# Return an FFI pointer to a string allocated from the global allocator
|
|
15
|
+
# so that it can be released by another library.
|
|
16
|
+
#
|
|
17
|
+
# @private
|
|
18
|
+
#
|
|
19
|
+
# @param string [String] Ruby string that will be copied into the new
|
|
20
|
+
# string pointer.
|
|
21
|
+
#
|
|
22
|
+
# @return [FFI::Pointer]
|
|
23
|
+
def string_pointer(string)
|
|
24
|
+
bytes = string.to_s
|
|
25
|
+
pointer = malloc(bytes.bytesize + 1)
|
|
26
|
+
Kernel.raise(NoMemoryError, "malloc failed") if pointer.null?
|
|
27
|
+
|
|
28
|
+
pointer.put_bytes(0, bytes)
|
|
29
|
+
pointer.put_char(bytes.bytesize, 0) # NUL terminator
|
|
30
|
+
|
|
31
|
+
pointer.autorelease = false
|
|
32
|
+
|
|
33
|
+
pointer
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -160,10 +160,9 @@ module Rdkafka
|
|
|
160
160
|
|
|
161
161
|
if p.metadata
|
|
162
162
|
part = Rdkafka::Bindings::TopicPartition.new(ref)
|
|
163
|
-
str_ptr = FFI::MemoryPointer.from_string(p.metadata)
|
|
164
163
|
# The metadata string is owned by librdkafka once handed over and released here:
|
|
165
164
|
# https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
|
|
166
|
-
str_ptr
|
|
165
|
+
str_ptr = Allocator.string_pointer(p.metadata)
|
|
167
166
|
part[:metadata] = str_ptr
|
|
168
167
|
part[:metadata_size] = p.metadata.bytesize
|
|
169
168
|
end
|
data/lib/rdkafka/consumer.rb
CHANGED
|
@@ -916,33 +916,35 @@ module Rdkafka
|
|
|
916
916
|
begin
|
|
917
917
|
while i < count
|
|
918
918
|
ptr = buffer.get_pointer(i * FFI::Pointer.size)
|
|
919
|
+
# Take ownership of this index before processing the pointer: advancing `i` up front means
|
|
920
|
+
# the cleanup loop in the `ensure` below starts past it, so a raise mid-iteration cannot
|
|
921
|
+
# double-free the same message.
|
|
922
|
+
i += 1
|
|
919
923
|
|
|
920
|
-
if ptr.null?
|
|
921
|
-
i += 1
|
|
922
|
-
next
|
|
923
|
-
end
|
|
924
|
+
next if ptr.null?
|
|
924
925
|
|
|
925
|
-
|
|
926
|
+
# One `ensure` around the whole body guarantees `ptr` is destroyed exactly once on every
|
|
927
|
+
# path - the error branch, a successful build, a rescued `RdkafkaError`, or any other
|
|
928
|
+
# exception unwinding through here - so nothing leaks in the window before the build.
|
|
929
|
+
begin
|
|
930
|
+
native_message = Rdkafka::Bindings::Message.new(ptr)
|
|
926
931
|
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
next
|
|
932
|
-
end
|
|
932
|
+
if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
933
|
+
results << Rdkafka::RdkafkaError.new(native_message[:err])
|
|
934
|
+
next
|
|
935
|
+
end
|
|
933
936
|
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
937
|
+
begin
|
|
938
|
+
results << Rdkafka::Consumer::Message.new(native_message)
|
|
939
|
+
rescue Rdkafka::RdkafkaError => e
|
|
940
|
+
# A message that fails to build (e.g. a header read error) is surfaced inline as an
|
|
941
|
+
# error event rather than discarding the whole batch - including the messages already
|
|
942
|
+
# built - and raising, which silently lost them once their offsets had been stored.
|
|
943
|
+
results << e
|
|
944
|
+
end
|
|
941
945
|
ensure
|
|
942
946
|
Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
|
|
943
947
|
end
|
|
944
|
-
|
|
945
|
-
i += 1
|
|
946
948
|
end
|
|
947
949
|
ensure
|
|
948
950
|
while i < count
|
|
@@ -991,33 +993,35 @@ module Rdkafka
|
|
|
991
993
|
begin
|
|
992
994
|
while i < count
|
|
993
995
|
ptr = buffer.get_pointer(i * FFI::Pointer.size)
|
|
996
|
+
# Take ownership of this index before processing the pointer: advancing `i` up front means
|
|
997
|
+
# the cleanup loop in the `ensure` below starts past it, so a raise mid-iteration cannot
|
|
998
|
+
# double-free the same message.
|
|
999
|
+
i += 1
|
|
994
1000
|
|
|
995
|
-
if ptr.null?
|
|
996
|
-
i += 1
|
|
997
|
-
next
|
|
998
|
-
end
|
|
1001
|
+
next if ptr.null?
|
|
999
1002
|
|
|
1000
|
-
|
|
1003
|
+
# One `ensure` around the whole body guarantees `ptr` is destroyed exactly once on every
|
|
1004
|
+
# path - the error branch, a successful build, a rescued `RdkafkaError`, or any other
|
|
1005
|
+
# exception unwinding through here - so nothing leaks in the window before the build.
|
|
1006
|
+
begin
|
|
1007
|
+
native_message = Rdkafka::Bindings::Message.new(ptr)
|
|
1001
1008
|
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
next
|
|
1007
|
-
end
|
|
1009
|
+
if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
1010
|
+
results << Rdkafka::RdkafkaError.new(native_message[:err])
|
|
1011
|
+
next
|
|
1012
|
+
end
|
|
1008
1013
|
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1014
|
+
begin
|
|
1015
|
+
results << Rdkafka::Consumer::Message.new(native_message)
|
|
1016
|
+
rescue Rdkafka::RdkafkaError => e
|
|
1017
|
+
# A message that fails to build (e.g. a header read error) is surfaced inline as an
|
|
1018
|
+
# error event rather than discarding the whole batch - including the messages already
|
|
1019
|
+
# built - and raising, which silently lost them once their offsets had been stored.
|
|
1020
|
+
results << e
|
|
1021
|
+
end
|
|
1016
1022
|
ensure
|
|
1017
1023
|
Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
|
|
1018
1024
|
end
|
|
1019
|
-
|
|
1020
|
-
i += 1
|
|
1021
1025
|
end
|
|
1022
1026
|
ensure
|
|
1023
1027
|
while i < count
|
data/lib/rdkafka/version.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
4
|
# Current rdkafka-ruby gem version
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.30.0"
|
|
6
6
|
# Target librdkafka version to be used
|
|
7
|
-
LIBRDKAFKA_VERSION = "2.
|
|
7
|
+
LIBRDKAFKA_VERSION = "2.15.0"
|
|
8
8
|
# SHA256 hash of the librdkafka source tarball for verification
|
|
9
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
|
9
|
+
LIBRDKAFKA_SOURCE_SHA256 = "259015220cdca708afe838b5aa79ebf1a5fb710fb4179cf918d390aed85d5dbc"
|
|
10
10
|
end
|
data/lib/rdkafka.rb
CHANGED
|
@@ -13,6 +13,10 @@ require "rdkafka/helpers/list_offsets"
|
|
|
13
13
|
require "rdkafka/defaults"
|
|
14
14
|
require "rdkafka/abstract_handle"
|
|
15
15
|
require "rdkafka/admin"
|
|
16
|
+
require "rdkafka/admin/alter_consumer_group_offsets_handle"
|
|
17
|
+
require "rdkafka/admin/alter_consumer_group_offsets_report"
|
|
18
|
+
require "rdkafka/admin/delete_consumer_group_offsets_handle"
|
|
19
|
+
require "rdkafka/admin/delete_consumer_group_offsets_report"
|
|
16
20
|
require "rdkafka/admin/create_topic_handle"
|
|
17
21
|
require "rdkafka/admin/create_topic_report"
|
|
18
22
|
require "rdkafka/admin/delete_groups_handle"
|
|
@@ -46,6 +50,8 @@ require "rdkafka/callbacks/base_handler"
|
|
|
46
50
|
require "rdkafka/callbacks/create_topic_handler"
|
|
47
51
|
require "rdkafka/callbacks/delete_topic_handler"
|
|
48
52
|
require "rdkafka/callbacks/create_partitions_handler"
|
|
53
|
+
require "rdkafka/callbacks/alter_consumer_group_offsets_handler"
|
|
54
|
+
require "rdkafka/callbacks/delete_consumer_group_offsets_handler"
|
|
49
55
|
require "rdkafka/callbacks/delete_groups_handler"
|
|
50
56
|
require "rdkafka/callbacks/delete_records_handler"
|
|
51
57
|
require "rdkafka/callbacks/create_acl_handler"
|
|
@@ -60,6 +66,7 @@ require "rdkafka/consumer"
|
|
|
60
66
|
require "rdkafka/consumer/headers"
|
|
61
67
|
require "rdkafka/consumer/message"
|
|
62
68
|
require "rdkafka/consumer/partition"
|
|
69
|
+
require "rdkafka/consumer/allocator"
|
|
63
70
|
require "rdkafka/consumer/topic_partition_list"
|
|
64
71
|
require "rdkafka/error"
|
|
65
72
|
require "rdkafka/metadata"
|
|
@@ -68,9 +75,12 @@ require "rdkafka/producer/partitions_count_cache"
|
|
|
68
75
|
require "rdkafka/producer"
|
|
69
76
|
require "rdkafka/producer/delivery_handle"
|
|
70
77
|
require "rdkafka/producer/delivery_report"
|
|
78
|
+
require "rdkafka/clients"
|
|
71
79
|
|
|
72
80
|
# Main Rdkafka namespace of this gem
|
|
73
81
|
module Rdkafka
|
|
74
82
|
end
|
|
75
83
|
|
|
76
84
|
Rdkafka::Bindings.rd_kafka_global_init
|
|
85
|
+
|
|
86
|
+
at_exit { Rdkafka::Clients.close_all }
|
data/package-lock.json
CHANGED
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
}
|
|
104
104
|
},
|
|
105
105
|
"node_modules/fastq": {
|
|
106
|
-
"version": "1.20.
|
|
107
|
-
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.
|
|
108
|
-
"integrity": "sha512-
|
|
106
|
+
"version": "1.20.2",
|
|
107
|
+
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
|
|
108
|
+
"integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
|
|
109
109
|
"dev": true,
|
|
110
110
|
"license": "ISC",
|
|
111
111
|
"dependencies": {
|
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.8.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
|
|
291
|
+
"integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
data/renovate.json
CHANGED