rdkafka 0.22.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +101 -3
- data/README.md +21 -14
- data/Rakefile +21 -21
- data/bin/verify_kafka_warnings +39 -0
- data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.2.tar.gz} +0 -0
- data/docker-compose-ssl.yml +35 -0
- data/docker-compose.yml +2 -2
- data/ext/Rakefile +27 -27
- data/lib/rdkafka/abstract_handle.rb +54 -7
- data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
- data/lib/rdkafka/admin/config_binding_result.rb +2 -2
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +10 -6
- data/lib/rdkafka/admin/create_acl_report.rb +3 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +8 -7
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +8 -7
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +9 -8
- data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +10 -7
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +8 -7
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +9 -8
- data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +7 -10
- data/lib/rdkafka/admin/describe_configs_report.rb +8 -6
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -10
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +8 -6
- data/lib/rdkafka/admin/list_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
- data/lib/rdkafka/admin.rb +357 -159
- data/lib/rdkafka/bindings.rb +205 -111
- data/lib/rdkafka/callbacks/base_handler.rb +62 -0
- data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
- data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
- data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
- data/lib/rdkafka/callbacks.rb +125 -216
- data/lib/rdkafka/config.rb +126 -66
- data/lib/rdkafka/consumer/headers.rb +22 -7
- data/lib/rdkafka/consumer/message.rb +12 -11
- data/lib/rdkafka/consumer/partition.rb +15 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +64 -38
- data/lib/rdkafka/consumer.rb +513 -59
- data/lib/rdkafka/defaults.rb +125 -0
- data/lib/rdkafka/error.rb +40 -14
- data/lib/rdkafka/helpers/metadata.rb +29 -0
- data/lib/rdkafka/helpers/oauth.rb +45 -13
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +128 -37
- data/lib/rdkafka/native_kafka.rb +91 -8
- data/lib/rdkafka/producer/delivery_handle.rb +6 -6
- data/lib/rdkafka/producer/delivery_report.rb +10 -6
- data/lib/rdkafka/producer/partitions_count_cache.rb +51 -55
- data/lib/rdkafka/producer.rb +193 -95
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +15 -0
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +58 -36
- data/renovate.json +39 -23
- metadata +39 -125
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
- data/.github/workflows/ci_macos_arm64.yml +0 -284
- data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
- data/.github/workflows/push_macos_arm64.yml +0 -54
- data/.github/workflows/push_ruby.yml +0 -37
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.gitignore +0 -14
- data/.rspec +0 -2
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.yardopts +0 -2
- data/Gemfile +0 -5
- data/ext/README.md +0 -19
- data/ext/build_common.sh +0 -361
- data/ext/build_linux_x86_64_gnu.sh +0 -306
- data/ext/build_linux_x86_64_musl.sh +0 -763
- data/ext/build_macos_arm64.sh +0 -550
- data/spec/rdkafka/abstract_handle_spec.rb +0 -117
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
- data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
- data/spec/rdkafka/admin_spec.rb +0 -971
- data/spec/rdkafka/bindings_spec.rb +0 -199
- data/spec/rdkafka/callbacks_spec.rb +0 -20
- data/spec/rdkafka/config_spec.rb +0 -258
- data/spec/rdkafka/consumer/headers_spec.rb +0 -73
- data/spec/rdkafka/consumer/message_spec.rb +0 -139
- data/spec/rdkafka/consumer/partition_spec.rb +0 -57
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
- data/spec/rdkafka/consumer_spec.rb +0 -1274
- data/spec/rdkafka/error_spec.rb +0 -89
- data/spec/rdkafka/metadata_spec.rb +0 -79
- data/spec/rdkafka/native_kafka_spec.rb +0 -130
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
- data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
- data/spec/rdkafka/producer_spec.rb +0 -1345
- data/spec/spec_helper.rb +0 -195
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
+
# Admin client for Kafka administrative operations
|
|
4
5
|
class Admin
|
|
5
6
|
include Helpers::OAuth
|
|
7
|
+
include Helpers::Metadata
|
|
6
8
|
|
|
7
9
|
class << self
|
|
8
10
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
9
11
|
# Useful for debugging and building UIs, etc.
|
|
10
12
|
#
|
|
11
|
-
# @return [Hash
|
|
13
|
+
# @return [Hash{Integer => Hash}] hash with errors mapped by code
|
|
12
14
|
def describe_errors
|
|
13
15
|
# Memory pointers for the array of structures and count
|
|
14
16
|
p_error_descs = FFI::MemoryPointer.new(:pointer)
|
|
@@ -35,8 +37,8 @@ module Rdkafka
|
|
|
35
37
|
# Read values from the struct
|
|
36
38
|
code = error_desc[:code]
|
|
37
39
|
|
|
38
|
-
name =
|
|
39
|
-
desc =
|
|
40
|
+
name = ""
|
|
41
|
+
desc = ""
|
|
40
42
|
|
|
41
43
|
name = error_desc[:name].read_string unless error_desc[:name].null?
|
|
42
44
|
desc = error_desc[:desc].read_string unless error_desc[:desc].null?
|
|
@@ -49,6 +51,7 @@ module Rdkafka
|
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
# @private
|
|
54
|
+
# @param native_kafka [NativeKafka] wrapper around the native Kafka handle
|
|
52
55
|
def initialize(native_kafka)
|
|
53
56
|
@native_kafka = native_kafka
|
|
54
57
|
|
|
@@ -69,20 +72,62 @@ module Rdkafka
|
|
|
69
72
|
end
|
|
70
73
|
end
|
|
71
74
|
|
|
72
|
-
|
|
73
|
-
|
|
75
|
+
# Enable IO event notifications for fiber scheduler integration
|
|
76
|
+
# When admin operations complete, librdkafka will write to your FD
|
|
77
|
+
#
|
|
78
|
+
# @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
|
|
79
|
+
# @param payload [String] data to write to fd (default: "\x01")
|
|
80
|
+
# @return [nil]
|
|
81
|
+
# @raise [ClosedInnerError] when the admin client is closed
|
|
82
|
+
def enable_queue_io_events(fd, payload = "\x01")
|
|
83
|
+
@native_kafka.enable_main_queue_io_events(fd, payload)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Enable IO event notifications for background events
|
|
87
|
+
# @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
|
|
88
|
+
# @param payload [String] data to write to fd (default: "\x01")
|
|
89
|
+
# @return [nil]
|
|
90
|
+
# @raise [ClosedInnerError] when the admin client is closed
|
|
91
|
+
def enable_background_queue_io_events(fd, payload = "\x01")
|
|
92
|
+
@native_kafka.enable_background_queue_io_events(fd, payload)
|
|
74
93
|
end
|
|
75
94
|
|
|
76
|
-
#
|
|
95
|
+
# Polls for events in a non-blocking loop, yielding the count after each iteration.
|
|
96
|
+
#
|
|
97
|
+
# This method processes events (stats, errors, etc.) in a single GVL/mutex session,
|
|
98
|
+
# which is more efficient than repeated individual polls. It uses non-blocking polls
|
|
99
|
+
# internally (no GVL release between polls).
|
|
100
|
+
#
|
|
101
|
+
# Yields the count of events processed after each poll iteration, allowing the caller
|
|
102
|
+
# to implement timeout or other termination logic by returning `:stop`.
|
|
103
|
+
#
|
|
104
|
+
# @yield [count] Called after each poll iteration
|
|
105
|
+
# @yieldparam count [Integer] Number of events processed in this iteration
|
|
106
|
+
# @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
|
|
107
|
+
# @return [nil]
|
|
108
|
+
# @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
|
|
109
|
+
#
|
|
110
|
+
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
111
|
+
# Other admin operations will wait until this method returns.
|
|
112
|
+
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
77
113
|
#
|
|
78
|
-
# @
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
|
|
114
|
+
# @example Drain all pending events
|
|
115
|
+
# admin.events_poll_nb_each { |_count| }
|
|
116
|
+
#
|
|
117
|
+
# @example With timeout control
|
|
118
|
+
# deadline = monotonic_now + timeout_ms
|
|
119
|
+
# admin.events_poll_nb_each do |_count|
|
|
120
|
+
# :stop if monotonic_now >= deadline
|
|
121
|
+
# end
|
|
122
|
+
def events_poll_nb_each
|
|
82
123
|
closed_admin_check(__method__)
|
|
83
124
|
|
|
84
125
|
@native_kafka.with_inner do |inner|
|
|
85
|
-
|
|
126
|
+
loop do
|
|
127
|
+
count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
|
|
128
|
+
break if count.zero?
|
|
129
|
+
break if yield(count) == :stop
|
|
130
|
+
end
|
|
86
131
|
end
|
|
87
132
|
end
|
|
88
133
|
|
|
@@ -100,13 +145,16 @@ module Rdkafka
|
|
|
100
145
|
|
|
101
146
|
# Create a topic with the given partition count and replication factor
|
|
102
147
|
#
|
|
148
|
+
# @param topic_name [String] name of the topic to create
|
|
149
|
+
# @param partition_count [Integer] number of partitions for the topic
|
|
150
|
+
# @param replication_factor [Integer] replication factor for the topic
|
|
151
|
+
# @param topic_config [Hash] optional topic configuration settings
|
|
103
152
|
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
|
|
104
153
|
# creating the topic
|
|
105
|
-
#
|
|
106
154
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
107
155
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
108
156
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
109
|
-
def create_topic(topic_name, partition_count, replication_factor, topic_config={})
|
|
157
|
+
def create_topic(topic_name, partition_count, replication_factor, topic_config = {})
|
|
110
158
|
closed_admin_check(__method__)
|
|
111
159
|
|
|
112
160
|
# Create a rd_kafka_NewTopic_t representing the new topic
|
|
@@ -122,14 +170,12 @@ module Rdkafka
|
|
|
122
170
|
raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
|
|
123
171
|
end
|
|
124
172
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
)
|
|
132
|
-
end
|
|
173
|
+
topic_config&.each do |key, value|
|
|
174
|
+
Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
|
|
175
|
+
new_topic_ptr,
|
|
176
|
+
key.to_s,
|
|
177
|
+
value.to_s
|
|
178
|
+
)
|
|
133
179
|
end
|
|
134
180
|
|
|
135
181
|
# Note that rd_kafka_CreateTopics can create more than one topic at a time
|
|
@@ -149,7 +195,7 @@ module Rdkafka
|
|
|
149
195
|
# Create and register the handle we will return to the caller
|
|
150
196
|
create_topic_handle = CreateTopicHandle.new
|
|
151
197
|
create_topic_handle[:pending] = true
|
|
152
|
-
create_topic_handle[:response] =
|
|
198
|
+
create_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
153
199
|
CreateTopicHandle.register(create_topic_handle)
|
|
154
200
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
155
201
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
|
|
@@ -178,6 +224,11 @@ module Rdkafka
|
|
|
178
224
|
create_topic_handle
|
|
179
225
|
end
|
|
180
226
|
|
|
227
|
+
# Deletes a consumer group
|
|
228
|
+
#
|
|
229
|
+
# @param group_id [String] the group id to delete
|
|
230
|
+
# @return [DeleteGroupsHandle] delete groups handle that can be used to wait for the result
|
|
231
|
+
# @raise [RdkafkaError] When deleting the group fails
|
|
181
232
|
def delete_group(group_id)
|
|
182
233
|
closed_admin_check(__method__)
|
|
183
234
|
|
|
@@ -195,14 +246,14 @@ module Rdkafka
|
|
|
195
246
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
196
247
|
end
|
|
197
248
|
if queue_ptr.null?
|
|
198
|
-
Rdkafka::Bindings.
|
|
249
|
+
Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
|
|
199
250
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
200
251
|
end
|
|
201
252
|
|
|
202
253
|
# Create and register the handle we will return to the caller
|
|
203
254
|
delete_groups_handle = DeleteGroupsHandle.new
|
|
204
255
|
delete_groups_handle[:pending] = true
|
|
205
|
-
delete_groups_handle[:response] =
|
|
256
|
+
delete_groups_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
206
257
|
DeleteGroupsHandle.register(delete_groups_handle)
|
|
207
258
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
208
259
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
|
|
@@ -233,6 +284,7 @@ module Rdkafka
|
|
|
233
284
|
|
|
234
285
|
# Deletes the named topic
|
|
235
286
|
#
|
|
287
|
+
# @param topic_name [String] name of the topic to delete
|
|
236
288
|
# @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
|
|
237
289
|
# deleting the topic
|
|
238
290
|
# @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
|
|
@@ -259,7 +311,7 @@ module Rdkafka
|
|
|
259
311
|
# Create and register the handle we will return to the caller
|
|
260
312
|
delete_topic_handle = DeleteTopicHandle.new
|
|
261
313
|
delete_topic_handle[:pending] = true
|
|
262
|
-
delete_topic_handle[:response] =
|
|
314
|
+
delete_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
263
315
|
DeleteTopicHandle.register(delete_topic_handle)
|
|
264
316
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
265
317
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
|
|
@@ -290,14 +342,12 @@ module Rdkafka
|
|
|
290
342
|
|
|
291
343
|
# Creates extra partitions for a given topic
|
|
292
344
|
#
|
|
293
|
-
# @param topic_name [String]
|
|
345
|
+
# @param topic_name [String] name of the topic
|
|
294
346
|
# @param partition_count [Integer] how many partitions we want to end up with for given topic
|
|
295
|
-
#
|
|
347
|
+
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the result
|
|
296
348
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
297
349
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
298
350
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
299
|
-
#
|
|
300
|
-
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
|
|
301
351
|
def create_partitions(topic_name, partition_count)
|
|
302
352
|
closed_admin_check(__method__)
|
|
303
353
|
|
|
@@ -327,7 +377,7 @@ module Rdkafka
|
|
|
327
377
|
# Create and register the handle we will return to the caller
|
|
328
378
|
create_partitions_handle = CreatePartitionsHandle.new
|
|
329
379
|
create_partitions_handle[:pending] = true
|
|
330
|
-
create_partitions_handle[:response] =
|
|
380
|
+
create_partitions_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
331
381
|
CreatePartitionsHandle.register(create_partitions_handle)
|
|
332
382
|
admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
|
|
333
383
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
|
|
@@ -354,40 +404,36 @@ module Rdkafka
|
|
|
354
404
|
end
|
|
355
405
|
|
|
356
406
|
# Create acl
|
|
357
|
-
# @param resource_type - values of type rd_kafka_ResourceType_t
|
|
358
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
359
|
-
# valid values are:
|
|
360
|
-
# RD_KAFKA_RESOURCE_TOPIC = 2
|
|
361
|
-
# RD_KAFKA_RESOURCE_GROUP = 3
|
|
362
|
-
# RD_KAFKA_RESOURCE_BROKER = 4
|
|
363
|
-
# @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
|
|
364
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
365
|
-
# valid values are:
|
|
366
|
-
# RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
367
|
-
# RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
368
|
-
# RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
369
|
-
# @param operation - values of type rd_kafka_AclOperation_t
|
|
370
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
371
|
-
# valid values are:
|
|
372
|
-
# RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
373
|
-
# RD_KAFKA_ACL_OPERATION_READ = 3
|
|
374
|
-
# RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
375
|
-
# RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
376
|
-
# RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
377
|
-
# RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
378
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
379
|
-
# RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
380
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
381
|
-
# RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
382
|
-
# RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
383
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
384
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
385
|
-
# valid values are:
|
|
386
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
387
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
388
407
|
#
|
|
408
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
409
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
410
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
411
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
412
|
+
# @param resource_name [String] name of the resource
|
|
413
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
414
|
+
# - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
|
|
415
|
+
# - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
416
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
417
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
418
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
419
|
+
# @param principal [String] principal (e.g., "User:alice")
|
|
420
|
+
# @param host [String] host address
|
|
421
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
422
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
423
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
424
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
425
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
426
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
427
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
428
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
429
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
430
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
431
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
432
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
433
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
434
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
435
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
389
436
|
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
|
|
390
|
-
#
|
|
391
437
|
# @raise [RdkafkaError]
|
|
392
438
|
def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
393
439
|
closed_admin_check(__method__)
|
|
@@ -427,7 +473,7 @@ module Rdkafka
|
|
|
427
473
|
# Create and register the handle that we will return to the caller
|
|
428
474
|
create_acl_handle = CreateAclHandle.new
|
|
429
475
|
create_acl_handle[:pending] = true
|
|
430
|
-
create_acl_handle[:response] =
|
|
476
|
+
create_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
431
477
|
CreateAclHandle.register(create_acl_handle)
|
|
432
478
|
|
|
433
479
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -460,39 +506,35 @@ module Rdkafka
|
|
|
460
506
|
|
|
461
507
|
# Delete acl
|
|
462
508
|
#
|
|
463
|
-
# @param resource_type
|
|
464
|
-
#
|
|
465
|
-
#
|
|
466
|
-
#
|
|
467
|
-
#
|
|
468
|
-
#
|
|
469
|
-
#
|
|
470
|
-
#
|
|
471
|
-
#
|
|
472
|
-
#
|
|
473
|
-
#
|
|
474
|
-
#
|
|
475
|
-
# @param
|
|
476
|
-
#
|
|
477
|
-
#
|
|
478
|
-
#
|
|
479
|
-
#
|
|
480
|
-
#
|
|
481
|
-
#
|
|
482
|
-
#
|
|
483
|
-
#
|
|
484
|
-
#
|
|
485
|
-
#
|
|
486
|
-
#
|
|
487
|
-
#
|
|
488
|
-
#
|
|
489
|
-
#
|
|
490
|
-
#
|
|
491
|
-
# valid values are:
|
|
492
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
493
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
509
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
510
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
511
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
512
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
513
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
514
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
515
|
+
# - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
|
|
516
|
+
# - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
517
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
518
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
519
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
520
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
521
|
+
# @param host [String, nil] host address or nil for any
|
|
522
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
523
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
524
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
525
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
526
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
527
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
528
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
529
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
530
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
531
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
532
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
533
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
534
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
535
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
536
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
494
537
|
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
|
|
495
|
-
#
|
|
496
538
|
# @raise [RdkafkaError]
|
|
497
539
|
def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
498
540
|
closed_admin_check(__method__)
|
|
@@ -527,14 +569,14 @@ module Rdkafka
|
|
|
527
569
|
end
|
|
528
570
|
|
|
529
571
|
if queue_ptr.null?
|
|
530
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
572
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
|
|
531
573
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
532
574
|
end
|
|
533
575
|
|
|
534
576
|
# Create and register the handle that we will return to the caller
|
|
535
577
|
delete_acl_handle = DeleteAclHandle.new
|
|
536
578
|
delete_acl_handle[:pending] = true
|
|
537
|
-
delete_acl_handle[:response] =
|
|
579
|
+
delete_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
538
580
|
DeleteAclHandle.register(delete_acl_handle)
|
|
539
581
|
|
|
540
582
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -567,39 +609,35 @@ module Rdkafka
|
|
|
567
609
|
|
|
568
610
|
# Describe acls
|
|
569
611
|
#
|
|
570
|
-
# @param resource_type
|
|
571
|
-
#
|
|
572
|
-
#
|
|
573
|
-
#
|
|
574
|
-
#
|
|
575
|
-
#
|
|
576
|
-
#
|
|
577
|
-
#
|
|
578
|
-
#
|
|
579
|
-
#
|
|
580
|
-
#
|
|
581
|
-
#
|
|
582
|
-
# @param
|
|
583
|
-
#
|
|
584
|
-
#
|
|
585
|
-
#
|
|
586
|
-
#
|
|
587
|
-
#
|
|
588
|
-
#
|
|
589
|
-
#
|
|
590
|
-
#
|
|
591
|
-
#
|
|
592
|
-
#
|
|
593
|
-
#
|
|
594
|
-
#
|
|
595
|
-
#
|
|
596
|
-
#
|
|
597
|
-
#
|
|
598
|
-
# valid values are:
|
|
599
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
600
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
612
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
613
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
614
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
615
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
616
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
617
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
618
|
+
# - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
|
|
619
|
+
# - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
620
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
621
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
622
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
623
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
624
|
+
# @param host [String, nil] host address or nil for any
|
|
625
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
626
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
627
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
628
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
629
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
630
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
631
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
632
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
633
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
634
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
635
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
636
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
637
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
638
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
639
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
601
640
|
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
|
|
602
|
-
#
|
|
603
641
|
# @raise [RdkafkaError]
|
|
604
642
|
def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
605
643
|
closed_admin_check(__method__)
|
|
@@ -627,14 +665,14 @@ module Rdkafka
|
|
|
627
665
|
end
|
|
628
666
|
|
|
629
667
|
if queue_ptr.null?
|
|
630
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
668
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
|
|
631
669
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
632
670
|
end
|
|
633
671
|
|
|
634
672
|
# Create and register the handle that we will return to the caller
|
|
635
673
|
describe_acl_handle = DescribeAclHandle.new
|
|
636
674
|
describe_acl_handle[:pending] = true
|
|
637
|
-
describe_acl_handle[:response] =
|
|
675
|
+
describe_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
638
676
|
DescribeAclHandle.register(describe_acl_handle)
|
|
639
677
|
|
|
640
678
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -664,7 +702,6 @@ module Rdkafka
|
|
|
664
702
|
describe_acl_handle
|
|
665
703
|
end
|
|
666
704
|
|
|
667
|
-
|
|
668
705
|
# Describe configs
|
|
669
706
|
#
|
|
670
707
|
# @param resources [Array<Hash>] Array where elements are hashes with two keys:
|
|
@@ -679,9 +716,17 @@ module Rdkafka
|
|
|
679
716
|
def describe_configs(resources)
|
|
680
717
|
closed_admin_check(__method__)
|
|
681
718
|
|
|
719
|
+
# Parse user input before allocating any native resources or registering the handle, so a
|
|
720
|
+
# missing :resource_type / :resource_name key raises with nothing to clean up. Previously the
|
|
721
|
+
# KeyError leaked the background queue, the AdminOptions, the registered handle and any
|
|
722
|
+
# ConfigResources already built.
|
|
723
|
+
parsed_resources = resources.map do |resource_details|
|
|
724
|
+
[resource_details.fetch(:resource_type), resource_details.fetch(:resource_name)]
|
|
725
|
+
end
|
|
726
|
+
|
|
682
727
|
handle = DescribeConfigsHandle.new
|
|
683
728
|
handle[:pending] = true
|
|
684
|
-
handle[:response] =
|
|
729
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
685
730
|
|
|
686
731
|
queue_ptr = @native_kafka.with_inner do |inner|
|
|
687
732
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
@@ -701,12 +746,10 @@ module Rdkafka
|
|
|
701
746
|
DescribeConfigsHandle.register(handle)
|
|
702
747
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
703
748
|
|
|
704
|
-
pointer_array =
|
|
749
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name|
|
|
705
750
|
Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
706
|
-
|
|
707
|
-
FFI::MemoryPointer.from_string(
|
|
708
|
-
resource_details.fetch(:resource_name)
|
|
709
|
-
)
|
|
751
|
+
resource_type,
|
|
752
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
710
753
|
)
|
|
711
754
|
end
|
|
712
755
|
|
|
@@ -728,10 +771,15 @@ module Rdkafka
|
|
|
728
771
|
|
|
729
772
|
raise
|
|
730
773
|
ensure
|
|
731
|
-
Rdkafka::Bindings.
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
774
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
775
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
776
|
+
|
|
777
|
+
if configs_array_ptr
|
|
778
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
|
779
|
+
configs_array_ptr,
|
|
780
|
+
pointer_array.size
|
|
781
|
+
)
|
|
782
|
+
end
|
|
735
783
|
end
|
|
736
784
|
|
|
737
785
|
handle
|
|
@@ -752,9 +800,22 @@ module Rdkafka
|
|
|
752
800
|
def incremental_alter_configs(resources_with_configs)
|
|
753
801
|
closed_admin_check(__method__)
|
|
754
802
|
|
|
803
|
+
# Parse user input before allocating native resources or registering the handle, so a missing
|
|
804
|
+
# key raises with nothing to clean up. Previously the KeyError leaked the background queue,
|
|
805
|
+
# the AdminOptions, the registered handle and any ConfigResources already built.
|
|
806
|
+
parsed_resources = resources_with_configs.map do |resource_details|
|
|
807
|
+
[
|
|
808
|
+
resource_details.fetch(:resource_type),
|
|
809
|
+
resource_details.fetch(:resource_name),
|
|
810
|
+
resource_details.fetch(:configs).map do |config|
|
|
811
|
+
[config.fetch(:name), config.fetch(:op_type), config.fetch(:value)]
|
|
812
|
+
end
|
|
813
|
+
]
|
|
814
|
+
end
|
|
815
|
+
|
|
755
816
|
handle = IncrementalAlterConfigsHandle.new
|
|
756
817
|
handle[:pending] = true
|
|
757
|
-
handle[:response] =
|
|
818
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
758
819
|
|
|
759
820
|
queue_ptr = @native_kafka.with_inner do |inner|
|
|
760
821
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
@@ -774,23 +835,39 @@ module Rdkafka
|
|
|
774
835
|
IncrementalAlterConfigsHandle.register(handle)
|
|
775
836
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
776
837
|
|
|
777
|
-
|
|
778
|
-
|
|
838
|
+
add_error = nil
|
|
839
|
+
|
|
840
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name, configs|
|
|
779
841
|
# First build the appropriate resource representation
|
|
780
842
|
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
781
|
-
|
|
782
|
-
FFI::MemoryPointer.from_string(
|
|
783
|
-
resource_details.fetch(:resource_name)
|
|
784
|
-
)
|
|
843
|
+
resource_type,
|
|
844
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
785
845
|
)
|
|
786
846
|
|
|
787
|
-
|
|
788
|
-
|
|
847
|
+
configs.each do |name, op_type, value|
|
|
848
|
+
# rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
|
|
849
|
+
# an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
|
|
850
|
+
# used to be ignored: the entry was silently dropped, the alter request still reported
|
|
851
|
+
# success, and the error object leaked. Capture the first error (always destroying the
|
|
852
|
+
# native error object) and raise it below, before the request is sent.
|
|
853
|
+
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
789
854
|
resource_ptr,
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
855
|
+
name,
|
|
856
|
+
op_type,
|
|
857
|
+
value
|
|
793
858
|
)
|
|
859
|
+
|
|
860
|
+
unless error_ptr.null?
|
|
861
|
+
code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
|
|
862
|
+
Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
|
|
863
|
+
|
|
864
|
+
unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
865
|
+
add_error ||= Rdkafka::RdkafkaError.new(
|
|
866
|
+
code,
|
|
867
|
+
"rd_kafka_ConfigResource_add_incremental_config"
|
|
868
|
+
)
|
|
869
|
+
end
|
|
870
|
+
end
|
|
794
871
|
end
|
|
795
872
|
|
|
796
873
|
resource_ptr
|
|
@@ -799,8 +876,11 @@ module Rdkafka
|
|
|
799
876
|
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
800
877
|
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
801
878
|
|
|
802
|
-
|
|
803
879
|
begin
|
|
880
|
+
# Raise only after the full array is built so the ensure below frees every ConfigResource
|
|
881
|
+
# we created, and before the request is sent so a rejected entry can't report success.
|
|
882
|
+
raise add_error if add_error
|
|
883
|
+
|
|
804
884
|
@native_kafka.with_inner do |inner|
|
|
805
885
|
Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
|
|
806
886
|
inner,
|
|
@@ -815,10 +895,124 @@ module Rdkafka
|
|
|
815
895
|
|
|
816
896
|
raise
|
|
817
897
|
ensure
|
|
818
|
-
Rdkafka::Bindings.
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
898
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
899
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
900
|
+
|
|
901
|
+
if configs_array_ptr
|
|
902
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
|
903
|
+
configs_array_ptr,
|
|
904
|
+
pointer_array.size
|
|
905
|
+
)
|
|
906
|
+
end
|
|
907
|
+
end
|
|
908
|
+
|
|
909
|
+
handle
|
|
910
|
+
end
|
|
911
|
+
|
|
912
|
+
# Queries partition offsets by specification (earliest, latest, max_timestamp, or by
|
|
913
|
+
# timestamp) without requiring a consumer group.
|
|
914
|
+
#
|
|
915
|
+
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
916
|
+
# arrays of partition offset specifications. Each specification is a hash with:
|
|
917
|
+
# - `:partition` [Integer] partition number
|
|
918
|
+
# - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
|
|
919
|
+
# `:max_timestamp`, or an integer timestamp in milliseconds
|
|
920
|
+
# @param isolation_level [Integer, nil] optional isolation level:
|
|
921
|
+
# - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
|
|
922
|
+
# - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
|
|
923
|
+
#
|
|
924
|
+
# @return [ListOffsetsHandle] handle that can be used to wait for the result
|
|
925
|
+
#
|
|
926
|
+
# @raise [ClosedAdminError] when the admin is closed
|
|
927
|
+
# @raise [ConfigError] when the background queue is unavailable
|
|
928
|
+
#
|
|
929
|
+
# @example Query earliest and latest offsets
|
|
930
|
+
# handle = admin.list_offsets(
|
|
931
|
+
# { "my_topic" => [
|
|
932
|
+
# { partition: 0, offset: :earliest },
|
|
933
|
+
# { partition: 1, offset: :latest }
|
|
934
|
+
# ] }
|
|
935
|
+
# )
|
|
936
|
+
# report = handle.wait(max_wait_timeout_ms: 15_000)
|
|
937
|
+
# report.offsets
|
|
938
|
+
# # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
|
|
939
|
+
def list_offsets(topic_partition_offsets, isolation_level: nil)
|
|
940
|
+
closed_admin_check(__method__)
|
|
941
|
+
|
|
942
|
+
# Parse and validate every offset spec before allocating the native list, so a missing key or
|
|
943
|
+
# an unknown offset specification raises with nothing to clean up. Previously the
|
|
944
|
+
# ArgumentError (or KeyError) was raised after `rd_kafka_topic_partition_list_new`, leaking
|
|
945
|
+
# the native list.
|
|
946
|
+
parsed = topic_partition_offsets.flat_map do |topic, partitions|
|
|
947
|
+
partitions.map do |spec|
|
|
948
|
+
offset = spec.fetch(:offset)
|
|
949
|
+
|
|
950
|
+
native_offset = case offset
|
|
951
|
+
when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
|
|
952
|
+
when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
|
|
953
|
+
when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
|
|
954
|
+
when Integer then offset
|
|
955
|
+
else
|
|
956
|
+
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
[topic, spec.fetch(:partition), native_offset]
|
|
960
|
+
end
|
|
961
|
+
end
|
|
962
|
+
|
|
963
|
+
# Build native topic partition list
|
|
964
|
+
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
|
|
965
|
+
|
|
966
|
+
parsed.each do |topic, partition, native_offset|
|
|
967
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
968
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
969
|
+
end
|
|
970
|
+
|
|
971
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
972
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
973
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
974
|
+
end
|
|
975
|
+
|
|
976
|
+
if queue_ptr.null?
|
|
977
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
978
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
979
|
+
end
|
|
980
|
+
|
|
981
|
+
# Create and register the handle we will return to the caller
|
|
982
|
+
handle = ListOffsetsHandle.new
|
|
983
|
+
handle[:pending] = true
|
|
984
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
985
|
+
|
|
986
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
987
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
988
|
+
inner,
|
|
989
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
|
|
990
|
+
)
|
|
991
|
+
end
|
|
992
|
+
|
|
993
|
+
if isolation_level
|
|
994
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
|
|
995
|
+
end
|
|
996
|
+
|
|
997
|
+
ListOffsetsHandle.register(handle)
|
|
998
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
999
|
+
|
|
1000
|
+
begin
|
|
1001
|
+
@native_kafka.with_inner do |inner|
|
|
1002
|
+
Rdkafka::Bindings.rd_kafka_ListOffsets(
|
|
1003
|
+
inner,
|
|
1004
|
+
tpl,
|
|
1005
|
+
admin_options_ptr,
|
|
1006
|
+
queue_ptr
|
|
1007
|
+
)
|
|
1008
|
+
end
|
|
1009
|
+
rescue Exception
|
|
1010
|
+
ListOffsetsHandle.remove(handle.to_ptr.address)
|
|
1011
|
+
raise
|
|
1012
|
+
ensure
|
|
1013
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
1014
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
1015
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
822
1016
|
end
|
|
823
1017
|
|
|
824
1018
|
handle
|
|
@@ -826,8 +1020,12 @@ module Rdkafka
|
|
|
826
1020
|
|
|
827
1021
|
private
|
|
828
1022
|
|
|
1023
|
+
# Checks if the admin is closed and raises an error if so
|
|
1024
|
+
# @param method [Symbol] name of the calling method for error context
|
|
1025
|
+
# @raise [ClosedAdminError] when the admin is closed
|
|
829
1026
|
def closed_admin_check(method)
|
|
830
1027
|
raise Rdkafka::ClosedAdminError.new(method) if closed?
|
|
831
1028
|
end
|
|
1029
|
+
alias_method :closed_check, :closed_admin_check
|
|
832
1030
|
end
|
|
833
1031
|
end
|