rdkafka 0.28.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +3 -2
- data/bin/verify_kafka_warnings +5 -0
- data/docker-compose-ssl.yml +4 -1
- data/docker-compose.yml +4 -1
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/abstract_handle.rb +31 -2
- data/lib/rdkafka/admin/config_binding_result.rb +2 -2
- data/lib/rdkafka/admin/create_acl_handle.rb +6 -5
- data/lib/rdkafka/admin/create_acl_report.rb +2 -1
- data/lib/rdkafka/admin/create_partitions_handle.rb +4 -6
- data/lib/rdkafka/admin/create_topic_handle.rb +4 -6
- data/lib/rdkafka/admin/delete_acl_handle.rb +5 -7
- data/lib/rdkafka/admin/delete_groups_handle.rb +4 -6
- data/lib/rdkafka/admin/delete_records_handle.rb +31 -0
- data/lib/rdkafka/admin/delete_records_report.rb +25 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +4 -6
- data/lib/rdkafka/admin/describe_acl_handle.rb +5 -7
- data/lib/rdkafka/admin/describe_acl_report.rb +2 -1
- data/lib/rdkafka/admin/describe_configs_handle.rb +4 -10
- data/lib/rdkafka/admin/describe_configs_report.rb +1 -5
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +4 -10
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +1 -5
- data/lib/rdkafka/admin/list_consumer_groups_handle.rb +31 -0
- data/lib/rdkafka/admin/list_consumer_groups_report.rb +83 -0
- data/lib/rdkafka/admin/list_offsets_handle.rb +5 -10
- data/lib/rdkafka/admin/list_offsets_report.rb +5 -2
- data/lib/rdkafka/admin.rb +238 -145
- data/lib/rdkafka/bindings.rb +61 -16
- data/lib/rdkafka/callbacks/base_handler.rb +62 -0
- data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
- data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_records_handler.rb +38 -0
- data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
- data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/list_consumer_groups_handler.rb +39 -0
- data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
- data/lib/rdkafka/callbacks.rb +54 -246
- data/lib/rdkafka/config.rb +69 -50
- data/lib/rdkafka/consumer/headers.rb +19 -5
- data/lib/rdkafka/consumer/partition.rb +8 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +58 -30
- data/lib/rdkafka/consumer.rb +211 -52
- data/lib/rdkafka/defaults.rb +24 -7
- data/lib/rdkafka/helpers/list_offsets.rb +127 -0
- data/lib/rdkafka/helpers/metadata.rb +29 -0
- data/lib/rdkafka/helpers/oauth.rb +10 -5
- data/lib/rdkafka/metadata.rb +86 -19
- data/lib/rdkafka/native_kafka.rb +22 -11
- data/lib/rdkafka/producer/delivery_handle.rb +4 -5
- data/lib/rdkafka/producer/partitions_count_cache.rb +24 -38
- data/lib/rdkafka/producer.rb +81 -61
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +19 -0
- data/package-lock.json +3 -3
- data/rdkafka.gemspec +1 -0
- metadata +21 -5
- data/Gemfile +0 -13
- data/Gemfile.lint +0 -14
- data/Gemfile.lint.lock +0 -123
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
class Admin
|
|
5
|
+
# Handle for list consumer groups operation
|
|
6
|
+
class ListConsumerGroupsHandle < AbstractHandle
|
|
7
|
+
layout :pending, :bool,
|
|
8
|
+
:response, :int
|
|
9
|
+
|
|
10
|
+
# @return [String] the name of the operation
|
|
11
|
+
def operation_name
|
|
12
|
+
"list consumer groups"
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @return [ListConsumerGroupsReport] report prepared by the background event callback, with
|
|
16
|
+
# the listed consumer groups.
|
|
17
|
+
def create_result
|
|
18
|
+
prepared_result
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Raises an error if the operation failed
|
|
22
|
+
# @raise [RdkafkaError]
|
|
23
|
+
def raise_error
|
|
24
|
+
raise RdkafkaError.new(
|
|
25
|
+
self[:response],
|
|
26
|
+
broker_message: broker_message
|
|
27
|
+
)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
class Admin
|
|
5
|
+
# Report for list consumer groups operation result
|
|
6
|
+
class ListConsumerGroupsReport
|
|
7
|
+
# Consumer groups listed cluster-wide. Each entry is a hash with:
|
|
8
|
+
# - `:group_id` [String] the consumer group id
|
|
9
|
+
# - `:is_simple_consumer_group` [Boolean] `true` for a "simple" consumer group - one that
|
|
10
|
+
# assigns partitions manually (via `assign`) and uses Kafka only for offset storage,
|
|
11
|
+
# rather than joining the group-management protocol and letting Kafka assign partitions
|
|
12
|
+
# and rebalance automatically (`subscribe`). Simple groups have no members from the
|
|
13
|
+
# broker's point of view, so they never rebalance.
|
|
14
|
+
# - `:state` [Integer] the group state as a `Bindings::RD_KAFKA_CONSUMER_GROUP_STATE_*`
|
|
15
|
+
# code, one of: `RD_KAFKA_CONSUMER_GROUP_STATE_UNKNOWN`,
|
|
16
|
+
# `RD_KAFKA_CONSUMER_GROUP_STATE_PREPARING_REBALANCE`,
|
|
17
|
+
# `RD_KAFKA_CONSUMER_GROUP_STATE_COMPLETING_REBALANCE`,
|
|
18
|
+
# `RD_KAFKA_CONSUMER_GROUP_STATE_STABLE`, `RD_KAFKA_CONSUMER_GROUP_STATE_DEAD`,
|
|
19
|
+
# `RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY`
|
|
20
|
+
# - `:state_name` [String] human-readable name of that state (e.g. `"Stable"`, `"Empty"`)
|
|
21
|
+
# @return [Array<Hash>]
|
|
22
|
+
attr_reader :groups
|
|
23
|
+
|
|
24
|
+
# Per-broker errors reported alongside the (partial) group listing. `ListConsumerGroups`
|
|
25
|
+
# fans out to every broker and returns valid groups and errors separately, so a broker
|
|
26
|
+
# being unreachable does not discard the groups the reachable brokers returned.
|
|
27
|
+
# @return [Array<RdkafkaError>]
|
|
28
|
+
attr_reader :errors
|
|
29
|
+
|
|
30
|
+
# @param result_ptr [FFI::Pointer] pointer to the `rd_kafka_ListConsumerGroups_result_t`
|
|
31
|
+
def initialize(result_ptr)
|
|
32
|
+
@groups = []
|
|
33
|
+
@errors = []
|
|
34
|
+
|
|
35
|
+
return if result_ptr.null?
|
|
36
|
+
|
|
37
|
+
extract_groups(result_ptr)
|
|
38
|
+
extract_errors(result_ptr)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# @param result_ptr [FFI::Pointer] pointer to the result
|
|
44
|
+
def extract_groups(result_ptr)
|
|
45
|
+
count_ptr = FFI::MemoryPointer.new(:size_t)
|
|
46
|
+
array_ptr = Bindings.rd_kafka_ListConsumerGroups_result_valid(result_ptr, count_ptr)
|
|
47
|
+
|
|
48
|
+
return if array_ptr.null?
|
|
49
|
+
|
|
50
|
+
array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |listing_ptr|
|
|
51
|
+
state = Bindings.rd_kafka_ConsumerGroupListing_state(listing_ptr)
|
|
52
|
+
group_id_ptr = Bindings.rd_kafka_ConsumerGroupListing_group_id(listing_ptr)
|
|
53
|
+
state_name_ptr = Bindings.rd_kafka_consumer_group_state_name(state)
|
|
54
|
+
|
|
55
|
+
@groups << {
|
|
56
|
+
group_id: group_id_ptr.null? ? nil : group_id_ptr.read_string,
|
|
57
|
+
is_simple_consumer_group:
|
|
58
|
+
Bindings.rd_kafka_ConsumerGroupListing_is_simple_consumer_group(listing_ptr) != 0,
|
|
59
|
+
state: state,
|
|
60
|
+
state_name: state_name_ptr.null? ? nil : state_name_ptr.read_string
|
|
61
|
+
}
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @param result_ptr [FFI::Pointer] pointer to the result
|
|
66
|
+
def extract_errors(result_ptr)
|
|
67
|
+
count_ptr = FFI::MemoryPointer.new(:size_t)
|
|
68
|
+
array_ptr = Bindings.rd_kafka_ListConsumerGroups_result_errors(result_ptr, count_ptr)
|
|
69
|
+
|
|
70
|
+
return if array_ptr.null?
|
|
71
|
+
|
|
72
|
+
array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |error_ptr|
|
|
73
|
+
string_ptr = Bindings.rd_kafka_error_string(error_ptr)
|
|
74
|
+
|
|
75
|
+
@errors << RdkafkaError.new(
|
|
76
|
+
Bindings.rd_kafka_error_code(error_ptr),
|
|
77
|
+
broker_message: string_ptr.null? ? nil : string_ptr.read_string
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -5,22 +5,17 @@ module Rdkafka
|
|
|
5
5
|
# Handle for list offsets operation
|
|
6
6
|
class ListOffsetsHandle < AbstractHandle
|
|
7
7
|
layout :pending, :bool,
|
|
8
|
-
:response, :int
|
|
9
|
-
:response_string, :pointer,
|
|
10
|
-
:result_infos, :pointer,
|
|
11
|
-
:result_count, :int
|
|
8
|
+
:response, :int
|
|
12
9
|
|
|
13
10
|
# @return [String] the name of the operation.
|
|
14
11
|
def operation_name
|
|
15
12
|
"list offsets"
|
|
16
13
|
end
|
|
17
14
|
|
|
18
|
-
# @return [ListOffsetsReport]
|
|
15
|
+
# @return [ListOffsetsReport] report prepared by the background event callback, with
|
|
16
|
+
# partition offset information.
|
|
19
17
|
def create_result
|
|
20
|
-
|
|
21
|
-
result_infos: self[:result_infos],
|
|
22
|
-
result_count: self[:result_count]
|
|
23
|
-
)
|
|
18
|
+
prepared_result
|
|
24
19
|
end
|
|
25
20
|
|
|
26
21
|
# Raises an error if the operation failed
|
|
@@ -28,7 +23,7 @@ module Rdkafka
|
|
|
28
23
|
def raise_error
|
|
29
24
|
raise RdkafkaError.new(
|
|
30
25
|
self[:response],
|
|
31
|
-
broker_message:
|
|
26
|
+
broker_message: broker_message
|
|
32
27
|
)
|
|
33
28
|
end
|
|
34
29
|
end
|
|
@@ -36,7 +36,7 @@ module Rdkafka
|
|
|
36
36
|
|
|
37
37
|
# Validates the partition result and raises an error if invalid
|
|
38
38
|
# @param result_info_ptr [FFI::Pointer] pointer to the result info
|
|
39
|
-
# @raise [RdkafkaError] when the partition has an error
|
|
39
|
+
# @raise [RdkafkaError] when the partition has an error, naming the failing partition
|
|
40
40
|
def validate!(result_info_ptr)
|
|
41
41
|
tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
|
|
42
42
|
tp = Bindings::TopicPartition.new(tp_ptr)
|
|
@@ -44,7 +44,10 @@ module Rdkafka
|
|
|
44
44
|
|
|
45
45
|
return if code.zero?
|
|
46
46
|
|
|
47
|
-
raise RdkafkaError.new(
|
|
47
|
+
raise RdkafkaError.new(
|
|
48
|
+
code,
|
|
49
|
+
"Error querying offsets for partition #{tp[:partition]} of '#{tp[:topic]}'"
|
|
50
|
+
)
|
|
48
51
|
end
|
|
49
52
|
end
|
|
50
53
|
end
|
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -4,6 +4,8 @@ module Rdkafka
|
|
|
4
4
|
# Admin client for Kafka administrative operations
|
|
5
5
|
class Admin
|
|
6
6
|
include Helpers::OAuth
|
|
7
|
+
include Helpers::Metadata
|
|
8
|
+
include Helpers::ListOffsets
|
|
7
9
|
|
|
8
10
|
class << self
|
|
9
11
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
@@ -106,10 +108,6 @@ module Rdkafka
|
|
|
106
108
|
# @return [nil]
|
|
107
109
|
# @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
|
|
108
110
|
#
|
|
109
|
-
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
110
|
-
# Other admin operations will wait until this method returns.
|
|
111
|
-
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
112
|
-
#
|
|
113
111
|
# @example Drain all pending events
|
|
114
112
|
# admin.events_poll_nb_each { |_count| }
|
|
115
113
|
#
|
|
@@ -118,6 +116,9 @@ module Rdkafka
|
|
|
118
116
|
# admin.events_poll_nb_each do |_count|
|
|
119
117
|
# :stop if monotonic_now >= deadline
|
|
120
118
|
# end
|
|
119
|
+
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
120
|
+
# Other admin operations will wait until this method returns.
|
|
121
|
+
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
121
122
|
def events_poll_nb_each
|
|
122
123
|
closed_admin_check(__method__)
|
|
123
124
|
|
|
@@ -130,19 +131,6 @@ module Rdkafka
|
|
|
130
131
|
end
|
|
131
132
|
end
|
|
132
133
|
|
|
133
|
-
# Performs the metadata request using admin
|
|
134
|
-
#
|
|
135
|
-
# @param topic_name [String, nil] metadat about particular topic or all if nil
|
|
136
|
-
# @param timeout_ms [Integer] metadata request timeout
|
|
137
|
-
# @return [Metadata] requested metadata
|
|
138
|
-
def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
|
|
139
|
-
closed_admin_check(__method__)
|
|
140
|
-
|
|
141
|
-
@native_kafka.with_inner do |inner|
|
|
142
|
-
Metadata.new(inner, topic_name, timeout_ms)
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
|
|
146
134
|
# Close this admin instance
|
|
147
135
|
def close
|
|
148
136
|
return if closed?
|
|
@@ -258,7 +246,7 @@ module Rdkafka
|
|
|
258
246
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
259
247
|
end
|
|
260
248
|
if queue_ptr.null?
|
|
261
|
-
Rdkafka::Bindings.
|
|
249
|
+
Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
|
|
262
250
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
263
251
|
end
|
|
264
252
|
|
|
@@ -294,6 +282,172 @@ module Rdkafka
|
|
|
294
282
|
delete_groups_handle
|
|
295
283
|
end
|
|
296
284
|
|
|
285
|
+
# Deletes all messages in the given partitions up to (but not including) the given offset.
|
|
286
|
+
#
|
|
287
|
+
# The programmatic equivalent of `kafka-delete-records.sh`. Useful for GDPR/right-to-erasure
|
|
288
|
+
# compliance, clearing out poison messages, or retention cleanup outside of the configured
|
|
289
|
+
# time/size-based retention policy.
|
|
290
|
+
#
|
|
291
|
+
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
292
|
+
# arrays of partition delete specifications. Each specification is a hash with:
|
|
293
|
+
# - `:partition` [Integer] partition number
|
|
294
|
+
# - `:offset` [Symbol, Integer] delete all messages before this offset (exclusive) - an
|
|
295
|
+
# integer offset, or `:end` to delete all data currently in the partition
|
|
296
|
+
#
|
|
297
|
+
# @return [DeleteRecordsHandle] handle that can be used to wait for the result
|
|
298
|
+
# @raise [RdkafkaError] when deleting records fails
|
|
299
|
+
#
|
|
300
|
+
# @example Delete all messages before offset 100 in one partition, and all data in another
|
|
301
|
+
# report = admin.delete_records(
|
|
302
|
+
# "my_topic" => [
|
|
303
|
+
# { partition: 0, offset: 100 },
|
|
304
|
+
# { partition: 1, offset: :end }
|
|
305
|
+
# ]
|
|
306
|
+
# ).wait(max_wait_timeout_ms: 15_000)
|
|
307
|
+
#
|
|
308
|
+
# report.offsets.to_h
|
|
309
|
+
# # => { "my_topic" => [#<Partition @partition=0, @offset=100, @err=0>, ...] }
|
|
310
|
+
def delete_records(topic_partition_offsets)
|
|
311
|
+
closed_admin_check(__method__)
|
|
312
|
+
|
|
313
|
+
parsed = topic_partition_offsets.flat_map do |topic, partitions|
|
|
314
|
+
partitions.map do |spec|
|
|
315
|
+
offset = spec.fetch(:offset)
|
|
316
|
+
|
|
317
|
+
native_offset = case offset
|
|
318
|
+
when :end then Rdkafka::Bindings::RD_KAFKA_OFFSET_END
|
|
319
|
+
when Integer then offset
|
|
320
|
+
else
|
|
321
|
+
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
[topic, spec.fetch(:partition), native_offset]
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
|
|
329
|
+
|
|
330
|
+
parsed.each do |topic, partition, native_offset|
|
|
331
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
332
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# rd_kafka_DeleteRecords_new copies the tpl it is given, so it does not need to (and must
|
|
336
|
+
# not) outlive this call - it is destroyed below, independently of the wrapping
|
|
337
|
+
# DeleteRecords_t object.
|
|
338
|
+
delete_records_ptr = Rdkafka::Bindings.rd_kafka_DeleteRecords_new(tpl)
|
|
339
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
340
|
+
|
|
341
|
+
pointer_array = [delete_records_ptr]
|
|
342
|
+
records_array_ptr = FFI::MemoryPointer.new(:pointer)
|
|
343
|
+
records_array_ptr.write_array_of_pointer(pointer_array)
|
|
344
|
+
|
|
345
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
346
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
347
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
348
|
+
end
|
|
349
|
+
if queue_ptr.null?
|
|
350
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
|
|
351
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Create and register the handle we will return to the caller
|
|
355
|
+
handle = DeleteRecordsHandle.new
|
|
356
|
+
handle[:pending] = true
|
|
357
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
358
|
+
DeleteRecordsHandle.register(handle)
|
|
359
|
+
|
|
360
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
361
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETERECORDS)
|
|
362
|
+
end
|
|
363
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
364
|
+
|
|
365
|
+
begin
|
|
366
|
+
@native_kafka.with_inner do |inner|
|
|
367
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords(
|
|
368
|
+
inner,
|
|
369
|
+
records_array_ptr,
|
|
370
|
+
1,
|
|
371
|
+
admin_options_ptr,
|
|
372
|
+
queue_ptr
|
|
373
|
+
)
|
|
374
|
+
end
|
|
375
|
+
rescue Exception
|
|
376
|
+
DeleteRecordsHandle.remove(handle.to_ptr.address)
|
|
377
|
+
raise
|
|
378
|
+
ensure
|
|
379
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
380
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
381
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
handle
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Lists consumer groups cluster-wide.
|
|
388
|
+
#
|
|
389
|
+
# librdkafka issues a single `ListConsumerGroups` request that is fanned out to every broker
|
|
390
|
+
# internally, so the result covers all consumer groups in the cluster, not only those
|
|
391
|
+
# coordinated by the connected broker.
|
|
392
|
+
#
|
|
393
|
+
# @return [ListConsumerGroupsHandle] handle that can be used to wait for the result
|
|
394
|
+
# @raise [RdkafkaError] when listing the consumer groups fails
|
|
395
|
+
#
|
|
396
|
+
# @example List every consumer group in the cluster and print its name and attributes
|
|
397
|
+
# report = admin.list_consumer_groups.wait(max_wait_timeout_ms: 15_000)
|
|
398
|
+
#
|
|
399
|
+
# report.groups.each do |group|
|
|
400
|
+
# puts "#{group[:group_id]} - #{group[:state_name]} " \
|
|
401
|
+
# "(simple: #{group[:is_simple_consumer_group]})"
|
|
402
|
+
# end
|
|
403
|
+
#
|
|
404
|
+
# # Any brokers that could not be reached are reported separately
|
|
405
|
+
# report.errors.each { |error| warn "partial listing error: #{error.message}" }
|
|
406
|
+
def list_consumer_groups
|
|
407
|
+
closed_admin_check(__method__)
|
|
408
|
+
|
|
409
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
410
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
411
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
if queue_ptr.null?
|
|
415
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Create and register the handle we will return to the caller
|
|
419
|
+
handle = ListConsumerGroupsHandle.new
|
|
420
|
+
handle[:pending] = true
|
|
421
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
422
|
+
ListConsumerGroupsHandle.register(handle)
|
|
423
|
+
|
|
424
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
425
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
426
|
+
inner,
|
|
427
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS
|
|
428
|
+
)
|
|
429
|
+
end
|
|
430
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
431
|
+
|
|
432
|
+
begin
|
|
433
|
+
@native_kafka.with_inner do |inner|
|
|
434
|
+
Rdkafka::Bindings.rd_kafka_ListConsumerGroups(
|
|
435
|
+
inner,
|
|
436
|
+
admin_options_ptr,
|
|
437
|
+
queue_ptr
|
|
438
|
+
)
|
|
439
|
+
end
|
|
440
|
+
rescue Exception
|
|
441
|
+
ListConsumerGroupsHandle.remove(handle.to_ptr.address)
|
|
442
|
+
raise
|
|
443
|
+
ensure
|
|
444
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
445
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
handle
|
|
449
|
+
end
|
|
450
|
+
|
|
297
451
|
# Deletes the named topic
|
|
298
452
|
#
|
|
299
453
|
# @param topic_name [String] name of the topic to delete
|
|
@@ -356,7 +510,8 @@ module Rdkafka
|
|
|
356
510
|
#
|
|
357
511
|
# @param topic_name [String] name of the topic
|
|
358
512
|
# @param partition_count [Integer] how many partitions we want to end up with for given topic
|
|
359
|
-
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
|
|
513
|
+
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
|
|
514
|
+
# result
|
|
360
515
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
361
516
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
362
517
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
@@ -445,7 +600,8 @@ module Rdkafka
|
|
|
445
600
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
446
601
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
447
602
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
448
|
-
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of
|
|
603
|
+
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of
|
|
604
|
+
# creating the acl
|
|
449
605
|
# @raise [RdkafkaError]
|
|
450
606
|
def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
451
607
|
closed_admin_check(__method__)
|
|
@@ -546,7 +702,8 @@ module Rdkafka
|
|
|
546
702
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
547
703
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
548
704
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
549
|
-
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of
|
|
705
|
+
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of
|
|
706
|
+
# deleting the acl
|
|
550
707
|
# @raise [RdkafkaError]
|
|
551
708
|
def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
552
709
|
closed_admin_check(__method__)
|
|
@@ -581,7 +738,7 @@ module Rdkafka
|
|
|
581
738
|
end
|
|
582
739
|
|
|
583
740
|
if queue_ptr.null?
|
|
584
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
741
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
|
|
585
742
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
586
743
|
end
|
|
587
744
|
|
|
@@ -649,7 +806,8 @@ module Rdkafka
|
|
|
649
806
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
650
807
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
651
808
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
652
|
-
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of
|
|
809
|
+
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of
|
|
810
|
+
# fetching acls
|
|
653
811
|
# @raise [RdkafkaError]
|
|
654
812
|
def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
655
813
|
closed_admin_check(__method__)
|
|
@@ -677,7 +835,7 @@ module Rdkafka
|
|
|
677
835
|
end
|
|
678
836
|
|
|
679
837
|
if queue_ptr.null?
|
|
680
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
838
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
|
|
681
839
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
682
840
|
end
|
|
683
841
|
|
|
@@ -728,6 +886,14 @@ module Rdkafka
|
|
|
728
886
|
def describe_configs(resources)
|
|
729
887
|
closed_admin_check(__method__)
|
|
730
888
|
|
|
889
|
+
# Parse user input before allocating any native resources or registering the handle, so a
|
|
890
|
+
# missing :resource_type / :resource_name key raises with nothing to clean up. Previously the
|
|
891
|
+
# KeyError leaked the background queue, the AdminOptions, the registered handle and any
|
|
892
|
+
# ConfigResources already built.
|
|
893
|
+
parsed_resources = resources.map do |resource_details|
|
|
894
|
+
[resource_details.fetch(:resource_type), resource_details.fetch(:resource_name)]
|
|
895
|
+
end
|
|
896
|
+
|
|
731
897
|
handle = DescribeConfigsHandle.new
|
|
732
898
|
handle[:pending] = true
|
|
733
899
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
@@ -750,12 +916,10 @@ module Rdkafka
|
|
|
750
916
|
DescribeConfigsHandle.register(handle)
|
|
751
917
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
752
918
|
|
|
753
|
-
pointer_array =
|
|
919
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name|
|
|
754
920
|
Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
755
|
-
|
|
756
|
-
FFI::MemoryPointer.from_string(
|
|
757
|
-
resource_details.fetch(:resource_name)
|
|
758
|
-
)
|
|
921
|
+
resource_type,
|
|
922
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
759
923
|
)
|
|
760
924
|
end
|
|
761
925
|
|
|
@@ -806,6 +970,19 @@ module Rdkafka
|
|
|
806
970
|
def incremental_alter_configs(resources_with_configs)
|
|
807
971
|
closed_admin_check(__method__)
|
|
808
972
|
|
|
973
|
+
# Parse user input before allocating native resources or registering the handle, so a missing
|
|
974
|
+
# key raises with nothing to clean up. Previously the KeyError leaked the background queue,
|
|
975
|
+
# the AdminOptions, the registered handle and any ConfigResources already built.
|
|
976
|
+
parsed_resources = resources_with_configs.map do |resource_details|
|
|
977
|
+
[
|
|
978
|
+
resource_details.fetch(:resource_type),
|
|
979
|
+
resource_details.fetch(:resource_name),
|
|
980
|
+
resource_details.fetch(:configs).map do |config|
|
|
981
|
+
[config.fetch(:name), config.fetch(:op_type), config.fetch(:value)]
|
|
982
|
+
end
|
|
983
|
+
]
|
|
984
|
+
end
|
|
985
|
+
|
|
809
986
|
handle = IncrementalAlterConfigsHandle.new
|
|
810
987
|
handle[:pending] = true
|
|
811
988
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
@@ -828,23 +1005,39 @@ module Rdkafka
|
|
|
828
1005
|
IncrementalAlterConfigsHandle.register(handle)
|
|
829
1006
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
830
1007
|
|
|
831
|
-
|
|
832
|
-
|
|
1008
|
+
add_error = nil
|
|
1009
|
+
|
|
1010
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name, configs|
|
|
833
1011
|
# First build the appropriate resource representation
|
|
834
1012
|
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
835
|
-
|
|
836
|
-
FFI::MemoryPointer.from_string(
|
|
837
|
-
resource_details.fetch(:resource_name)
|
|
838
|
-
)
|
|
1013
|
+
resource_type,
|
|
1014
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
839
1015
|
)
|
|
840
1016
|
|
|
841
|
-
|
|
842
|
-
|
|
1017
|
+
configs.each do |name, op_type, value|
|
|
1018
|
+
# rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
|
|
1019
|
+
# an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
|
|
1020
|
+
# used to be ignored: the entry was silently dropped, the alter request still reported
|
|
1021
|
+
# success, and the error object leaked. Capture the first error (always destroying the
|
|
1022
|
+
# native error object) and raise it below, before the request is sent.
|
|
1023
|
+
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
843
1024
|
resource_ptr,
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1025
|
+
name,
|
|
1026
|
+
op_type,
|
|
1027
|
+
value
|
|
847
1028
|
)
|
|
1029
|
+
|
|
1030
|
+
unless error_ptr.null?
|
|
1031
|
+
code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
|
|
1032
|
+
Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
|
|
1033
|
+
|
|
1034
|
+
unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
1035
|
+
add_error ||= Rdkafka::RdkafkaError.new(
|
|
1036
|
+
code,
|
|
1037
|
+
"rd_kafka_ConfigResource_add_incremental_config"
|
|
1038
|
+
)
|
|
1039
|
+
end
|
|
1040
|
+
end
|
|
848
1041
|
end
|
|
849
1042
|
|
|
850
1043
|
resource_ptr
|
|
@@ -854,6 +1047,10 @@ module Rdkafka
|
|
|
854
1047
|
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
855
1048
|
|
|
856
1049
|
begin
|
|
1050
|
+
# Raise only after the full array is built so the ensure below frees every ConfigResource
|
|
1051
|
+
# we created, and before the request is sent so a rejected entry can't report success.
|
|
1052
|
+
raise add_error if add_error
|
|
1053
|
+
|
|
857
1054
|
@native_kafka.with_inner do |inner|
|
|
858
1055
|
Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
|
|
859
1056
|
inner,
|
|
@@ -882,111 +1079,6 @@ module Rdkafka
|
|
|
882
1079
|
handle
|
|
883
1080
|
end
|
|
884
1081
|
|
|
885
|
-
# Queries partition offsets by specification (earliest, latest, max_timestamp, or by
|
|
886
|
-
# timestamp) without requiring a consumer group.
|
|
887
|
-
#
|
|
888
|
-
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
889
|
-
# arrays of partition offset specifications. Each specification is a hash with:
|
|
890
|
-
# - `:partition` [Integer] partition number
|
|
891
|
-
# - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
|
|
892
|
-
# `:max_timestamp`, or an integer timestamp in milliseconds
|
|
893
|
-
# @param isolation_level [Integer, nil] optional isolation level:
|
|
894
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
|
|
895
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
|
|
896
|
-
#
|
|
897
|
-
# @return [ListOffsetsHandle] handle that can be used to wait for the result
|
|
898
|
-
#
|
|
899
|
-
# @raise [ClosedAdminError] when the admin is closed
|
|
900
|
-
# @raise [ConfigError] when the background queue is unavailable
|
|
901
|
-
#
|
|
902
|
-
# @example Query earliest and latest offsets
|
|
903
|
-
# handle = admin.list_offsets(
|
|
904
|
-
# { "my_topic" => [
|
|
905
|
-
# { partition: 0, offset: :earliest },
|
|
906
|
-
# { partition: 1, offset: :latest }
|
|
907
|
-
# ] }
|
|
908
|
-
# )
|
|
909
|
-
# report = handle.wait(max_wait_timeout_ms: 15_000)
|
|
910
|
-
# report.offsets
|
|
911
|
-
# # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
|
|
912
|
-
def list_offsets(topic_partition_offsets, isolation_level: nil)
|
|
913
|
-
closed_admin_check(__method__)
|
|
914
|
-
|
|
915
|
-
# Count total partitions for pre-allocation
|
|
916
|
-
total = topic_partition_offsets.sum { |_, partitions| partitions.size }
|
|
917
|
-
|
|
918
|
-
# Build native topic partition list
|
|
919
|
-
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
|
|
920
|
-
|
|
921
|
-
topic_partition_offsets.each do |topic, partitions|
|
|
922
|
-
partitions.each do |spec|
|
|
923
|
-
partition = spec.fetch(:partition)
|
|
924
|
-
offset = spec.fetch(:offset)
|
|
925
|
-
|
|
926
|
-
native_offset = case offset
|
|
927
|
-
when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
|
|
928
|
-
when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
|
|
929
|
-
when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
|
|
930
|
-
when Integer then offset
|
|
931
|
-
else
|
|
932
|
-
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
933
|
-
end
|
|
934
|
-
|
|
935
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
936
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
937
|
-
end
|
|
938
|
-
end
|
|
939
|
-
|
|
940
|
-
# Get a pointer to the queue that our request will be enqueued on
|
|
941
|
-
queue_ptr = @native_kafka.with_inner do |inner|
|
|
942
|
-
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
943
|
-
end
|
|
944
|
-
|
|
945
|
-
if queue_ptr.null?
|
|
946
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
947
|
-
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
948
|
-
end
|
|
949
|
-
|
|
950
|
-
# Create and register the handle we will return to the caller
|
|
951
|
-
handle = ListOffsetsHandle.new
|
|
952
|
-
handle[:pending] = true
|
|
953
|
-
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
954
|
-
|
|
955
|
-
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
956
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
957
|
-
inner,
|
|
958
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
|
|
959
|
-
)
|
|
960
|
-
end
|
|
961
|
-
|
|
962
|
-
if isolation_level
|
|
963
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
|
|
964
|
-
end
|
|
965
|
-
|
|
966
|
-
ListOffsetsHandle.register(handle)
|
|
967
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
968
|
-
|
|
969
|
-
begin
|
|
970
|
-
@native_kafka.with_inner do |inner|
|
|
971
|
-
Rdkafka::Bindings.rd_kafka_ListOffsets(
|
|
972
|
-
inner,
|
|
973
|
-
tpl,
|
|
974
|
-
admin_options_ptr,
|
|
975
|
-
queue_ptr
|
|
976
|
-
)
|
|
977
|
-
end
|
|
978
|
-
rescue Exception
|
|
979
|
-
ListOffsetsHandle.remove(handle.to_ptr.address)
|
|
980
|
-
raise
|
|
981
|
-
ensure
|
|
982
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
983
|
-
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
984
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
985
|
-
end
|
|
986
|
-
|
|
987
|
-
handle
|
|
988
|
-
end
|
|
989
|
-
|
|
990
1082
|
private
|
|
991
1083
|
|
|
992
1084
|
# Checks if the admin is closed and raises an error if so
|
|
@@ -995,5 +1087,6 @@ module Rdkafka
|
|
|
995
1087
|
def closed_admin_check(method)
|
|
996
1088
|
raise Rdkafka::ClosedAdminError.new(method) if closed?
|
|
997
1089
|
end
|
|
1090
|
+
alias_method :closed_check, :closed_admin_check
|
|
998
1091
|
end
|
|
999
1092
|
end
|