rdkafka 0.29.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -11
- data/README.md +1 -0
- data/dist/{librdkafka-2.14.2.tar.gz → librdkafka-2.15.0.tar.gz} +0 -0
- data/lib/rdkafka/admin/alter_consumer_group_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/alter_consumer_group_offsets_report.rb +50 -0
- data/lib/rdkafka/admin/delete_consumer_group_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/delete_consumer_group_offsets_report.rb +50 -0
- data/lib/rdkafka/admin.rb +263 -85
- data/lib/rdkafka/bindings.rb +46 -2
- data/lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb +65 -0
- data/lib/rdkafka/callbacks/delete_consumer_group_offsets_handler.rb +65 -0
- data/lib/rdkafka/callbacks.rb +14 -16
- data/lib/rdkafka/clients.rb +114 -0
- data/lib/rdkafka/config.rb +118 -39
- data/lib/rdkafka/consumer/allocator.rb +38 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +1 -2
- data/lib/rdkafka/consumer.rb +44 -40
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +10 -0
- data/package-lock.json +6 -6
- data/renovate.json +1 -1
- metadata +11 -3
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -224,6 +224,78 @@ module Rdkafka
|
|
|
224
224
|
create_topic_handle
|
|
225
225
|
end
|
|
226
226
|
|
|
227
|
+
# Sets the committed offsets of a consumer group from outside the group.
|
|
228
|
+
#
|
|
229
|
+
# Kafka requires the group to be empty (no active members) for this to be accepted; when it
|
|
230
|
+
# is not, the broker rejects the request and the handle raises with `group_not_empty` rather
|
|
231
|
+
# than reporting success.
|
|
232
|
+
#
|
|
233
|
+
# @param group_id [String] the group whose offsets we want to set
|
|
234
|
+
# @param topic_partition_list [TopicPartitionList] partitions with the offsets to commit
|
|
235
|
+
# @return [AlterConsumerGroupOffsetsHandle] handle that can be used to wait for the result
|
|
236
|
+
# @raise [RdkafkaError] when altering the offsets fails
|
|
237
|
+
def alter_consumer_group_offsets(group_id, topic_partition_list)
|
|
238
|
+
closed_admin_check(__method__)
|
|
239
|
+
|
|
240
|
+
native_tpl = topic_partition_list.to_native_tpl
|
|
241
|
+
|
|
242
|
+
begin
|
|
243
|
+
alter_ptr = Rdkafka::Bindings.rd_kafka_AlterConsumerGroupOffsets_new(
|
|
244
|
+
FFI::MemoryPointer.from_string(group_id),
|
|
245
|
+
native_tpl
|
|
246
|
+
)
|
|
247
|
+
rescue Exception
|
|
248
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl)
|
|
249
|
+
raise
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# librdkafka copies the list into the request object, so ours is no longer needed
|
|
253
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl)
|
|
254
|
+
|
|
255
|
+
enqueue_group_offsets_op(
|
|
256
|
+
request_ptr: alter_ptr,
|
|
257
|
+
handle: AlterConsumerGroupOffsetsHandle.new,
|
|
258
|
+
admin_op: Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS,
|
|
259
|
+
destroy: Rdkafka::Bindings.method(:rd_kafka_AlterConsumerGroupOffsets_destroy),
|
|
260
|
+
enqueue: Rdkafka::Bindings.method(:rd_kafka_AlterConsumerGroupOffsets)
|
|
261
|
+
)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Deletes the committed offsets of specific partitions of a consumer group, leaving the
|
|
265
|
+
# group and its remaining offsets in place.
|
|
266
|
+
#
|
|
267
|
+
# This is the partition scoped counterpart of {#delete_group}, which removes the whole group.
|
|
268
|
+
#
|
|
269
|
+
# @param group_id [String] the group whose offsets we want to delete
|
|
270
|
+
# @param topic_partition_list [TopicPartitionList] partitions whose offsets to delete
|
|
271
|
+
# @return [DeleteConsumerGroupOffsetsHandle] handle that can be used to wait for the result
|
|
272
|
+
# @raise [RdkafkaError] when deleting the offsets fails
|
|
273
|
+
def delete_consumer_group_offsets(group_id, topic_partition_list)
|
|
274
|
+
closed_admin_check(__method__)
|
|
275
|
+
|
|
276
|
+
native_tpl = topic_partition_list.to_native_tpl
|
|
277
|
+
|
|
278
|
+
begin
|
|
279
|
+
delete_ptr = Rdkafka::Bindings.rd_kafka_DeleteConsumerGroupOffsets_new(
|
|
280
|
+
FFI::MemoryPointer.from_string(group_id),
|
|
281
|
+
native_tpl
|
|
282
|
+
)
|
|
283
|
+
rescue Exception
|
|
284
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl)
|
|
285
|
+
raise
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl)
|
|
289
|
+
|
|
290
|
+
enqueue_group_offsets_op(
|
|
291
|
+
request_ptr: delete_ptr,
|
|
292
|
+
handle: DeleteConsumerGroupOffsetsHandle.new,
|
|
293
|
+
admin_op: Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETECONSUMERGROUPOFFSETS,
|
|
294
|
+
destroy: Rdkafka::Bindings.method(:rd_kafka_DeleteConsumerGroupOffsets_destroy),
|
|
295
|
+
enqueue: Rdkafka::Bindings.method(:rd_kafka_DeleteConsumerGroupOffsets)
|
|
296
|
+
)
|
|
297
|
+
end
|
|
298
|
+
|
|
227
299
|
# Deletes a consumer group
|
|
228
300
|
#
|
|
229
301
|
# @param group_id [String] the group id to delete
|
|
@@ -898,35 +970,64 @@ module Rdkafka
|
|
|
898
970
|
handle[:pending] = true
|
|
899
971
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
900
972
|
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
973
|
+
# All native allocation happens inside the begin so a raise at any point (a null background
|
|
974
|
+
# queue, a non-String resource name making `from_string` raise mid-build, etc.) is cleaned up
|
|
975
|
+
# by the ensure rather than leaking the queue, the AdminOptions, the handle and the
|
|
976
|
+
# ConfigResources already built.
|
|
977
|
+
queue_ptr = nil
|
|
978
|
+
admin_options_ptr = nil
|
|
979
|
+
pointer_array = []
|
|
980
|
+
registered = false
|
|
904
981
|
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
982
|
+
begin
|
|
983
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
984
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
985
|
+
end
|
|
908
986
|
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
|
913
|
-
)
|
|
914
|
-
end
|
|
987
|
+
if queue_ptr.null?
|
|
988
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
989
|
+
end
|
|
915
990
|
|
|
916
|
-
|
|
917
|
-
|
|
991
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
992
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
993
|
+
inner,
|
|
994
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
|
995
|
+
)
|
|
996
|
+
end
|
|
918
997
|
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
end
|
|
998
|
+
# A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
|
|
999
|
+
# before registering the handle (the ensure frees the queue already acquired).
|
|
1000
|
+
if admin_options_ptr.null?
|
|
1001
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
|
|
1002
|
+
end
|
|
925
1003
|
|
|
926
|
-
|
|
927
|
-
|
|
1004
|
+
DescribeConfigsHandle.register(handle)
|
|
1005
|
+
registered = true
|
|
1006
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
1007
|
+
|
|
1008
|
+
# Build resources one at a time so a raise on a later element still leaves the earlier ones
|
|
1009
|
+
# in pointer_array for the ensure to destroy.
|
|
1010
|
+
parsed_resources.each do |resource_type, resource_name|
|
|
1011
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
1012
|
+
resource_type,
|
|
1013
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
1014
|
+
)
|
|
1015
|
+
|
|
1016
|
+
# librdkafka returns NULL for an empty resource name or a negative resource type. Passing
|
|
1017
|
+
# NULL on to DescribeConfigs and then destroying it both dereference NULL and segfault, so
|
|
1018
|
+
# reject it here (the ensure frees everything already built).
|
|
1019
|
+
if resource_ptr.null?
|
|
1020
|
+
raise Rdkafka::Config::ConfigError.new(
|
|
1021
|
+
"rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
|
|
1022
|
+
)
|
|
1023
|
+
end
|
|
1024
|
+
|
|
1025
|
+
pointer_array << resource_ptr
|
|
1026
|
+
end
|
|
1027
|
+
|
|
1028
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
1029
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
928
1030
|
|
|
929
|
-
begin
|
|
930
1031
|
@native_kafka.with_inner do |inner|
|
|
931
1032
|
Rdkafka::Bindings.rd_kafka_DescribeConfigs(
|
|
932
1033
|
inner,
|
|
@@ -937,18 +1038,20 @@ module Rdkafka
|
|
|
937
1038
|
)
|
|
938
1039
|
end
|
|
939
1040
|
rescue Exception
|
|
940
|
-
DescribeConfigsHandle.remove(handle.to_ptr.address)
|
|
1041
|
+
DescribeConfigsHandle.remove(handle.to_ptr.address) if registered
|
|
941
1042
|
|
|
942
1043
|
raise
|
|
943
1044
|
ensure
|
|
944
|
-
|
|
945
|
-
|
|
1045
|
+
if admin_options_ptr && !admin_options_ptr.null?
|
|
1046
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
1047
|
+
end
|
|
946
1048
|
|
|
947
|
-
if
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
1049
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
|
|
1050
|
+
|
|
1051
|
+
pointer_array.each do |config_resource_ptr|
|
|
1052
|
+
next if config_resource_ptr.null?
|
|
1053
|
+
|
|
1054
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
|
|
952
1055
|
end
|
|
953
1056
|
end
|
|
954
1057
|
|
|
@@ -987,66 +1090,91 @@ module Rdkafka
|
|
|
987
1090
|
handle[:pending] = true
|
|
988
1091
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
989
1092
|
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1093
|
+
# All native allocation happens inside the begin so a raise at any point (a null background
|
|
1094
|
+
# queue, a non-String resource name, or FFI marshaling of a config name/value failing while
|
|
1095
|
+
# adding configs) is cleaned up by the ensure rather than leaking the queue, the AdminOptions,
|
|
1096
|
+
# the handle and the ConfigResources already built.
|
|
1097
|
+
queue_ptr = nil
|
|
1098
|
+
admin_options_ptr = nil
|
|
1099
|
+
pointer_array = []
|
|
1100
|
+
registered = false
|
|
1101
|
+
add_error = nil
|
|
993
1102
|
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1103
|
+
begin
|
|
1104
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
1105
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
1106
|
+
end
|
|
997
1107
|
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
|
1002
|
-
)
|
|
1003
|
-
end
|
|
1108
|
+
if queue_ptr.null?
|
|
1109
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
1110
|
+
end
|
|
1004
1111
|
|
|
1005
|
-
|
|
1006
|
-
|
|
1112
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
1113
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
1114
|
+
inner,
|
|
1115
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
|
1116
|
+
)
|
|
1117
|
+
end
|
|
1007
1118
|
|
|
1008
|
-
|
|
1119
|
+
# A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
|
|
1120
|
+
# before registering the handle (the ensure frees the queue already acquired).
|
|
1121
|
+
if admin_options_ptr.null?
|
|
1122
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
|
|
1123
|
+
end
|
|
1009
1124
|
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
resource_type,
|
|
1014
|
-
FFI::MemoryPointer.from_string(resource_name)
|
|
1015
|
-
)
|
|
1125
|
+
IncrementalAlterConfigsHandle.register(handle)
|
|
1126
|
+
registered = true
|
|
1127
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
1016
1128
|
|
|
1017
|
-
|
|
1018
|
-
#
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
# native error object) and raise it below, before the request is sent.
|
|
1023
|
-
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
1024
|
-
resource_ptr,
|
|
1025
|
-
name,
|
|
1026
|
-
op_type,
|
|
1027
|
-
value
|
|
1129
|
+
parsed_resources.each do |resource_type, resource_name, configs|
|
|
1130
|
+
# First build the appropriate resource representation
|
|
1131
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
1132
|
+
resource_type,
|
|
1133
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
1028
1134
|
)
|
|
1029
1135
|
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1136
|
+
# librdkafka returns NULL for an empty resource name or a negative resource type. Passing
|
|
1137
|
+
# NULL on to add_incremental_config/IncrementalAlterConfigs and then destroying it both
|
|
1138
|
+
# dereference NULL and segfault, so reject it here (the ensure frees everything built).
|
|
1139
|
+
if resource_ptr.null?
|
|
1140
|
+
raise Rdkafka::Config::ConfigError.new(
|
|
1141
|
+
"rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
|
|
1142
|
+
)
|
|
1143
|
+
end
|
|
1033
1144
|
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1145
|
+
# Track it immediately so a raise while adding its configs still frees it via the ensure.
|
|
1146
|
+
pointer_array << resource_ptr
|
|
1147
|
+
|
|
1148
|
+
configs.each do |name, op_type, value|
|
|
1149
|
+
# rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
|
|
1150
|
+
# an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
|
|
1151
|
+
# used to be ignored: the entry was silently dropped, the alter request still reported
|
|
1152
|
+
# success, and the error object leaked. Capture the first error (always destroying the
|
|
1153
|
+
# native error object) and raise it below, before the request is sent.
|
|
1154
|
+
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
1155
|
+
resource_ptr,
|
|
1156
|
+
name,
|
|
1157
|
+
op_type,
|
|
1158
|
+
value
|
|
1159
|
+
)
|
|
1160
|
+
|
|
1161
|
+
unless error_ptr.null?
|
|
1162
|
+
code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
|
|
1163
|
+
Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
|
|
1164
|
+
|
|
1165
|
+
unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
1166
|
+
add_error ||= Rdkafka::RdkafkaError.new(
|
|
1167
|
+
code,
|
|
1168
|
+
"rd_kafka_ConfigResource_add_incremental_config"
|
|
1169
|
+
)
|
|
1170
|
+
end
|
|
1039
1171
|
end
|
|
1040
1172
|
end
|
|
1041
1173
|
end
|
|
1042
1174
|
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
1047
|
-
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
1175
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
1176
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
1048
1177
|
|
|
1049
|
-
begin
|
|
1050
1178
|
# Raise only after the full array is built so the ensure below frees every ConfigResource
|
|
1051
1179
|
# we created, and before the request is sent so a rejected entry can't report success.
|
|
1052
1180
|
raise add_error if add_error
|
|
@@ -1061,18 +1189,20 @@ module Rdkafka
|
|
|
1061
1189
|
)
|
|
1062
1190
|
end
|
|
1063
1191
|
rescue Exception
|
|
1064
|
-
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
|
|
1192
|
+
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address) if registered
|
|
1065
1193
|
|
|
1066
1194
|
raise
|
|
1067
1195
|
ensure
|
|
1068
|
-
|
|
1069
|
-
|
|
1196
|
+
if admin_options_ptr && !admin_options_ptr.null?
|
|
1197
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
1198
|
+
end
|
|
1070
1199
|
|
|
1071
|
-
if
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1200
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
|
|
1201
|
+
|
|
1202
|
+
pointer_array.each do |config_resource_ptr|
|
|
1203
|
+
next if config_resource_ptr.null?
|
|
1204
|
+
|
|
1205
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
|
|
1076
1206
|
end
|
|
1077
1207
|
end
|
|
1078
1208
|
|
|
@@ -1081,6 +1211,54 @@ module Rdkafka
|
|
|
1081
1211
|
|
|
1082
1212
|
private
|
|
1083
1213
|
|
|
1214
|
+
# Shared enqueue path for the two consumer group offset admin operations. They differ only
|
|
1215
|
+
# in their C entry points, so keeping one body avoids two near identical copies of the
|
|
1216
|
+
# queue/handle/options dance and its cleanup.
|
|
1217
|
+
#
|
|
1218
|
+
# @param request_ptr [FFI::Pointer] the op specific request object
|
|
1219
|
+
# @param handle [AbstractHandle] the handle to register and return
|
|
1220
|
+
# @param admin_op [Integer] the RD_KAFKA_ADMIN_OP_* constant for this operation
|
|
1221
|
+
# @param destroy [Method] bound function used to free the request object
|
|
1222
|
+
# @param enqueue [Method] bound function used to enqueue the request
|
|
1223
|
+
# @return [AbstractHandle] the registered handle
|
|
1224
|
+
def enqueue_group_offsets_op(request_ptr:, handle:, admin_op:, destroy:, enqueue:)
|
|
1225
|
+
array_ptr = FFI::MemoryPointer.new(:pointer)
|
|
1226
|
+
array_ptr.write_array_of_pointer([request_ptr])
|
|
1227
|
+
|
|
1228
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
1229
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
1230
|
+
end
|
|
1231
|
+
|
|
1232
|
+
if queue_ptr.null?
|
|
1233
|
+
destroy.call(request_ptr)
|
|
1234
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
1235
|
+
end
|
|
1236
|
+
|
|
1237
|
+
handle[:pending] = true
|
|
1238
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
1239
|
+
handle.class.register(handle)
|
|
1240
|
+
|
|
1241
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
1242
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, admin_op)
|
|
1243
|
+
end
|
|
1244
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
1245
|
+
|
|
1246
|
+
begin
|
|
1247
|
+
@native_kafka.with_inner do |inner|
|
|
1248
|
+
enqueue.call(inner, array_ptr, 1, admin_options_ptr, queue_ptr)
|
|
1249
|
+
end
|
|
1250
|
+
rescue Exception
|
|
1251
|
+
handle.class.remove(handle.to_ptr.address)
|
|
1252
|
+
raise
|
|
1253
|
+
ensure
|
|
1254
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
1255
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
1256
|
+
destroy.call(request_ptr)
|
|
1257
|
+
end
|
|
1258
|
+
|
|
1259
|
+
handle
|
|
1260
|
+
end
|
|
1261
|
+
|
|
1084
1262
|
# Checks if the admin is closed and raises an error if so
|
|
1085
1263
|
# @param method [Symbol] name of the calling method for error context
|
|
1086
1264
|
# @raise [ClosedAdminError] when the admin is closed
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -161,6 +161,7 @@ module Rdkafka
|
|
|
161
161
|
|
|
162
162
|
attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
|
163
163
|
attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
|
|
164
|
+
attach_function :rd_kafka_ConfigResource_destroy, [:pointer], :void
|
|
164
165
|
attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
|
|
165
166
|
attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
|
|
166
167
|
attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
|
|
@@ -191,7 +192,7 @@ module Rdkafka
|
|
|
191
192
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
|
192
193
|
|
|
193
194
|
# List Offsets
|
|
194
|
-
RD_KAFKA_ADMIN_OP_LISTOFFSETS =
|
|
195
|
+
RD_KAFKA_ADMIN_OP_LISTOFFSETS = 21
|
|
195
196
|
RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
|
|
196
197
|
|
|
197
198
|
# rd_kafka_IsolationLevel_t
|
|
@@ -502,9 +503,31 @@ module Rdkafka
|
|
|
502
503
|
raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
|
|
503
504
|
end
|
|
504
505
|
|
|
505
|
-
public_send(method_name, topic_ptr, str_ptr, str
|
|
506
|
+
public_send(method_name, topic_ptr, str_ptr, partition_key_length(str), partition_count, nil, nil)
|
|
506
507
|
end
|
|
507
508
|
|
|
509
|
+
# Partition key length as the character count (legacy default).
|
|
510
|
+
#
|
|
511
|
+
# @param str [String] the partition key string
|
|
512
|
+
# @return [Integer]
|
|
513
|
+
def self.partition_key_size(str)
|
|
514
|
+
str.size
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
# Partition key length as the byte count, which is what librdkafka hashes (UTF-8 bytes copied
|
|
518
|
+
# into the key pointer) and what other Kafka clients use.
|
|
519
|
+
#
|
|
520
|
+
# @param str [String] the partition key string
|
|
521
|
+
# @return [Integer]
|
|
522
|
+
def self.partition_key_bytesize(str)
|
|
523
|
+
str.bytesize
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
# Length of the partition key passed to librdkafka. Aliased to one of the methods above by
|
|
527
|
+
# `Rdkafka::Config.partitioner_key_uses_bytesize=`, so the per-message path does not check the
|
|
528
|
+
# setting on every call.
|
|
529
|
+
singleton_class.alias_method :partition_key_length, :partition_key_size
|
|
530
|
+
|
|
508
531
|
# Create Topics
|
|
509
532
|
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
|
510
533
|
RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
|
|
@@ -546,6 +569,26 @@ module Rdkafka
|
|
|
546
569
|
attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
|
|
547
570
|
attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
|
|
548
571
|
|
|
572
|
+
# Alter Consumer Group Offsets
|
|
573
|
+
RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS = 15 # rd_kafka_admin_op_t
|
|
574
|
+
RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT = 0x10000 # rd_kafka_event_type_t
|
|
575
|
+
|
|
576
|
+
attach_function :rd_kafka_AlterConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
|
577
|
+
attach_function :rd_kafka_AlterConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_AlterConsumerGroupOffsets_t*
|
|
578
|
+
attach_function :rd_kafka_AlterConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
|
|
579
|
+
attach_function :rd_kafka_event_AlterConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_AlterConsumerGroupOffsets_result_t*
|
|
580
|
+
attach_function :rd_kafka_AlterConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_AlterConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
|
|
581
|
+
|
|
582
|
+
# Delete Consumer Group Offsets
|
|
583
|
+
RD_KAFKA_ADMIN_OP_DELETECONSUMERGROUPOFFSETS = 8 # rd_kafka_admin_op_t
|
|
584
|
+
RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT = 107 # rd_kafka_event_type_t
|
|
585
|
+
|
|
586
|
+
attach_function :rd_kafka_DeleteConsumerGroupOffsets, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
|
587
|
+
attach_function :rd_kafka_DeleteConsumerGroupOffsets_new, [:pointer, :pointer], :pointer, blocking: true # const char*, const rd_kafka_topic_partition_list_t* => rd_kafka_DeleteConsumerGroupOffsets_t*
|
|
588
|
+
attach_function :rd_kafka_DeleteConsumerGroupOffsets_destroy, [:pointer], :void, blocking: true
|
|
589
|
+
attach_function :rd_kafka_event_DeleteConsumerGroupOffsets_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteConsumerGroupOffsets_result_t*
|
|
590
|
+
attach_function :rd_kafka_DeleteConsumerGroupOffsets_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteConsumerGroupOffsets_result_t*, size_t* => rd_kafka_group_result_t**
|
|
591
|
+
|
|
549
592
|
# Delete Records
|
|
550
593
|
RD_KAFKA_ADMIN_OP_DELETERECORDS = 6 # rd_kafka_admin_op_t
|
|
551
594
|
RD_KAFKA_EVENT_DELETERECORDS_RESULT = 105 # rd_kafka_event_type_t
|
|
@@ -695,5 +738,6 @@ module Rdkafka
|
|
|
695
738
|
|
|
696
739
|
attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
|
|
697
740
|
attach_function :rd_kafka_group_result_name, [:pointer], :pointer
|
|
741
|
+
attach_function :rd_kafka_group_result_partitions, [:pointer], :pointer # rd_kafka_group_result_t* => rd_kafka_topic_partition_list_t*
|
|
698
742
|
end
|
|
699
743
|
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_ALTERCONSUMERGROUPOFFSETS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class AlterConsumerGroupOffsetsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the alter consumer group offsets handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
result = Rdkafka::Bindings.rd_kafka_event_AlterConsumerGroupOffsets_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
16
|
+
group_result_array = Rdkafka::Bindings.rd_kafka_AlterConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
|
|
17
|
+
group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
|
|
18
|
+
handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
19
|
+
|
|
20
|
+
if (handle = Rdkafka::Admin::AlterConsumerGroupOffsetsHandle.remove(handle_ptr.address))
|
|
21
|
+
unless resolve_operation_error(event_ptr, handle)
|
|
22
|
+
group_result = group_results[0]
|
|
23
|
+
|
|
24
|
+
# librdkafka only supports one group per invocation, so a missing group result
|
|
25
|
+
# means the broker returned nothing to act on rather than a partial success.
|
|
26
|
+
if group_result.nil?
|
|
27
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
28
|
+
handle.result = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
|
|
29
|
+
FFI::Pointer::NULL,
|
|
30
|
+
FFI::Pointer::NULL
|
|
31
|
+
)
|
|
32
|
+
else
|
|
33
|
+
report = Rdkafka::Admin::AlterConsumerGroupOffsetsReport.new(
|
|
34
|
+
group_result.error_string,
|
|
35
|
+
group_result.result_name,
|
|
36
|
+
Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
|
|
37
|
+
)
|
|
38
|
+
handle.result = report
|
|
39
|
+
|
|
40
|
+
# Kafka reports a rejected offset change per partition, leaving the group level
|
|
41
|
+
# error unset - a group with live members comes back with the group error NULL
|
|
42
|
+
# and `unknown_member_id` on each partition. Resolving on the group error alone
|
|
43
|
+
# would hand back a report that looks like success, so any failed partition has
|
|
44
|
+
# to fail the handle too.
|
|
45
|
+
failed = report.partitions.find { |partition| partition[:error] }
|
|
46
|
+
|
|
47
|
+
if !group_result.result_error.zero?
|
|
48
|
+
handle[:response] = group_result.result_error
|
|
49
|
+
handle.broker_message = report.error_string
|
|
50
|
+
elsif failed
|
|
51
|
+
handle[:response] = failed[:error].rdkafka_response
|
|
52
|
+
handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
|
|
53
|
+
else
|
|
54
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
handle.unlock
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DELETECONSUMERGROUPOFFSETS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DeleteConsumerGroupOffsetsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the delete consumer group offsets handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
result = Rdkafka::Bindings.rd_kafka_event_DeleteConsumerGroupOffsets_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
16
|
+
group_result_array = Rdkafka::Bindings.rd_kafka_DeleteConsumerGroupOffsets_result_groups(result, pointer_to_size_t)
|
|
17
|
+
group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, group_result_array)
|
|
18
|
+
handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
19
|
+
|
|
20
|
+
if (handle = Rdkafka::Admin::DeleteConsumerGroupOffsetsHandle.remove(handle_ptr.address))
|
|
21
|
+
unless resolve_operation_error(event_ptr, handle)
|
|
22
|
+
group_result = group_results[0]
|
|
23
|
+
|
|
24
|
+
# librdkafka only supports one group per invocation, so a missing group result
|
|
25
|
+
# means the broker returned nothing to act on rather than a partial success.
|
|
26
|
+
if group_result.nil?
|
|
27
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
28
|
+
handle.result = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
|
|
29
|
+
FFI::Pointer::NULL,
|
|
30
|
+
FFI::Pointer::NULL
|
|
31
|
+
)
|
|
32
|
+
else
|
|
33
|
+
report = Rdkafka::Admin::DeleteConsumerGroupOffsetsReport.new(
|
|
34
|
+
group_result.error_string,
|
|
35
|
+
group_result.result_name,
|
|
36
|
+
Rdkafka::Bindings.rd_kafka_group_result_partitions(group_result_array.read_pointer)
|
|
37
|
+
)
|
|
38
|
+
handle.result = report
|
|
39
|
+
|
|
40
|
+
# Kafka reports a rejected offset change per partition, leaving the group level
|
|
41
|
+
# error unset - a group with live members comes back with the group error NULL
|
|
42
|
+
# and `unknown_member_id` on each partition. Resolving on the group error alone
|
|
43
|
+
# would hand back a report that looks like success, so any failed partition has
|
|
44
|
+
# to fail the handle too.
|
|
45
|
+
failed = report.partitions.find { |partition| partition[:error] }
|
|
46
|
+
|
|
47
|
+
if !group_result.result_error.zero?
|
|
48
|
+
handle[:response] = group_result.result_error
|
|
49
|
+
handle.broker_message = report.error_string
|
|
50
|
+
elsif failed
|
|
51
|
+
handle[:response] = failed[:error].rdkafka_response
|
|
52
|
+
handle.broker_message = "Failed for partition #{failed[:partition]} of '#{failed[:topic]}'"
|
|
53
|
+
else
|
|
54
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
handle.unlock
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|