rdkafka 0.29.0-aarch64-linux-gnu → 0.29.2-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +41 -24
- data/README.md +2 -2
- data/bin/verify_kafka_warnings +5 -0
- data/docker-compose-ssl.yml +3 -0
- data/docker-compose.yml +3 -0
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/admin/create_acl_report.rb +2 -1
- data/lib/rdkafka/admin/delete_records_handle.rb +31 -0
- data/lib/rdkafka/admin/delete_records_report.rb +25 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +2 -1
- data/lib/rdkafka/admin/list_consumer_groups_handle.rb +31 -0
- data/lib/rdkafka/admin/list_consumer_groups_report.rb +83 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +5 -2
- data/lib/rdkafka/admin.rb +318 -199
- data/lib/rdkafka/bindings.rb +53 -12
- data/lib/rdkafka/callbacks/delete_records_handler.rb +38 -0
- data/lib/rdkafka/callbacks/list_consumer_groups_handler.rb +39 -0
- data/lib/rdkafka/callbacks.rb +4 -2
- data/lib/rdkafka/clients.rb +114 -0
- data/lib/rdkafka/config.rb +108 -56
- data/lib/rdkafka/consumer/allocator.rb +38 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +15 -10
- data/lib/rdkafka/consumer.rb +147 -74
- data/lib/rdkafka/defaults.rb +5 -7
- data/lib/rdkafka/helpers/list_offsets.rb +127 -0
- data/lib/rdkafka/helpers/oauth.rb +10 -5
- data/lib/rdkafka/native_kafka.rb +20 -7
- data/lib/rdkafka/producer/delivery_handle.rb +1 -2
- data/lib/rdkafka/producer.rb +25 -17
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +11 -0
- data/package-lock.json +6 -6
- data/renovate.json +1 -1
- metadata +11 -2
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -5,6 +5,7 @@ module Rdkafka
|
|
|
5
5
|
class Admin
|
|
6
6
|
include Helpers::OAuth
|
|
7
7
|
include Helpers::Metadata
|
|
8
|
+
include Helpers::ListOffsets
|
|
8
9
|
|
|
9
10
|
class << self
|
|
10
11
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
@@ -107,10 +108,6 @@ module Rdkafka
|
|
|
107
108
|
# @return [nil]
|
|
108
109
|
# @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
|
|
109
110
|
#
|
|
110
|
-
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
111
|
-
# Other admin operations will wait until this method returns.
|
|
112
|
-
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
113
|
-
#
|
|
114
111
|
# @example Drain all pending events
|
|
115
112
|
# admin.events_poll_nb_each { |_count| }
|
|
116
113
|
#
|
|
@@ -119,6 +116,9 @@ module Rdkafka
|
|
|
119
116
|
# admin.events_poll_nb_each do |_count|
|
|
120
117
|
# :stop if monotonic_now >= deadline
|
|
121
118
|
# end
|
|
119
|
+
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
120
|
+
# Other admin operations will wait until this method returns.
|
|
121
|
+
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
122
122
|
def events_poll_nb_each
|
|
123
123
|
closed_admin_check(__method__)
|
|
124
124
|
|
|
@@ -282,6 +282,172 @@ module Rdkafka
|
|
|
282
282
|
delete_groups_handle
|
|
283
283
|
end
|
|
284
284
|
|
|
285
|
+
# Deletes all messages in the given partitions up to (but not including) the given offset.
|
|
286
|
+
#
|
|
287
|
+
# The programmatic equivalent of `kafka-delete-records.sh`. Useful for GDPR/right-to-erasure
|
|
288
|
+
# compliance, clearing out poison messages, or retention cleanup outside of the configured
|
|
289
|
+
# time/size-based retention policy.
|
|
290
|
+
#
|
|
291
|
+
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
292
|
+
# arrays of partition delete specifications. Each specification is a hash with:
|
|
293
|
+
# - `:partition` [Integer] partition number
|
|
294
|
+
# - `:offset` [Symbol, Integer] delete all messages before this offset (exclusive) - an
|
|
295
|
+
# integer offset, or `:end` to delete all data currently in the partition
|
|
296
|
+
#
|
|
297
|
+
# @return [DeleteRecordsHandle] handle that can be used to wait for the result
|
|
298
|
+
# @raise [RdkafkaError] when deleting records fails
|
|
299
|
+
#
|
|
300
|
+
# @example Delete all messages before offset 100 in one partition, and all data in another
|
|
301
|
+
# report = admin.delete_records(
|
|
302
|
+
# "my_topic" => [
|
|
303
|
+
# { partition: 0, offset: 100 },
|
|
304
|
+
# { partition: 1, offset: :end }
|
|
305
|
+
# ]
|
|
306
|
+
# ).wait(max_wait_timeout_ms: 15_000)
|
|
307
|
+
#
|
|
308
|
+
# report.offsets.to_h
|
|
309
|
+
# # => { "my_topic" => [#<Partition @partition=0, @offset=100, @err=0>, ...] }
|
|
310
|
+
def delete_records(topic_partition_offsets)
|
|
311
|
+
closed_admin_check(__method__)
|
|
312
|
+
|
|
313
|
+
parsed = topic_partition_offsets.flat_map do |topic, partitions|
|
|
314
|
+
partitions.map do |spec|
|
|
315
|
+
offset = spec.fetch(:offset)
|
|
316
|
+
|
|
317
|
+
native_offset = case offset
|
|
318
|
+
when :end then Rdkafka::Bindings::RD_KAFKA_OFFSET_END
|
|
319
|
+
when Integer then offset
|
|
320
|
+
else
|
|
321
|
+
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
[topic, spec.fetch(:partition), native_offset]
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
|
|
329
|
+
|
|
330
|
+
parsed.each do |topic, partition, native_offset|
|
|
331
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
332
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# rd_kafka_DeleteRecords_new copies the tpl it is given, so it does not need to (and must
|
|
336
|
+
# not) outlive this call - it is destroyed below, independently of the wrapping
|
|
337
|
+
# DeleteRecords_t object.
|
|
338
|
+
delete_records_ptr = Rdkafka::Bindings.rd_kafka_DeleteRecords_new(tpl)
|
|
339
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
340
|
+
|
|
341
|
+
pointer_array = [delete_records_ptr]
|
|
342
|
+
records_array_ptr = FFI::MemoryPointer.new(:pointer)
|
|
343
|
+
records_array_ptr.write_array_of_pointer(pointer_array)
|
|
344
|
+
|
|
345
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
346
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
347
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
348
|
+
end
|
|
349
|
+
if queue_ptr.null?
|
|
350
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
|
|
351
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Create and register the handle we will return to the caller
|
|
355
|
+
handle = DeleteRecordsHandle.new
|
|
356
|
+
handle[:pending] = true
|
|
357
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
358
|
+
DeleteRecordsHandle.register(handle)
|
|
359
|
+
|
|
360
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
361
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETERECORDS)
|
|
362
|
+
end
|
|
363
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
364
|
+
|
|
365
|
+
begin
|
|
366
|
+
@native_kafka.with_inner do |inner|
|
|
367
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords(
|
|
368
|
+
inner,
|
|
369
|
+
records_array_ptr,
|
|
370
|
+
1,
|
|
371
|
+
admin_options_ptr,
|
|
372
|
+
queue_ptr
|
|
373
|
+
)
|
|
374
|
+
end
|
|
375
|
+
rescue Exception
|
|
376
|
+
DeleteRecordsHandle.remove(handle.to_ptr.address)
|
|
377
|
+
raise
|
|
378
|
+
ensure
|
|
379
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
380
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
381
|
+
Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
handle
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Lists consumer groups cluster-wide.
|
|
388
|
+
#
|
|
389
|
+
# librdkafka issues a single `ListConsumerGroups` request that is fanned out to every broker
|
|
390
|
+
# internally, so the result covers all consumer groups in the cluster, not only those
|
|
391
|
+
# coordinated by the connected broker.
|
|
392
|
+
#
|
|
393
|
+
# @return [ListConsumerGroupsHandle] handle that can be used to wait for the result
|
|
394
|
+
# @raise [RdkafkaError] when listing the consumer groups fails
|
|
395
|
+
#
|
|
396
|
+
# @example List every consumer group in the cluster and print its name and attributes
|
|
397
|
+
# report = admin.list_consumer_groups.wait(max_wait_timeout_ms: 15_000)
|
|
398
|
+
#
|
|
399
|
+
# report.groups.each do |group|
|
|
400
|
+
# puts "#{group[:group_id]} - #{group[:state_name]} " \
|
|
401
|
+
# "(simple: #{group[:is_simple_consumer_group]})"
|
|
402
|
+
# end
|
|
403
|
+
#
|
|
404
|
+
# # Any brokers that could not be reached are reported separately
|
|
405
|
+
# report.errors.each { |error| warn "partial listing error: #{error.message}" }
|
|
406
|
+
def list_consumer_groups
|
|
407
|
+
closed_admin_check(__method__)
|
|
408
|
+
|
|
409
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
410
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
411
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
if queue_ptr.null?
|
|
415
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Create and register the handle we will return to the caller
|
|
419
|
+
handle = ListConsumerGroupsHandle.new
|
|
420
|
+
handle[:pending] = true
|
|
421
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
422
|
+
ListConsumerGroupsHandle.register(handle)
|
|
423
|
+
|
|
424
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
425
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
426
|
+
inner,
|
|
427
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS
|
|
428
|
+
)
|
|
429
|
+
end
|
|
430
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
431
|
+
|
|
432
|
+
begin
|
|
433
|
+
@native_kafka.with_inner do |inner|
|
|
434
|
+
Rdkafka::Bindings.rd_kafka_ListConsumerGroups(
|
|
435
|
+
inner,
|
|
436
|
+
admin_options_ptr,
|
|
437
|
+
queue_ptr
|
|
438
|
+
)
|
|
439
|
+
end
|
|
440
|
+
rescue Exception
|
|
441
|
+
ListConsumerGroupsHandle.remove(handle.to_ptr.address)
|
|
442
|
+
raise
|
|
443
|
+
ensure
|
|
444
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
445
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
handle
|
|
449
|
+
end
|
|
450
|
+
|
|
285
451
|
# Deletes the named topic
|
|
286
452
|
#
|
|
287
453
|
# @param topic_name [String] name of the topic to delete
|
|
@@ -344,7 +510,8 @@ module Rdkafka
|
|
|
344
510
|
#
|
|
345
511
|
# @param topic_name [String] name of the topic
|
|
346
512
|
# @param partition_count [Integer] how many partitions we want to end up with for given topic
|
|
347
|
-
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
|
|
513
|
+
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
|
|
514
|
+
# result
|
|
348
515
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
349
516
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
350
517
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
@@ -433,7 +600,8 @@ module Rdkafka
|
|
|
433
600
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
434
601
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
435
602
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
436
|
-
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of
|
|
603
|
+
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of
|
|
604
|
+
# creating the acl
|
|
437
605
|
# @raise [RdkafkaError]
|
|
438
606
|
def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
439
607
|
closed_admin_check(__method__)
|
|
@@ -534,7 +702,8 @@ module Rdkafka
|
|
|
534
702
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
535
703
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
536
704
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
537
|
-
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of
|
|
705
|
+
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of
|
|
706
|
+
# deleting the acl
|
|
538
707
|
# @raise [RdkafkaError]
|
|
539
708
|
def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
540
709
|
closed_admin_check(__method__)
|
|
@@ -637,7 +806,8 @@ module Rdkafka
|
|
|
637
806
|
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
638
807
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
639
808
|
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
640
|
-
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of
|
|
809
|
+
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of
|
|
810
|
+
# fetching acls
|
|
641
811
|
# @raise [RdkafkaError]
|
|
642
812
|
def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
643
813
|
closed_admin_check(__method__)
|
|
@@ -728,35 +898,64 @@ module Rdkafka
|
|
|
728
898
|
handle[:pending] = true
|
|
729
899
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
730
900
|
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
901
|
+
# All native allocation happens inside the begin so a raise at any point (a null background
|
|
902
|
+
# queue, a non-String resource name making `from_string` raise mid-build, etc.) is cleaned up
|
|
903
|
+
# by the ensure rather than leaking the queue, the AdminOptions, the handle and the
|
|
904
|
+
# ConfigResources already built.
|
|
905
|
+
queue_ptr = nil
|
|
906
|
+
admin_options_ptr = nil
|
|
907
|
+
pointer_array = []
|
|
908
|
+
registered = false
|
|
734
909
|
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
910
|
+
begin
|
|
911
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
912
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
913
|
+
end
|
|
738
914
|
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
|
743
|
-
)
|
|
744
|
-
end
|
|
915
|
+
if queue_ptr.null?
|
|
916
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
917
|
+
end
|
|
745
918
|
|
|
746
|
-
|
|
747
|
-
|
|
919
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
920
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
921
|
+
inner,
|
|
922
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
|
923
|
+
)
|
|
924
|
+
end
|
|
748
925
|
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
end
|
|
926
|
+
# A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
|
|
927
|
+
# before registering the handle (the ensure frees the queue already acquired).
|
|
928
|
+
if admin_options_ptr.null?
|
|
929
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
|
|
930
|
+
end
|
|
755
931
|
|
|
756
|
-
|
|
757
|
-
|
|
932
|
+
DescribeConfigsHandle.register(handle)
|
|
933
|
+
registered = true
|
|
934
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
935
|
+
|
|
936
|
+
# Build resources one at a time so a raise on a later element still leaves the earlier ones
|
|
937
|
+
# in pointer_array for the ensure to destroy.
|
|
938
|
+
parsed_resources.each do |resource_type, resource_name|
|
|
939
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
940
|
+
resource_type,
|
|
941
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
942
|
+
)
|
|
943
|
+
|
|
944
|
+
# librdkafka returns NULL for an empty resource name or a negative resource type. Passing
|
|
945
|
+
# NULL on to DescribeConfigs and then destroying it both dereference NULL and segfault, so
|
|
946
|
+
# reject it here (the ensure frees everything already built).
|
|
947
|
+
if resource_ptr.null?
|
|
948
|
+
raise Rdkafka::Config::ConfigError.new(
|
|
949
|
+
"rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
|
|
950
|
+
)
|
|
951
|
+
end
|
|
952
|
+
|
|
953
|
+
pointer_array << resource_ptr
|
|
954
|
+
end
|
|
955
|
+
|
|
956
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
957
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
758
958
|
|
|
759
|
-
begin
|
|
760
959
|
@native_kafka.with_inner do |inner|
|
|
761
960
|
Rdkafka::Bindings.rd_kafka_DescribeConfigs(
|
|
762
961
|
inner,
|
|
@@ -767,18 +966,20 @@ module Rdkafka
|
|
|
767
966
|
)
|
|
768
967
|
end
|
|
769
968
|
rescue Exception
|
|
770
|
-
DescribeConfigsHandle.remove(handle.to_ptr.address)
|
|
969
|
+
DescribeConfigsHandle.remove(handle.to_ptr.address) if registered
|
|
771
970
|
|
|
772
971
|
raise
|
|
773
972
|
ensure
|
|
774
|
-
|
|
775
|
-
|
|
973
|
+
if admin_options_ptr && !admin_options_ptr.null?
|
|
974
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
975
|
+
end
|
|
776
976
|
|
|
777
|
-
if
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
977
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
|
|
978
|
+
|
|
979
|
+
pointer_array.each do |config_resource_ptr|
|
|
980
|
+
next if config_resource_ptr.null?
|
|
981
|
+
|
|
982
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
|
|
782
983
|
end
|
|
783
984
|
end
|
|
784
985
|
|
|
@@ -817,66 +1018,91 @@ module Rdkafka
|
|
|
817
1018
|
handle[:pending] = true
|
|
818
1019
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
819
1020
|
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
1021
|
+
# All native allocation happens inside the begin so a raise at any point (a null background
|
|
1022
|
+
# queue, a non-String resource name, or FFI marshaling of a config name/value failing while
|
|
1023
|
+
# adding configs) is cleaned up by the ensure rather than leaking the queue, the AdminOptions,
|
|
1024
|
+
# the handle and the ConfigResources already built.
|
|
1025
|
+
queue_ptr = nil
|
|
1026
|
+
admin_options_ptr = nil
|
|
1027
|
+
pointer_array = []
|
|
1028
|
+
registered = false
|
|
1029
|
+
add_error = nil
|
|
823
1030
|
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
1031
|
+
begin
|
|
1032
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
1033
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
1034
|
+
end
|
|
827
1035
|
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
|
832
|
-
)
|
|
833
|
-
end
|
|
1036
|
+
if queue_ptr.null?
|
|
1037
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
1038
|
+
end
|
|
834
1039
|
|
|
835
|
-
|
|
836
|
-
|
|
1040
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
1041
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
1042
|
+
inner,
|
|
1043
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
|
1044
|
+
)
|
|
1045
|
+
end
|
|
837
1046
|
|
|
838
|
-
|
|
1047
|
+
# A NULL AdminOptions would segfault the moment we set its opaque pointer, so reject it
|
|
1048
|
+
# before registering the handle (the ensure frees the queue already acquired).
|
|
1049
|
+
if admin_options_ptr.null?
|
|
1050
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_AdminOptions_new was NULL")
|
|
1051
|
+
end
|
|
839
1052
|
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
resource_type,
|
|
844
|
-
FFI::MemoryPointer.from_string(resource_name)
|
|
845
|
-
)
|
|
1053
|
+
IncrementalAlterConfigsHandle.register(handle)
|
|
1054
|
+
registered = true
|
|
1055
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
846
1056
|
|
|
847
|
-
|
|
848
|
-
#
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
# native error object) and raise it below, before the request is sent.
|
|
853
|
-
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
854
|
-
resource_ptr,
|
|
855
|
-
name,
|
|
856
|
-
op_type,
|
|
857
|
-
value
|
|
1057
|
+
parsed_resources.each do |resource_type, resource_name, configs|
|
|
1058
|
+
# First build the appropriate resource representation
|
|
1059
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
1060
|
+
resource_type,
|
|
1061
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
858
1062
|
)
|
|
859
1063
|
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1064
|
+
# librdkafka returns NULL for an empty resource name or a negative resource type. Passing
|
|
1065
|
+
# NULL on to add_incremental_config/IncrementalAlterConfigs and then destroying it both
|
|
1066
|
+
# dereference NULL and segfault, so reject it here (the ensure frees everything built).
|
|
1067
|
+
if resource_ptr.null?
|
|
1068
|
+
raise Rdkafka::Config::ConfigError.new(
|
|
1069
|
+
"rd_kafka_ConfigResource_new was NULL for #{resource_name.inspect} (type #{resource_type})"
|
|
1070
|
+
)
|
|
1071
|
+
end
|
|
863
1072
|
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
1073
|
+
# Track it immediately so a raise while adding its configs still frees it via the ensure.
|
|
1074
|
+
pointer_array << resource_ptr
|
|
1075
|
+
|
|
1076
|
+
configs.each do |name, op_type, value|
|
|
1077
|
+
# rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
|
|
1078
|
+
# an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
|
|
1079
|
+
# used to be ignored: the entry was silently dropped, the alter request still reported
|
|
1080
|
+
# success, and the error object leaked. Capture the first error (always destroying the
|
|
1081
|
+
# native error object) and raise it below, before the request is sent.
|
|
1082
|
+
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
1083
|
+
resource_ptr,
|
|
1084
|
+
name,
|
|
1085
|
+
op_type,
|
|
1086
|
+
value
|
|
1087
|
+
)
|
|
1088
|
+
|
|
1089
|
+
unless error_ptr.null?
|
|
1090
|
+
code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
|
|
1091
|
+
Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
|
|
1092
|
+
|
|
1093
|
+
unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
1094
|
+
add_error ||= Rdkafka::RdkafkaError.new(
|
|
1095
|
+
code,
|
|
1096
|
+
"rd_kafka_ConfigResource_add_incremental_config"
|
|
1097
|
+
)
|
|
1098
|
+
end
|
|
869
1099
|
end
|
|
870
1100
|
end
|
|
871
1101
|
end
|
|
872
1102
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
877
|
-
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
1103
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
|
1104
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
878
1105
|
|
|
879
|
-
begin
|
|
880
1106
|
# Raise only after the full array is built so the ensure below frees every ConfigResource
|
|
881
1107
|
# we created, and before the request is sent so a rejected entry can't report success.
|
|
882
1108
|
raise add_error if add_error
|
|
@@ -891,128 +1117,21 @@ module Rdkafka
|
|
|
891
1117
|
)
|
|
892
1118
|
end
|
|
893
1119
|
rescue Exception
|
|
894
|
-
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
|
|
1120
|
+
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address) if registered
|
|
895
1121
|
|
|
896
1122
|
raise
|
|
897
1123
|
ensure
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
if configs_array_ptr
|
|
902
|
-
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
|
903
|
-
configs_array_ptr,
|
|
904
|
-
pointer_array.size
|
|
905
|
-
)
|
|
906
|
-
end
|
|
907
|
-
end
|
|
908
|
-
|
|
909
|
-
handle
|
|
910
|
-
end
|
|
911
|
-
|
|
912
|
-
# Queries partition offsets by specification (earliest, latest, max_timestamp, or by
|
|
913
|
-
# timestamp) without requiring a consumer group.
|
|
914
|
-
#
|
|
915
|
-
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
916
|
-
# arrays of partition offset specifications. Each specification is a hash with:
|
|
917
|
-
# - `:partition` [Integer] partition number
|
|
918
|
-
# - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
|
|
919
|
-
# `:max_timestamp`, or an integer timestamp in milliseconds
|
|
920
|
-
# @param isolation_level [Integer, nil] optional isolation level:
|
|
921
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
|
|
922
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
|
|
923
|
-
#
|
|
924
|
-
# @return [ListOffsetsHandle] handle that can be used to wait for the result
|
|
925
|
-
#
|
|
926
|
-
# @raise [ClosedAdminError] when the admin is closed
|
|
927
|
-
# @raise [ConfigError] when the background queue is unavailable
|
|
928
|
-
#
|
|
929
|
-
# @example Query earliest and latest offsets
|
|
930
|
-
# handle = admin.list_offsets(
|
|
931
|
-
# { "my_topic" => [
|
|
932
|
-
# { partition: 0, offset: :earliest },
|
|
933
|
-
# { partition: 1, offset: :latest }
|
|
934
|
-
# ] }
|
|
935
|
-
# )
|
|
936
|
-
# report = handle.wait(max_wait_timeout_ms: 15_000)
|
|
937
|
-
# report.offsets
|
|
938
|
-
# # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
|
|
939
|
-
def list_offsets(topic_partition_offsets, isolation_level: nil)
|
|
940
|
-
closed_admin_check(__method__)
|
|
941
|
-
|
|
942
|
-
# Parse and validate every offset spec before allocating the native list, so a missing key or
|
|
943
|
-
# an unknown offset specification raises with nothing to clean up. Previously the
|
|
944
|
-
# ArgumentError (or KeyError) was raised after `rd_kafka_topic_partition_list_new`, leaking
|
|
945
|
-
# the native list.
|
|
946
|
-
parsed = topic_partition_offsets.flat_map do |topic, partitions|
|
|
947
|
-
partitions.map do |spec|
|
|
948
|
-
offset = spec.fetch(:offset)
|
|
949
|
-
|
|
950
|
-
native_offset = case offset
|
|
951
|
-
when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
|
|
952
|
-
when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
|
|
953
|
-
when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
|
|
954
|
-
when Integer then offset
|
|
955
|
-
else
|
|
956
|
-
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
957
|
-
end
|
|
958
|
-
|
|
959
|
-
[topic, spec.fetch(:partition), native_offset]
|
|
1124
|
+
if admin_options_ptr && !admin_options_ptr.null?
|
|
1125
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
960
1126
|
end
|
|
961
|
-
end
|
|
962
|
-
|
|
963
|
-
# Build native topic partition list
|
|
964
|
-
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
|
|
965
|
-
|
|
966
|
-
parsed.each do |topic, partition, native_offset|
|
|
967
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
968
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
969
|
-
end
|
|
970
1127
|
|
|
971
|
-
|
|
972
|
-
queue_ptr = @native_kafka.with_inner do |inner|
|
|
973
|
-
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
974
|
-
end
|
|
975
|
-
|
|
976
|
-
if queue_ptr.null?
|
|
977
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
978
|
-
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
979
|
-
end
|
|
980
|
-
|
|
981
|
-
# Create and register the handle we will return to the caller
|
|
982
|
-
handle = ListOffsetsHandle.new
|
|
983
|
-
handle[:pending] = true
|
|
984
|
-
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
985
|
-
|
|
986
|
-
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
987
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
988
|
-
inner,
|
|
989
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
|
|
990
|
-
)
|
|
991
|
-
end
|
|
992
|
-
|
|
993
|
-
if isolation_level
|
|
994
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
|
|
995
|
-
end
|
|
1128
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr) if queue_ptr && !queue_ptr.null?
|
|
996
1129
|
|
|
997
|
-
|
|
998
|
-
|
|
1130
|
+
pointer_array.each do |config_resource_ptr|
|
|
1131
|
+
next if config_resource_ptr.null?
|
|
999
1132
|
|
|
1000
|
-
|
|
1001
|
-
@native_kafka.with_inner do |inner|
|
|
1002
|
-
Rdkafka::Bindings.rd_kafka_ListOffsets(
|
|
1003
|
-
inner,
|
|
1004
|
-
tpl,
|
|
1005
|
-
admin_options_ptr,
|
|
1006
|
-
queue_ptr
|
|
1007
|
-
)
|
|
1133
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy(config_resource_ptr)
|
|
1008
1134
|
end
|
|
1009
|
-
rescue Exception
|
|
1010
|
-
ListOffsetsHandle.remove(handle.to_ptr.address)
|
|
1011
|
-
raise
|
|
1012
|
-
ensure
|
|
1013
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
1014
|
-
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
1015
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
1016
1135
|
end
|
|
1017
1136
|
|
|
1018
1137
|
handle
|