rdkafka 0.28.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +42 -0
  3. data/README.md +3 -2
  4. data/bin/verify_kafka_warnings +5 -0
  5. data/docker-compose-ssl.yml +4 -1
  6. data/docker-compose.yml +4 -1
  7. data/ext/librdkafka.so +0 -0
  8. data/lib/rdkafka/abstract_handle.rb +31 -2
  9. data/lib/rdkafka/admin/config_binding_result.rb +2 -2
  10. data/lib/rdkafka/admin/create_acl_handle.rb +6 -5
  11. data/lib/rdkafka/admin/create_acl_report.rb +2 -1
  12. data/lib/rdkafka/admin/create_partitions_handle.rb +4 -6
  13. data/lib/rdkafka/admin/create_topic_handle.rb +4 -6
  14. data/lib/rdkafka/admin/delete_acl_handle.rb +5 -7
  15. data/lib/rdkafka/admin/delete_groups_handle.rb +4 -6
  16. data/lib/rdkafka/admin/delete_records_handle.rb +31 -0
  17. data/lib/rdkafka/admin/delete_records_report.rb +25 -0
  18. data/lib/rdkafka/admin/delete_topic_handle.rb +4 -6
  19. data/lib/rdkafka/admin/describe_acl_handle.rb +5 -7
  20. data/lib/rdkafka/admin/describe_acl_report.rb +2 -1
  21. data/lib/rdkafka/admin/describe_configs_handle.rb +4 -10
  22. data/lib/rdkafka/admin/describe_configs_report.rb +1 -5
  23. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +4 -10
  24. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +1 -5
  25. data/lib/rdkafka/admin/list_consumer_groups_handle.rb +31 -0
  26. data/lib/rdkafka/admin/list_consumer_groups_report.rb +83 -0
  27. data/lib/rdkafka/admin/list_offsets_handle.rb +5 -10
  28. data/lib/rdkafka/admin/list_offsets_report.rb +5 -2
  29. data/lib/rdkafka/admin.rb +238 -145
  30. data/lib/rdkafka/bindings.rb +61 -16
  31. data/lib/rdkafka/callbacks/base_handler.rb +62 -0
  32. data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
  33. data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
  34. data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
  35. data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
  36. data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
  37. data/lib/rdkafka/callbacks/delete_records_handler.rb +38 -0
  38. data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
  39. data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
  40. data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
  41. data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
  42. data/lib/rdkafka/callbacks/list_consumer_groups_handler.rb +39 -0
  43. data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
  44. data/lib/rdkafka/callbacks.rb +54 -246
  45. data/lib/rdkafka/config.rb +69 -50
  46. data/lib/rdkafka/consumer/headers.rb +19 -5
  47. data/lib/rdkafka/consumer/partition.rb +8 -1
  48. data/lib/rdkafka/consumer/topic_partition_list.rb +58 -30
  49. data/lib/rdkafka/consumer.rb +211 -52
  50. data/lib/rdkafka/defaults.rb +24 -7
  51. data/lib/rdkafka/helpers/list_offsets.rb +127 -0
  52. data/lib/rdkafka/helpers/metadata.rb +29 -0
  53. data/lib/rdkafka/helpers/oauth.rb +10 -5
  54. data/lib/rdkafka/metadata.rb +86 -19
  55. data/lib/rdkafka/native_kafka.rb +22 -11
  56. data/lib/rdkafka/producer/delivery_handle.rb +4 -5
  57. data/lib/rdkafka/producer/partitions_count_cache.rb +24 -38
  58. data/lib/rdkafka/producer.rb +81 -61
  59. data/lib/rdkafka/version.rb +3 -3
  60. data/lib/rdkafka.rb +19 -0
  61. data/package-lock.json +3 -3
  62. data/rdkafka.gemspec +1 -0
  63. metadata +21 -5
  64. data/Gemfile +0 -13
  65. data/Gemfile.lint +0 -14
  66. data/Gemfile.lint.lock +0 -123
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ # Handle for list consumer groups operation
6
+ class ListConsumerGroupsHandle < AbstractHandle
7
+ layout :pending, :bool,
8
+ :response, :int
9
+
10
+ # @return [String] the name of the operation
11
+ def operation_name
12
+ "list consumer groups"
13
+ end
14
+
15
+ # @return [ListConsumerGroupsReport] report prepared by the background event callback, with
16
+ # the listed consumer groups.
17
+ def create_result
18
+ prepared_result
19
+ end
20
+
21
+ # Raises an error if the operation failed
22
+ # @raise [RdkafkaError]
23
+ def raise_error
24
+ raise RdkafkaError.new(
25
+ self[:response],
26
+ broker_message: broker_message
27
+ )
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ # Report for list consumer groups operation result
6
+ class ListConsumerGroupsReport
7
+ # Consumer groups listed cluster-wide. Each entry is a hash with:
8
+ # - `:group_id` [String] the consumer group id
9
+ # - `:is_simple_consumer_group` [Boolean] `true` for a "simple" consumer group - one that
10
+ # assigns partitions manually (via `assign`) and uses Kafka only for offset storage,
11
+ # rather than joining the group-management protocol and letting Kafka assign partitions
12
+ # and rebalance automatically (`subscribe`). Simple groups have no members from the
13
+ # broker's point of view, so they never rebalance.
14
+ # - `:state` [Integer] the group state as a `Bindings::RD_KAFKA_CONSUMER_GROUP_STATE_*`
15
+ # code, one of: `RD_KAFKA_CONSUMER_GROUP_STATE_UNKNOWN`,
16
+ # `RD_KAFKA_CONSUMER_GROUP_STATE_PREPARING_REBALANCE`,
17
+ # `RD_KAFKA_CONSUMER_GROUP_STATE_COMPLETING_REBALANCE`,
18
+ # `RD_KAFKA_CONSUMER_GROUP_STATE_STABLE`, `RD_KAFKA_CONSUMER_GROUP_STATE_DEAD`,
19
+ # `RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY`
20
+ # - `:state_name` [String] human-readable name of that state (e.g. `"Stable"`, `"Empty"`)
21
+ # @return [Array<Hash>]
22
+ attr_reader :groups
23
+
24
+ # Per-broker errors reported alongside the (partial) group listing. `ListConsumerGroups`
25
+ # fans out to every broker and returns valid groups and errors separately, so a broker
26
+ # being unreachable does not discard the groups the reachable brokers returned.
27
+ # @return [Array<RdkafkaError>]
28
+ attr_reader :errors
29
+
30
+ # @param result_ptr [FFI::Pointer] pointer to the `rd_kafka_ListConsumerGroups_result_t`
31
+ def initialize(result_ptr)
32
+ @groups = []
33
+ @errors = []
34
+
35
+ return if result_ptr.null?
36
+
37
+ extract_groups(result_ptr)
38
+ extract_errors(result_ptr)
39
+ end
40
+
41
+ private
42
+
43
+ # @param result_ptr [FFI::Pointer] pointer to the result
44
+ def extract_groups(result_ptr)
45
+ count_ptr = FFI::MemoryPointer.new(:size_t)
46
+ array_ptr = Bindings.rd_kafka_ListConsumerGroups_result_valid(result_ptr, count_ptr)
47
+
48
+ return if array_ptr.null?
49
+
50
+ array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |listing_ptr|
51
+ state = Bindings.rd_kafka_ConsumerGroupListing_state(listing_ptr)
52
+ group_id_ptr = Bindings.rd_kafka_ConsumerGroupListing_group_id(listing_ptr)
53
+ state_name_ptr = Bindings.rd_kafka_consumer_group_state_name(state)
54
+
55
+ @groups << {
56
+ group_id: group_id_ptr.null? ? nil : group_id_ptr.read_string,
57
+ is_simple_consumer_group:
58
+ Bindings.rd_kafka_ConsumerGroupListing_is_simple_consumer_group(listing_ptr) != 0,
59
+ state: state,
60
+ state_name: state_name_ptr.null? ? nil : state_name_ptr.read_string
61
+ }
62
+ end
63
+ end
64
+
65
+ # @param result_ptr [FFI::Pointer] pointer to the result
66
+ def extract_errors(result_ptr)
67
+ count_ptr = FFI::MemoryPointer.new(:size_t)
68
+ array_ptr = Bindings.rd_kafka_ListConsumerGroups_result_errors(result_ptr, count_ptr)
69
+
70
+ return if array_ptr.null?
71
+
72
+ array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |error_ptr|
73
+ string_ptr = Bindings.rd_kafka_error_string(error_ptr)
74
+
75
+ @errors << RdkafkaError.new(
76
+ Bindings.rd_kafka_error_code(error_ptr),
77
+ broker_message: string_ptr.null? ? nil : string_ptr.read_string
78
+ )
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -5,22 +5,17 @@ module Rdkafka
5
5
  # Handle for list offsets operation
6
6
  class ListOffsetsHandle < AbstractHandle
7
7
  layout :pending, :bool,
8
- :response, :int,
9
- :response_string, :pointer,
10
- :result_infos, :pointer,
11
- :result_count, :int
8
+ :response, :int
12
9
 
13
10
  # @return [String] the name of the operation.
14
11
  def operation_name
15
12
  "list offsets"
16
13
  end
17
14
 
18
- # @return [ListOffsetsReport] instance with partition offset information.
15
+ # @return [ListOffsetsReport] report prepared by the background event callback, with
16
+ # partition offset information.
19
17
  def create_result
20
- ListOffsetsReport.new(
21
- result_infos: self[:result_infos],
22
- result_count: self[:result_count]
23
- )
18
+ prepared_result
24
19
  end
25
20
 
26
21
  # Raises an error if the operation failed
@@ -28,7 +23,7 @@ module Rdkafka
28
23
  def raise_error
29
24
  raise RdkafkaError.new(
30
25
  self[:response],
31
- broker_message: self[:response_string].read_string
26
+ broker_message: broker_message
32
27
  )
33
28
  end
34
29
  end
@@ -36,7 +36,7 @@ module Rdkafka
36
36
 
37
37
  # Validates the partition result and raises an error if invalid
38
38
  # @param result_info_ptr [FFI::Pointer] pointer to the result info
39
- # @raise [RdkafkaError] when the partition has an error
39
+ # @raise [RdkafkaError] when the partition has an error, naming the failing partition
40
40
  def validate!(result_info_ptr)
41
41
  tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
42
42
  tp = Bindings::TopicPartition.new(tp_ptr)
@@ -44,7 +44,10 @@ module Rdkafka
44
44
 
45
45
  return if code.zero?
46
46
 
47
- raise RdkafkaError.new(code)
47
+ raise RdkafkaError.new(
48
+ code,
49
+ "Error querying offsets for partition #{tp[:partition]} of '#{tp[:topic]}'"
50
+ )
48
51
  end
49
52
  end
50
53
  end
data/lib/rdkafka/admin.rb CHANGED
@@ -4,6 +4,8 @@ module Rdkafka
4
4
  # Admin client for Kafka administrative operations
5
5
  class Admin
6
6
  include Helpers::OAuth
7
+ include Helpers::Metadata
8
+ include Helpers::ListOffsets
7
9
 
8
10
  class << self
9
11
  # Allows us to retrieve librdkafka errors with descriptions
@@ -106,10 +108,6 @@ module Rdkafka
106
108
  # @return [nil]
107
109
  # @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
108
110
  #
109
- # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
110
- # Other admin operations will wait until this method returns.
111
- # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
112
- #
113
111
  # @example Drain all pending events
114
112
  # admin.events_poll_nb_each { |_count| }
115
113
  #
@@ -118,6 +116,9 @@ module Rdkafka
118
116
  # admin.events_poll_nb_each do |_count|
119
117
  # :stop if monotonic_now >= deadline
120
118
  # end
119
+ # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
120
+ # Other admin operations will wait until this method returns.
121
+ # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
121
122
  def events_poll_nb_each
122
123
  closed_admin_check(__method__)
123
124
 
@@ -130,19 +131,6 @@ module Rdkafka
130
131
  end
131
132
  end
132
133
 
133
- # Performs the metadata request using admin
134
- #
135
- # @param topic_name [String, nil] metadat about particular topic or all if nil
136
- # @param timeout_ms [Integer] metadata request timeout
137
- # @return [Metadata] requested metadata
138
- def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
139
- closed_admin_check(__method__)
140
-
141
- @native_kafka.with_inner do |inner|
142
- Metadata.new(inner, topic_name, timeout_ms)
143
- end
144
- end
145
-
146
134
  # Close this admin instance
147
135
  def close
148
136
  return if closed?
@@ -258,7 +246,7 @@ module Rdkafka
258
246
  Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
259
247
  end
260
248
  if queue_ptr.null?
261
- Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
249
+ Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
262
250
  raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
263
251
  end
264
252
 
@@ -294,6 +282,172 @@ module Rdkafka
294
282
  delete_groups_handle
295
283
  end
296
284
 
285
+ # Deletes all messages in the given partitions up to (but not including) the given offset.
286
+ #
287
+ # The programmatic equivalent of `kafka-delete-records.sh`. Useful for GDPR/right-to-erasure
288
+ # compliance, clearing out poison messages, or retention cleanup outside of the configured
289
+ # time/size-based retention policy.
290
+ #
291
+ # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
292
+ # arrays of partition delete specifications. Each specification is a hash with:
293
+ # - `:partition` [Integer] partition number
294
+ # - `:offset` [Symbol, Integer] delete all messages before this offset (exclusive) - an
295
+ # integer offset, or `:end` to delete all data currently in the partition
296
+ #
297
+ # @return [DeleteRecordsHandle] handle that can be used to wait for the result
298
+ # @raise [RdkafkaError] when deleting records fails
299
+ #
300
+ # @example Delete all messages before offset 100 in one partition, and all data in another
301
+ # report = admin.delete_records(
302
+ # "my_topic" => [
303
+ # { partition: 0, offset: 100 },
304
+ # { partition: 1, offset: :end }
305
+ # ]
306
+ # ).wait(max_wait_timeout_ms: 15_000)
307
+ #
308
+ # report.offsets.to_h
309
+ # # => { "my_topic" => [#<Partition @partition=0, @offset=100, @err=0>, ...] }
310
+ def delete_records(topic_partition_offsets)
311
+ closed_admin_check(__method__)
312
+
313
+ parsed = topic_partition_offsets.flat_map do |topic, partitions|
314
+ partitions.map do |spec|
315
+ offset = spec.fetch(:offset)
316
+
317
+ native_offset = case offset
318
+ when :end then Rdkafka::Bindings::RD_KAFKA_OFFSET_END
319
+ when Integer then offset
320
+ else
321
+ raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
322
+ end
323
+
324
+ [topic, spec.fetch(:partition), native_offset]
325
+ end
326
+ end
327
+
328
+ tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(parsed.size)
329
+
330
+ parsed.each do |topic, partition, native_offset|
331
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
332
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
333
+ end
334
+
335
+ # rd_kafka_DeleteRecords_new copies the tpl it is given, so it does not need to (and must
336
+ # not) outlive this call - it is destroyed below, independently of the wrapping
337
+ # DeleteRecords_t object.
338
+ delete_records_ptr = Rdkafka::Bindings.rd_kafka_DeleteRecords_new(tpl)
339
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
340
+
341
+ pointer_array = [delete_records_ptr]
342
+ records_array_ptr = FFI::MemoryPointer.new(:pointer)
343
+ records_array_ptr.write_array_of_pointer(pointer_array)
344
+
345
+ # Get a pointer to the queue that our request will be enqueued on
346
+ queue_ptr = @native_kafka.with_inner do |inner|
347
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
348
+ end
349
+ if queue_ptr.null?
350
+ Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
351
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
352
+ end
353
+
354
+ # Create and register the handle we will return to the caller
355
+ handle = DeleteRecordsHandle.new
356
+ handle[:pending] = true
357
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
358
+ DeleteRecordsHandle.register(handle)
359
+
360
+ admin_options_ptr = @native_kafka.with_inner do |inner|
361
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETERECORDS)
362
+ end
363
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
364
+
365
+ begin
366
+ @native_kafka.with_inner do |inner|
367
+ Rdkafka::Bindings.rd_kafka_DeleteRecords(
368
+ inner,
369
+ records_array_ptr,
370
+ 1,
371
+ admin_options_ptr,
372
+ queue_ptr
373
+ )
374
+ end
375
+ rescue Exception
376
+ DeleteRecordsHandle.remove(handle.to_ptr.address)
377
+ raise
378
+ ensure
379
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
380
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
381
+ Rdkafka::Bindings.rd_kafka_DeleteRecords_destroy(delete_records_ptr)
382
+ end
383
+
384
+ handle
385
+ end
386
+
387
+ # Lists consumer groups cluster-wide.
388
+ #
389
+ # librdkafka issues a single `ListConsumerGroups` request that is fanned out to every broker
390
+ # internally, so the result covers all consumer groups in the cluster, not only those
391
+ # coordinated by the connected broker.
392
+ #
393
+ # @return [ListConsumerGroupsHandle] handle that can be used to wait for the result
394
+ # @raise [RdkafkaError] when listing the consumer groups fails
395
+ #
396
+ # @example List every consumer group in the cluster and print its name and attributes
397
+ # report = admin.list_consumer_groups.wait(max_wait_timeout_ms: 15_000)
398
+ #
399
+ # report.groups.each do |group|
400
+ # puts "#{group[:group_id]} - #{group[:state_name]} " \
401
+ # "(simple: #{group[:is_simple_consumer_group]})"
402
+ # end
403
+ #
404
+ # # Any brokers that could not be reached are reported separately
405
+ # report.errors.each { |error| warn "partial listing error: #{error.message}" }
406
+ def list_consumer_groups
407
+ closed_admin_check(__method__)
408
+
409
+ # Get a pointer to the queue that our request will be enqueued on
410
+ queue_ptr = @native_kafka.with_inner do |inner|
411
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
412
+ end
413
+
414
+ if queue_ptr.null?
415
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
416
+ end
417
+
418
+ # Create and register the handle we will return to the caller
419
+ handle = ListConsumerGroupsHandle.new
420
+ handle[:pending] = true
421
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
422
+ ListConsumerGroupsHandle.register(handle)
423
+
424
+ admin_options_ptr = @native_kafka.with_inner do |inner|
425
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
426
+ inner,
427
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS
428
+ )
429
+ end
430
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
431
+
432
+ begin
433
+ @native_kafka.with_inner do |inner|
434
+ Rdkafka::Bindings.rd_kafka_ListConsumerGroups(
435
+ inner,
436
+ admin_options_ptr,
437
+ queue_ptr
438
+ )
439
+ end
440
+ rescue Exception
441
+ ListConsumerGroupsHandle.remove(handle.to_ptr.address)
442
+ raise
443
+ ensure
444
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
445
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
446
+ end
447
+
448
+ handle
449
+ end
450
+
297
451
  # Deletes the named topic
298
452
  #
299
453
  # @param topic_name [String] name of the topic to delete
@@ -356,7 +510,8 @@ module Rdkafka
356
510
  #
357
511
  # @param topic_name [String] name of the topic
358
512
  # @param partition_count [Integer] how many partitions we want to end up with for given topic
359
- # @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the result
513
+ # @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
514
+ # result
360
515
  # @raise [ConfigError] When the partition count or replication factor are out of valid range
361
516
  # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
362
517
  # @raise [RdkafkaError] When the topic configuration is invalid
@@ -445,7 +600,8 @@ module Rdkafka
445
600
  # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
446
601
  # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
447
602
  # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
448
- # @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
603
+ # @return [CreateAclHandle] Create acl handle that can be used to wait for the result of
604
+ # creating the acl
449
605
  # @raise [RdkafkaError]
450
606
  def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
451
607
  closed_admin_check(__method__)
@@ -546,7 +702,8 @@ module Rdkafka
546
702
  # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
547
703
  # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
548
704
  # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
549
- # @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
705
+ # @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of
706
+ # deleting the acl
550
707
  # @raise [RdkafkaError]
551
708
  def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
552
709
  closed_admin_check(__method__)
@@ -581,7 +738,7 @@ module Rdkafka
581
738
  end
582
739
 
583
740
  if queue_ptr.null?
584
- Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
741
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
585
742
  raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
586
743
  end
587
744
 
@@ -649,7 +806,8 @@ module Rdkafka
649
806
  # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
650
807
  # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
651
808
  # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
652
- # @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
809
+ # @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of
810
+ # fetching acls
653
811
  # @raise [RdkafkaError]
654
812
  def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
655
813
  closed_admin_check(__method__)
@@ -677,7 +835,7 @@ module Rdkafka
677
835
  end
678
836
 
679
837
  if queue_ptr.null?
680
- Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
838
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
681
839
  raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
682
840
  end
683
841
 
@@ -728,6 +886,14 @@ module Rdkafka
728
886
  def describe_configs(resources)
729
887
  closed_admin_check(__method__)
730
888
 
889
+ # Parse user input before allocating any native resources or registering the handle, so a
890
+ # missing :resource_type / :resource_name key raises with nothing to clean up. Previously the
891
+ # KeyError leaked the background queue, the AdminOptions, the registered handle and any
892
+ # ConfigResources already built.
893
+ parsed_resources = resources.map do |resource_details|
894
+ [resource_details.fetch(:resource_type), resource_details.fetch(:resource_name)]
895
+ end
896
+
731
897
  handle = DescribeConfigsHandle.new
732
898
  handle[:pending] = true
733
899
  handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
@@ -750,12 +916,10 @@ module Rdkafka
750
916
  DescribeConfigsHandle.register(handle)
751
917
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
752
918
 
753
- pointer_array = resources.map do |resource_details|
919
+ pointer_array = parsed_resources.map do |resource_type, resource_name|
754
920
  Rdkafka::Bindings.rd_kafka_ConfigResource_new(
755
- resource_details.fetch(:resource_type),
756
- FFI::MemoryPointer.from_string(
757
- resource_details.fetch(:resource_name)
758
- )
921
+ resource_type,
922
+ FFI::MemoryPointer.from_string(resource_name)
759
923
  )
760
924
  end
761
925
 
@@ -806,6 +970,19 @@ module Rdkafka
806
970
  def incremental_alter_configs(resources_with_configs)
807
971
  closed_admin_check(__method__)
808
972
 
973
+ # Parse user input before allocating native resources or registering the handle, so a missing
974
+ # key raises with nothing to clean up. Previously the KeyError leaked the background queue,
975
+ # the AdminOptions, the registered handle and any ConfigResources already built.
976
+ parsed_resources = resources_with_configs.map do |resource_details|
977
+ [
978
+ resource_details.fetch(:resource_type),
979
+ resource_details.fetch(:resource_name),
980
+ resource_details.fetch(:configs).map do |config|
981
+ [config.fetch(:name), config.fetch(:op_type), config.fetch(:value)]
982
+ end
983
+ ]
984
+ end
985
+
809
986
  handle = IncrementalAlterConfigsHandle.new
810
987
  handle[:pending] = true
811
988
  handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
@@ -828,23 +1005,39 @@ module Rdkafka
828
1005
  IncrementalAlterConfigsHandle.register(handle)
829
1006
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
830
1007
 
831
- # Tu poprawnie tworzyc
832
- pointer_array = resources_with_configs.map do |resource_details|
1008
+ add_error = nil
1009
+
1010
+ pointer_array = parsed_resources.map do |resource_type, resource_name, configs|
833
1011
  # First build the appropriate resource representation
834
1012
  resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
835
- resource_details.fetch(:resource_type),
836
- FFI::MemoryPointer.from_string(
837
- resource_details.fetch(:resource_name)
838
- )
1013
+ resource_type,
1014
+ FFI::MemoryPointer.from_string(resource_name)
839
1015
  )
840
1016
 
841
- resource_details.fetch(:configs).each do |config|
842
- Bindings.rd_kafka_ConfigResource_add_incremental_config(
1017
+ configs.each do |name, op_type, value|
1018
+ # rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
1019
+ # an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
1020
+ # used to be ignored: the entry was silently dropped, the alter request still reported
1021
+ # success, and the error object leaked. Capture the first error (always destroying the
1022
+ # native error object) and raise it below, before the request is sent.
1023
+ error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
843
1024
  resource_ptr,
844
- config.fetch(:name),
845
- config.fetch(:op_type),
846
- config.fetch(:value)
1025
+ name,
1026
+ op_type,
1027
+ value
847
1028
  )
1029
+
1030
+ unless error_ptr.null?
1031
+ code = Rdkafka::Bindings.rd_kafka_error_code(error_ptr)
1032
+ Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
1033
+
1034
+ unless code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
1035
+ add_error ||= Rdkafka::RdkafkaError.new(
1036
+ code,
1037
+ "rd_kafka_ConfigResource_add_incremental_config"
1038
+ )
1039
+ end
1040
+ end
848
1041
  end
849
1042
 
850
1043
  resource_ptr
@@ -854,6 +1047,10 @@ module Rdkafka
854
1047
  configs_array_ptr.write_array_of_pointer(pointer_array)
855
1048
 
856
1049
  begin
1050
+ # Raise only after the full array is built so the ensure below frees every ConfigResource
1051
+ # we created, and before the request is sent so a rejected entry can't report success.
1052
+ raise add_error if add_error
1053
+
857
1054
  @native_kafka.with_inner do |inner|
858
1055
  Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
859
1056
  inner,
@@ -882,111 +1079,6 @@ module Rdkafka
882
1079
  handle
883
1080
  end
884
1081
 
885
- # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
886
- # timestamp) without requiring a consumer group.
887
- #
888
- # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
889
- # arrays of partition offset specifications. Each specification is a hash with:
890
- # - `:partition` [Integer] partition number
891
- # - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
892
- # `:max_timestamp`, or an integer timestamp in milliseconds
893
- # @param isolation_level [Integer, nil] optional isolation level:
894
- # - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
895
- # - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
896
- #
897
- # @return [ListOffsetsHandle] handle that can be used to wait for the result
898
- #
899
- # @raise [ClosedAdminError] when the admin is closed
900
- # @raise [ConfigError] when the background queue is unavailable
901
- #
902
- # @example Query earliest and latest offsets
903
- # handle = admin.list_offsets(
904
- # { "my_topic" => [
905
- # { partition: 0, offset: :earliest },
906
- # { partition: 1, offset: :latest }
907
- # ] }
908
- # )
909
- # report = handle.wait(max_wait_timeout_ms: 15_000)
910
- # report.offsets
911
- # # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
912
- def list_offsets(topic_partition_offsets, isolation_level: nil)
913
- closed_admin_check(__method__)
914
-
915
- # Count total partitions for pre-allocation
916
- total = topic_partition_offsets.sum { |_, partitions| partitions.size }
917
-
918
- # Build native topic partition list
919
- tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
920
-
921
- topic_partition_offsets.each do |topic, partitions|
922
- partitions.each do |spec|
923
- partition = spec.fetch(:partition)
924
- offset = spec.fetch(:offset)
925
-
926
- native_offset = case offset
927
- when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
928
- when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
929
- when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
930
- when Integer then offset
931
- else
932
- raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
933
- end
934
-
935
- Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
936
- Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
937
- end
938
- end
939
-
940
- # Get a pointer to the queue that our request will be enqueued on
941
- queue_ptr = @native_kafka.with_inner do |inner|
942
- Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
943
- end
944
-
945
- if queue_ptr.null?
946
- Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
947
- raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
948
- end
949
-
950
- # Create and register the handle we will return to the caller
951
- handle = ListOffsetsHandle.new
952
- handle[:pending] = true
953
- handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
954
-
955
- admin_options_ptr = @native_kafka.with_inner do |inner|
956
- Rdkafka::Bindings.rd_kafka_AdminOptions_new(
957
- inner,
958
- Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
959
- )
960
- end
961
-
962
- if isolation_level
963
- Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
964
- end
965
-
966
- ListOffsetsHandle.register(handle)
967
- Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
968
-
969
- begin
970
- @native_kafka.with_inner do |inner|
971
- Rdkafka::Bindings.rd_kafka_ListOffsets(
972
- inner,
973
- tpl,
974
- admin_options_ptr,
975
- queue_ptr
976
- )
977
- end
978
- rescue Exception
979
- ListOffsetsHandle.remove(handle.to_ptr.address)
980
- raise
981
- ensure
982
- Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
983
- Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
984
- Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
985
- end
986
-
987
- handle
988
- end
989
-
990
1082
  private
991
1083
 
992
1084
  # Checks if the admin is closed and raises an error if so
@@ -995,5 +1087,6 @@ module Rdkafka
995
1087
  def closed_admin_check(method)
996
1088
  raise Rdkafka::ClosedAdminError.new(method) if closed?
997
1089
  end
1090
+ alias_method :closed_check, :closed_admin_check
998
1091
  end
999
1092
  end