rdkafka 0.22.2 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +63 -3
  3. data/Gemfile +8 -0
  4. data/Gemfile.lint +14 -0
  5. data/Gemfile.lint.lock +123 -0
  6. data/README.md +19 -14
  7. data/Rakefile +21 -21
  8. data/bin/verify_kafka_warnings +39 -0
  9. data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.0.tar.gz} +0 -0
  10. data/docker-compose-ssl.yml +35 -0
  11. data/docker-compose.yml +2 -2
  12. data/ext/Rakefile +27 -27
  13. data/lib/rdkafka/abstract_handle.rb +23 -5
  14. data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
  15. data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
  16. data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
  17. data/lib/rdkafka/admin/create_acl_report.rb +3 -2
  18. data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
  19. data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
  20. data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
  21. data/lib/rdkafka/admin/create_topic_report.rb +3 -0
  22. data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
  23. data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
  24. data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
  25. data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
  26. data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
  27. data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
  28. data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
  29. data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
  30. data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
  31. data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
  32. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
  33. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
  34. data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
  35. data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
  36. data/lib/rdkafka/admin.rb +301 -135
  37. data/lib/rdkafka/bindings.rb +199 -110
  38. data/lib/rdkafka/callbacks.rb +124 -21
  39. data/lib/rdkafka/config.rb +81 -33
  40. data/lib/rdkafka/consumer/headers.rb +3 -2
  41. data/lib/rdkafka/consumer/message.rb +12 -11
  42. data/lib/rdkafka/consumer/partition.rb +8 -4
  43. data/lib/rdkafka/consumer/topic_partition_list.rb +21 -17
  44. data/lib/rdkafka/consumer.rb +397 -45
  45. data/lib/rdkafka/defaults.rb +106 -0
  46. data/lib/rdkafka/error.rb +40 -14
  47. data/lib/rdkafka/helpers/oauth.rb +45 -13
  48. data/lib/rdkafka/helpers/time.rb +5 -0
  49. data/lib/rdkafka/metadata.rb +45 -21
  50. data/lib/rdkafka/native_kafka.rb +89 -4
  51. data/lib/rdkafka/producer/delivery_handle.rb +5 -5
  52. data/lib/rdkafka/producer/delivery_report.rb +10 -6
  53. data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
  54. data/lib/rdkafka/producer.rb +168 -82
  55. data/lib/rdkafka/version.rb +6 -3
  56. data/lib/rdkafka.rb +3 -0
  57. data/package-lock.json +331 -0
  58. data/package.json +9 -0
  59. data/rdkafka.gemspec +57 -36
  60. data/renovate.json +29 -24
  61. metadata +29 -124
  62. data/.github/CODEOWNERS +0 -3
  63. data/.github/FUNDING.yml +0 -1
  64. data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
  65. data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
  66. data/.github/workflows/ci_macos_arm64.yml +0 -284
  67. data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
  68. data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
  69. data/.github/workflows/push_macos_arm64.yml +0 -54
  70. data/.github/workflows/push_ruby.yml +0 -37
  71. data/.github/workflows/verify-action-pins.yml +0 -16
  72. data/.gitignore +0 -14
  73. data/.rspec +0 -2
  74. data/.ruby-gemset +0 -1
  75. data/.ruby-version +0 -1
  76. data/.yardopts +0 -2
  77. data/ext/README.md +0 -19
  78. data/ext/build_common.sh +0 -361
  79. data/ext/build_linux_x86_64_gnu.sh +0 -306
  80. data/ext/build_linux_x86_64_musl.sh +0 -763
  81. data/ext/build_macos_arm64.sh +0 -550
  82. data/spec/rdkafka/abstract_handle_spec.rb +0 -117
  83. data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
  84. data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
  85. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
  86. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
  87. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
  88. data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
  89. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
  90. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
  91. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
  92. data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
  93. data/spec/rdkafka/admin_spec.rb +0 -971
  94. data/spec/rdkafka/bindings_spec.rb +0 -199
  95. data/spec/rdkafka/callbacks_spec.rb +0 -20
  96. data/spec/rdkafka/config_spec.rb +0 -258
  97. data/spec/rdkafka/consumer/headers_spec.rb +0 -73
  98. data/spec/rdkafka/consumer/message_spec.rb +0 -139
  99. data/spec/rdkafka/consumer/partition_spec.rb +0 -57
  100. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
  101. data/spec/rdkafka/consumer_spec.rb +0 -1274
  102. data/spec/rdkafka/error_spec.rb +0 -89
  103. data/spec/rdkafka/metadata_spec.rb +0 -79
  104. data/spec/rdkafka/native_kafka_spec.rb +0 -130
  105. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
  106. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
  107. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
  108. data/spec/rdkafka/producer_spec.rb +0 -1345
  109. data/spec/spec_helper.rb +0 -195
data/lib/rdkafka/admin.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
+ # Admin client for Kafka administrative operations
4
5
  class Admin
5
6
  include Helpers::OAuth
6
7
 
@@ -8,7 +9,7 @@ module Rdkafka
8
9
  # Allows us to retrieve librdkafka errors with descriptions
9
10
  # Useful for debugging and building UIs, etc.
10
11
  #
11
- # @return [Hash<Integer, Hash>] hash with errors mapped by code
12
+ # @return [Hash{Integer => Hash}] hash with errors mapped by code
12
13
  def describe_errors
13
14
  # Memory pointers for the array of structures and count
14
15
  p_error_descs = FFI::MemoryPointer.new(:pointer)
@@ -35,8 +36,8 @@ module Rdkafka
35
36
  # Read values from the struct
36
37
  code = error_desc[:code]
37
38
 
38
- name = ''
39
- desc = ''
39
+ name = ""
40
+ desc = ""
40
41
 
41
42
  name = error_desc[:name].read_string unless error_desc[:name].null?
42
43
  desc = error_desc[:desc].read_string unless error_desc[:desc].null?
@@ -49,6 +50,7 @@ module Rdkafka
49
50
  end
50
51
 
51
52
  # @private
53
+ # @param native_kafka [NativeKafka] wrapper around the native Kafka handle
52
54
  def initialize(native_kafka)
53
55
  @native_kafka = native_kafka
54
56
 
@@ -69,8 +71,63 @@ module Rdkafka
69
71
  end
70
72
  end
71
73
 
72
- def finalizer
73
- ->(_) { close }
74
+ # Enable IO event notifications for fiber scheduler integration
75
+ # When admin operations complete, librdkafka will write to your FD
76
+ #
77
+ # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
78
+ # @param payload [String] data to write to fd (default: "\x01")
79
+ # @return [nil]
80
+ # @raise [ClosedInnerError] when the admin client is closed
81
+ def enable_queue_io_events(fd, payload = "\x01")
82
+ @native_kafka.enable_main_queue_io_events(fd, payload)
83
+ end
84
+
85
+ # Enable IO event notifications for background events
86
+ # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
87
+ # @param payload [String] data to write to fd (default: "\x01")
88
+ # @return [nil]
89
+ # @raise [ClosedInnerError] when the admin client is closed
90
+ def enable_background_queue_io_events(fd, payload = "\x01")
91
+ @native_kafka.enable_background_queue_io_events(fd, payload)
92
+ end
93
+
94
+ # Polls for events in a non-blocking loop, yielding the count after each iteration.
95
+ #
96
+ # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
97
+ # which is more efficient than repeated individual polls. It uses non-blocking polls
98
+ # internally (no GVL release between polls).
99
+ #
100
+ # Yields the count of events processed after each poll iteration, allowing the caller
101
+ # to implement timeout or other termination logic by returning `:stop`.
102
+ #
103
+ # @yield [count] Called after each poll iteration
104
+ # @yieldparam count [Integer] Number of events processed in this iteration
105
+ # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
106
+ # @return [nil]
107
+ # @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
108
+ #
109
+ # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
110
+ # Other admin operations will wait until this method returns.
111
+ # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
112
+ #
113
+ # @example Drain all pending events
114
+ # admin.events_poll_nb_each { |_count| }
115
+ #
116
+ # @example With timeout control
117
+ # deadline = monotonic_now + timeout_ms
118
+ # admin.events_poll_nb_each do |_count|
119
+ # :stop if monotonic_now >= deadline
120
+ # end
121
+ def events_poll_nb_each
122
+ closed_admin_check(__method__)
123
+
124
+ @native_kafka.with_inner do |inner|
125
+ loop do
126
+ count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
127
+ break if count.zero?
128
+ break if yield(count) == :stop
129
+ end
130
+ end
74
131
  end
75
132
 
76
133
  # Performs the metadata request using admin
@@ -78,7 +135,7 @@ module Rdkafka
78
135
  # @param topic_name [String, nil] metadat about particular topic or all if nil
79
136
  # @param timeout_ms [Integer] metadata request timeout
80
137
  # @return [Metadata] requested metadata
81
- def metadata(topic_name = nil, timeout_ms = 2_000)
138
+ def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
82
139
  closed_admin_check(__method__)
83
140
 
84
141
  @native_kafka.with_inner do |inner|
@@ -100,13 +157,16 @@ module Rdkafka
100
157
 
101
158
  # Create a topic with the given partition count and replication factor
102
159
  #
160
+ # @param topic_name [String] name of the topic to create
161
+ # @param partition_count [Integer] number of partitions for the topic
162
+ # @param replication_factor [Integer] replication factor for the topic
163
+ # @param topic_config [Hash] optional topic configuration settings
103
164
  # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
104
165
  # creating the topic
105
- #
106
166
  # @raise [ConfigError] When the partition count or replication factor are out of valid range
107
167
  # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
108
168
  # @raise [RdkafkaError] When the topic configuration is invalid
109
- def create_topic(topic_name, partition_count, replication_factor, topic_config={})
169
+ def create_topic(topic_name, partition_count, replication_factor, topic_config = {})
110
170
  closed_admin_check(__method__)
111
171
 
112
172
  # Create a rd_kafka_NewTopic_t representing the new topic
@@ -122,14 +182,12 @@ module Rdkafka
122
182
  raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
123
183
  end
124
184
 
125
- unless topic_config.nil?
126
- topic_config.each do |key, value|
127
- Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
128
- new_topic_ptr,
129
- key.to_s,
130
- value.to_s
131
- )
132
- end
185
+ topic_config&.each do |key, value|
186
+ Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
187
+ new_topic_ptr,
188
+ key.to_s,
189
+ value.to_s
190
+ )
133
191
  end
134
192
 
135
193
  # Note that rd_kafka_CreateTopics can create more than one topic at a time
@@ -149,7 +207,7 @@ module Rdkafka
149
207
  # Create and register the handle we will return to the caller
150
208
  create_topic_handle = CreateTopicHandle.new
151
209
  create_topic_handle[:pending] = true
152
- create_topic_handle[:response] = -1
210
+ create_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
153
211
  CreateTopicHandle.register(create_topic_handle)
154
212
  admin_options_ptr = @native_kafka.with_inner do |inner|
155
213
  Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
@@ -178,6 +236,11 @@ module Rdkafka
178
236
  create_topic_handle
179
237
  end
180
238
 
239
+ # Deletes a consumer group
240
+ #
241
+ # @param group_id [String] the group id to delete
242
+ # @return [DeleteGroupsHandle] delete groups handle that can be used to wait for the result
243
+ # @raise [RdkafkaError] When deleting the group fails
181
244
  def delete_group(group_id)
182
245
  closed_admin_check(__method__)
183
246
 
@@ -202,7 +265,7 @@ module Rdkafka
202
265
  # Create and register the handle we will return to the caller
203
266
  delete_groups_handle = DeleteGroupsHandle.new
204
267
  delete_groups_handle[:pending] = true
205
- delete_groups_handle[:response] = -1
268
+ delete_groups_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
206
269
  DeleteGroupsHandle.register(delete_groups_handle)
207
270
  admin_options_ptr = @native_kafka.with_inner do |inner|
208
271
  Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
@@ -233,6 +296,7 @@ module Rdkafka
233
296
 
234
297
  # Deletes the named topic
235
298
  #
299
+ # @param topic_name [String] name of the topic to delete
236
300
  # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
237
301
  # deleting the topic
238
302
  # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
@@ -259,7 +323,7 @@ module Rdkafka
259
323
  # Create and register the handle we will return to the caller
260
324
  delete_topic_handle = DeleteTopicHandle.new
261
325
  delete_topic_handle[:pending] = true
262
- delete_topic_handle[:response] = -1
326
+ delete_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
263
327
  DeleteTopicHandle.register(delete_topic_handle)
264
328
  admin_options_ptr = @native_kafka.with_inner do |inner|
265
329
  Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
@@ -290,14 +354,12 @@ module Rdkafka
290
354
 
291
355
  # Creates extra partitions for a given topic
292
356
  #
293
- # @param topic_name [String]
357
+ # @param topic_name [String] name of the topic
294
358
  # @param partition_count [Integer] how many partitions we want to end up with for given topic
295
- #
359
+ # @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the result
296
360
  # @raise [ConfigError] When the partition count or replication factor are out of valid range
297
361
  # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
298
362
  # @raise [RdkafkaError] When the topic configuration is invalid
299
- #
300
- # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
301
363
  def create_partitions(topic_name, partition_count)
302
364
  closed_admin_check(__method__)
303
365
 
@@ -327,7 +389,7 @@ module Rdkafka
327
389
  # Create and register the handle we will return to the caller
328
390
  create_partitions_handle = CreatePartitionsHandle.new
329
391
  create_partitions_handle[:pending] = true
330
- create_partitions_handle[:response] = -1
392
+ create_partitions_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
331
393
  CreatePartitionsHandle.register(create_partitions_handle)
332
394
  admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
333
395
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
@@ -354,40 +416,36 @@ module Rdkafka
354
416
  end
355
417
 
356
418
  # Create acl
357
- # @param resource_type - values of type rd_kafka_ResourceType_t
358
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
359
- # valid values are:
360
- # RD_KAFKA_RESOURCE_TOPIC = 2
361
- # RD_KAFKA_RESOURCE_GROUP = 3
362
- # RD_KAFKA_RESOURCE_BROKER = 4
363
- # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
364
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
365
- # valid values are:
366
- # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
367
- # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
368
- # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
369
- # @param operation - values of type rd_kafka_AclOperation_t
370
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
371
- # valid values are:
372
- # RD_KAFKA_ACL_OPERATION_ALL = 2
373
- # RD_KAFKA_ACL_OPERATION_READ = 3
374
- # RD_KAFKA_ACL_OPERATION_WRITE = 4
375
- # RD_KAFKA_ACL_OPERATION_CREATE = 5
376
- # RD_KAFKA_ACL_OPERATION_DELETE = 6
377
- # RD_KAFKA_ACL_OPERATION_ALTER = 7
378
- # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
379
- # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
380
- # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
381
- # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
382
- # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
383
- # @param permission_type - values of type rd_kafka_AclPermissionType_t
384
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
385
- # valid values are:
386
- # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
387
- # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
388
419
  #
420
+ # @param resource_type [Integer] rd_kafka_ResourceType_t value:
421
+ # - RD_KAFKA_RESOURCE_TOPIC = 2
422
+ # - RD_KAFKA_RESOURCE_GROUP = 3
423
+ # - RD_KAFKA_RESOURCE_BROKER = 4
424
+ # @param resource_name [String] name of the resource
425
+ # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
426
+ # - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
427
+ # - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
428
+ # - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
429
+ # - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
430
+ # - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
431
+ # @param principal [String] principal (e.g., "User:alice")
432
+ # @param host [String] host address
433
+ # @param operation [Integer] rd_kafka_AclOperation_t value:
434
+ # - RD_KAFKA_ACL_OPERATION_ALL = 2
435
+ # - RD_KAFKA_ACL_OPERATION_READ = 3
436
+ # - RD_KAFKA_ACL_OPERATION_WRITE = 4
437
+ # - RD_KAFKA_ACL_OPERATION_CREATE = 5
438
+ # - RD_KAFKA_ACL_OPERATION_DELETE = 6
439
+ # - RD_KAFKA_ACL_OPERATION_ALTER = 7
440
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
441
+ # - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
442
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
443
+ # - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
444
+ # - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
445
+ # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
446
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
447
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
389
448
  # @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
390
- #
391
449
  # @raise [RdkafkaError]
392
450
  def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
393
451
  closed_admin_check(__method__)
@@ -427,7 +485,7 @@ module Rdkafka
427
485
  # Create and register the handle that we will return to the caller
428
486
  create_acl_handle = CreateAclHandle.new
429
487
  create_acl_handle[:pending] = true
430
- create_acl_handle[:response] = -1
488
+ create_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
431
489
  CreateAclHandle.register(create_acl_handle)
432
490
 
433
491
  admin_options_ptr = @native_kafka.with_inner do |inner|
@@ -460,39 +518,35 @@ module Rdkafka
460
518
 
461
519
  # Delete acl
462
520
  #
463
- # @param resource_type - values of type rd_kafka_ResourceType_t
464
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
465
- # valid values are:
466
- # RD_KAFKA_RESOURCE_TOPIC = 2
467
- # RD_KAFKA_RESOURCE_GROUP = 3
468
- # RD_KAFKA_RESOURCE_BROKER = 4
469
- # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
470
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
471
- # valid values are:
472
- # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
473
- # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
474
- # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
475
- # @param operation - values of type rd_kafka_AclOperation_t
476
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
477
- # valid values are:
478
- # RD_KAFKA_ACL_OPERATION_ALL = 2
479
- # RD_KAFKA_ACL_OPERATION_READ = 3
480
- # RD_KAFKA_ACL_OPERATION_WRITE = 4
481
- # RD_KAFKA_ACL_OPERATION_CREATE = 5
482
- # RD_KAFKA_ACL_OPERATION_DELETE = 6
483
- # RD_KAFKA_ACL_OPERATION_ALTER = 7
484
- # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
485
- # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
486
- # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
487
- # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
488
- # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
489
- # @param permission_type - values of type rd_kafka_AclPermissionType_t
490
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
491
- # valid values are:
492
- # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
493
- # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
521
+ # @param resource_type [Integer] rd_kafka_ResourceType_t value:
522
+ # - RD_KAFKA_RESOURCE_TOPIC = 2
523
+ # - RD_KAFKA_RESOURCE_GROUP = 3
524
+ # - RD_KAFKA_RESOURCE_BROKER = 4
525
+ # @param resource_name [String, nil] name of the resource or nil for any
526
+ # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
527
+ # - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
528
+ # - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
529
+ # - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
530
+ # - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
531
+ # - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
532
+ # @param principal [String, nil] principal (e.g., "User:alice") or nil for any
533
+ # @param host [String, nil] host address or nil for any
534
+ # @param operation [Integer] rd_kafka_AclOperation_t value:
535
+ # - RD_KAFKA_ACL_OPERATION_ALL = 2
536
+ # - RD_KAFKA_ACL_OPERATION_READ = 3
537
+ # - RD_KAFKA_ACL_OPERATION_WRITE = 4
538
+ # - RD_KAFKA_ACL_OPERATION_CREATE = 5
539
+ # - RD_KAFKA_ACL_OPERATION_DELETE = 6
540
+ # - RD_KAFKA_ACL_OPERATION_ALTER = 7
541
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
542
+ # - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
543
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
544
+ # - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
545
+ # - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
546
+ # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
547
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
548
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
494
549
  # @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
495
- #
496
550
  # @raise [RdkafkaError]
497
551
  def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
498
552
  closed_admin_check(__method__)
@@ -534,7 +588,7 @@ module Rdkafka
534
588
  # Create and register the handle that we will return to the caller
535
589
  delete_acl_handle = DeleteAclHandle.new
536
590
  delete_acl_handle[:pending] = true
537
- delete_acl_handle[:response] = -1
591
+ delete_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
538
592
  DeleteAclHandle.register(delete_acl_handle)
539
593
 
540
594
  admin_options_ptr = @native_kafka.with_inner do |inner|
@@ -567,39 +621,35 @@ module Rdkafka
567
621
 
568
622
  # Describe acls
569
623
  #
570
- # @param resource_type - values of type rd_kafka_ResourceType_t
571
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
572
- # valid values are:
573
- # RD_KAFKA_RESOURCE_TOPIC = 2
574
- # RD_KAFKA_RESOURCE_GROUP = 3
575
- # RD_KAFKA_RESOURCE_BROKER = 4
576
- # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
577
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
578
- # valid values are:
579
- # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
580
- # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
581
- # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
582
- # @param operation - values of type rd_kafka_AclOperation_t
583
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
584
- # valid values are:
585
- # RD_KAFKA_ACL_OPERATION_ALL = 2
586
- # RD_KAFKA_ACL_OPERATION_READ = 3
587
- # RD_KAFKA_ACL_OPERATION_WRITE = 4
588
- # RD_KAFKA_ACL_OPERATION_CREATE = 5
589
- # RD_KAFKA_ACL_OPERATION_DELETE = 6
590
- # RD_KAFKA_ACL_OPERATION_ALTER = 7
591
- # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
592
- # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
593
- # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
594
- # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
595
- # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
596
- # @param permission_type - values of type rd_kafka_AclPermissionType_t
597
- # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
598
- # valid values are:
599
- # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
600
- # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
624
+ # @param resource_type [Integer] rd_kafka_ResourceType_t value:
625
+ # - RD_KAFKA_RESOURCE_TOPIC = 2
626
+ # - RD_KAFKA_RESOURCE_GROUP = 3
627
+ # - RD_KAFKA_RESOURCE_BROKER = 4
628
+ # @param resource_name [String, nil] name of the resource or nil for any
629
+ # @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
630
+ # - RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
631
+ # - RD_KAFKA_RESOURCE_PATTERN_ANY = 1
632
+ # - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
633
+ # - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
634
+ # - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
635
+ # @param principal [String, nil] principal (e.g., "User:alice") or nil for any
636
+ # @param host [String, nil] host address or nil for any
637
+ # @param operation [Integer] rd_kafka_AclOperation_t value:
638
+ # - RD_KAFKA_ACL_OPERATION_ALL = 2
639
+ # - RD_KAFKA_ACL_OPERATION_READ = 3
640
+ # - RD_KAFKA_ACL_OPERATION_WRITE = 4
641
+ # - RD_KAFKA_ACL_OPERATION_CREATE = 5
642
+ # - RD_KAFKA_ACL_OPERATION_DELETE = 6
643
+ # - RD_KAFKA_ACL_OPERATION_ALTER = 7
644
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
645
+ # - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
646
+ # - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
647
+ # - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
648
+ # - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
649
+ # @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
650
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
651
+ # - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
601
652
  # @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
602
- #
603
653
  # @raise [RdkafkaError]
604
654
  def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
605
655
  closed_admin_check(__method__)
@@ -634,7 +684,7 @@ module Rdkafka
634
684
  # Create and register the handle that we will return to the caller
635
685
  describe_acl_handle = DescribeAclHandle.new
636
686
  describe_acl_handle[:pending] = true
637
- describe_acl_handle[:response] = -1
687
+ describe_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
638
688
  DescribeAclHandle.register(describe_acl_handle)
639
689
 
640
690
  admin_options_ptr = @native_kafka.with_inner do |inner|
@@ -664,7 +714,6 @@ module Rdkafka
664
714
  describe_acl_handle
665
715
  end
666
716
 
667
-
668
717
  # Describe configs
669
718
  #
670
719
  # @param resources [Array<Hash>] Array where elements are hashes with two keys:
@@ -681,7 +730,7 @@ module Rdkafka
681
730
 
682
731
  handle = DescribeConfigsHandle.new
683
732
  handle[:pending] = true
684
- handle[:response] = -1
733
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
685
734
 
686
735
  queue_ptr = @native_kafka.with_inner do |inner|
687
736
  Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
@@ -728,10 +777,15 @@ module Rdkafka
728
777
 
729
778
  raise
730
779
  ensure
731
- Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
732
- configs_array_ptr,
733
- pointer_array.size
734
- ) if configs_array_ptr
780
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
781
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
782
+
783
+ if configs_array_ptr
784
+ Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
785
+ configs_array_ptr,
786
+ pointer_array.size
787
+ )
788
+ end
735
789
  end
736
790
 
737
791
  handle
@@ -754,7 +808,7 @@ module Rdkafka
754
808
 
755
809
  handle = IncrementalAlterConfigsHandle.new
756
810
  handle[:pending] = true
757
- handle[:response] = -1
811
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
758
812
 
759
813
  queue_ptr = @native_kafka.with_inner do |inner|
760
814
  Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
@@ -799,7 +853,6 @@ module Rdkafka
799
853
  configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
800
854
  configs_array_ptr.write_array_of_pointer(pointer_array)
801
855
 
802
-
803
856
  begin
804
857
  @native_kafka.with_inner do |inner|
805
858
  Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
@@ -815,10 +868,120 @@ module Rdkafka
815
868
 
816
869
  raise
817
870
  ensure
818
- Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
819
- configs_array_ptr,
820
- pointer_array.size
821
- ) if configs_array_ptr
871
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
872
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
873
+
874
+ if configs_array_ptr
875
+ Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
876
+ configs_array_ptr,
877
+ pointer_array.size
878
+ )
879
+ end
880
+ end
881
+
882
+ handle
883
+ end
884
+
885
+ # Queries partition offsets by specification (earliest, latest, max_timestamp, or by
886
+ # timestamp) without requiring a consumer group.
887
+ #
888
+ # @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
889
+ # arrays of partition offset specifications. Each specification is a hash with:
890
+ # - `:partition` [Integer] partition number
891
+ # - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
892
+ # `:max_timestamp`, or an integer timestamp in milliseconds
893
+ # @param isolation_level [Integer, nil] optional isolation level:
894
+ # - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
895
+ # - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
896
+ #
897
+ # @return [ListOffsetsHandle] handle that can be used to wait for the result
898
+ #
899
+ # @raise [ClosedAdminError] when the admin is closed
900
+ # @raise [ConfigError] when the background queue is unavailable
901
+ #
902
+ # @example Query earliest and latest offsets
903
+ # handle = admin.list_offsets(
904
+ # { "my_topic" => [
905
+ # { partition: 0, offset: :earliest },
906
+ # { partition: 1, offset: :latest }
907
+ # ] }
908
+ # )
909
+ # report = handle.wait(max_wait_timeout_ms: 15_000)
910
+ # report.offsets
911
+ # # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
912
+ def list_offsets(topic_partition_offsets, isolation_level: nil)
913
+ closed_admin_check(__method__)
914
+
915
+ # Count total partitions for pre-allocation
916
+ total = topic_partition_offsets.sum { |_, partitions| partitions.size }
917
+
918
+ # Build native topic partition list
919
+ tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
920
+
921
+ topic_partition_offsets.each do |topic, partitions|
922
+ partitions.each do |spec|
923
+ partition = spec.fetch(:partition)
924
+ offset = spec.fetch(:offset)
925
+
926
+ native_offset = case offset
927
+ when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
928
+ when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
929
+ when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
930
+ when Integer then offset
931
+ else
932
+ raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
933
+ end
934
+
935
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
936
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
937
+ end
938
+ end
939
+
940
+ # Get a pointer to the queue that our request will be enqueued on
941
+ queue_ptr = @native_kafka.with_inner do |inner|
942
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
943
+ end
944
+
945
+ if queue_ptr.null?
946
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
947
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
948
+ end
949
+
950
+ # Create and register the handle we will return to the caller
951
+ handle = ListOffsetsHandle.new
952
+ handle[:pending] = true
953
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
954
+
955
+ admin_options_ptr = @native_kafka.with_inner do |inner|
956
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
957
+ inner,
958
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
959
+ )
960
+ end
961
+
962
+ if isolation_level
963
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
964
+ end
965
+
966
+ ListOffsetsHandle.register(handle)
967
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
968
+
969
+ begin
970
+ @native_kafka.with_inner do |inner|
971
+ Rdkafka::Bindings.rd_kafka_ListOffsets(
972
+ inner,
973
+ tpl,
974
+ admin_options_ptr,
975
+ queue_ptr
976
+ )
977
+ end
978
+ rescue Exception
979
+ ListOffsetsHandle.remove(handle.to_ptr.address)
980
+ raise
981
+ ensure
982
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
983
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
984
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
822
985
  end
823
986
 
824
987
  handle
@@ -826,6 +989,9 @@ module Rdkafka
826
989
 
827
990
  private
828
991
 
992
+ # Checks if the admin is closed and raises an error if so
993
+ # @param method [Symbol] name of the calling method for error context
994
+ # @raise [ClosedAdminError] when the admin is closed
829
995
  def closed_admin_check(method)
830
996
  raise Rdkafka::ClosedAdminError.new(method) if closed?
831
997
  end