karafka-rdkafka 0.20.0.rc3-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_x86_64_gnu.yml +248 -0
  5. data/.github/workflows/ci_macos_arm64.yml +301 -0
  6. data/.github/workflows/push_linux_x86_64_gnu.yml +60 -0
  7. data/.github/workflows/push_ruby.yml +37 -0
  8. data/.github/workflows/verify-action-pins.yml +16 -0
  9. data/.gitignore +15 -0
  10. data/.rspec +2 -0
  11. data/.ruby-gemset +1 -0
  12. data/.ruby-version +1 -0
  13. data/.yardopts +2 -0
  14. data/CHANGELOG.md +323 -0
  15. data/Gemfile +5 -0
  16. data/MIT-LICENSE +22 -0
  17. data/README.md +177 -0
  18. data/Rakefile +96 -0
  19. data/docker-compose.yml +25 -0
  20. data/ext/README.md +19 -0
  21. data/ext/Rakefile +131 -0
  22. data/ext/build_common.sh +361 -0
  23. data/ext/build_linux_x86_64_gnu.sh +306 -0
  24. data/ext/build_macos_arm64.sh +550 -0
  25. data/ext/librdkafka.so +0 -0
  26. data/karafka-rdkafka.gemspec +61 -0
  27. data/lib/rdkafka/abstract_handle.rb +116 -0
  28. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  29. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  30. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  31. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  32. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  33. data/lib/rdkafka/admin/create_partitions_handle.rb +30 -0
  34. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  35. data/lib/rdkafka/admin/create_topic_handle.rb +32 -0
  36. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  37. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  38. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  39. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  40. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  41. data/lib/rdkafka/admin/delete_topic_handle.rb +32 -0
  42. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  43. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  44. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  45. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  46. data/lib/rdkafka/admin/describe_configs_report.rb +48 -0
  47. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  48. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +48 -0
  49. data/lib/rdkafka/admin.rb +832 -0
  50. data/lib/rdkafka/bindings.rb +582 -0
  51. data/lib/rdkafka/callbacks.rb +415 -0
  52. data/lib/rdkafka/config.rb +398 -0
  53. data/lib/rdkafka/consumer/headers.rb +79 -0
  54. data/lib/rdkafka/consumer/message.rb +86 -0
  55. data/lib/rdkafka/consumer/partition.rb +57 -0
  56. data/lib/rdkafka/consumer/topic_partition_list.rb +190 -0
  57. data/lib/rdkafka/consumer.rb +663 -0
  58. data/lib/rdkafka/error.rb +201 -0
  59. data/lib/rdkafka/helpers/oauth.rb +58 -0
  60. data/lib/rdkafka/helpers/time.rb +14 -0
  61. data/lib/rdkafka/metadata.rb +115 -0
  62. data/lib/rdkafka/native_kafka.rb +139 -0
  63. data/lib/rdkafka/producer/delivery_handle.rb +48 -0
  64. data/lib/rdkafka/producer/delivery_report.rb +45 -0
  65. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  66. data/lib/rdkafka/producer.rb +492 -0
  67. data/lib/rdkafka/version.rb +7 -0
  68. data/lib/rdkafka.rb +54 -0
  69. data/renovate.json +92 -0
  70. data/spec/rdkafka/abstract_handle_spec.rb +117 -0
  71. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  72. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  73. data/spec/rdkafka/admin/create_topic_handle_spec.rb +54 -0
  74. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  75. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  76. data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  77. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +54 -0
  78. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  79. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  80. data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  81. data/spec/rdkafka/admin_spec.rb +769 -0
  82. data/spec/rdkafka/bindings_spec.rb +222 -0
  83. data/spec/rdkafka/callbacks_spec.rb +20 -0
  84. data/spec/rdkafka/config_spec.rb +258 -0
  85. data/spec/rdkafka/consumer/headers_spec.rb +73 -0
  86. data/spec/rdkafka/consumer/message_spec.rb +139 -0
  87. data/spec/rdkafka/consumer/partition_spec.rb +57 -0
  88. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  89. data/spec/rdkafka/consumer_spec.rb +1299 -0
  90. data/spec/rdkafka/error_spec.rb +95 -0
  91. data/spec/rdkafka/metadata_spec.rb +79 -0
  92. data/spec/rdkafka/native_kafka_spec.rb +130 -0
  93. data/spec/rdkafka/producer/delivery_handle_spec.rb +60 -0
  94. data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
  95. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  96. data/spec/rdkafka/producer/partitions_count_spec.rb +359 -0
  97. data/spec/rdkafka/producer_spec.rb +1234 -0
  98. data/spec/spec_helper.rb +181 -0
  99. metadata +244 -0
@@ -0,0 +1,832 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ class Admin
5
+ include Helpers::OAuth
6
+
7
+ class << self
8
+ # Allows us to retrieve librdkafka errors with descriptions
9
+ # Useful for debugging and building UIs, etc.
10
+ #
11
+ # @return [Hash<Integer, Hash>] hash with errors mapped by code
12
+ def describe_errors
13
+ # Memory pointers for the array of structures and count
14
+ p_error_descs = FFI::MemoryPointer.new(:pointer)
15
+ p_count = FFI::MemoryPointer.new(:size_t)
16
+
17
+ # Call the attached function
18
+ Bindings.rd_kafka_get_err_descs(p_error_descs, p_count)
19
+
20
+ # Retrieve the number of items in the array
21
+ count = p_count.read_uint
22
+
23
+ # Get the pointer to the array of error descriptions
24
+ array_of_errors = FFI::Pointer.new(Bindings::NativeErrorDesc, p_error_descs.read_pointer)
25
+
26
+ errors = {}
27
+
28
+ count.times do |i|
29
+ # Get the pointer to each struct
30
+ error_ptr = array_of_errors[i]
31
+
32
+ # Create a new instance of NativeErrorDesc for each item
33
+ error_desc = Bindings::NativeErrorDesc.new(error_ptr)
34
+
35
+ # Read values from the struct
36
+ code = error_desc[:code]
37
+
38
+ name = ''
39
+ desc = ''
40
+
41
+ name = error_desc[:name].read_string unless error_desc[:name].null?
42
+ desc = error_desc[:desc].read_string unless error_desc[:desc].null?
43
+
44
+ errors[code] = { code: code, name: name, description: desc }
45
+ end
46
+
47
+ errors
48
+ end
49
+ end
50
+
51
+ # @private
52
+ def initialize(native_kafka)
53
+ @native_kafka = native_kafka
54
+
55
+ # Makes sure, that native kafka gets closed before it gets GCed by Ruby
56
+ ObjectSpace.define_finalizer(self, native_kafka.finalizer)
57
+ end
58
+
59
+ # Starts the native Kafka polling thread and kicks off the init polling
60
+ # @note Not needed to run unless explicit start was disabled
61
+ def start
62
+ @native_kafka.start
63
+ end
64
+
65
+ # @return [String] admin name
66
+ def name
67
+ @name ||= @native_kafka.with_inner do |inner|
68
+ ::Rdkafka::Bindings.rd_kafka_name(inner)
69
+ end
70
+ end
71
+
72
+ def finalizer
73
+ ->(_) { close }
74
+ end
75
+
76
+ # Performs the metadata request using admin
77
+ #
78
+ # @param topic_name [String, nil] metadat about particular topic or all if nil
79
+ # @param timeout_ms [Integer] metadata request timeout
80
+ # @return [Metadata] requested metadata
81
+ def metadata(topic_name = nil, timeout_ms = 2_000)
82
+ closed_admin_check(__method__)
83
+
84
+ @native_kafka.with_inner do |inner|
85
+ Metadata.new(inner, topic_name, timeout_ms)
86
+ end
87
+ end
88
+
89
+ # Close this admin instance
90
+ def close
91
+ return if closed?
92
+ ObjectSpace.undefine_finalizer(self)
93
+ @native_kafka.close
94
+ end
95
+
96
+ # Whether this admin has closed
97
+ def closed?
98
+ @native_kafka.closed?
99
+ end
100
+
101
+ # Create a topic with the given partition count and replication factor
102
+ #
103
+ # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
104
+ # creating the topic
105
+ #
106
+ # @raise [ConfigError] When the partition count or replication factor are out of valid range
107
+ # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
108
+ # @raise [RdkafkaError] When the topic configuration is invalid
109
+ def create_topic(topic_name, partition_count, replication_factor, topic_config={})
110
+ closed_admin_check(__method__)
111
+
112
+ # Create a rd_kafka_NewTopic_t representing the new topic
113
+ error_buffer = FFI::MemoryPointer.from_string(" " * 256)
114
+ new_topic_ptr = Rdkafka::Bindings.rd_kafka_NewTopic_new(
115
+ FFI::MemoryPointer.from_string(topic_name),
116
+ partition_count,
117
+ replication_factor,
118
+ error_buffer,
119
+ 256
120
+ )
121
+ if new_topic_ptr.null?
122
+ raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
123
+ end
124
+
125
+ unless topic_config.nil?
126
+ topic_config.each do |key, value|
127
+ Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
128
+ new_topic_ptr,
129
+ key.to_s,
130
+ value.to_s
131
+ )
132
+ end
133
+ end
134
+
135
+ # Note that rd_kafka_CreateTopics can create more than one topic at a time
136
+ pointer_array = [new_topic_ptr]
137
+ topics_array_ptr = FFI::MemoryPointer.new(:pointer)
138
+ topics_array_ptr.write_array_of_pointer(pointer_array)
139
+
140
+ # Get a pointer to the queue that our request will be enqueued on
141
+ queue_ptr = @native_kafka.with_inner do |inner|
142
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
143
+ end
144
+ if queue_ptr.null?
145
+ Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
146
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
147
+ end
148
+
149
+ # Create and register the handle we will return to the caller
150
+ create_topic_handle = CreateTopicHandle.new
151
+ create_topic_handle[:pending] = true
152
+ create_topic_handle[:response] = -1
153
+ CreateTopicHandle.register(create_topic_handle)
154
+ admin_options_ptr = @native_kafka.with_inner do |inner|
155
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
156
+ end
157
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
158
+
159
+ begin
160
+ @native_kafka.with_inner do |inner|
161
+ Rdkafka::Bindings.rd_kafka_CreateTopics(
162
+ inner,
163
+ topics_array_ptr,
164
+ 1,
165
+ admin_options_ptr,
166
+ queue_ptr
167
+ )
168
+ end
169
+ rescue Exception
170
+ CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
171
+ raise
172
+ ensure
173
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
174
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
175
+ Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
176
+ end
177
+
178
+ create_topic_handle
179
+ end
180
+
181
+ def delete_group(group_id)
182
+ closed_admin_check(__method__)
183
+
184
+ # Create a rd_kafka_DeleteGroup_t representing the new topic
185
+ delete_groups_ptr = Rdkafka::Bindings.rd_kafka_DeleteGroup_new(
186
+ FFI::MemoryPointer.from_string(group_id)
187
+ )
188
+
189
+ pointer_array = [delete_groups_ptr]
190
+ groups_array_ptr = FFI::MemoryPointer.new(:pointer)
191
+ groups_array_ptr.write_array_of_pointer(pointer_array)
192
+
193
+ # Get a pointer to the queue that our request will be enqueued on
194
+ queue_ptr = @native_kafka.with_inner do |inner|
195
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
196
+ end
197
+ if queue_ptr.null?
198
+ Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
199
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
200
+ end
201
+
202
+ # Create and register the handle we will return to the caller
203
+ delete_groups_handle = DeleteGroupsHandle.new
204
+ delete_groups_handle[:pending] = true
205
+ delete_groups_handle[:response] = -1
206
+ DeleteGroupsHandle.register(delete_groups_handle)
207
+ admin_options_ptr = @native_kafka.with_inner do |inner|
208
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
209
+ end
210
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_groups_handle.to_ptr)
211
+
212
+ begin
213
+ @native_kafka.with_inner do |inner|
214
+ Rdkafka::Bindings.rd_kafka_DeleteGroups(
215
+ inner,
216
+ groups_array_ptr,
217
+ 1,
218
+ admin_options_ptr,
219
+ queue_ptr
220
+ )
221
+ end
222
+ rescue Exception
223
+ DeleteGroupsHandle.remove(delete_groups_handle.to_ptr.address)
224
+ raise
225
+ ensure
226
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
227
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
228
+ Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
229
+ end
230
+
231
+ delete_groups_handle
232
+ end
233
+
234
+ # Deletes the named topic
235
+ #
236
+ # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
237
+ # deleting the topic
238
+ # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
239
+ def delete_topic(topic_name)
240
+ closed_admin_check(__method__)
241
+
242
+ # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
243
+ delete_topic_ptr = Rdkafka::Bindings.rd_kafka_DeleteTopic_new(FFI::MemoryPointer.from_string(topic_name))
244
+
245
+ # Note that rd_kafka_DeleteTopics can create more than one topic at a time
246
+ pointer_array = [delete_topic_ptr]
247
+ topics_array_ptr = FFI::MemoryPointer.new(:pointer)
248
+ topics_array_ptr.write_array_of_pointer(pointer_array)
249
+
250
+ # Get a pointer to the queue that our request will be enqueued on
251
+ queue_ptr = @native_kafka.with_inner do |inner|
252
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
253
+ end
254
+ if queue_ptr.null?
255
+ Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
256
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
257
+ end
258
+
259
+ # Create and register the handle we will return to the caller
260
+ delete_topic_handle = DeleteTopicHandle.new
261
+ delete_topic_handle[:pending] = true
262
+ delete_topic_handle[:response] = -1
263
+ DeleteTopicHandle.register(delete_topic_handle)
264
+ admin_options_ptr = @native_kafka.with_inner do |inner|
265
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
266
+ end
267
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
268
+
269
+ begin
270
+ @native_kafka.with_inner do |inner|
271
+ Rdkafka::Bindings.rd_kafka_DeleteTopics(
272
+ inner,
273
+ topics_array_ptr,
274
+ 1,
275
+ admin_options_ptr,
276
+ queue_ptr
277
+ )
278
+ end
279
+ rescue Exception
280
+ DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
281
+ raise
282
+ ensure
283
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
284
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
285
+ Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
286
+ end
287
+
288
+ delete_topic_handle
289
+ end
290
+
291
+ # Creates extra partitions for a given topic
292
+ #
293
+ # @param topic_name [String]
294
+ # @param partition_count [Integer] how many partitions we want to end up with for given topic
295
+ #
296
+ # @raise [ConfigError] When the partition count or replication factor are out of valid range
297
+ # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
298
+ # @raise [RdkafkaError] When the topic configuration is invalid
299
+ #
300
+ # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
301
+ def create_partitions(topic_name, partition_count)
302
+ closed_admin_check(__method__)
303
+
304
+ @native_kafka.with_inner do |inner|
305
+ error_buffer = FFI::MemoryPointer.from_string(" " * 256)
306
+ new_partitions_ptr = Rdkafka::Bindings.rd_kafka_NewPartitions_new(
307
+ FFI::MemoryPointer.from_string(topic_name),
308
+ partition_count,
309
+ error_buffer,
310
+ 256
311
+ )
312
+ if new_partitions_ptr.null?
313
+ raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
314
+ end
315
+
316
+ pointer_array = [new_partitions_ptr]
317
+ topics_array_ptr = FFI::MemoryPointer.new(:pointer)
318
+ topics_array_ptr.write_array_of_pointer(pointer_array)
319
+
320
+ # Get a pointer to the queue that our request will be enqueued on
321
+ queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
322
+ if queue_ptr.null?
323
+ Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
324
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
325
+ end
326
+
327
+ # Create and register the handle we will return to the caller
328
+ create_partitions_handle = CreatePartitionsHandle.new
329
+ create_partitions_handle[:pending] = true
330
+ create_partitions_handle[:response] = -1
331
+ CreatePartitionsHandle.register(create_partitions_handle)
332
+ admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
333
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
334
+
335
+ begin
336
+ Rdkafka::Bindings.rd_kafka_CreatePartitions(
337
+ inner,
338
+ topics_array_ptr,
339
+ 1,
340
+ admin_options_ptr,
341
+ queue_ptr
342
+ )
343
+ rescue Exception
344
+ CreatePartitionsHandle.remove(create_partitions_handle.to_ptr.address)
345
+ raise
346
+ ensure
347
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
348
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
349
+ Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
350
+ end
351
+
352
+ create_partitions_handle
353
+ end
354
+ end
355
+
356
+ # Create acl
357
+ # @param resource_type - values of type rd_kafka_ResourceType_t
358
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
359
+ # valid values are:
360
+ # RD_KAFKA_RESOURCE_TOPIC = 2
361
+ # RD_KAFKA_RESOURCE_GROUP = 3
362
+ # RD_KAFKA_RESOURCE_BROKER = 4
363
+ # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
364
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
365
+ # valid values are:
366
+ # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
367
+ # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
368
+ # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
369
+ # @param operation - values of type rd_kafka_AclOperation_t
370
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
371
+ # valid values are:
372
+ # RD_KAFKA_ACL_OPERATION_ALL = 2
373
+ # RD_KAFKA_ACL_OPERATION_READ = 3
374
+ # RD_KAFKA_ACL_OPERATION_WRITE = 4
375
+ # RD_KAFKA_ACL_OPERATION_CREATE = 5
376
+ # RD_KAFKA_ACL_OPERATION_DELETE = 6
377
+ # RD_KAFKA_ACL_OPERATION_ALTER = 7
378
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
379
+ # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
380
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
381
+ # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
382
+ # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
383
+ # @param permission_type - values of type rd_kafka_AclPermissionType_t
384
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
385
+ # valid values are:
386
+ # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
387
+ # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
388
+ #
389
+ # @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
390
+ #
391
+ # @raise [RdkafkaError]
392
+ def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
393
+ closed_admin_check(__method__)
394
+
395
+ # Create a rd_kafka_AclBinding_t representing the new acl
396
+ error_buffer = FFI::MemoryPointer.from_string(" " * 256)
397
+ new_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
398
+ resource_type,
399
+ FFI::MemoryPointer.from_string(resource_name),
400
+ resource_pattern_type,
401
+ FFI::MemoryPointer.from_string(principal),
402
+ FFI::MemoryPointer.from_string(host),
403
+ operation,
404
+ permission_type,
405
+ error_buffer,
406
+ 256
407
+ )
408
+ if new_acl_ptr.null?
409
+ raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
410
+ end
411
+
412
+ # Note that rd_kafka_CreateAcls can create more than one acl at a time
413
+ pointer_array = [new_acl_ptr]
414
+ acls_array_ptr = FFI::MemoryPointer.new(:pointer)
415
+ acls_array_ptr.write_array_of_pointer(pointer_array)
416
+
417
+ # Get a pointer to the queue that our request will be enqueued on
418
+ queue_ptr = @native_kafka.with_inner do |inner|
419
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
420
+ end
421
+
422
+ if queue_ptr.null?
423
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
424
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
425
+ end
426
+
427
+ # Create and register the handle that we will return to the caller
428
+ create_acl_handle = CreateAclHandle.new
429
+ create_acl_handle[:pending] = true
430
+ create_acl_handle[:response] = -1
431
+ CreateAclHandle.register(create_acl_handle)
432
+
433
+ admin_options_ptr = @native_kafka.with_inner do |inner|
434
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEACLS)
435
+ end
436
+
437
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_acl_handle.to_ptr)
438
+
439
+ begin
440
+ @native_kafka.with_inner do |inner|
441
+ Rdkafka::Bindings.rd_kafka_CreateAcls(
442
+ inner,
443
+ acls_array_ptr,
444
+ 1,
445
+ admin_options_ptr,
446
+ queue_ptr
447
+ )
448
+ end
449
+ rescue Exception
450
+ CreateAclHandle.remove(create_acl_handle.to_ptr.address)
451
+ raise
452
+ ensure
453
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
454
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
455
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
456
+ end
457
+
458
+ create_acl_handle
459
+ end
460
+
461
+ # Delete acl
462
+ #
463
+ # @param resource_type - values of type rd_kafka_ResourceType_t
464
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
465
+ # valid values are:
466
+ # RD_KAFKA_RESOURCE_TOPIC = 2
467
+ # RD_KAFKA_RESOURCE_GROUP = 3
468
+ # RD_KAFKA_RESOURCE_BROKER = 4
469
+ # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
470
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
471
+ # valid values are:
472
+ # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
473
+ # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
474
+ # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
475
+ # @param operation - values of type rd_kafka_AclOperation_t
476
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
477
+ # valid values are:
478
+ # RD_KAFKA_ACL_OPERATION_ALL = 2
479
+ # RD_KAFKA_ACL_OPERATION_READ = 3
480
+ # RD_KAFKA_ACL_OPERATION_WRITE = 4
481
+ # RD_KAFKA_ACL_OPERATION_CREATE = 5
482
+ # RD_KAFKA_ACL_OPERATION_DELETE = 6
483
+ # RD_KAFKA_ACL_OPERATION_ALTER = 7
484
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
485
+ # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
486
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
487
+ # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
488
+ # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
489
+ # @param permission_type - values of type rd_kafka_AclPermissionType_t
490
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
491
+ # valid values are:
492
+ # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
493
+ # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
494
+ # @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
495
+ #
496
+ # @raise [RdkafkaError]
497
+ def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
498
+ closed_admin_check(__method__)
499
+
500
+ # Create a rd_kafka_AclBinding_t representing the acl to be deleted
501
+ error_buffer = FFI::MemoryPointer.from_string(" " * 256)
502
+
503
+ delete_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBindingFilter_new(
504
+ resource_type,
505
+ resource_name ? FFI::MemoryPointer.from_string(resource_name) : nil,
506
+ resource_pattern_type,
507
+ principal ? FFI::MemoryPointer.from_string(principal) : nil,
508
+ host ? FFI::MemoryPointer.from_string(host) : nil,
509
+ operation,
510
+ permission_type,
511
+ error_buffer,
512
+ 256
513
+ )
514
+
515
+ if delete_acl_ptr.null?
516
+ raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
517
+ end
518
+
519
+ # Note that rd_kafka_DeleteAcls can delete more than one acl at a time
520
+ pointer_array = [delete_acl_ptr]
521
+ acls_array_ptr = FFI::MemoryPointer.new(:pointer)
522
+ acls_array_ptr.write_array_of_pointer(pointer_array)
523
+
524
+ # Get a pointer to the queue that our request will be enqueued on
525
+ queue_ptr = @native_kafka.with_inner do |inner|
526
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
527
+ end
528
+
529
+ if queue_ptr.null?
530
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
531
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
532
+ end
533
+
534
+ # Create and register the handle that we will return to the caller
535
+ delete_acl_handle = DeleteAclHandle.new
536
+ delete_acl_handle[:pending] = true
537
+ delete_acl_handle[:response] = -1
538
+ DeleteAclHandle.register(delete_acl_handle)
539
+
540
+ admin_options_ptr = @native_kafka.with_inner do |inner|
541
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETEACLS)
542
+ end
543
+
544
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_acl_handle.to_ptr)
545
+
546
+ begin
547
+ @native_kafka.with_inner do |inner|
548
+ Rdkafka::Bindings.rd_kafka_DeleteAcls(
549
+ inner,
550
+ acls_array_ptr,
551
+ 1,
552
+ admin_options_ptr,
553
+ queue_ptr
554
+ )
555
+ end
556
+ rescue Exception
557
+ DeleteAclHandle.remove(delete_acl_handle.to_ptr.address)
558
+ raise
559
+ ensure
560
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
561
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
562
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
563
+ end
564
+
565
+ delete_acl_handle
566
+ end
567
+
568
+ # Describe acls
569
+ #
570
+ # @param resource_type - values of type rd_kafka_ResourceType_t
571
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
572
+ # valid values are:
573
+ # RD_KAFKA_RESOURCE_TOPIC = 2
574
+ # RD_KAFKA_RESOURCE_GROUP = 3
575
+ # RD_KAFKA_RESOURCE_BROKER = 4
576
+ # @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
577
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
578
+ # valid values are:
579
+ # RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
580
+ # RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
581
+ # RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
582
+ # @param operation - values of type rd_kafka_AclOperation_t
583
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
584
+ # valid values are:
585
+ # RD_KAFKA_ACL_OPERATION_ALL = 2
586
+ # RD_KAFKA_ACL_OPERATION_READ = 3
587
+ # RD_KAFKA_ACL_OPERATION_WRITE = 4
588
+ # RD_KAFKA_ACL_OPERATION_CREATE = 5
589
+ # RD_KAFKA_ACL_OPERATION_DELETE = 6
590
+ # RD_KAFKA_ACL_OPERATION_ALTER = 7
591
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
592
+ # RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
593
+ # RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
594
+ # RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
595
+ # RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
596
+ # @param permission_type - values of type rd_kafka_AclPermissionType_t
597
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
598
+ # valid values are:
599
+ # RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
600
+ # RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
601
+ # @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
602
+ #
603
+ # @raise [RdkafkaError]
604
+ def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
605
+ closed_admin_check(__method__)
606
+
607
+ # Create a rd_kafka_AclBinding_t with the filters to fetch existing acls
608
+ error_buffer = FFI::MemoryPointer.from_string(" " * 256)
609
+ describe_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBindingFilter_new(
610
+ resource_type,
611
+ resource_name ? FFI::MemoryPointer.from_string(resource_name) : nil,
612
+ resource_pattern_type,
613
+ principal ? FFI::MemoryPointer.from_string(principal) : nil,
614
+ host ? FFI::MemoryPointer.from_string(host) : nil,
615
+ operation,
616
+ permission_type,
617
+ error_buffer,
618
+ 256
619
+ )
620
+ if describe_acl_ptr.null?
621
+ raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
622
+ end
623
+
624
+ # Get a pointer to the queue that our request will be enqueued on
625
+ queue_ptr = @native_kafka.with_inner do |inner|
626
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
627
+ end
628
+
629
+ if queue_ptr.null?
630
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(new_acl_ptr)
631
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
632
+ end
633
+
634
+ # Create and register the handle that we will return to the caller
635
+ describe_acl_handle = DescribeAclHandle.new
636
+ describe_acl_handle[:pending] = true
637
+ describe_acl_handle[:response] = -1
638
+ DescribeAclHandle.register(describe_acl_handle)
639
+
640
+ admin_options_ptr = @native_kafka.with_inner do |inner|
641
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBEACLS)
642
+ end
643
+
644
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, describe_acl_handle.to_ptr)
645
+
646
+ begin
647
+ @native_kafka.with_inner do |inner|
648
+ Rdkafka::Bindings.rd_kafka_DescribeAcls(
649
+ inner,
650
+ describe_acl_ptr,
651
+ admin_options_ptr,
652
+ queue_ptr
653
+ )
654
+ end
655
+ rescue Exception
656
+ DescribeAclHandle.remove(describe_acl_handle.to_ptr.address)
657
+ raise
658
+ ensure
659
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
660
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
661
+ Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
662
+ end
663
+
664
+ describe_acl_handle
665
+ end
666
+
667
+ # Describe configs
668
+ #
669
+ # @param resources [Array<Hash>] Array where elements are hashes with two keys:
670
+ # - `:resource_type` - numerical resource type based on Kafka API
671
+ # - `:resource_name` - string with resource name
672
+ # @return [DescribeConfigsHandle] Describe config handle that can be used to wait for the
673
+ # result of fetching resources with their appropriate configs
674
+ #
675
+ # @raise [RdkafkaError]
676
+ #
677
+ # @note Several resources can be requested at one go, but only one broker at a time
678
+ def describe_configs(resources)
679
+ closed_admin_check(__method__)
680
+
681
+ handle = DescribeConfigsHandle.new
682
+ handle[:pending] = true
683
+ handle[:response] = -1
684
+
685
+ queue_ptr = @native_kafka.with_inner do |inner|
686
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
687
+ end
688
+
689
+ if queue_ptr.null?
690
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
691
+ end
692
+
693
+ admin_options_ptr = @native_kafka.with_inner do |inner|
694
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
695
+ inner,
696
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
697
+ )
698
+ end
699
+
700
+ DescribeConfigsHandle.register(handle)
701
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
702
+
703
+ pointer_array = resources.map do |resource_details|
704
+ Rdkafka::Bindings.rd_kafka_ConfigResource_new(
705
+ resource_details.fetch(:resource_type),
706
+ FFI::MemoryPointer.from_string(
707
+ resource_details.fetch(:resource_name)
708
+ )
709
+ )
710
+ end
711
+
712
+ configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
713
+ configs_array_ptr.write_array_of_pointer(pointer_array)
714
+
715
+ begin
716
+ @native_kafka.with_inner do |inner|
717
+ Rdkafka::Bindings.rd_kafka_DescribeConfigs(
718
+ inner,
719
+ configs_array_ptr,
720
+ pointer_array.size,
721
+ admin_options_ptr,
722
+ queue_ptr
723
+ )
724
+ end
725
+ rescue Exception
726
+ DescribeConfigsHandle.remove(handle.to_ptr.address)
727
+
728
+ raise
729
+ ensure
730
+ Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
731
+ configs_array_ptr,
732
+ pointer_array.size
733
+ ) if configs_array_ptr
734
+ end
735
+
736
+ handle
737
+ end
738
+
739
+ # Alters in an incremental way all the configs provided for given resources
740
+ #
741
+ # @param resources_with_configs [Array<Hash>] resources with the configs key that contains
742
+ # name, value and the proper op_type to perform on this value.
743
+ #
744
+ # @return [IncrementalAlterConfigsHandle] Incremental alter configs handle that can be used to
745
+ # wait for the result of altering resources with their appropriate configs
746
+ #
747
+ # @raise [RdkafkaError]
748
+ #
749
+ # @note Several resources can be requested at one go, but only one broker at a time
750
+ # @note The results won't contain altered values but only the altered resources
751
+ def incremental_alter_configs(resources_with_configs)
752
+ closed_admin_check(__method__)
753
+
754
+ handle = IncrementalAlterConfigsHandle.new
755
+ handle[:pending] = true
756
+ handle[:response] = -1
757
+
758
+ queue_ptr = @native_kafka.with_inner do |inner|
759
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
760
+ end
761
+
762
+ if queue_ptr.null?
763
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
764
+ end
765
+
766
+ admin_options_ptr = @native_kafka.with_inner do |inner|
767
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
768
+ inner,
769
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
770
+ )
771
+ end
772
+
773
+ IncrementalAlterConfigsHandle.register(handle)
774
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
775
+
776
+ # Tu poprawnie tworzyc
777
+ pointer_array = resources_with_configs.map do |resource_details|
778
+ # First build the appropriate resource representation
779
+ resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
780
+ resource_details.fetch(:resource_type),
781
+ FFI::MemoryPointer.from_string(
782
+ resource_details.fetch(:resource_name)
783
+ )
784
+ )
785
+
786
+ resource_details.fetch(:configs).each do |config|
787
+ Bindings.rd_kafka_ConfigResource_add_incremental_config(
788
+ resource_ptr,
789
+ config.fetch(:name),
790
+ config.fetch(:op_type),
791
+ config.fetch(:value)
792
+ )
793
+ end
794
+
795
+ resource_ptr
796
+ end
797
+
798
+ configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
799
+ configs_array_ptr.write_array_of_pointer(pointer_array)
800
+
801
+
802
+ begin
803
+ @native_kafka.with_inner do |inner|
804
+ Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
805
+ inner,
806
+ configs_array_ptr,
807
+ pointer_array.size,
808
+ admin_options_ptr,
809
+ queue_ptr
810
+ )
811
+ end
812
+ rescue Exception
813
+ IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
814
+
815
+ raise
816
+ ensure
817
+ Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
818
+ configs_array_ptr,
819
+ pointer_array.size
820
+ ) if configs_array_ptr
821
+ end
822
+
823
+ handle
824
+ end
825
+
826
+ private
827
+
828
+ def closed_admin_check(method)
829
+ raise Rdkafka::ClosedAdminError.new(method) if closed?
830
+ end
831
+ end
832
+ end