karafka-rdkafka 0.23.1.rc1-arm64-darwin → 0.24.0.rc1-arm64-darwin
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -1
- data/Gemfile +9 -0
- data/docker-compose-ssl.yml +1 -1
- data/docker-compose.yml +1 -1
- data/ext/librdkafka.dylib +0 -0
- data/karafka-rdkafka.gemspec +0 -7
- data/lib/rdkafka/abstract_handle.rb +23 -5
- data/lib/rdkafka/admin/acl_binding_result.rb +1 -1
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/create_acl_report.rb +3 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +3 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +3 -0
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +3 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +5 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +3 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +3 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +6 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +3 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +6 -0
- data/lib/rdkafka/admin.rb +107 -111
- data/lib/rdkafka/bindings.rb +58 -28
- data/lib/rdkafka/callbacks.rb +63 -3
- data/lib/rdkafka/config.rb +20 -8
- data/lib/rdkafka/consumer/headers.rb +3 -2
- data/lib/rdkafka/consumer/message.rb +7 -3
- data/lib/rdkafka/consumer/partition.rb +6 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +7 -7
- data/lib/rdkafka/consumer.rb +24 -12
- data/lib/rdkafka/defaults.rb +84 -0
- data/lib/rdkafka/error.rb +49 -1
- data/lib/rdkafka/helpers/oauth.rb +11 -5
- data/lib/rdkafka/metadata.rb +29 -5
- data/lib/rdkafka/native_kafka.rb +26 -2
- data/lib/rdkafka/producer/delivery_report.rb +6 -2
- data/lib/rdkafka/producer/partitions_count_cache.rb +24 -14
- data/lib/rdkafka/producer/testing.rb +3 -3
- data/lib/rdkafka/producer.rb +63 -19
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +2 -0
- data/renovate.json +1 -8
- metadata +3 -86
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
+
# Admin client for Kafka administrative operations
|
|
4
5
|
class Admin
|
|
5
6
|
include Helpers::OAuth
|
|
6
7
|
|
|
@@ -8,7 +9,7 @@ module Rdkafka
|
|
|
8
9
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
9
10
|
# Useful for debugging and building UIs, etc.
|
|
10
11
|
#
|
|
11
|
-
# @return [Hash
|
|
12
|
+
# @return [Hash{Integer => Hash}] hash with errors mapped by code
|
|
12
13
|
def describe_errors
|
|
13
14
|
# Memory pointers for the array of structures and count
|
|
14
15
|
p_error_descs = FFI::MemoryPointer.new(:pointer)
|
|
@@ -49,6 +50,7 @@ module Rdkafka
|
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
# @private
|
|
53
|
+
# @param native_kafka [NativeKafka] wrapper around the native Kafka handle
|
|
52
54
|
def initialize(native_kafka)
|
|
53
55
|
@native_kafka = native_kafka
|
|
54
56
|
|
|
@@ -69,6 +71,8 @@ module Rdkafka
|
|
|
69
71
|
end
|
|
70
72
|
end
|
|
71
73
|
|
|
74
|
+
# @return [Proc] finalizer proc for closing the admin
|
|
75
|
+
# @private
|
|
72
76
|
def finalizer
|
|
73
77
|
->(_) { close }
|
|
74
78
|
end
|
|
@@ -78,7 +82,7 @@ module Rdkafka
|
|
|
78
82
|
# @param topic_name [String, nil] metadat about particular topic or all if nil
|
|
79
83
|
# @param timeout_ms [Integer] metadata request timeout
|
|
80
84
|
# @return [Metadata] requested metadata
|
|
81
|
-
def metadata(topic_name = nil, timeout_ms =
|
|
85
|
+
def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
|
|
82
86
|
closed_admin_check(__method__)
|
|
83
87
|
|
|
84
88
|
@native_kafka.with_inner do |inner|
|
|
@@ -100,9 +104,12 @@ module Rdkafka
|
|
|
100
104
|
|
|
101
105
|
# Create a topic with the given partition count and replication factor
|
|
102
106
|
#
|
|
107
|
+
# @param topic_name [String] name of the topic to create
|
|
108
|
+
# @param partition_count [Integer] number of partitions for the topic
|
|
109
|
+
# @param replication_factor [Integer] replication factor for the topic
|
|
110
|
+
# @param topic_config [Hash] optional topic configuration settings
|
|
103
111
|
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
|
|
104
112
|
# creating the topic
|
|
105
|
-
#
|
|
106
113
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
107
114
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
108
115
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
@@ -149,7 +156,7 @@ module Rdkafka
|
|
|
149
156
|
# Create and register the handle we will return to the caller
|
|
150
157
|
create_topic_handle = CreateTopicHandle.new
|
|
151
158
|
create_topic_handle[:pending] = true
|
|
152
|
-
create_topic_handle[:response] =
|
|
159
|
+
create_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
153
160
|
CreateTopicHandle.register(create_topic_handle)
|
|
154
161
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
155
162
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
|
|
@@ -178,6 +185,11 @@ module Rdkafka
|
|
|
178
185
|
create_topic_handle
|
|
179
186
|
end
|
|
180
187
|
|
|
188
|
+
# Deletes a consumer group
|
|
189
|
+
#
|
|
190
|
+
# @param group_id [String] the group id to delete
|
|
191
|
+
# @return [DeleteGroupsHandle] delete groups handle that can be used to wait for the result
|
|
192
|
+
# @raise [RdkafkaError] When deleting the group fails
|
|
181
193
|
def delete_group(group_id)
|
|
182
194
|
closed_admin_check(__method__)
|
|
183
195
|
|
|
@@ -202,7 +214,7 @@ module Rdkafka
|
|
|
202
214
|
# Create and register the handle we will return to the caller
|
|
203
215
|
delete_groups_handle = DeleteGroupsHandle.new
|
|
204
216
|
delete_groups_handle[:pending] = true
|
|
205
|
-
delete_groups_handle[:response] =
|
|
217
|
+
delete_groups_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
206
218
|
DeleteGroupsHandle.register(delete_groups_handle)
|
|
207
219
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
208
220
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
|
|
@@ -233,6 +245,7 @@ module Rdkafka
|
|
|
233
245
|
|
|
234
246
|
# Deletes the named topic
|
|
235
247
|
#
|
|
248
|
+
# @param topic_name [String] name of the topic to delete
|
|
236
249
|
# @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
|
|
237
250
|
# deleting the topic
|
|
238
251
|
# @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
|
|
@@ -259,7 +272,7 @@ module Rdkafka
|
|
|
259
272
|
# Create and register the handle we will return to the caller
|
|
260
273
|
delete_topic_handle = DeleteTopicHandle.new
|
|
261
274
|
delete_topic_handle[:pending] = true
|
|
262
|
-
delete_topic_handle[:response] =
|
|
275
|
+
delete_topic_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
263
276
|
DeleteTopicHandle.register(delete_topic_handle)
|
|
264
277
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
265
278
|
Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
|
|
@@ -290,14 +303,12 @@ module Rdkafka
|
|
|
290
303
|
|
|
291
304
|
# Creates extra partitions for a given topic
|
|
292
305
|
#
|
|
293
|
-
# @param topic_name [String]
|
|
306
|
+
# @param topic_name [String] name of the topic
|
|
294
307
|
# @param partition_count [Integer] how many partitions we want to end up with for given topic
|
|
295
|
-
#
|
|
308
|
+
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the result
|
|
296
309
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
297
310
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
298
311
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
299
|
-
#
|
|
300
|
-
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
|
|
301
312
|
def create_partitions(topic_name, partition_count)
|
|
302
313
|
closed_admin_check(__method__)
|
|
303
314
|
|
|
@@ -354,40 +365,34 @@ module Rdkafka
|
|
|
354
365
|
end
|
|
355
366
|
|
|
356
367
|
# Create acl
|
|
357
|
-
# @param resource_type - values of type rd_kafka_ResourceType_t
|
|
358
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
359
|
-
# valid values are:
|
|
360
|
-
# RD_KAFKA_RESOURCE_TOPIC = 2
|
|
361
|
-
# RD_KAFKA_RESOURCE_GROUP = 3
|
|
362
|
-
# RD_KAFKA_RESOURCE_BROKER = 4
|
|
363
|
-
# @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
|
|
364
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
365
|
-
# valid values are:
|
|
366
|
-
# RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
367
|
-
# RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
368
|
-
# RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
369
|
-
# @param operation - values of type rd_kafka_AclOperation_t
|
|
370
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
371
|
-
# valid values are:
|
|
372
|
-
# RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
373
|
-
# RD_KAFKA_ACL_OPERATION_READ = 3
|
|
374
|
-
# RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
375
|
-
# RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
376
|
-
# RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
377
|
-
# RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
378
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
379
|
-
# RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
380
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
381
|
-
# RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
382
|
-
# RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
383
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
384
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
385
|
-
# valid values are:
|
|
386
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
387
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
388
368
|
#
|
|
369
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
370
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
371
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
372
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
373
|
+
# @param resource_name [String] name of the resource
|
|
374
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
375
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
376
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
377
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
378
|
+
# @param principal [String] principal (e.g., "User:alice")
|
|
379
|
+
# @param host [String] host address
|
|
380
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
381
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
382
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
383
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
384
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
385
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
386
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
387
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
388
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
389
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
390
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
391
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
392
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
393
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
394
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
389
395
|
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
|
|
390
|
-
#
|
|
391
396
|
# @raise [RdkafkaError]
|
|
392
397
|
def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
393
398
|
closed_admin_check(__method__)
|
|
@@ -427,7 +432,7 @@ module Rdkafka
|
|
|
427
432
|
# Create and register the handle that we will return to the caller
|
|
428
433
|
create_acl_handle = CreateAclHandle.new
|
|
429
434
|
create_acl_handle[:pending] = true
|
|
430
|
-
create_acl_handle[:response] =
|
|
435
|
+
create_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
431
436
|
CreateAclHandle.register(create_acl_handle)
|
|
432
437
|
|
|
433
438
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -460,39 +465,33 @@ module Rdkafka
|
|
|
460
465
|
|
|
461
466
|
# Delete acl
|
|
462
467
|
#
|
|
463
|
-
# @param resource_type
|
|
464
|
-
#
|
|
465
|
-
#
|
|
466
|
-
#
|
|
467
|
-
#
|
|
468
|
-
#
|
|
469
|
-
#
|
|
470
|
-
#
|
|
471
|
-
#
|
|
472
|
-
#
|
|
473
|
-
#
|
|
474
|
-
#
|
|
475
|
-
#
|
|
476
|
-
#
|
|
477
|
-
#
|
|
478
|
-
#
|
|
479
|
-
#
|
|
480
|
-
#
|
|
481
|
-
#
|
|
482
|
-
#
|
|
483
|
-
#
|
|
484
|
-
#
|
|
485
|
-
#
|
|
486
|
-
#
|
|
487
|
-
#
|
|
488
|
-
#
|
|
489
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
490
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
491
|
-
# valid values are:
|
|
492
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
493
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
468
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
469
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
470
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
471
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
472
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
473
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
474
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
475
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
476
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
477
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
478
|
+
# @param host [String, nil] host address or nil for any
|
|
479
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
480
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
481
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
482
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
483
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
484
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
485
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
486
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
487
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
488
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
489
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
490
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
491
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
492
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
493
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
494
494
|
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
|
|
495
|
-
#
|
|
496
495
|
# @raise [RdkafkaError]
|
|
497
496
|
def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
498
497
|
closed_admin_check(__method__)
|
|
@@ -534,7 +533,7 @@ module Rdkafka
|
|
|
534
533
|
# Create and register the handle that we will return to the caller
|
|
535
534
|
delete_acl_handle = DeleteAclHandle.new
|
|
536
535
|
delete_acl_handle[:pending] = true
|
|
537
|
-
delete_acl_handle[:response] =
|
|
536
|
+
delete_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
538
537
|
DeleteAclHandle.register(delete_acl_handle)
|
|
539
538
|
|
|
540
539
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -567,39 +566,33 @@ module Rdkafka
|
|
|
567
566
|
|
|
568
567
|
# Describe acls
|
|
569
568
|
#
|
|
570
|
-
# @param resource_type
|
|
571
|
-
#
|
|
572
|
-
#
|
|
573
|
-
#
|
|
574
|
-
#
|
|
575
|
-
#
|
|
576
|
-
#
|
|
577
|
-
#
|
|
578
|
-
#
|
|
579
|
-
#
|
|
580
|
-
#
|
|
581
|
-
#
|
|
582
|
-
#
|
|
583
|
-
#
|
|
584
|
-
#
|
|
585
|
-
#
|
|
586
|
-
#
|
|
587
|
-
#
|
|
588
|
-
#
|
|
589
|
-
#
|
|
590
|
-
#
|
|
591
|
-
#
|
|
592
|
-
#
|
|
593
|
-
#
|
|
594
|
-
#
|
|
595
|
-
#
|
|
596
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
597
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
598
|
-
# valid values are:
|
|
599
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
600
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
569
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
570
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
571
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
572
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
573
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
574
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
575
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
576
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
577
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
578
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
579
|
+
# @param host [String, nil] host address or nil for any
|
|
580
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
581
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
582
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
583
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
584
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
585
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
586
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
587
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
588
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
589
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
590
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
591
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
592
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
593
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
594
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
601
595
|
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
|
|
602
|
-
#
|
|
603
596
|
# @raise [RdkafkaError]
|
|
604
597
|
def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
605
598
|
closed_admin_check(__method__)
|
|
@@ -634,7 +627,7 @@ module Rdkafka
|
|
|
634
627
|
# Create and register the handle that we will return to the caller
|
|
635
628
|
describe_acl_handle = DescribeAclHandle.new
|
|
636
629
|
describe_acl_handle[:pending] = true
|
|
637
|
-
describe_acl_handle[:response] =
|
|
630
|
+
describe_acl_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
638
631
|
DescribeAclHandle.register(describe_acl_handle)
|
|
639
632
|
|
|
640
633
|
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
@@ -680,7 +673,7 @@ module Rdkafka
|
|
|
680
673
|
|
|
681
674
|
handle = DescribeConfigsHandle.new
|
|
682
675
|
handle[:pending] = true
|
|
683
|
-
handle[:response] =
|
|
676
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
684
677
|
|
|
685
678
|
queue_ptr = @native_kafka.with_inner do |inner|
|
|
686
679
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
@@ -753,7 +746,7 @@ module Rdkafka
|
|
|
753
746
|
|
|
754
747
|
handle = IncrementalAlterConfigsHandle.new
|
|
755
748
|
handle[:pending] = true
|
|
756
|
-
handle[:response] =
|
|
749
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
757
750
|
|
|
758
751
|
queue_ptr = @native_kafka.with_inner do |inner|
|
|
759
752
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
@@ -825,6 +818,9 @@ module Rdkafka
|
|
|
825
818
|
|
|
826
819
|
private
|
|
827
820
|
|
|
821
|
+
# Checks if the admin is closed and raises an error if so
|
|
822
|
+
# @param method [Symbol] name of the calling method for error context
|
|
823
|
+
# @raise [ClosedAdminError] when the admin is closed
|
|
828
824
|
def closed_admin_check(method)
|
|
829
825
|
raise Rdkafka::ClosedAdminError.new(method) if closed?
|
|
830
826
|
end
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -14,6 +14,8 @@ module Rdkafka
|
|
|
14
14
|
module Bindings
|
|
15
15
|
extend FFI::Library
|
|
16
16
|
|
|
17
|
+
# Returns the library extension based on the host OS
|
|
18
|
+
# @return [String] 'dylib' on macOS, 'so' on other systems
|
|
17
19
|
def self.lib_extension
|
|
18
20
|
if RbConfig::CONFIG['host_os'] =~ /darwin/
|
|
19
21
|
'dylib'
|
|
@@ -22,7 +24,39 @@ module Rdkafka
|
|
|
22
24
|
end
|
|
23
25
|
end
|
|
24
26
|
|
|
25
|
-
ffi_lib
|
|
27
|
+
# Wrap ffi_lib to provide better error messages for glibc compatibility issues
|
|
28
|
+
begin
|
|
29
|
+
ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
|
|
30
|
+
rescue LoadError => e
|
|
31
|
+
error_message = e.message
|
|
32
|
+
|
|
33
|
+
# Check if this is a glibc version mismatch error
|
|
34
|
+
if error_message =~ /GLIBC_[\d.]+['"` ]?\s*not found/i
|
|
35
|
+
glibc_version = error_message[/GLIBC_([\d.]+)/, 1] || 'unknown'
|
|
36
|
+
|
|
37
|
+
raise Rdkafka::LibraryLoadError, <<~ERROR_MSG.strip
|
|
38
|
+
Failed to load librdkafka due to glibc compatibility issue.
|
|
39
|
+
|
|
40
|
+
The precompiled librdkafka binary requires glibc version #{glibc_version} or higher,
|
|
41
|
+
but your system has an older version installed.
|
|
42
|
+
|
|
43
|
+
To resolve this issue, you have two options:
|
|
44
|
+
|
|
45
|
+
1. Upgrade your system to a supported platform (recommended)
|
|
46
|
+
|
|
47
|
+
2. Force compilation from source by reinstalling without the precompiled binary:
|
|
48
|
+
gem install rdkafka --platform=ruby
|
|
49
|
+
|
|
50
|
+
Or if using Bundler, add to your Gemfile:
|
|
51
|
+
gem 'rdkafka', force_ruby_platform: true
|
|
52
|
+
|
|
53
|
+
Original error: #{error_message}
|
|
54
|
+
ERROR_MSG
|
|
55
|
+
else
|
|
56
|
+
# Re-raise the original error if it's not a glibc issue
|
|
57
|
+
raise
|
|
58
|
+
end
|
|
59
|
+
end
|
|
26
60
|
|
|
27
61
|
RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
|
|
28
62
|
RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
|
|
@@ -37,7 +71,7 @@ module Rdkafka
|
|
|
37
71
|
# Unassigned partition
|
|
38
72
|
RD_KAFKA_PARTITION_UA = -1
|
|
39
73
|
# String representation of unassigned partition (used in stats hash keys)
|
|
40
|
-
RD_KAFKA_PARTITION_UA_STR =
|
|
74
|
+
RD_KAFKA_PARTITION_UA_STR = RD_KAFKA_PARTITION_UA.to_s.freeze
|
|
41
75
|
|
|
42
76
|
RD_KAFKA_OFFSET_END = -1
|
|
43
77
|
RD_KAFKA_OFFSET_BEGINNING = -2
|
|
@@ -46,6 +80,7 @@ module Rdkafka
|
|
|
46
80
|
|
|
47
81
|
EMPTY_HASH = {}.freeze
|
|
48
82
|
|
|
83
|
+
# FFI struct for size_t pointer wrapper
|
|
49
84
|
class SizePtr < FFI::Struct
|
|
50
85
|
layout :value, :size_t
|
|
51
86
|
end
|
|
@@ -69,8 +104,7 @@ module Rdkafka
|
|
|
69
104
|
attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
|
|
70
105
|
attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
|
|
71
106
|
|
|
72
|
-
#
|
|
73
|
-
|
|
107
|
+
# FFI struct representing a Kafka message (rd_kafka_message_t)
|
|
74
108
|
class Message < FFI::Struct
|
|
75
109
|
layout :err, :int,
|
|
76
110
|
:rkt, :pointer,
|
|
@@ -89,8 +123,7 @@ module Rdkafka
|
|
|
89
123
|
attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
|
|
90
124
|
attach_function :rd_kafka_topic_name, [:pointer], :string
|
|
91
125
|
|
|
92
|
-
#
|
|
93
|
-
|
|
126
|
+
# FFI struct representing a topic partition (rd_kafka_topic_partition_t)
|
|
94
127
|
class TopicPartition < FFI::Struct
|
|
95
128
|
layout :topic, :string,
|
|
96
129
|
:partition, :int32,
|
|
@@ -102,6 +135,7 @@ module Rdkafka
|
|
|
102
135
|
:_private, :pointer
|
|
103
136
|
end
|
|
104
137
|
|
|
138
|
+
# FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
|
|
105
139
|
class TopicPartitionList < FFI::Struct
|
|
106
140
|
layout :cnt, :int,
|
|
107
141
|
:size, :int,
|
|
@@ -114,12 +148,10 @@ module Rdkafka
|
|
|
114
148
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
|
115
149
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
|
116
150
|
|
|
117
|
-
#
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
# details. Each resource will also have separate errors results if obtaining configuration
|
|
122
|
-
# was not possible for any reason
|
|
151
|
+
# FFI struct representing a config resource (rd_kafka_ConfigResource_t)
|
|
152
|
+
# Structs for management of configurations. Each configuration is attached to a resource
|
|
153
|
+
# and one resource can have many configuration details. Each resource will also have
|
|
154
|
+
# separate errors results if obtaining configuration was not possible for any reason
|
|
123
155
|
class ConfigResource < FFI::Struct
|
|
124
156
|
layout :type, :int,
|
|
125
157
|
:name, :string
|
|
@@ -156,7 +188,7 @@ module Rdkafka
|
|
|
156
188
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
|
157
189
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
|
158
190
|
|
|
159
|
-
#
|
|
191
|
+
# FFI struct for error description (rd_kafka_err_desc)
|
|
160
192
|
class NativeErrorDesc < FFI::Struct
|
|
161
193
|
layout :code, :int,
|
|
162
194
|
:name, :pointer,
|
|
@@ -256,7 +288,7 @@ module Rdkafka
|
|
|
256
288
|
end
|
|
257
289
|
|
|
258
290
|
# Return 0 so librdkafka frees the json string
|
|
259
|
-
|
|
291
|
+
RD_KAFKA_RESP_ERR_NO_ERROR
|
|
260
292
|
end
|
|
261
293
|
|
|
262
294
|
# Retrieves fatal error details from a kafka client handle.
|
|
@@ -416,7 +448,6 @@ module Rdkafka
|
|
|
416
448
|
attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
|
|
417
449
|
|
|
418
450
|
# Producer
|
|
419
|
-
|
|
420
451
|
RD_KAFKA_VTYPE_END = 0
|
|
421
452
|
RD_KAFKA_VTYPE_TOPIC = 1
|
|
422
453
|
RD_KAFKA_VTYPE_RKT = 2
|
|
@@ -443,13 +474,22 @@ module Rdkafka
|
|
|
443
474
|
attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
|
|
444
475
|
attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
|
|
445
476
|
|
|
446
|
-
#
|
|
477
|
+
# Hash mapping partitioner names to their FFI function symbols
|
|
478
|
+
# @return [Hash{String => Symbol}]
|
|
447
479
|
PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
|
|
448
480
|
method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
|
|
449
481
|
attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
|
|
450
482
|
hsh[name] = method_name
|
|
451
483
|
end
|
|
452
484
|
|
|
485
|
+
# Calculates the partition for a message based on the partitioner
|
|
486
|
+
#
|
|
487
|
+
# @param topic_ptr [FFI::Pointer] pointer to the topic handle
|
|
488
|
+
# @param str [String] the partition key string
|
|
489
|
+
# @param partition_count [Integer, nil] number of partitions
|
|
490
|
+
# @param partitioner [String] name of the partitioner to use
|
|
491
|
+
# @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
|
|
492
|
+
# @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
|
|
453
493
|
def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
|
|
454
494
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
|
455
495
|
return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
|
|
@@ -463,7 +503,6 @@ module Rdkafka
|
|
|
463
503
|
end
|
|
464
504
|
|
|
465
505
|
# Create Topics
|
|
466
|
-
|
|
467
506
|
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
|
468
507
|
RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
|
|
469
508
|
|
|
@@ -475,7 +514,6 @@ module Rdkafka
|
|
|
475
514
|
attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
|
|
476
515
|
|
|
477
516
|
# Delete Topics
|
|
478
|
-
|
|
479
517
|
RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
|
|
480
518
|
RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
|
|
481
519
|
|
|
@@ -496,7 +534,6 @@ module Rdkafka
|
|
|
496
534
|
attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
|
|
497
535
|
|
|
498
536
|
# Delete Group
|
|
499
|
-
|
|
500
537
|
RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
|
|
501
538
|
RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
|
|
502
539
|
|
|
@@ -530,7 +567,6 @@ module Rdkafka
|
|
|
530
567
|
attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
|
|
531
568
|
|
|
532
569
|
# Create Acls
|
|
533
|
-
|
|
534
570
|
RD_KAFKA_ADMIN_OP_CREATEACLS = 9
|
|
535
571
|
RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
|
|
536
572
|
|
|
@@ -539,7 +575,6 @@ module Rdkafka
|
|
|
539
575
|
attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
|
|
540
576
|
|
|
541
577
|
# Delete Acls
|
|
542
|
-
|
|
543
578
|
RD_KAFKA_ADMIN_OP_DELETEACLS = 11
|
|
544
579
|
RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
|
|
545
580
|
|
|
@@ -550,7 +585,6 @@ module Rdkafka
|
|
|
550
585
|
attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
|
|
551
586
|
|
|
552
587
|
# Describe Acls
|
|
553
|
-
|
|
554
588
|
RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
|
|
555
589
|
RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
|
|
556
590
|
|
|
@@ -572,7 +606,6 @@ module Rdkafka
|
|
|
572
606
|
attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
|
|
573
607
|
|
|
574
608
|
# rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
575
|
-
|
|
576
609
|
RD_KAFKA_RESOURCE_ANY = 1
|
|
577
610
|
RD_KAFKA_RESOURCE_TOPIC = 2
|
|
578
611
|
RD_KAFKA_RESOURCE_GROUP = 3
|
|
@@ -580,14 +613,12 @@ module Rdkafka
|
|
|
580
613
|
RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
|
|
581
614
|
|
|
582
615
|
# rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
583
|
-
|
|
584
616
|
RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
585
617
|
RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
586
618
|
RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
587
619
|
RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
588
620
|
|
|
589
621
|
# rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
590
|
-
|
|
591
622
|
RD_KAFKA_ACL_OPERATION_ANY = 1
|
|
592
623
|
RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
593
624
|
RD_KAFKA_ACL_OPERATION_READ = 3
|
|
@@ -602,7 +633,6 @@ module Rdkafka
|
|
|
602
633
|
RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
603
634
|
|
|
604
635
|
# rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
605
|
-
|
|
606
636
|
RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
|
|
607
637
|
RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
608
638
|
RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
@@ -616,8 +646,8 @@ module Rdkafka
|
|
|
616
646
|
attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
|
|
617
647
|
|
|
618
648
|
|
|
619
|
-
#
|
|
620
|
-
class NativeError < FFI::Struct
|
|
649
|
+
# FFI struct for native error (rd_kafka_error_t)
|
|
650
|
+
class NativeError < FFI::Struct
|
|
621
651
|
layout :code, :int32,
|
|
622
652
|
:errstr, :pointer,
|
|
623
653
|
:fatal, :u_int8_t,
|