karafka-rdkafka 0.23.1.rc2-x86_64-linux-musl → 0.24.0.rc1-x86_64-linux-musl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -1
- data/Gemfile +9 -0
- data/docker-compose-ssl.yml +1 -1
- data/docker-compose.yml +1 -1
- data/ext/librdkafka.so +0 -0
- data/karafka-rdkafka.gemspec +0 -7
- data/lib/rdkafka/abstract_handle.rb +23 -5
- data/lib/rdkafka/admin/acl_binding_result.rb +1 -1
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/create_acl_report.rb +3 -0
- data/lib/rdkafka/admin/create_partitions_handle.rb +3 -0
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +3 -0
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_report.rb +3 -0
- data/lib/rdkafka/admin/delete_groups_handle.rb +5 -0
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +3 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +3 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +6 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +3 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +6 -0
- data/lib/rdkafka/admin.rb +99 -103
- data/lib/rdkafka/bindings.rb +23 -25
- data/lib/rdkafka/callbacks.rb +62 -2
- data/lib/rdkafka/config.rb +20 -8
- data/lib/rdkafka/consumer/headers.rb +3 -2
- data/lib/rdkafka/consumer/message.rb +7 -3
- data/lib/rdkafka/consumer/partition.rb +6 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +5 -5
- data/lib/rdkafka/consumer.rb +24 -12
- data/lib/rdkafka/defaults.rb +84 -0
- data/lib/rdkafka/error.rb +46 -1
- data/lib/rdkafka/helpers/oauth.rb +11 -5
- data/lib/rdkafka/metadata.rb +29 -5
- data/lib/rdkafka/native_kafka.rb +26 -2
- data/lib/rdkafka/producer/delivery_report.rb +6 -2
- data/lib/rdkafka/producer/partitions_count_cache.rb +24 -14
- data/lib/rdkafka/producer/testing.rb +3 -3
- data/lib/rdkafka/producer.rb +60 -16
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +2 -0
- data/renovate.json +1 -8
- metadata +3 -86
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
+
# Admin client for Kafka administrative operations
|
|
4
5
|
class Admin
|
|
5
6
|
include Helpers::OAuth
|
|
6
7
|
|
|
@@ -8,7 +9,7 @@ module Rdkafka
|
|
|
8
9
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
9
10
|
# Useful for debugging and building UIs, etc.
|
|
10
11
|
#
|
|
11
|
-
# @return [Hash
|
|
12
|
+
# @return [Hash{Integer => Hash}] hash with errors mapped by code
|
|
12
13
|
def describe_errors
|
|
13
14
|
# Memory pointers for the array of structures and count
|
|
14
15
|
p_error_descs = FFI::MemoryPointer.new(:pointer)
|
|
@@ -49,6 +50,7 @@ module Rdkafka
|
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
# @private
|
|
53
|
+
# @param native_kafka [NativeKafka] wrapper around the native Kafka handle
|
|
52
54
|
def initialize(native_kafka)
|
|
53
55
|
@native_kafka = native_kafka
|
|
54
56
|
|
|
@@ -69,6 +71,8 @@ module Rdkafka
|
|
|
69
71
|
end
|
|
70
72
|
end
|
|
71
73
|
|
|
74
|
+
# @return [Proc] finalizer proc for closing the admin
|
|
75
|
+
# @private
|
|
72
76
|
def finalizer
|
|
73
77
|
->(_) { close }
|
|
74
78
|
end
|
|
@@ -78,7 +82,7 @@ module Rdkafka
|
|
|
78
82
|
# @param topic_name [String, nil] metadat about particular topic or all if nil
|
|
79
83
|
# @param timeout_ms [Integer] metadata request timeout
|
|
80
84
|
# @return [Metadata] requested metadata
|
|
81
|
-
def metadata(topic_name = nil, timeout_ms =
|
|
85
|
+
def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
|
|
82
86
|
closed_admin_check(__method__)
|
|
83
87
|
|
|
84
88
|
@native_kafka.with_inner do |inner|
|
|
@@ -100,9 +104,12 @@ module Rdkafka
|
|
|
100
104
|
|
|
101
105
|
# Create a topic with the given partition count and replication factor
|
|
102
106
|
#
|
|
107
|
+
# @param topic_name [String] name of the topic to create
|
|
108
|
+
# @param partition_count [Integer] number of partitions for the topic
|
|
109
|
+
# @param replication_factor [Integer] replication factor for the topic
|
|
110
|
+
# @param topic_config [Hash] optional topic configuration settings
|
|
103
111
|
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
|
|
104
112
|
# creating the topic
|
|
105
|
-
#
|
|
106
113
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
107
114
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
108
115
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
@@ -178,6 +185,11 @@ module Rdkafka
|
|
|
178
185
|
create_topic_handle
|
|
179
186
|
end
|
|
180
187
|
|
|
188
|
+
# Deletes a consumer group
|
|
189
|
+
#
|
|
190
|
+
# @param group_id [String] the group id to delete
|
|
191
|
+
# @return [DeleteGroupsHandle] delete groups handle that can be used to wait for the result
|
|
192
|
+
# @raise [RdkafkaError] When deleting the group fails
|
|
181
193
|
def delete_group(group_id)
|
|
182
194
|
closed_admin_check(__method__)
|
|
183
195
|
|
|
@@ -233,6 +245,7 @@ module Rdkafka
|
|
|
233
245
|
|
|
234
246
|
# Deletes the named topic
|
|
235
247
|
#
|
|
248
|
+
# @param topic_name [String] name of the topic to delete
|
|
236
249
|
# @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
|
|
237
250
|
# deleting the topic
|
|
238
251
|
# @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
|
|
@@ -290,14 +303,12 @@ module Rdkafka
|
|
|
290
303
|
|
|
291
304
|
# Creates extra partitions for a given topic
|
|
292
305
|
#
|
|
293
|
-
# @param topic_name [String]
|
|
306
|
+
# @param topic_name [String] name of the topic
|
|
294
307
|
# @param partition_count [Integer] how many partitions we want to end up with for given topic
|
|
295
|
-
#
|
|
308
|
+
# @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the result
|
|
296
309
|
# @raise [ConfigError] When the partition count or replication factor are out of valid range
|
|
297
310
|
# @raise [RdkafkaError] When the topic name is invalid or the topic already exists
|
|
298
311
|
# @raise [RdkafkaError] When the topic configuration is invalid
|
|
299
|
-
#
|
|
300
|
-
# @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
|
|
301
312
|
def create_partitions(topic_name, partition_count)
|
|
302
313
|
closed_admin_check(__method__)
|
|
303
314
|
|
|
@@ -354,40 +365,34 @@ module Rdkafka
|
|
|
354
365
|
end
|
|
355
366
|
|
|
356
367
|
# Create acl
|
|
357
|
-
# @param resource_type - values of type rd_kafka_ResourceType_t
|
|
358
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
359
|
-
# valid values are:
|
|
360
|
-
# RD_KAFKA_RESOURCE_TOPIC = 2
|
|
361
|
-
# RD_KAFKA_RESOURCE_GROUP = 3
|
|
362
|
-
# RD_KAFKA_RESOURCE_BROKER = 4
|
|
363
|
-
# @param resource_pattern_type - values of type rd_kafka_ResourcePatternType_t
|
|
364
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
365
|
-
# valid values are:
|
|
366
|
-
# RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
367
|
-
# RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
368
|
-
# RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
369
|
-
# @param operation - values of type rd_kafka_AclOperation_t
|
|
370
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
371
|
-
# valid values are:
|
|
372
|
-
# RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
373
|
-
# RD_KAFKA_ACL_OPERATION_READ = 3
|
|
374
|
-
# RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
375
|
-
# RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
376
|
-
# RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
377
|
-
# RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
378
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
379
|
-
# RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
380
|
-
# RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
381
|
-
# RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
382
|
-
# RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
383
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
384
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
385
|
-
# valid values are:
|
|
386
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
387
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
388
368
|
#
|
|
369
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
370
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
371
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
372
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
373
|
+
# @param resource_name [String] name of the resource
|
|
374
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
375
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
376
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
377
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
378
|
+
# @param principal [String] principal (e.g., "User:alice")
|
|
379
|
+
# @param host [String] host address
|
|
380
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
381
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
382
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
383
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
384
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
385
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
386
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
387
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
388
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
389
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
390
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
391
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
392
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
393
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
394
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
389
395
|
# @return [CreateAclHandle] Create acl handle that can be used to wait for the result of creating the acl
|
|
390
|
-
#
|
|
391
396
|
# @raise [RdkafkaError]
|
|
392
397
|
def create_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
393
398
|
closed_admin_check(__method__)
|
|
@@ -460,39 +465,33 @@ module Rdkafka
|
|
|
460
465
|
|
|
461
466
|
# Delete acl
|
|
462
467
|
#
|
|
463
|
-
# @param resource_type
|
|
464
|
-
#
|
|
465
|
-
#
|
|
466
|
-
#
|
|
467
|
-
#
|
|
468
|
-
#
|
|
469
|
-
#
|
|
470
|
-
#
|
|
471
|
-
#
|
|
472
|
-
#
|
|
473
|
-
#
|
|
474
|
-
#
|
|
475
|
-
#
|
|
476
|
-
#
|
|
477
|
-
#
|
|
478
|
-
#
|
|
479
|
-
#
|
|
480
|
-
#
|
|
481
|
-
#
|
|
482
|
-
#
|
|
483
|
-
#
|
|
484
|
-
#
|
|
485
|
-
#
|
|
486
|
-
#
|
|
487
|
-
#
|
|
488
|
-
#
|
|
489
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
490
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
491
|
-
# valid values are:
|
|
492
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
493
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
468
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
469
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
470
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
471
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
472
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
473
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
474
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
475
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
476
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
477
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
478
|
+
# @param host [String, nil] host address or nil for any
|
|
479
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
480
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
481
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
482
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
483
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
484
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
485
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
486
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
487
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
488
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
489
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
490
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
491
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
492
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
493
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
494
494
|
# @return [DeleteAclHandle] Delete acl handle that can be used to wait for the result of deleting the acl
|
|
495
|
-
#
|
|
496
495
|
# @raise [RdkafkaError]
|
|
497
496
|
def delete_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
498
497
|
closed_admin_check(__method__)
|
|
@@ -567,39 +566,33 @@ module Rdkafka
|
|
|
567
566
|
|
|
568
567
|
# Describe acls
|
|
569
568
|
#
|
|
570
|
-
# @param resource_type
|
|
571
|
-
#
|
|
572
|
-
#
|
|
573
|
-
#
|
|
574
|
-
#
|
|
575
|
-
#
|
|
576
|
-
#
|
|
577
|
-
#
|
|
578
|
-
#
|
|
579
|
-
#
|
|
580
|
-
#
|
|
581
|
-
#
|
|
582
|
-
#
|
|
583
|
-
#
|
|
584
|
-
#
|
|
585
|
-
#
|
|
586
|
-
#
|
|
587
|
-
#
|
|
588
|
-
#
|
|
589
|
-
#
|
|
590
|
-
#
|
|
591
|
-
#
|
|
592
|
-
#
|
|
593
|
-
#
|
|
594
|
-
#
|
|
595
|
-
#
|
|
596
|
-
# @param permission_type - values of type rd_kafka_AclPermissionType_t
|
|
597
|
-
# https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
598
|
-
# valid values are:
|
|
599
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
600
|
-
# RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
569
|
+
# @param resource_type [Integer] rd_kafka_ResourceType_t value:
|
|
570
|
+
# - RD_KAFKA_RESOURCE_TOPIC = 2
|
|
571
|
+
# - RD_KAFKA_RESOURCE_GROUP = 3
|
|
572
|
+
# - RD_KAFKA_RESOURCE_BROKER = 4
|
|
573
|
+
# @param resource_name [String, nil] name of the resource or nil for any
|
|
574
|
+
# @param resource_pattern_type [Integer] rd_kafka_ResourcePatternType_t value:
|
|
575
|
+
# - RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
576
|
+
# - RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
577
|
+
# - RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
578
|
+
# @param principal [String, nil] principal (e.g., "User:alice") or nil for any
|
|
579
|
+
# @param host [String, nil] host address or nil for any
|
|
580
|
+
# @param operation [Integer] rd_kafka_AclOperation_t value:
|
|
581
|
+
# - RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
582
|
+
# - RD_KAFKA_ACL_OPERATION_READ = 3
|
|
583
|
+
# - RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
584
|
+
# - RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
585
|
+
# - RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
586
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
587
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
588
|
+
# - RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
589
|
+
# - RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
590
|
+
# - RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
591
|
+
# - RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
592
|
+
# @param permission_type [Integer] rd_kafka_AclPermissionType_t value:
|
|
593
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
594
|
+
# - RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
601
595
|
# @return [DescribeAclHandle] Describe acl handle that can be used to wait for the result of fetching acls
|
|
602
|
-
#
|
|
603
596
|
# @raise [RdkafkaError]
|
|
604
597
|
def describe_acl(resource_type:, resource_name:, resource_pattern_type:, principal:, host:, operation:, permission_type:)
|
|
605
598
|
closed_admin_check(__method__)
|
|
@@ -825,6 +818,9 @@ module Rdkafka
|
|
|
825
818
|
|
|
826
819
|
private
|
|
827
820
|
|
|
821
|
+
# Checks if the admin is closed and raises an error if so
|
|
822
|
+
# @param method [Symbol] name of the calling method for error context
|
|
823
|
+
# @raise [ClosedAdminError] when the admin is closed
|
|
828
824
|
def closed_admin_check(method)
|
|
829
825
|
raise Rdkafka::ClosedAdminError.new(method) if closed?
|
|
830
826
|
end
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -14,6 +14,8 @@ module Rdkafka
|
|
|
14
14
|
module Bindings
|
|
15
15
|
extend FFI::Library
|
|
16
16
|
|
|
17
|
+
# Returns the library extension based on the host OS
|
|
18
|
+
# @return [String] 'dylib' on macOS, 'so' on other systems
|
|
17
19
|
def self.lib_extension
|
|
18
20
|
if RbConfig::CONFIG['host_os'] =~ /darwin/
|
|
19
21
|
'dylib'
|
|
@@ -78,6 +80,7 @@ module Rdkafka
|
|
|
78
80
|
|
|
79
81
|
EMPTY_HASH = {}.freeze
|
|
80
82
|
|
|
83
|
+
# FFI struct for size_t pointer wrapper
|
|
81
84
|
class SizePtr < FFI::Struct
|
|
82
85
|
layout :value, :size_t
|
|
83
86
|
end
|
|
@@ -101,8 +104,7 @@ module Rdkafka
|
|
|
101
104
|
attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
|
|
102
105
|
attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
|
|
103
106
|
|
|
104
|
-
#
|
|
105
|
-
|
|
107
|
+
# FFI struct representing a Kafka message (rd_kafka_message_t)
|
|
106
108
|
class Message < FFI::Struct
|
|
107
109
|
layout :err, :int,
|
|
108
110
|
:rkt, :pointer,
|
|
@@ -121,8 +123,7 @@ module Rdkafka
|
|
|
121
123
|
attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
|
|
122
124
|
attach_function :rd_kafka_topic_name, [:pointer], :string
|
|
123
125
|
|
|
124
|
-
#
|
|
125
|
-
|
|
126
|
+
# FFI struct representing a topic partition (rd_kafka_topic_partition_t)
|
|
126
127
|
class TopicPartition < FFI::Struct
|
|
127
128
|
layout :topic, :string,
|
|
128
129
|
:partition, :int32,
|
|
@@ -134,6 +135,7 @@ module Rdkafka
|
|
|
134
135
|
:_private, :pointer
|
|
135
136
|
end
|
|
136
137
|
|
|
138
|
+
# FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
|
|
137
139
|
class TopicPartitionList < FFI::Struct
|
|
138
140
|
layout :cnt, :int,
|
|
139
141
|
:size, :int,
|
|
@@ -146,12 +148,10 @@ module Rdkafka
|
|
|
146
148
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
|
147
149
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
|
148
150
|
|
|
149
|
-
#
|
|
150
|
-
#
|
|
151
|
-
#
|
|
152
|
-
#
|
|
153
|
-
# details. Each resource will also have separate errors results if obtaining configuration
|
|
154
|
-
# was not possible for any reason
|
|
151
|
+
# FFI struct representing a config resource (rd_kafka_ConfigResource_t)
|
|
152
|
+
# Structs for management of configurations. Each configuration is attached to a resource
|
|
153
|
+
# and one resource can have many configuration details. Each resource will also have
|
|
154
|
+
# separate errors results if obtaining configuration was not possible for any reason
|
|
155
155
|
class ConfigResource < FFI::Struct
|
|
156
156
|
layout :type, :int,
|
|
157
157
|
:name, :string
|
|
@@ -188,7 +188,7 @@ module Rdkafka
|
|
|
188
188
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
|
189
189
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
|
190
190
|
|
|
191
|
-
#
|
|
191
|
+
# FFI struct for error description (rd_kafka_err_desc)
|
|
192
192
|
class NativeErrorDesc < FFI::Struct
|
|
193
193
|
layout :code, :int,
|
|
194
194
|
:name, :pointer,
|
|
@@ -448,7 +448,6 @@ module Rdkafka
|
|
|
448
448
|
attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
|
|
449
449
|
|
|
450
450
|
# Producer
|
|
451
|
-
|
|
452
451
|
RD_KAFKA_VTYPE_END = 0
|
|
453
452
|
RD_KAFKA_VTYPE_TOPIC = 1
|
|
454
453
|
RD_KAFKA_VTYPE_RKT = 2
|
|
@@ -475,13 +474,22 @@ module Rdkafka
|
|
|
475
474
|
attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
|
|
476
475
|
attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
|
|
477
476
|
|
|
478
|
-
#
|
|
477
|
+
# Hash mapping partitioner names to their FFI function symbols
|
|
478
|
+
# @return [Hash{String => Symbol}]
|
|
479
479
|
PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
|
|
480
480
|
method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
|
|
481
481
|
attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
|
|
482
482
|
hsh[name] = method_name
|
|
483
483
|
end
|
|
484
484
|
|
|
485
|
+
# Calculates the partition for a message based on the partitioner
|
|
486
|
+
#
|
|
487
|
+
# @param topic_ptr [FFI::Pointer] pointer to the topic handle
|
|
488
|
+
# @param str [String] the partition key string
|
|
489
|
+
# @param partition_count [Integer, nil] number of partitions
|
|
490
|
+
# @param partitioner [String] name of the partitioner to use
|
|
491
|
+
# @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
|
|
492
|
+
# @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
|
|
485
493
|
def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
|
|
486
494
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
|
487
495
|
return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
|
|
@@ -495,7 +503,6 @@ module Rdkafka
|
|
|
495
503
|
end
|
|
496
504
|
|
|
497
505
|
# Create Topics
|
|
498
|
-
|
|
499
506
|
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
|
500
507
|
RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
|
|
501
508
|
|
|
@@ -507,7 +514,6 @@ module Rdkafka
|
|
|
507
514
|
attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
|
|
508
515
|
|
|
509
516
|
# Delete Topics
|
|
510
|
-
|
|
511
517
|
RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
|
|
512
518
|
RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
|
|
513
519
|
|
|
@@ -528,7 +534,6 @@ module Rdkafka
|
|
|
528
534
|
attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
|
|
529
535
|
|
|
530
536
|
# Delete Group
|
|
531
|
-
|
|
532
537
|
RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
|
|
533
538
|
RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
|
|
534
539
|
|
|
@@ -562,7 +567,6 @@ module Rdkafka
|
|
|
562
567
|
attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
|
|
563
568
|
|
|
564
569
|
# Create Acls
|
|
565
|
-
|
|
566
570
|
RD_KAFKA_ADMIN_OP_CREATEACLS = 9
|
|
567
571
|
RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
|
|
568
572
|
|
|
@@ -571,7 +575,6 @@ module Rdkafka
|
|
|
571
575
|
attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
|
|
572
576
|
|
|
573
577
|
# Delete Acls
|
|
574
|
-
|
|
575
578
|
RD_KAFKA_ADMIN_OP_DELETEACLS = 11
|
|
576
579
|
RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
|
|
577
580
|
|
|
@@ -582,7 +585,6 @@ module Rdkafka
|
|
|
582
585
|
attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
|
|
583
586
|
|
|
584
587
|
# Describe Acls
|
|
585
|
-
|
|
586
588
|
RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
|
|
587
589
|
RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
|
|
588
590
|
|
|
@@ -604,7 +606,6 @@ module Rdkafka
|
|
|
604
606
|
attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
|
|
605
607
|
|
|
606
608
|
# rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
607
|
-
|
|
608
609
|
RD_KAFKA_RESOURCE_ANY = 1
|
|
609
610
|
RD_KAFKA_RESOURCE_TOPIC = 2
|
|
610
611
|
RD_KAFKA_RESOURCE_GROUP = 3
|
|
@@ -612,14 +613,12 @@ module Rdkafka
|
|
|
612
613
|
RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
|
|
613
614
|
|
|
614
615
|
# rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
615
|
-
|
|
616
616
|
RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
617
617
|
RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
618
618
|
RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
619
619
|
RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
620
620
|
|
|
621
621
|
# rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
622
|
-
|
|
623
622
|
RD_KAFKA_ACL_OPERATION_ANY = 1
|
|
624
623
|
RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
625
624
|
RD_KAFKA_ACL_OPERATION_READ = 3
|
|
@@ -634,7 +633,6 @@ module Rdkafka
|
|
|
634
633
|
RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
635
634
|
|
|
636
635
|
# rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
637
|
-
|
|
638
636
|
RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
|
|
639
637
|
RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
640
638
|
RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
@@ -648,8 +646,8 @@ module Rdkafka
|
|
|
648
646
|
attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
|
|
649
647
|
|
|
650
648
|
|
|
651
|
-
#
|
|
652
|
-
class NativeError < FFI::Struct
|
|
649
|
+
# FFI struct for native error (rd_kafka_error_t)
|
|
650
|
+
class NativeError < FFI::Struct
|
|
653
651
|
layout :code, :int32,
|
|
654
652
|
:errstr, :pointer,
|
|
655
653
|
:fatal, :u_int8_t,
|