rdkafka 0.16.0.beta1 → 0.16.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +0 -1
- data/CHANGELOG.md +5 -0
- data/lib/rdkafka/admin/config_binding_result.rb +30 -0
- data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +1 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
- data/lib/rdkafka/admin.rb +204 -0
- data/lib/rdkafka/bindings.rb +51 -0
- data/lib/rdkafka/callbacks.rb +85 -9
- data/lib/rdkafka/producer.rb +94 -4
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +6 -0
- data/rdkafka.gemspec +1 -1
- data/spec/rdkafka/admin_spec.rb +283 -3
- data/spec/rdkafka/producer_spec.rb +42 -0
- data.tar.gz.sig +0 -0
- metadata +9 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3132a3d7af1fc531a6259fe3a027e555bfb59183737c3fcffdb5f7e9dc746320
|
4
|
+
data.tar.gz: 3f1c36a78196660ec0c1b522ff084f63c4c27fa53383864579107a937136cf72
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54d9c9b2cb2f788f8b5134241ff312c7afba6b7d260b4bfe2782fc492023d21b93498fe3af5024e0cc2482741bc70969a7219c85fa8cf35c0c65bdd3a76a73ee
|
7
|
+
data.tar.gz: 9af50d5fc373faf18c5bc1a71d080812bd230a637a40900e3025f97fcfa70df8b9de945b06332c813e9d03316a8eaac6217005f1d733c54be2b8e4b3045c3e9f
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
# Rdkafka Changelog
|
2
2
|
|
3
3
|
## 0.16.0 (Unreleased)
|
4
|
+
- **[Breaking]** Retire support for Ruby 2.7.
|
5
|
+
- **[Feature]** Support incremental config describe + alter API.
|
4
6
|
- **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
|
7
|
+
- **[Feature]** Provide ability to use topic config on a producer for custom behaviors per dispatch.
|
8
|
+
- [Enhancement] Use topic config reference cache for messages production to prevent topic objects allocation with each message.
|
9
|
+
- [Enhancement] Provide `Rrdkafka::Admin#describe_errors` to get errors descriptions (mensfeld)
|
5
10
|
- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
|
6
11
|
- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
|
7
12
|
- [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
# A single config binding result that represents its values extracted from C
|
6
|
+
class ConfigBindingResult
|
7
|
+
attr_reader :name, :value, :read_only, :default, :sensitive, :synonym, :synonyms
|
8
|
+
|
9
|
+
# @param config_ptr [FFI::Pointer] config pointer
|
10
|
+
def initialize(config_ptr)
|
11
|
+
@name = Bindings.rd_kafka_ConfigEntry_name(config_ptr)
|
12
|
+
@value = Bindings.rd_kafka_ConfigEntry_value(config_ptr)
|
13
|
+
@read_only = Bindings.rd_kafka_ConfigEntry_is_read_only(config_ptr)
|
14
|
+
@default = Bindings.rd_kafka_ConfigEntry_is_default(config_ptr)
|
15
|
+
@sensitive = Bindings.rd_kafka_ConfigEntry_is_sensitive(config_ptr)
|
16
|
+
@synonym = Bindings.rd_kafka_ConfigEntry_is_synonym(config_ptr)
|
17
|
+
@synonyms = []
|
18
|
+
|
19
|
+
# The code below builds up the config synonyms using same config binding
|
20
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
21
|
+
synonym_ptr = Bindings.rd_kafka_ConfigEntry_synonyms(config_ptr, pointer_to_size_t)
|
22
|
+
synonyms_ptr = synonym_ptr.read_array_of_pointer(pointer_to_size_t.read_int)
|
23
|
+
|
24
|
+
(1..pointer_to_size_t.read_int).map do |ar|
|
25
|
+
self.class.new synonyms_ptr[ar - 1]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
# A simple binding that represents the requested config resource
|
6
|
+
class ConfigResourceBindingResult
|
7
|
+
attr_reader :name, :type, :configs, :configs_count
|
8
|
+
|
9
|
+
def initialize(config_resource_ptr)
|
10
|
+
ffi_binding = Bindings::ConfigResource.new(config_resource_ptr)
|
11
|
+
|
12
|
+
@name = ffi_binding[:name]
|
13
|
+
@type = ffi_binding[:type]
|
14
|
+
@configs = []
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class DescribeConfigsHandle < AbstractHandle
|
6
|
+
layout :pending, :bool,
|
7
|
+
:response, :int,
|
8
|
+
:response_string, :pointer,
|
9
|
+
:config_entries, :pointer,
|
10
|
+
:entry_count, :int
|
11
|
+
|
12
|
+
# @return [String] the name of the operation.
|
13
|
+
def operation_name
|
14
|
+
"describe configs"
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [DescribeAclReport] instance with an array of acls that matches the request filters.
|
18
|
+
def create_result
|
19
|
+
DescribeConfigsReport.new(
|
20
|
+
config_entries: self[:config_entries],
|
21
|
+
entry_count: self[:entry_count]
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def raise_error
|
26
|
+
raise RdkafkaError.new(
|
27
|
+
self[:response],
|
28
|
+
broker_message: self[:response_string].read_string
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class DescribeConfigsReport
|
6
|
+
attr_reader :resources
|
7
|
+
|
8
|
+
def initialize(config_entries:, entry_count:)
|
9
|
+
@resources=[]
|
10
|
+
|
11
|
+
return if config_entries == FFI::Pointer::NULL
|
12
|
+
|
13
|
+
config_entries
|
14
|
+
.read_array_of_pointer(entry_count)
|
15
|
+
.each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
|
16
|
+
.each do |config_resource_result_ptr|
|
17
|
+
config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
|
18
|
+
|
19
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
20
|
+
configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
|
21
|
+
config_resource_result_ptr,
|
22
|
+
pointer_to_size_t
|
23
|
+
)
|
24
|
+
|
25
|
+
configs_ptr
|
26
|
+
.read_array_of_pointer(pointer_to_size_t.read_int)
|
27
|
+
.map { |config_ptr| ConfigBindingResult.new(config_ptr) }
|
28
|
+
.each { |config_binding| config_resource_result.configs << config_binding }
|
29
|
+
|
30
|
+
@resources << config_resource_result
|
31
|
+
end
|
32
|
+
ensure
|
33
|
+
return if config_entries == FFI::Pointer::NULL
|
34
|
+
|
35
|
+
Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def validate!(config_resource_result_ptr)
|
41
|
+
code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
|
42
|
+
|
43
|
+
return if code.zero?
|
44
|
+
|
45
|
+
raise(
|
46
|
+
RdkafkaError.new(
|
47
|
+
code,
|
48
|
+
Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
|
49
|
+
)
|
50
|
+
)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class IncrementalAlterConfigsHandle < AbstractHandle
|
6
|
+
layout :pending, :bool,
|
7
|
+
:response, :int,
|
8
|
+
:response_string, :pointer,
|
9
|
+
:config_entries, :pointer,
|
10
|
+
:entry_count, :int
|
11
|
+
|
12
|
+
# @return [String] the name of the operation.
|
13
|
+
def operation_name
|
14
|
+
"incremental alter configs"
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [DescribeAclReport] instance with an array of acls that matches the request filters.
|
18
|
+
def create_result
|
19
|
+
IncrementalAlterConfigsReport.new(
|
20
|
+
config_entries: self[:config_entries],
|
21
|
+
entry_count: self[:entry_count]
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def raise_error
|
26
|
+
raise RdkafkaError.new(
|
27
|
+
self[:response],
|
28
|
+
broker_message: self[:response_string].read_string
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class IncrementalAlterConfigsReport
|
6
|
+
attr_reader :resources
|
7
|
+
|
8
|
+
def initialize(config_entries:, entry_count:)
|
9
|
+
@resources=[]
|
10
|
+
|
11
|
+
return if config_entries == FFI::Pointer::NULL
|
12
|
+
|
13
|
+
config_entries
|
14
|
+
.read_array_of_pointer(entry_count)
|
15
|
+
.each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
|
16
|
+
.each do |config_resource_result_ptr|
|
17
|
+
config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
|
18
|
+
|
19
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
20
|
+
configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
|
21
|
+
config_resource_result_ptr,
|
22
|
+
pointer_to_size_t
|
23
|
+
)
|
24
|
+
|
25
|
+
configs_ptr
|
26
|
+
.read_array_of_pointer(pointer_to_size_t.read_int)
|
27
|
+
.map { |config_ptr| ConfigBindingResult.new(config_ptr) }
|
28
|
+
.each { |config_binding| config_resource_result.configs << config_binding }
|
29
|
+
|
30
|
+
@resources << config_resource_result
|
31
|
+
end
|
32
|
+
ensure
|
33
|
+
return if config_entries == FFI::Pointer::NULL
|
34
|
+
|
35
|
+
Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def validate!(config_resource_result_ptr)
|
41
|
+
code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
|
42
|
+
|
43
|
+
return if code.zero?
|
44
|
+
|
45
|
+
raise(
|
46
|
+
RdkafkaError.new(
|
47
|
+
code,
|
48
|
+
Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
|
49
|
+
)
|
50
|
+
)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -4,6 +4,50 @@ module Rdkafka
|
|
4
4
|
class Admin
|
5
5
|
include Helpers::OAuth
|
6
6
|
|
7
|
+
class << self
|
8
|
+
# Allows us to retrieve librdkafka errors with descriptions
|
9
|
+
# Useful for debugging and building UIs, etc.
|
10
|
+
#
|
11
|
+
# @return [Hash<Integer, Hash>] hash with errors mapped by code
|
12
|
+
def describe_errors
|
13
|
+
# Memory pointers for the array of structures and count
|
14
|
+
p_error_descs = FFI::MemoryPointer.new(:pointer)
|
15
|
+
p_count = FFI::MemoryPointer.new(:size_t)
|
16
|
+
|
17
|
+
# Call the attached function
|
18
|
+
Bindings.rd_kafka_get_err_descs(p_error_descs, p_count)
|
19
|
+
|
20
|
+
# Retrieve the number of items in the array
|
21
|
+
count = p_count.read_uint
|
22
|
+
|
23
|
+
# Get the pointer to the array of error descriptions
|
24
|
+
array_of_errors = FFI::Pointer.new(Bindings::NativeErrorDesc, p_error_descs.read_pointer)
|
25
|
+
|
26
|
+
errors = {}
|
27
|
+
|
28
|
+
count.times do |i|
|
29
|
+
# Get the pointer to each struct
|
30
|
+
error_ptr = array_of_errors[i]
|
31
|
+
|
32
|
+
# Create a new instance of NativeErrorDesc for each item
|
33
|
+
error_desc = Bindings::NativeErrorDesc.new(error_ptr)
|
34
|
+
|
35
|
+
# Read values from the struct
|
36
|
+
code = error_desc[:code]
|
37
|
+
|
38
|
+
name = ''
|
39
|
+
desc = ''
|
40
|
+
|
41
|
+
name = error_desc[:name].read_string unless error_desc[:name].null?
|
42
|
+
desc = error_desc[:desc].read_string unless error_desc[:desc].null?
|
43
|
+
|
44
|
+
errors[code] = { code: code, name: name, description: desc }
|
45
|
+
end
|
46
|
+
|
47
|
+
errors
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
7
51
|
# @private
|
8
52
|
def initialize(native_kafka)
|
9
53
|
@native_kafka = native_kafka
|
@@ -620,6 +664,166 @@ module Rdkafka
|
|
620
664
|
describe_acl_handle
|
621
665
|
end
|
622
666
|
|
667
|
+
|
668
|
+
# Describe configs
|
669
|
+
#
|
670
|
+
# @param resources [Array<Hash>] Array where elements are hashes with two keys:
|
671
|
+
# - `:resource_type` - numerical resource type based on Kafka API
|
672
|
+
# - `:resource_name` - string with resource name
|
673
|
+
# @return [DescribeConfigsHandle] Describe config handle that can be used to wait for the
|
674
|
+
# result of fetching resources with their appropriate configs
|
675
|
+
#
|
676
|
+
# @raise [RdkafkaError]
|
677
|
+
#
|
678
|
+
# @note Several resources can be requested at one go, but only one broker at a time
|
679
|
+
def describe_configs(resources)
|
680
|
+
closed_admin_check(__method__)
|
681
|
+
|
682
|
+
handle = DescribeConfigsHandle.new
|
683
|
+
handle[:pending] = true
|
684
|
+
handle[:response] = -1
|
685
|
+
|
686
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
687
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
688
|
+
end
|
689
|
+
|
690
|
+
if queue_ptr.null?
|
691
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
692
|
+
end
|
693
|
+
|
694
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
695
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
696
|
+
inner,
|
697
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
698
|
+
)
|
699
|
+
end
|
700
|
+
|
701
|
+
DescribeConfigsHandle.register(handle)
|
702
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
703
|
+
|
704
|
+
pointer_array = resources.map do |resource_details|
|
705
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
706
|
+
resource_details.fetch(:resource_type),
|
707
|
+
FFI::MemoryPointer.from_string(
|
708
|
+
resource_details.fetch(:resource_name)
|
709
|
+
)
|
710
|
+
)
|
711
|
+
end
|
712
|
+
|
713
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
714
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
715
|
+
|
716
|
+
begin
|
717
|
+
@native_kafka.with_inner do |inner|
|
718
|
+
Rdkafka::Bindings.rd_kafka_DescribeConfigs(
|
719
|
+
inner,
|
720
|
+
configs_array_ptr,
|
721
|
+
pointer_array.size,
|
722
|
+
admin_options_ptr,
|
723
|
+
queue_ptr
|
724
|
+
)
|
725
|
+
end
|
726
|
+
rescue Exception
|
727
|
+
DescribeConfigsHandle.remove(handle.to_ptr.address)
|
728
|
+
|
729
|
+
raise
|
730
|
+
ensure
|
731
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
732
|
+
configs_array_ptr,
|
733
|
+
pointer_array.size
|
734
|
+
) if configs_array_ptr
|
735
|
+
end
|
736
|
+
|
737
|
+
handle
|
738
|
+
end
|
739
|
+
|
740
|
+
# Alters in an incremental way all the configs provided for given resources
|
741
|
+
#
|
742
|
+
# @param resources_with_configs [Array<Hash>] resources with the configs key that contains
|
743
|
+
# name, value and the proper op_type to perform on this value.
|
744
|
+
#
|
745
|
+
# @return [IncrementalAlterConfigsHandle] Incremental alter configs handle that can be used to
|
746
|
+
# wait for the result of altering resources with their appropriate configs
|
747
|
+
#
|
748
|
+
# @raise [RdkafkaError]
|
749
|
+
#
|
750
|
+
# @note Several resources can be requested at one go, but only one broker at a time
|
751
|
+
# @note The results won't contain altered values but only the altered resources
|
752
|
+
def incremental_alter_configs(resources_with_configs)
|
753
|
+
closed_admin_check(__method__)
|
754
|
+
|
755
|
+
handle = IncrementalAlterConfigsHandle.new
|
756
|
+
handle[:pending] = true
|
757
|
+
handle[:response] = -1
|
758
|
+
|
759
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
760
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
761
|
+
end
|
762
|
+
|
763
|
+
if queue_ptr.null?
|
764
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
765
|
+
end
|
766
|
+
|
767
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
768
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
769
|
+
inner,
|
770
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
771
|
+
)
|
772
|
+
end
|
773
|
+
|
774
|
+
IncrementalAlterConfigsHandle.register(handle)
|
775
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
776
|
+
|
777
|
+
# Tu poprawnie tworzyc
|
778
|
+
pointer_array = resources_with_configs.map do |resource_details|
|
779
|
+
# First build the appropriate resource representation
|
780
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
781
|
+
resource_details.fetch(:resource_type),
|
782
|
+
FFI::MemoryPointer.from_string(
|
783
|
+
resource_details.fetch(:resource_name)
|
784
|
+
)
|
785
|
+
)
|
786
|
+
|
787
|
+
resource_details.fetch(:configs).each do |config|
|
788
|
+
Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
789
|
+
resource_ptr,
|
790
|
+
config.fetch(:name),
|
791
|
+
config.fetch(:op_type),
|
792
|
+
config.fetch(:value)
|
793
|
+
)
|
794
|
+
end
|
795
|
+
|
796
|
+
resource_ptr
|
797
|
+
end
|
798
|
+
|
799
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
800
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
801
|
+
|
802
|
+
|
803
|
+
begin
|
804
|
+
@native_kafka.with_inner do |inner|
|
805
|
+
Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
|
806
|
+
inner,
|
807
|
+
configs_array_ptr,
|
808
|
+
pointer_array.size,
|
809
|
+
admin_options_ptr,
|
810
|
+
queue_ptr
|
811
|
+
)
|
812
|
+
end
|
813
|
+
rescue Exception
|
814
|
+
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
|
815
|
+
|
816
|
+
raise
|
817
|
+
ensure
|
818
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
819
|
+
configs_array_ptr,
|
820
|
+
pointer_array.size
|
821
|
+
) if configs_array_ptr
|
822
|
+
end
|
823
|
+
|
824
|
+
handle
|
825
|
+
end
|
826
|
+
|
623
827
|
private
|
624
828
|
|
625
829
|
def closed_admin_check(method)
|
data/lib/rdkafka/bindings.rb
CHANGED
@@ -89,10 +89,58 @@ module Rdkafka
|
|
89
89
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
90
90
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
91
91
|
|
92
|
+
# Configs management
|
93
|
+
#
|
94
|
+
# Structs for management of configurations
|
95
|
+
# Each configuration is attached to a resource and one resource can have many configuration
|
96
|
+
# details. Each resource will also have separate errors results if obtaining configuration
|
97
|
+
# was not possible for any reason
|
98
|
+
class ConfigResource < FFI::Struct
|
99
|
+
layout :type, :int,
|
100
|
+
:name, :string
|
101
|
+
end
|
102
|
+
|
103
|
+
attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
104
|
+
attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
|
105
|
+
attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
|
106
|
+
attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
|
107
|
+
attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
|
108
|
+
attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
|
109
|
+
attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
|
110
|
+
attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
|
111
|
+
attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
|
112
|
+
attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
|
113
|
+
attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
|
114
|
+
attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
|
115
|
+
attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
|
116
|
+
attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
|
117
|
+
attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
|
118
|
+
attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
119
|
+
attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
|
120
|
+
attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
|
121
|
+
attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
|
122
|
+
|
123
|
+
RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
|
124
|
+
RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
|
125
|
+
|
126
|
+
RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
|
127
|
+
RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
|
128
|
+
|
129
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
|
130
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
|
131
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
132
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
133
|
+
|
92
134
|
# Errors
|
135
|
+
class NativeErrorDesc < FFI::Struct
|
136
|
+
layout :code, :int,
|
137
|
+
:name, :pointer,
|
138
|
+
:desc, :pointer
|
139
|
+
end
|
93
140
|
|
94
141
|
attach_function :rd_kafka_err2name, [:int], :string
|
95
142
|
attach_function :rd_kafka_err2str, [:int], :string
|
143
|
+
attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
|
96
144
|
|
97
145
|
# Configuration
|
98
146
|
|
@@ -119,6 +167,9 @@ module Rdkafka
|
|
119
167
|
# Log queue
|
120
168
|
attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
|
121
169
|
attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
|
170
|
+
# Per topic configs
|
171
|
+
attach_function :rd_kafka_topic_conf_new, [], :pointer
|
172
|
+
attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
122
173
|
|
123
174
|
LogCallback = FFI::Function.new(
|
124
175
|
:void, [:pointer, :int, :string, :string]
|
data/lib/rdkafka/callbacks.rb
CHANGED
@@ -113,6 +113,42 @@ module Rdkafka
|
|
113
113
|
end
|
114
114
|
end
|
115
115
|
|
116
|
+
class DescribeConfigsResult
|
117
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
118
|
+
|
119
|
+
def initialize(event_ptr)
|
120
|
+
@results=[]
|
121
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
122
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
123
|
+
|
124
|
+
if @result_error == 0
|
125
|
+
configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
|
126
|
+
# Get the number of matching acls
|
127
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
128
|
+
@results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
|
129
|
+
@results_count = pointer_to_size_t.read_int
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class IncrementalAlterConfigsResult
|
135
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
136
|
+
|
137
|
+
def initialize(event_ptr)
|
138
|
+
@results=[]
|
139
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
140
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
141
|
+
|
142
|
+
if @result_error == 0
|
143
|
+
incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
|
144
|
+
# Get the number of matching acls
|
145
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
146
|
+
@results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
|
147
|
+
@results_count = pointer_to_size_t.read_int
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
116
152
|
# FFI Function used for Create Topic and Delete Topic callbacks
|
117
153
|
BackgroundEventCallbackFunction = FFI::Function.new(
|
118
154
|
:void, [:pointer, :pointer, :pointer]
|
@@ -123,20 +159,24 @@ module Rdkafka
|
|
123
159
|
# @private
|
124
160
|
class BackgroundEventCallback
|
125
161
|
def self.call(_, event_ptr, _)
|
126
|
-
|
127
|
-
|
162
|
+
case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
163
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
|
128
164
|
process_create_topic(event_ptr)
|
129
|
-
|
165
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
|
166
|
+
process_describe_configs(event_ptr)
|
167
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
|
168
|
+
process_incremental_alter_configs(event_ptr)
|
169
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
|
130
170
|
process_delete_topic(event_ptr)
|
131
|
-
|
171
|
+
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
|
132
172
|
process_create_partitions(event_ptr)
|
133
|
-
|
173
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
|
134
174
|
process_create_acl(event_ptr)
|
135
|
-
|
175
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
|
136
176
|
process_delete_acl(event_ptr)
|
137
|
-
|
177
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
|
138
178
|
process_describe_acl(event_ptr)
|
139
|
-
|
179
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
|
140
180
|
process_delete_groups(event_ptr)
|
141
181
|
end
|
142
182
|
end
|
@@ -161,6 +201,42 @@ module Rdkafka
|
|
161
201
|
end
|
162
202
|
end
|
163
203
|
|
204
|
+
def self.process_describe_configs(event_ptr)
|
205
|
+
describe_configs = DescribeConfigsResult.new(event_ptr)
|
206
|
+
describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
207
|
+
|
208
|
+
if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
|
209
|
+
describe_configs_handle[:response] = describe_configs.result_error
|
210
|
+
describe_configs_handle[:response_string] = describe_configs.error_string
|
211
|
+
describe_configs_handle[:pending] = false
|
212
|
+
|
213
|
+
if describe_configs.result_error == 0
|
214
|
+
describe_configs_handle[:config_entries] = describe_configs.results
|
215
|
+
describe_configs_handle[:entry_count] = describe_configs.results_count
|
216
|
+
end
|
217
|
+
|
218
|
+
describe_configs_handle.unlock
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def self.process_incremental_alter_configs(event_ptr)
|
223
|
+
incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
|
224
|
+
incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
225
|
+
|
226
|
+
if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
|
227
|
+
incremental_alter_handle[:response] = incremental_alter.result_error
|
228
|
+
incremental_alter_handle[:response_string] = incremental_alter.error_string
|
229
|
+
incremental_alter_handle[:pending] = false
|
230
|
+
|
231
|
+
if incremental_alter.result_error == 0
|
232
|
+
incremental_alter_handle[:config_entries] = incremental_alter.results
|
233
|
+
incremental_alter_handle[:entry_count] = incremental_alter.results_count
|
234
|
+
end
|
235
|
+
|
236
|
+
incremental_alter_handle.unlock
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
164
240
|
def self.process_delete_groups(event_ptr)
|
165
241
|
delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
|
166
242
|
|
@@ -263,7 +339,7 @@ module Rdkafka
|
|
263
339
|
describe_acl_handle[:response_string] = describe_acl.error_string
|
264
340
|
|
265
341
|
if describe_acl.result_error == 0
|
266
|
-
describe_acl_handle[:acls]
|
342
|
+
describe_acl_handle[:acls] = describe_acl.matching_acls
|
267
343
|
describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
|
268
344
|
end
|
269
345
|
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -9,7 +9,15 @@ module Rdkafka
|
|
9
9
|
# Cache partitions count for 30 seconds
|
10
10
|
PARTITIONS_COUNT_TTL = 30
|
11
11
|
|
12
|
-
|
12
|
+
# Empty hash used as a default
|
13
|
+
EMPTY_HASH = {}.freeze
|
14
|
+
|
15
|
+
private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
|
16
|
+
|
17
|
+
# Raised when there was a critical issue when invoking rd_kafka_topic_new
|
18
|
+
# This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
|
19
|
+
# resolved and this is normalized in all the places
|
20
|
+
class TopicHandleCreationError < RuntimeError; end
|
13
21
|
|
14
22
|
# @private
|
15
23
|
# Returns the current delivery callback, by default this is nil.
|
@@ -28,6 +36,8 @@ module Rdkafka
|
|
28
36
|
# @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
|
29
37
|
# the "consistent_random" default
|
30
38
|
def initialize(native_kafka, partitioner_name)
|
39
|
+
@topics_refs_map = {}
|
40
|
+
@topics_configs = {}
|
31
41
|
@native_kafka = native_kafka
|
32
42
|
@partitioner_name = partitioner_name || "consistent_random"
|
33
43
|
|
@@ -54,6 +64,52 @@ module Rdkafka
|
|
54
64
|
end
|
55
65
|
end
|
56
66
|
|
67
|
+
# Sets alternative set of configuration details that can be set per topic
|
68
|
+
# @note It is not allowed to re-set the same topic config twice because of the underlying
|
69
|
+
# librdkafka caching
|
70
|
+
# @param topic [String] The topic name
|
71
|
+
# @param config [Hash] config we want to use per topic basis
|
72
|
+
# @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
|
73
|
+
# because it is already computed during the retrieval attempt in the `#produce` flow.
|
74
|
+
def set_topic_config(topic, config, config_hash)
|
75
|
+
# Ensure lock on topic reference just in case
|
76
|
+
@native_kafka.with_inner do |inner|
|
77
|
+
@topics_refs_map[topic] ||= {}
|
78
|
+
@topics_configs[topic] ||= {}
|
79
|
+
|
80
|
+
return if @topics_configs[topic].key?(config_hash)
|
81
|
+
|
82
|
+
# If config is empty, we create an empty reference that will be used with defaults
|
83
|
+
rd_topic_config = if config.empty?
|
84
|
+
nil
|
85
|
+
else
|
86
|
+
Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
|
87
|
+
config.each do |key, value|
|
88
|
+
error_buffer = FFI::MemoryPointer.new(:char, 256)
|
89
|
+
result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
|
90
|
+
topic_config,
|
91
|
+
key.to_s,
|
92
|
+
value.to_s,
|
93
|
+
error_buffer,
|
94
|
+
256
|
95
|
+
)
|
96
|
+
|
97
|
+
unless result == :config_ok
|
98
|
+
raise Config::ConfigError.new(error_buffer.read_string)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
|
105
|
+
|
106
|
+
raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
|
107
|
+
|
108
|
+
@topics_configs[topic][config_hash] = config
|
109
|
+
@topics_refs_map[topic][config_hash] = topic_handle
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
57
113
|
# Starts the native Kafka polling thread and kicks off the init polling
|
58
114
|
# @note Not needed to run unless explicit start was disabled
|
59
115
|
def start
|
@@ -83,7 +139,18 @@ module Rdkafka
|
|
83
139
|
def close
|
84
140
|
return if closed?
|
85
141
|
ObjectSpace.undefine_finalizer(self)
|
86
|
-
|
142
|
+
|
143
|
+
@native_kafka.close do
|
144
|
+
# We need to remove the topics references objects before we destroy the producer,
|
145
|
+
# otherwise they would leak out
|
146
|
+
@topics_refs_map.each_value do |refs|
|
147
|
+
refs.each_value do |ref|
|
148
|
+
Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
@topics_refs_map.clear
|
87
154
|
end
|
88
155
|
|
89
156
|
# Whether this producer has closed
|
@@ -182,11 +249,22 @@ module Rdkafka
|
|
182
249
|
# @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
|
183
250
|
# @param headers [Hash<String,String>] Optional message headers
|
184
251
|
# @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
|
252
|
+
# @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
|
185
253
|
#
|
186
254
|
# @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
|
187
255
|
#
|
188
256
|
# @raise [RdkafkaError] When adding the message to rdkafka's queue failed
|
189
|
-
def produce(
|
257
|
+
def produce(
|
258
|
+
topic:,
|
259
|
+
payload: nil,
|
260
|
+
key: nil,
|
261
|
+
partition: nil,
|
262
|
+
partition_key: nil,
|
263
|
+
timestamp: nil,
|
264
|
+
headers: nil,
|
265
|
+
label: nil,
|
266
|
+
topic_config: EMPTY_HASH
|
267
|
+
)
|
190
268
|
closed_producer_check(__method__)
|
191
269
|
|
192
270
|
# Start by checking and converting the input
|
@@ -205,8 +283,20 @@ module Rdkafka
|
|
205
283
|
key.bytesize
|
206
284
|
end
|
207
285
|
|
286
|
+
topic_config_hash = topic_config.hash
|
287
|
+
|
288
|
+
# Checks if we have the rdkafka topic reference object ready. It saves us on object
|
289
|
+
# allocation and allows to use custom config on demand.
|
290
|
+
set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
|
291
|
+
topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
|
292
|
+
|
208
293
|
if partition_key
|
209
294
|
partition_count = partition_count(topic)
|
295
|
+
|
296
|
+
# Check if there are no overrides for the partitioner and use the default one only when
|
297
|
+
# no per-topic is present.
|
298
|
+
partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
|
299
|
+
|
210
300
|
# If the topic is not present, set to -1
|
211
301
|
partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
|
212
302
|
end
|
@@ -236,7 +326,7 @@ module Rdkafka
|
|
236
326
|
DeliveryHandle.register(delivery_handle)
|
237
327
|
|
238
328
|
args = [
|
239
|
-
:int, Rdkafka::Bindings::
|
329
|
+
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
|
240
330
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
|
241
331
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
|
242
332
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
|
data/lib/rdkafka/version.rb
CHANGED
data/lib/rdkafka.rb
CHANGED
@@ -24,7 +24,13 @@ require "rdkafka/admin/delete_acl_handle"
|
|
24
24
|
require "rdkafka/admin/delete_acl_report"
|
25
25
|
require "rdkafka/admin/describe_acl_handle"
|
26
26
|
require "rdkafka/admin/describe_acl_report"
|
27
|
+
require "rdkafka/admin/describe_configs_handle"
|
28
|
+
require "rdkafka/admin/describe_configs_report"
|
29
|
+
require "rdkafka/admin/incremental_alter_configs_handle"
|
30
|
+
require "rdkafka/admin/incremental_alter_configs_report"
|
27
31
|
require "rdkafka/admin/acl_binding_result"
|
32
|
+
require "rdkafka/admin/config_binding_result"
|
33
|
+
require "rdkafka/admin/config_resource_binding_result"
|
28
34
|
require "rdkafka/bindings"
|
29
35
|
require "rdkafka/callbacks"
|
30
36
|
require "rdkafka/config"
|
data/rdkafka.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.name = 'rdkafka'
|
16
16
|
gem.require_paths = ['lib']
|
17
17
|
gem.version = Rdkafka::VERSION
|
18
|
-
gem.required_ruby_version = '>=
|
18
|
+
gem.required_ruby_version = '>= 3.0'
|
19
19
|
gem.extensions = %w(ext/Rakefile)
|
20
20
|
gem.cert_chain = %w[certs/cert_chain.pem]
|
21
21
|
|
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -16,12 +16,12 @@ describe Rdkafka::Admin do
|
|
16
16
|
admin.close
|
17
17
|
end
|
18
18
|
|
19
|
-
let(:topic_name) { "test-topic-#{
|
19
|
+
let(:topic_name) { "test-topic-#{SecureRandom.uuid}" }
|
20
20
|
let(:topic_partition_count) { 3 }
|
21
21
|
let(:topic_replication_factor) { 1 }
|
22
22
|
let(:topic_config) { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
|
23
23
|
let(:invalid_topic_config) { {"cleeeeenup.policee" => "campact"} }
|
24
|
-
let(:group_name) { "test-group-#{
|
24
|
+
let(:group_name) { "test-group-#{SecureRandom.uuid}" }
|
25
25
|
|
26
26
|
let(:resource_name) {"acl-test-topic"}
|
27
27
|
let(:resource_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
|
@@ -31,6 +31,14 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe '#describe_errors' do
|
35
|
+
let(:errors) { admin.class.describe_errors }
|
36
|
+
|
37
|
+
it { expect(errors.size).to eq(162) }
|
38
|
+
it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
|
39
|
+
it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
|
40
|
+
end
|
41
|
+
|
34
42
|
describe 'admin without auto-start' do
|
35
43
|
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
44
|
|
@@ -142,6 +150,275 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
142
150
|
end
|
143
151
|
end
|
144
152
|
|
153
|
+
describe "describe_configs" do
|
154
|
+
subject(:resources_results) { admin.describe_configs(resources).wait.resources }
|
155
|
+
|
156
|
+
before do
|
157
|
+
admin.create_topic(topic_name, 2, 1).wait
|
158
|
+
sleep(1)
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'when describing config of an existing topic' do
|
162
|
+
let(:resources) { [{ resource_type: 2, resource_name: topic_name }] }
|
163
|
+
|
164
|
+
it do
|
165
|
+
expect(resources_results.size).to eq(1)
|
166
|
+
expect(resources_results.first.type).to eq(2)
|
167
|
+
expect(resources_results.first.name).to eq(topic_name)
|
168
|
+
expect(resources_results.first.configs.size).to be > 25
|
169
|
+
expect(resources_results.first.configs.first.name).to eq('compression.type')
|
170
|
+
expect(resources_results.first.configs.first.value).to eq('producer')
|
171
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
context 'when describing config of a non-existing topic' do
|
176
|
+
let(:resources) { [{ resource_type: 2, resource_name: SecureRandom.uuid }] }
|
177
|
+
|
178
|
+
it 'expect to raise error' do
|
179
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
context 'when describing both existing and non-existing topics' do
|
184
|
+
let(:resources) do
|
185
|
+
[
|
186
|
+
{ resource_type: 2, resource_name: topic_name },
|
187
|
+
{ resource_type: 2, resource_name: SecureRandom.uuid }
|
188
|
+
]
|
189
|
+
end
|
190
|
+
|
191
|
+
it 'expect to raise error' do
|
192
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
context 'when describing multiple existing topics' do
|
197
|
+
let(:resources) do
|
198
|
+
[
|
199
|
+
{ resource_type: 2, resource_name: 'example_topic' },
|
200
|
+
{ resource_type: 2, resource_name: topic_name }
|
201
|
+
]
|
202
|
+
end
|
203
|
+
|
204
|
+
it do
|
205
|
+
expect(resources_results.size).to eq(2)
|
206
|
+
expect(resources_results.first.type).to eq(2)
|
207
|
+
expect(resources_results.first.name).to eq('example_topic')
|
208
|
+
expect(resources_results.last.type).to eq(2)
|
209
|
+
expect(resources_results.last.name).to eq(topic_name)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
context 'when trying to describe invalid resource type' do
|
214
|
+
let(:resources) { [{ resource_type: 0, resource_name: SecureRandom.uuid }] }
|
215
|
+
|
216
|
+
it 'expect to raise error' do
|
217
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_request/)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
context 'when trying to describe invalid broker' do
|
222
|
+
let(:resources) { [{ resource_type: 4, resource_name: 'non-existing' }] }
|
223
|
+
|
224
|
+
it 'expect to raise error' do
|
225
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
context 'when trying to describe valid broker' do
|
230
|
+
let(:resources) { [{ resource_type: 4, resource_name: '1' }] }
|
231
|
+
|
232
|
+
it do
|
233
|
+
expect(resources_results.size).to eq(1)
|
234
|
+
expect(resources_results.first.type).to eq(4)
|
235
|
+
expect(resources_results.first.name).to eq('1')
|
236
|
+
expect(resources_results.first.configs.size).to be > 230
|
237
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
238
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
239
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
context 'when describing valid broker with topics in one request' do
|
244
|
+
let(:resources) do
|
245
|
+
[
|
246
|
+
{ resource_type: 4, resource_name: '1' },
|
247
|
+
{ resource_type: 2, resource_name: topic_name }
|
248
|
+
]
|
249
|
+
end
|
250
|
+
|
251
|
+
it do
|
252
|
+
expect(resources_results.size).to eq(2)
|
253
|
+
expect(resources_results.first.type).to eq(4)
|
254
|
+
expect(resources_results.first.name).to eq('1')
|
255
|
+
expect(resources_results.first.configs.size).to be > 230
|
256
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
257
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
258
|
+
expect(resources_results.last.type).to eq(2)
|
259
|
+
expect(resources_results.last.name).to eq(topic_name)
|
260
|
+
expect(resources_results.last.configs.size).to be > 25
|
261
|
+
expect(resources_results.last.configs.first.name).to eq('compression.type')
|
262
|
+
expect(resources_results.last.configs.first.value).to eq('producer')
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
describe "incremental_alter_configs" do
|
268
|
+
subject(:resources_results) { admin.incremental_alter_configs(resources_with_configs).wait.resources }
|
269
|
+
|
270
|
+
before do
|
271
|
+
admin.create_topic(topic_name, 2, 1).wait
|
272
|
+
sleep(1)
|
273
|
+
end
|
274
|
+
|
275
|
+
context 'when altering one topic with one valid config via set' do
|
276
|
+
let(:target_retention) { (86400002 + rand(10_000)).to_s }
|
277
|
+
let(:resources_with_configs) do
|
278
|
+
[
|
279
|
+
{
|
280
|
+
resource_type: 2,
|
281
|
+
resource_name: topic_name,
|
282
|
+
configs: [
|
283
|
+
{
|
284
|
+
name: 'delete.retention.ms',
|
285
|
+
value: target_retention,
|
286
|
+
op_type: 0
|
287
|
+
}
|
288
|
+
]
|
289
|
+
}
|
290
|
+
]
|
291
|
+
end
|
292
|
+
|
293
|
+
it do
|
294
|
+
expect(resources_results.size).to eq(1)
|
295
|
+
expect(resources_results.first.type).to eq(2)
|
296
|
+
expect(resources_results.first.name).to eq(topic_name)
|
297
|
+
|
298
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
299
|
+
config.name == 'delete.retention.ms'
|
300
|
+
end
|
301
|
+
|
302
|
+
expect(ret_config.value).to eq(target_retention)
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
context 'when altering one topic with one valid config via delete' do
|
307
|
+
let(:target_retention) { (8640002 + rand(10_000)).to_s }
|
308
|
+
let(:resources_with_configs) do
|
309
|
+
[
|
310
|
+
{
|
311
|
+
resource_type: 2,
|
312
|
+
resource_name: topic_name,
|
313
|
+
configs: [
|
314
|
+
{
|
315
|
+
name: 'delete.retention.ms',
|
316
|
+
value: target_retention,
|
317
|
+
op_type: 1
|
318
|
+
}
|
319
|
+
]
|
320
|
+
}
|
321
|
+
]
|
322
|
+
end
|
323
|
+
|
324
|
+
it do
|
325
|
+
expect(resources_results.size).to eq(1)
|
326
|
+
expect(resources_results.first.type).to eq(2)
|
327
|
+
expect(resources_results.first.name).to eq(topic_name)
|
328
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
329
|
+
config.name == 'delete.retention.ms'
|
330
|
+
end
|
331
|
+
|
332
|
+
expect(ret_config.value).to eq('86400000')
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
context 'when altering one topic with one valid config via append' do
|
337
|
+
let(:target_policy) { 'compact' }
|
338
|
+
let(:resources_with_configs) do
|
339
|
+
[
|
340
|
+
{
|
341
|
+
resource_type: 2,
|
342
|
+
resource_name: topic_name,
|
343
|
+
configs: [
|
344
|
+
{
|
345
|
+
name: 'cleanup.policy',
|
346
|
+
value: target_policy,
|
347
|
+
op_type: 2
|
348
|
+
}
|
349
|
+
]
|
350
|
+
}
|
351
|
+
]
|
352
|
+
end
|
353
|
+
|
354
|
+
it do
|
355
|
+
expect(resources_results.size).to eq(1)
|
356
|
+
expect(resources_results.first.type).to eq(2)
|
357
|
+
expect(resources_results.first.name).to eq(topic_name)
|
358
|
+
|
359
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
360
|
+
config.name == 'cleanup.policy'
|
361
|
+
end
|
362
|
+
|
363
|
+
expect(ret_config.value).to eq("delete,#{target_policy}")
|
364
|
+
end
|
365
|
+
end
|
366
|
+
|
367
|
+
context 'when altering one topic with one valid config via subtrack' do
|
368
|
+
let(:target_policy) { 'delete' }
|
369
|
+
let(:resources_with_configs) do
|
370
|
+
[
|
371
|
+
{
|
372
|
+
resource_type: 2,
|
373
|
+
resource_name: topic_name,
|
374
|
+
configs: [
|
375
|
+
{
|
376
|
+
name: 'cleanup.policy',
|
377
|
+
value: target_policy,
|
378
|
+
op_type: 3
|
379
|
+
}
|
380
|
+
]
|
381
|
+
}
|
382
|
+
]
|
383
|
+
end
|
384
|
+
|
385
|
+
it do
|
386
|
+
expect(resources_results.size).to eq(1)
|
387
|
+
expect(resources_results.first.type).to eq(2)
|
388
|
+
expect(resources_results.first.name).to eq(topic_name)
|
389
|
+
|
390
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
391
|
+
config.name == 'cleanup.policy'
|
392
|
+
end
|
393
|
+
|
394
|
+
expect(ret_config.value).to eq('')
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
context 'when altering one topic with invalid config' do
|
399
|
+
let(:target_retention) { '-10' }
|
400
|
+
let(:resources_with_configs) do
|
401
|
+
[
|
402
|
+
{
|
403
|
+
resource_type: 2,
|
404
|
+
resource_name: topic_name,
|
405
|
+
configs: [
|
406
|
+
{
|
407
|
+
name: 'delete.retention.ms',
|
408
|
+
value: target_retention,
|
409
|
+
op_type: 0
|
410
|
+
}
|
411
|
+
]
|
412
|
+
}
|
413
|
+
]
|
414
|
+
end
|
415
|
+
|
416
|
+
it 'expect to raise error' do
|
417
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_config/)
|
418
|
+
end
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
145
422
|
describe "#delete_topic" do
|
146
423
|
describe "called with invalid input" do
|
147
424
|
# https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
|
@@ -412,7 +689,10 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
412
689
|
end
|
413
690
|
|
414
691
|
context 'when topic has less then desired number of partitions' do
|
415
|
-
before
|
692
|
+
before do
|
693
|
+
admin.create_topic(topic_name, 1, 1).wait
|
694
|
+
sleep(1)
|
695
|
+
end
|
416
696
|
|
417
697
|
it 'expect to change number of partitions' do
|
418
698
|
admin.create_partitions(topic_name, 10).wait
|
@@ -31,6 +31,48 @@ describe Rdkafka::Producer do
|
|
31
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
32
32
|
end
|
33
33
|
|
34
|
+
describe '#produce with topic config alterations' do
|
35
|
+
context 'when config is not valid' do
|
36
|
+
it 'expect to raise error' do
|
37
|
+
expect do
|
38
|
+
producer.produce(topic: 'test', payload: '', topic_config: { 'invalid': 'invalid' })
|
39
|
+
end.to raise_error(Rdkafka::Config::ConfigError)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'when config is valid' do
|
44
|
+
it 'expect to raise error' do
|
45
|
+
expect do
|
46
|
+
producer.produce(topic: 'test', payload: '', topic_config: { 'acks': 1 }).wait
|
47
|
+
end.not_to raise_error
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'when alteration should change behavior' do
|
51
|
+
# This is set incorrectly for a reason
|
52
|
+
# If alteration would not work, this will hang the spec suite
|
53
|
+
let(:producer) do
|
54
|
+
rdkafka_producer_config(
|
55
|
+
'message.timeout.ms': 1_000_000,
|
56
|
+
:"bootstrap.servers" => "localhost:9094",
|
57
|
+
).producer
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'expect to give up on delivery fast based on alteration config' do
|
61
|
+
expect do
|
62
|
+
producer.produce(
|
63
|
+
topic: 'produce_config_test',
|
64
|
+
payload: 'test',
|
65
|
+
topic_config: {
|
66
|
+
'compression.type': 'gzip',
|
67
|
+
'message.timeout.ms': 1
|
68
|
+
}
|
69
|
+
).wait
|
70
|
+
end.to raise_error(Rdkafka::RdkafkaError, /msg_timed_out/)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
34
76
|
context "delivery callback" do
|
35
77
|
context "with a proc/lambda" do
|
36
78
|
it "should set the callback" do
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.0.
|
4
|
+
version: 0.16.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-05-27 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
@@ -193,6 +193,8 @@ files:
|
|
193
193
|
- lib/rdkafka/abstract_handle.rb
|
194
194
|
- lib/rdkafka/admin.rb
|
195
195
|
- lib/rdkafka/admin/acl_binding_result.rb
|
196
|
+
- lib/rdkafka/admin/config_binding_result.rb
|
197
|
+
- lib/rdkafka/admin/config_resource_binding_result.rb
|
196
198
|
- lib/rdkafka/admin/create_acl_handle.rb
|
197
199
|
- lib/rdkafka/admin/create_acl_report.rb
|
198
200
|
- lib/rdkafka/admin/create_partitions_handle.rb
|
@@ -207,6 +209,10 @@ files:
|
|
207
209
|
- lib/rdkafka/admin/delete_topic_report.rb
|
208
210
|
- lib/rdkafka/admin/describe_acl_handle.rb
|
209
211
|
- lib/rdkafka/admin/describe_acl_report.rb
|
212
|
+
- lib/rdkafka/admin/describe_configs_handle.rb
|
213
|
+
- lib/rdkafka/admin/describe_configs_report.rb
|
214
|
+
- lib/rdkafka/admin/incremental_alter_configs_handle.rb
|
215
|
+
- lib/rdkafka/admin/incremental_alter_configs_report.rb
|
210
216
|
- lib/rdkafka/bindings.rb
|
211
217
|
- lib/rdkafka/callbacks.rb
|
212
218
|
- lib/rdkafka/config.rb
|
@@ -272,7 +278,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
272
278
|
requirements:
|
273
279
|
- - ">="
|
274
280
|
- !ruby/object:Gem::Version
|
275
|
-
version: '
|
281
|
+
version: '3.0'
|
276
282
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
277
283
|
requirements:
|
278
284
|
- - ">="
|
metadata.gz.sig
CHANGED
Binary file
|