rdkafka 0.16.0.beta1 → 0.16.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +0 -1
- data/CHANGELOG.md +5 -0
- data/lib/rdkafka/admin/config_binding_result.rb +30 -0
- data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +1 -0
- data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
- data/lib/rdkafka/admin.rb +204 -0
- data/lib/rdkafka/bindings.rb +51 -0
- data/lib/rdkafka/callbacks.rb +85 -9
- data/lib/rdkafka/producer.rb +94 -4
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +6 -0
- data/rdkafka.gemspec +1 -1
- data/spec/rdkafka/admin_spec.rb +283 -3
- data/spec/rdkafka/producer_spec.rb +42 -0
- data.tar.gz.sig +0 -0
- metadata +9 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3132a3d7af1fc531a6259fe3a027e555bfb59183737c3fcffdb5f7e9dc746320
|
4
|
+
data.tar.gz: 3f1c36a78196660ec0c1b522ff084f63c4c27fa53383864579107a937136cf72
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54d9c9b2cb2f788f8b5134241ff312c7afba6b7d260b4bfe2782fc492023d21b93498fe3af5024e0cc2482741bc70969a7219c85fa8cf35c0c65bdd3a76a73ee
|
7
|
+
data.tar.gz: 9af50d5fc373faf18c5bc1a71d080812bd230a637a40900e3025f97fcfa70df8b9de945b06332c813e9d03316a8eaac6217005f1d733c54be2b8e4b3045c3e9f
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
# Rdkafka Changelog
|
2
2
|
|
3
3
|
## 0.16.0 (Unreleased)
|
4
|
+
- **[Breaking]** Retire support for Ruby 2.7.
|
5
|
+
- **[Feature]** Support incremental config describe + alter API.
|
4
6
|
- **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
|
7
|
+
- **[Feature]** Provide ability to use topic config on a producer for custom behaviors per dispatch.
|
8
|
+
- [Enhancement] Use topic config reference cache for messages production to prevent topic objects allocation with each message.
|
9
|
+
- [Enhancement] Provide `Rrdkafka::Admin#describe_errors` to get errors descriptions (mensfeld)
|
5
10
|
- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
|
6
11
|
- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
|
7
12
|
- [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
# A single config binding result that represents its values extracted from C
|
6
|
+
class ConfigBindingResult
|
7
|
+
attr_reader :name, :value, :read_only, :default, :sensitive, :synonym, :synonyms
|
8
|
+
|
9
|
+
# @param config_ptr [FFI::Pointer] config pointer
|
10
|
+
def initialize(config_ptr)
|
11
|
+
@name = Bindings.rd_kafka_ConfigEntry_name(config_ptr)
|
12
|
+
@value = Bindings.rd_kafka_ConfigEntry_value(config_ptr)
|
13
|
+
@read_only = Bindings.rd_kafka_ConfigEntry_is_read_only(config_ptr)
|
14
|
+
@default = Bindings.rd_kafka_ConfigEntry_is_default(config_ptr)
|
15
|
+
@sensitive = Bindings.rd_kafka_ConfigEntry_is_sensitive(config_ptr)
|
16
|
+
@synonym = Bindings.rd_kafka_ConfigEntry_is_synonym(config_ptr)
|
17
|
+
@synonyms = []
|
18
|
+
|
19
|
+
# The code below builds up the config synonyms using same config binding
|
20
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
21
|
+
synonym_ptr = Bindings.rd_kafka_ConfigEntry_synonyms(config_ptr, pointer_to_size_t)
|
22
|
+
synonyms_ptr = synonym_ptr.read_array_of_pointer(pointer_to_size_t.read_int)
|
23
|
+
|
24
|
+
(1..pointer_to_size_t.read_int).map do |ar|
|
25
|
+
self.class.new synonyms_ptr[ar - 1]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
# A simple binding that represents the requested config resource
|
6
|
+
class ConfigResourceBindingResult
|
7
|
+
attr_reader :name, :type, :configs, :configs_count
|
8
|
+
|
9
|
+
def initialize(config_resource_ptr)
|
10
|
+
ffi_binding = Bindings::ConfigResource.new(config_resource_ptr)
|
11
|
+
|
12
|
+
@name = ffi_binding[:name]
|
13
|
+
@type = ffi_binding[:type]
|
14
|
+
@configs = []
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class DescribeConfigsHandle < AbstractHandle
|
6
|
+
layout :pending, :bool,
|
7
|
+
:response, :int,
|
8
|
+
:response_string, :pointer,
|
9
|
+
:config_entries, :pointer,
|
10
|
+
:entry_count, :int
|
11
|
+
|
12
|
+
# @return [String] the name of the operation.
|
13
|
+
def operation_name
|
14
|
+
"describe configs"
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [DescribeAclReport] instance with an array of acls that matches the request filters.
|
18
|
+
def create_result
|
19
|
+
DescribeConfigsReport.new(
|
20
|
+
config_entries: self[:config_entries],
|
21
|
+
entry_count: self[:entry_count]
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def raise_error
|
26
|
+
raise RdkafkaError.new(
|
27
|
+
self[:response],
|
28
|
+
broker_message: self[:response_string].read_string
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class DescribeConfigsReport
|
6
|
+
attr_reader :resources
|
7
|
+
|
8
|
+
def initialize(config_entries:, entry_count:)
|
9
|
+
@resources=[]
|
10
|
+
|
11
|
+
return if config_entries == FFI::Pointer::NULL
|
12
|
+
|
13
|
+
config_entries
|
14
|
+
.read_array_of_pointer(entry_count)
|
15
|
+
.each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
|
16
|
+
.each do |config_resource_result_ptr|
|
17
|
+
config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
|
18
|
+
|
19
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
20
|
+
configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
|
21
|
+
config_resource_result_ptr,
|
22
|
+
pointer_to_size_t
|
23
|
+
)
|
24
|
+
|
25
|
+
configs_ptr
|
26
|
+
.read_array_of_pointer(pointer_to_size_t.read_int)
|
27
|
+
.map { |config_ptr| ConfigBindingResult.new(config_ptr) }
|
28
|
+
.each { |config_binding| config_resource_result.configs << config_binding }
|
29
|
+
|
30
|
+
@resources << config_resource_result
|
31
|
+
end
|
32
|
+
ensure
|
33
|
+
return if config_entries == FFI::Pointer::NULL
|
34
|
+
|
35
|
+
Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def validate!(config_resource_result_ptr)
|
41
|
+
code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
|
42
|
+
|
43
|
+
return if code.zero?
|
44
|
+
|
45
|
+
raise(
|
46
|
+
RdkafkaError.new(
|
47
|
+
code,
|
48
|
+
Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
|
49
|
+
)
|
50
|
+
)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class IncrementalAlterConfigsHandle < AbstractHandle
|
6
|
+
layout :pending, :bool,
|
7
|
+
:response, :int,
|
8
|
+
:response_string, :pointer,
|
9
|
+
:config_entries, :pointer,
|
10
|
+
:entry_count, :int
|
11
|
+
|
12
|
+
# @return [String] the name of the operation.
|
13
|
+
def operation_name
|
14
|
+
"incremental alter configs"
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [DescribeAclReport] instance with an array of acls that matches the request filters.
|
18
|
+
def create_result
|
19
|
+
IncrementalAlterConfigsReport.new(
|
20
|
+
config_entries: self[:config_entries],
|
21
|
+
entry_count: self[:entry_count]
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def raise_error
|
26
|
+
raise RdkafkaError.new(
|
27
|
+
self[:response],
|
28
|
+
broker_message: self[:response_string].read_string
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
class Admin
|
5
|
+
class IncrementalAlterConfigsReport
|
6
|
+
attr_reader :resources
|
7
|
+
|
8
|
+
def initialize(config_entries:, entry_count:)
|
9
|
+
@resources=[]
|
10
|
+
|
11
|
+
return if config_entries == FFI::Pointer::NULL
|
12
|
+
|
13
|
+
config_entries
|
14
|
+
.read_array_of_pointer(entry_count)
|
15
|
+
.each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
|
16
|
+
.each do |config_resource_result_ptr|
|
17
|
+
config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
|
18
|
+
|
19
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
20
|
+
configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
|
21
|
+
config_resource_result_ptr,
|
22
|
+
pointer_to_size_t
|
23
|
+
)
|
24
|
+
|
25
|
+
configs_ptr
|
26
|
+
.read_array_of_pointer(pointer_to_size_t.read_int)
|
27
|
+
.map { |config_ptr| ConfigBindingResult.new(config_ptr) }
|
28
|
+
.each { |config_binding| config_resource_result.configs << config_binding }
|
29
|
+
|
30
|
+
@resources << config_resource_result
|
31
|
+
end
|
32
|
+
ensure
|
33
|
+
return if config_entries == FFI::Pointer::NULL
|
34
|
+
|
35
|
+
Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def validate!(config_resource_result_ptr)
|
41
|
+
code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
|
42
|
+
|
43
|
+
return if code.zero?
|
44
|
+
|
45
|
+
raise(
|
46
|
+
RdkafkaError.new(
|
47
|
+
code,
|
48
|
+
Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
|
49
|
+
)
|
50
|
+
)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -4,6 +4,50 @@ module Rdkafka
|
|
4
4
|
class Admin
|
5
5
|
include Helpers::OAuth
|
6
6
|
|
7
|
+
class << self
|
8
|
+
# Allows us to retrieve librdkafka errors with descriptions
|
9
|
+
# Useful for debugging and building UIs, etc.
|
10
|
+
#
|
11
|
+
# @return [Hash<Integer, Hash>] hash with errors mapped by code
|
12
|
+
def describe_errors
|
13
|
+
# Memory pointers for the array of structures and count
|
14
|
+
p_error_descs = FFI::MemoryPointer.new(:pointer)
|
15
|
+
p_count = FFI::MemoryPointer.new(:size_t)
|
16
|
+
|
17
|
+
# Call the attached function
|
18
|
+
Bindings.rd_kafka_get_err_descs(p_error_descs, p_count)
|
19
|
+
|
20
|
+
# Retrieve the number of items in the array
|
21
|
+
count = p_count.read_uint
|
22
|
+
|
23
|
+
# Get the pointer to the array of error descriptions
|
24
|
+
array_of_errors = FFI::Pointer.new(Bindings::NativeErrorDesc, p_error_descs.read_pointer)
|
25
|
+
|
26
|
+
errors = {}
|
27
|
+
|
28
|
+
count.times do |i|
|
29
|
+
# Get the pointer to each struct
|
30
|
+
error_ptr = array_of_errors[i]
|
31
|
+
|
32
|
+
# Create a new instance of NativeErrorDesc for each item
|
33
|
+
error_desc = Bindings::NativeErrorDesc.new(error_ptr)
|
34
|
+
|
35
|
+
# Read values from the struct
|
36
|
+
code = error_desc[:code]
|
37
|
+
|
38
|
+
name = ''
|
39
|
+
desc = ''
|
40
|
+
|
41
|
+
name = error_desc[:name].read_string unless error_desc[:name].null?
|
42
|
+
desc = error_desc[:desc].read_string unless error_desc[:desc].null?
|
43
|
+
|
44
|
+
errors[code] = { code: code, name: name, description: desc }
|
45
|
+
end
|
46
|
+
|
47
|
+
errors
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
7
51
|
# @private
|
8
52
|
def initialize(native_kafka)
|
9
53
|
@native_kafka = native_kafka
|
@@ -620,6 +664,166 @@ module Rdkafka
|
|
620
664
|
describe_acl_handle
|
621
665
|
end
|
622
666
|
|
667
|
+
|
668
|
+
# Describe configs
|
669
|
+
#
|
670
|
+
# @param resources [Array<Hash>] Array where elements are hashes with two keys:
|
671
|
+
# - `:resource_type` - numerical resource type based on Kafka API
|
672
|
+
# - `:resource_name` - string with resource name
|
673
|
+
# @return [DescribeConfigsHandle] Describe config handle that can be used to wait for the
|
674
|
+
# result of fetching resources with their appropriate configs
|
675
|
+
#
|
676
|
+
# @raise [RdkafkaError]
|
677
|
+
#
|
678
|
+
# @note Several resources can be requested at one go, but only one broker at a time
|
679
|
+
def describe_configs(resources)
|
680
|
+
closed_admin_check(__method__)
|
681
|
+
|
682
|
+
handle = DescribeConfigsHandle.new
|
683
|
+
handle[:pending] = true
|
684
|
+
handle[:response] = -1
|
685
|
+
|
686
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
687
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
688
|
+
end
|
689
|
+
|
690
|
+
if queue_ptr.null?
|
691
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
692
|
+
end
|
693
|
+
|
694
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
695
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
696
|
+
inner,
|
697
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
|
698
|
+
)
|
699
|
+
end
|
700
|
+
|
701
|
+
DescribeConfigsHandle.register(handle)
|
702
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
703
|
+
|
704
|
+
pointer_array = resources.map do |resource_details|
|
705
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
706
|
+
resource_details.fetch(:resource_type),
|
707
|
+
FFI::MemoryPointer.from_string(
|
708
|
+
resource_details.fetch(:resource_name)
|
709
|
+
)
|
710
|
+
)
|
711
|
+
end
|
712
|
+
|
713
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
714
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
715
|
+
|
716
|
+
begin
|
717
|
+
@native_kafka.with_inner do |inner|
|
718
|
+
Rdkafka::Bindings.rd_kafka_DescribeConfigs(
|
719
|
+
inner,
|
720
|
+
configs_array_ptr,
|
721
|
+
pointer_array.size,
|
722
|
+
admin_options_ptr,
|
723
|
+
queue_ptr
|
724
|
+
)
|
725
|
+
end
|
726
|
+
rescue Exception
|
727
|
+
DescribeConfigsHandle.remove(handle.to_ptr.address)
|
728
|
+
|
729
|
+
raise
|
730
|
+
ensure
|
731
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
732
|
+
configs_array_ptr,
|
733
|
+
pointer_array.size
|
734
|
+
) if configs_array_ptr
|
735
|
+
end
|
736
|
+
|
737
|
+
handle
|
738
|
+
end
|
739
|
+
|
740
|
+
# Alters in an incremental way all the configs provided for given resources
|
741
|
+
#
|
742
|
+
# @param resources_with_configs [Array<Hash>] resources with the configs key that contains
|
743
|
+
# name, value and the proper op_type to perform on this value.
|
744
|
+
#
|
745
|
+
# @return [IncrementalAlterConfigsHandle] Incremental alter configs handle that can be used to
|
746
|
+
# wait for the result of altering resources with their appropriate configs
|
747
|
+
#
|
748
|
+
# @raise [RdkafkaError]
|
749
|
+
#
|
750
|
+
# @note Several resources can be requested at one go, but only one broker at a time
|
751
|
+
# @note The results won't contain altered values but only the altered resources
|
752
|
+
def incremental_alter_configs(resources_with_configs)
|
753
|
+
closed_admin_check(__method__)
|
754
|
+
|
755
|
+
handle = IncrementalAlterConfigsHandle.new
|
756
|
+
handle[:pending] = true
|
757
|
+
handle[:response] = -1
|
758
|
+
|
759
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
760
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
761
|
+
end
|
762
|
+
|
763
|
+
if queue_ptr.null?
|
764
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
765
|
+
end
|
766
|
+
|
767
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
768
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
769
|
+
inner,
|
770
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
|
771
|
+
)
|
772
|
+
end
|
773
|
+
|
774
|
+
IncrementalAlterConfigsHandle.register(handle)
|
775
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
776
|
+
|
777
|
+
# Tu poprawnie tworzyc
|
778
|
+
pointer_array = resources_with_configs.map do |resource_details|
|
779
|
+
# First build the appropriate resource representation
|
780
|
+
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
781
|
+
resource_details.fetch(:resource_type),
|
782
|
+
FFI::MemoryPointer.from_string(
|
783
|
+
resource_details.fetch(:resource_name)
|
784
|
+
)
|
785
|
+
)
|
786
|
+
|
787
|
+
resource_details.fetch(:configs).each do |config|
|
788
|
+
Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
789
|
+
resource_ptr,
|
790
|
+
config.fetch(:name),
|
791
|
+
config.fetch(:op_type),
|
792
|
+
config.fetch(:value)
|
793
|
+
)
|
794
|
+
end
|
795
|
+
|
796
|
+
resource_ptr
|
797
|
+
end
|
798
|
+
|
799
|
+
configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
|
800
|
+
configs_array_ptr.write_array_of_pointer(pointer_array)
|
801
|
+
|
802
|
+
|
803
|
+
begin
|
804
|
+
@native_kafka.with_inner do |inner|
|
805
|
+
Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
|
806
|
+
inner,
|
807
|
+
configs_array_ptr,
|
808
|
+
pointer_array.size,
|
809
|
+
admin_options_ptr,
|
810
|
+
queue_ptr
|
811
|
+
)
|
812
|
+
end
|
813
|
+
rescue Exception
|
814
|
+
IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
|
815
|
+
|
816
|
+
raise
|
817
|
+
ensure
|
818
|
+
Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
|
819
|
+
configs_array_ptr,
|
820
|
+
pointer_array.size
|
821
|
+
) if configs_array_ptr
|
822
|
+
end
|
823
|
+
|
824
|
+
handle
|
825
|
+
end
|
826
|
+
|
623
827
|
private
|
624
828
|
|
625
829
|
def closed_admin_check(method)
|
data/lib/rdkafka/bindings.rb
CHANGED
@@ -89,10 +89,58 @@ module Rdkafka
|
|
89
89
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
90
90
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
91
91
|
|
92
|
+
# Configs management
|
93
|
+
#
|
94
|
+
# Structs for management of configurations
|
95
|
+
# Each configuration is attached to a resource and one resource can have many configuration
|
96
|
+
# details. Each resource will also have separate errors results if obtaining configuration
|
97
|
+
# was not possible for any reason
|
98
|
+
class ConfigResource < FFI::Struct
|
99
|
+
layout :type, :int,
|
100
|
+
:name, :string
|
101
|
+
end
|
102
|
+
|
103
|
+
attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
104
|
+
attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
|
105
|
+
attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
|
106
|
+
attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
|
107
|
+
attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
|
108
|
+
attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
|
109
|
+
attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
|
110
|
+
attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
|
111
|
+
attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
|
112
|
+
attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
|
113
|
+
attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
|
114
|
+
attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
|
115
|
+
attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
|
116
|
+
attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
|
117
|
+
attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
|
118
|
+
attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
119
|
+
attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
|
120
|
+
attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
|
121
|
+
attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
|
122
|
+
|
123
|
+
RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
|
124
|
+
RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
|
125
|
+
|
126
|
+
RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
|
127
|
+
RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
|
128
|
+
|
129
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
|
130
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
|
131
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
132
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
133
|
+
|
92
134
|
# Errors
|
135
|
+
class NativeErrorDesc < FFI::Struct
|
136
|
+
layout :code, :int,
|
137
|
+
:name, :pointer,
|
138
|
+
:desc, :pointer
|
139
|
+
end
|
93
140
|
|
94
141
|
attach_function :rd_kafka_err2name, [:int], :string
|
95
142
|
attach_function :rd_kafka_err2str, [:int], :string
|
143
|
+
attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
|
96
144
|
|
97
145
|
# Configuration
|
98
146
|
|
@@ -119,6 +167,9 @@ module Rdkafka
|
|
119
167
|
# Log queue
|
120
168
|
attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
|
121
169
|
attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
|
170
|
+
# Per topic configs
|
171
|
+
attach_function :rd_kafka_topic_conf_new, [], :pointer
|
172
|
+
attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
122
173
|
|
123
174
|
LogCallback = FFI::Function.new(
|
124
175
|
:void, [:pointer, :int, :string, :string]
|
data/lib/rdkafka/callbacks.rb
CHANGED
@@ -113,6 +113,42 @@ module Rdkafka
|
|
113
113
|
end
|
114
114
|
end
|
115
115
|
|
116
|
+
class DescribeConfigsResult
|
117
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
118
|
+
|
119
|
+
def initialize(event_ptr)
|
120
|
+
@results=[]
|
121
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
122
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
123
|
+
|
124
|
+
if @result_error == 0
|
125
|
+
configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
|
126
|
+
# Get the number of matching acls
|
127
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
128
|
+
@results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
|
129
|
+
@results_count = pointer_to_size_t.read_int
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class IncrementalAlterConfigsResult
|
135
|
+
attr_reader :result_error, :error_string, :results, :results_count
|
136
|
+
|
137
|
+
def initialize(event_ptr)
|
138
|
+
@results=[]
|
139
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
140
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
141
|
+
|
142
|
+
if @result_error == 0
|
143
|
+
incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
|
144
|
+
# Get the number of matching acls
|
145
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
146
|
+
@results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
|
147
|
+
@results_count = pointer_to_size_t.read_int
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
116
152
|
# FFI Function used for Create Topic and Delete Topic callbacks
|
117
153
|
BackgroundEventCallbackFunction = FFI::Function.new(
|
118
154
|
:void, [:pointer, :pointer, :pointer]
|
@@ -123,20 +159,24 @@ module Rdkafka
|
|
123
159
|
# @private
|
124
160
|
class BackgroundEventCallback
|
125
161
|
def self.call(_, event_ptr, _)
|
126
|
-
|
127
|
-
|
162
|
+
case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
163
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
|
128
164
|
process_create_topic(event_ptr)
|
129
|
-
|
165
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
|
166
|
+
process_describe_configs(event_ptr)
|
167
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
|
168
|
+
process_incremental_alter_configs(event_ptr)
|
169
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
|
130
170
|
process_delete_topic(event_ptr)
|
131
|
-
|
171
|
+
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
|
132
172
|
process_create_partitions(event_ptr)
|
133
|
-
|
173
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
|
134
174
|
process_create_acl(event_ptr)
|
135
|
-
|
175
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
|
136
176
|
process_delete_acl(event_ptr)
|
137
|
-
|
177
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
|
138
178
|
process_describe_acl(event_ptr)
|
139
|
-
|
179
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
|
140
180
|
process_delete_groups(event_ptr)
|
141
181
|
end
|
142
182
|
end
|
@@ -161,6 +201,42 @@ module Rdkafka
|
|
161
201
|
end
|
162
202
|
end
|
163
203
|
|
204
|
+
def self.process_describe_configs(event_ptr)
|
205
|
+
describe_configs = DescribeConfigsResult.new(event_ptr)
|
206
|
+
describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
207
|
+
|
208
|
+
if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
|
209
|
+
describe_configs_handle[:response] = describe_configs.result_error
|
210
|
+
describe_configs_handle[:response_string] = describe_configs.error_string
|
211
|
+
describe_configs_handle[:pending] = false
|
212
|
+
|
213
|
+
if describe_configs.result_error == 0
|
214
|
+
describe_configs_handle[:config_entries] = describe_configs.results
|
215
|
+
describe_configs_handle[:entry_count] = describe_configs.results_count
|
216
|
+
end
|
217
|
+
|
218
|
+
describe_configs_handle.unlock
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def self.process_incremental_alter_configs(event_ptr)
|
223
|
+
incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
|
224
|
+
incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
225
|
+
|
226
|
+
if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
|
227
|
+
incremental_alter_handle[:response] = incremental_alter.result_error
|
228
|
+
incremental_alter_handle[:response_string] = incremental_alter.error_string
|
229
|
+
incremental_alter_handle[:pending] = false
|
230
|
+
|
231
|
+
if incremental_alter.result_error == 0
|
232
|
+
incremental_alter_handle[:config_entries] = incremental_alter.results
|
233
|
+
incremental_alter_handle[:entry_count] = incremental_alter.results_count
|
234
|
+
end
|
235
|
+
|
236
|
+
incremental_alter_handle.unlock
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
164
240
|
def self.process_delete_groups(event_ptr)
|
165
241
|
delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
|
166
242
|
|
@@ -263,7 +339,7 @@ module Rdkafka
|
|
263
339
|
describe_acl_handle[:response_string] = describe_acl.error_string
|
264
340
|
|
265
341
|
if describe_acl.result_error == 0
|
266
|
-
describe_acl_handle[:acls]
|
342
|
+
describe_acl_handle[:acls] = describe_acl.matching_acls
|
267
343
|
describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
|
268
344
|
end
|
269
345
|
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -9,7 +9,15 @@ module Rdkafka
|
|
9
9
|
# Cache partitions count for 30 seconds
|
10
10
|
PARTITIONS_COUNT_TTL = 30
|
11
11
|
|
12
|
-
|
12
|
+
# Empty hash used as a default
|
13
|
+
EMPTY_HASH = {}.freeze
|
14
|
+
|
15
|
+
private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
|
16
|
+
|
17
|
+
# Raised when there was a critical issue when invoking rd_kafka_topic_new
|
18
|
+
# This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
|
19
|
+
# resolved and this is normalized in all the places
|
20
|
+
class TopicHandleCreationError < RuntimeError; end
|
13
21
|
|
14
22
|
# @private
|
15
23
|
# Returns the current delivery callback, by default this is nil.
|
@@ -28,6 +36,8 @@ module Rdkafka
|
|
28
36
|
# @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
|
29
37
|
# the "consistent_random" default
|
30
38
|
def initialize(native_kafka, partitioner_name)
|
39
|
+
@topics_refs_map = {}
|
40
|
+
@topics_configs = {}
|
31
41
|
@native_kafka = native_kafka
|
32
42
|
@partitioner_name = partitioner_name || "consistent_random"
|
33
43
|
|
@@ -54,6 +64,52 @@ module Rdkafka
|
|
54
64
|
end
|
55
65
|
end
|
56
66
|
|
67
|
+
# Sets alternative set of configuration details that can be set per topic
|
68
|
+
# @note It is not allowed to re-set the same topic config twice because of the underlying
|
69
|
+
# librdkafka caching
|
70
|
+
# @param topic [String] The topic name
|
71
|
+
# @param config [Hash] config we want to use per topic basis
|
72
|
+
# @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
|
73
|
+
# because it is already computed during the retrieval attempt in the `#produce` flow.
|
74
|
+
def set_topic_config(topic, config, config_hash)
|
75
|
+
# Ensure lock on topic reference just in case
|
76
|
+
@native_kafka.with_inner do |inner|
|
77
|
+
@topics_refs_map[topic] ||= {}
|
78
|
+
@topics_configs[topic] ||= {}
|
79
|
+
|
80
|
+
return if @topics_configs[topic].key?(config_hash)
|
81
|
+
|
82
|
+
# If config is empty, we create an empty reference that will be used with defaults
|
83
|
+
rd_topic_config = if config.empty?
|
84
|
+
nil
|
85
|
+
else
|
86
|
+
Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
|
87
|
+
config.each do |key, value|
|
88
|
+
error_buffer = FFI::MemoryPointer.new(:char, 256)
|
89
|
+
result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
|
90
|
+
topic_config,
|
91
|
+
key.to_s,
|
92
|
+
value.to_s,
|
93
|
+
error_buffer,
|
94
|
+
256
|
95
|
+
)
|
96
|
+
|
97
|
+
unless result == :config_ok
|
98
|
+
raise Config::ConfigError.new(error_buffer.read_string)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
|
105
|
+
|
106
|
+
raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
|
107
|
+
|
108
|
+
@topics_configs[topic][config_hash] = config
|
109
|
+
@topics_refs_map[topic][config_hash] = topic_handle
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
57
113
|
# Starts the native Kafka polling thread and kicks off the init polling
|
58
114
|
# @note Not needed to run unless explicit start was disabled
|
59
115
|
def start
|
@@ -83,7 +139,18 @@ module Rdkafka
|
|
83
139
|
def close
|
84
140
|
return if closed?
|
85
141
|
ObjectSpace.undefine_finalizer(self)
|
86
|
-
|
142
|
+
|
143
|
+
@native_kafka.close do
|
144
|
+
# We need to remove the topics references objects before we destroy the producer,
|
145
|
+
# otherwise they would leak out
|
146
|
+
@topics_refs_map.each_value do |refs|
|
147
|
+
refs.each_value do |ref|
|
148
|
+
Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
@topics_refs_map.clear
|
87
154
|
end
|
88
155
|
|
89
156
|
# Whether this producer has closed
|
@@ -182,11 +249,22 @@ module Rdkafka
|
|
182
249
|
# @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
|
183
250
|
# @param headers [Hash<String,String>] Optional message headers
|
184
251
|
# @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
|
252
|
+
# @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
|
185
253
|
#
|
186
254
|
# @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
|
187
255
|
#
|
188
256
|
# @raise [RdkafkaError] When adding the message to rdkafka's queue failed
|
189
|
-
def produce(
|
257
|
+
def produce(
|
258
|
+
topic:,
|
259
|
+
payload: nil,
|
260
|
+
key: nil,
|
261
|
+
partition: nil,
|
262
|
+
partition_key: nil,
|
263
|
+
timestamp: nil,
|
264
|
+
headers: nil,
|
265
|
+
label: nil,
|
266
|
+
topic_config: EMPTY_HASH
|
267
|
+
)
|
190
268
|
closed_producer_check(__method__)
|
191
269
|
|
192
270
|
# Start by checking and converting the input
|
@@ -205,8 +283,20 @@ module Rdkafka
|
|
205
283
|
key.bytesize
|
206
284
|
end
|
207
285
|
|
286
|
+
topic_config_hash = topic_config.hash
|
287
|
+
|
288
|
+
# Checks if we have the rdkafka topic reference object ready. It saves us on object
|
289
|
+
# allocation and allows to use custom config on demand.
|
290
|
+
set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
|
291
|
+
topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
|
292
|
+
|
208
293
|
if partition_key
|
209
294
|
partition_count = partition_count(topic)
|
295
|
+
|
296
|
+
# Check if there are no overrides for the partitioner and use the default one only when
|
297
|
+
# no per-topic is present.
|
298
|
+
partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
|
299
|
+
|
210
300
|
# If the topic is not present, set to -1
|
211
301
|
partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
|
212
302
|
end
|
@@ -236,7 +326,7 @@ module Rdkafka
|
|
236
326
|
DeliveryHandle.register(delivery_handle)
|
237
327
|
|
238
328
|
args = [
|
239
|
-
:int, Rdkafka::Bindings::
|
329
|
+
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
|
240
330
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
|
241
331
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
|
242
332
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
|
data/lib/rdkafka/version.rb
CHANGED
data/lib/rdkafka.rb
CHANGED
@@ -24,7 +24,13 @@ require "rdkafka/admin/delete_acl_handle"
|
|
24
24
|
require "rdkafka/admin/delete_acl_report"
|
25
25
|
require "rdkafka/admin/describe_acl_handle"
|
26
26
|
require "rdkafka/admin/describe_acl_report"
|
27
|
+
require "rdkafka/admin/describe_configs_handle"
|
28
|
+
require "rdkafka/admin/describe_configs_report"
|
29
|
+
require "rdkafka/admin/incremental_alter_configs_handle"
|
30
|
+
require "rdkafka/admin/incremental_alter_configs_report"
|
27
31
|
require "rdkafka/admin/acl_binding_result"
|
32
|
+
require "rdkafka/admin/config_binding_result"
|
33
|
+
require "rdkafka/admin/config_resource_binding_result"
|
28
34
|
require "rdkafka/bindings"
|
29
35
|
require "rdkafka/callbacks"
|
30
36
|
require "rdkafka/config"
|
data/rdkafka.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.name = 'rdkafka'
|
16
16
|
gem.require_paths = ['lib']
|
17
17
|
gem.version = Rdkafka::VERSION
|
18
|
-
gem.required_ruby_version = '>=
|
18
|
+
gem.required_ruby_version = '>= 3.0'
|
19
19
|
gem.extensions = %w(ext/Rakefile)
|
20
20
|
gem.cert_chain = %w[certs/cert_chain.pem]
|
21
21
|
|
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -16,12 +16,12 @@ describe Rdkafka::Admin do
|
|
16
16
|
admin.close
|
17
17
|
end
|
18
18
|
|
19
|
-
let(:topic_name) { "test-topic-#{
|
19
|
+
let(:topic_name) { "test-topic-#{SecureRandom.uuid}" }
|
20
20
|
let(:topic_partition_count) { 3 }
|
21
21
|
let(:topic_replication_factor) { 1 }
|
22
22
|
let(:topic_config) { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
|
23
23
|
let(:invalid_topic_config) { {"cleeeeenup.policee" => "campact"} }
|
24
|
-
let(:group_name) { "test-group-#{
|
24
|
+
let(:group_name) { "test-group-#{SecureRandom.uuid}" }
|
25
25
|
|
26
26
|
let(:resource_name) {"acl-test-topic"}
|
27
27
|
let(:resource_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
|
@@ -31,6 +31,14 @@ describe Rdkafka::Admin do
|
|
31
31
|
let(:operation) {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
|
32
32
|
let(:permission_type) {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
|
33
33
|
|
34
|
+
describe '#describe_errors' do
|
35
|
+
let(:errors) { admin.class.describe_errors }
|
36
|
+
|
37
|
+
it { expect(errors.size).to eq(162) }
|
38
|
+
it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
|
39
|
+
it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
|
40
|
+
end
|
41
|
+
|
34
42
|
describe 'admin without auto-start' do
|
35
43
|
let(:admin) { config.admin(native_kafka_auto_start: false) }
|
36
44
|
|
@@ -142,6 +150,275 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
142
150
|
end
|
143
151
|
end
|
144
152
|
|
153
|
+
describe "describe_configs" do
|
154
|
+
subject(:resources_results) { admin.describe_configs(resources).wait.resources }
|
155
|
+
|
156
|
+
before do
|
157
|
+
admin.create_topic(topic_name, 2, 1).wait
|
158
|
+
sleep(1)
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'when describing config of an existing topic' do
|
162
|
+
let(:resources) { [{ resource_type: 2, resource_name: topic_name }] }
|
163
|
+
|
164
|
+
it do
|
165
|
+
expect(resources_results.size).to eq(1)
|
166
|
+
expect(resources_results.first.type).to eq(2)
|
167
|
+
expect(resources_results.first.name).to eq(topic_name)
|
168
|
+
expect(resources_results.first.configs.size).to be > 25
|
169
|
+
expect(resources_results.first.configs.first.name).to eq('compression.type')
|
170
|
+
expect(resources_results.first.configs.first.value).to eq('producer')
|
171
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
context 'when describing config of a non-existing topic' do
|
176
|
+
let(:resources) { [{ resource_type: 2, resource_name: SecureRandom.uuid }] }
|
177
|
+
|
178
|
+
it 'expect to raise error' do
|
179
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
context 'when describing both existing and non-existing topics' do
|
184
|
+
let(:resources) do
|
185
|
+
[
|
186
|
+
{ resource_type: 2, resource_name: topic_name },
|
187
|
+
{ resource_type: 2, resource_name: SecureRandom.uuid }
|
188
|
+
]
|
189
|
+
end
|
190
|
+
|
191
|
+
it 'expect to raise error' do
|
192
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
context 'when describing multiple existing topics' do
|
197
|
+
let(:resources) do
|
198
|
+
[
|
199
|
+
{ resource_type: 2, resource_name: 'example_topic' },
|
200
|
+
{ resource_type: 2, resource_name: topic_name }
|
201
|
+
]
|
202
|
+
end
|
203
|
+
|
204
|
+
it do
|
205
|
+
expect(resources_results.size).to eq(2)
|
206
|
+
expect(resources_results.first.type).to eq(2)
|
207
|
+
expect(resources_results.first.name).to eq('example_topic')
|
208
|
+
expect(resources_results.last.type).to eq(2)
|
209
|
+
expect(resources_results.last.name).to eq(topic_name)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
context 'when trying to describe invalid resource type' do
|
214
|
+
let(:resources) { [{ resource_type: 0, resource_name: SecureRandom.uuid }] }
|
215
|
+
|
216
|
+
it 'expect to raise error' do
|
217
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_request/)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
context 'when trying to describe invalid broker' do
|
222
|
+
let(:resources) { [{ resource_type: 4, resource_name: 'non-existing' }] }
|
223
|
+
|
224
|
+
it 'expect to raise error' do
|
225
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
context 'when trying to describe valid broker' do
|
230
|
+
let(:resources) { [{ resource_type: 4, resource_name: '1' }] }
|
231
|
+
|
232
|
+
it do
|
233
|
+
expect(resources_results.size).to eq(1)
|
234
|
+
expect(resources_results.first.type).to eq(4)
|
235
|
+
expect(resources_results.first.name).to eq('1')
|
236
|
+
expect(resources_results.first.configs.size).to be > 230
|
237
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
238
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
239
|
+
expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
context 'when describing valid broker with topics in one request' do
|
244
|
+
let(:resources) do
|
245
|
+
[
|
246
|
+
{ resource_type: 4, resource_name: '1' },
|
247
|
+
{ resource_type: 2, resource_name: topic_name }
|
248
|
+
]
|
249
|
+
end
|
250
|
+
|
251
|
+
it do
|
252
|
+
expect(resources_results.size).to eq(2)
|
253
|
+
expect(resources_results.first.type).to eq(4)
|
254
|
+
expect(resources_results.first.name).to eq('1')
|
255
|
+
expect(resources_results.first.configs.size).to be > 230
|
256
|
+
expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
|
257
|
+
expect(resources_results.first.configs.first.value).to eq('0')
|
258
|
+
expect(resources_results.last.type).to eq(2)
|
259
|
+
expect(resources_results.last.name).to eq(topic_name)
|
260
|
+
expect(resources_results.last.configs.size).to be > 25
|
261
|
+
expect(resources_results.last.configs.first.name).to eq('compression.type')
|
262
|
+
expect(resources_results.last.configs.first.value).to eq('producer')
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
describe "incremental_alter_configs" do
|
268
|
+
subject(:resources_results) { admin.incremental_alter_configs(resources_with_configs).wait.resources }
|
269
|
+
|
270
|
+
before do
|
271
|
+
admin.create_topic(topic_name, 2, 1).wait
|
272
|
+
sleep(1)
|
273
|
+
end
|
274
|
+
|
275
|
+
context 'when altering one topic with one valid config via set' do
|
276
|
+
let(:target_retention) { (86400002 + rand(10_000)).to_s }
|
277
|
+
let(:resources_with_configs) do
|
278
|
+
[
|
279
|
+
{
|
280
|
+
resource_type: 2,
|
281
|
+
resource_name: topic_name,
|
282
|
+
configs: [
|
283
|
+
{
|
284
|
+
name: 'delete.retention.ms',
|
285
|
+
value: target_retention,
|
286
|
+
op_type: 0
|
287
|
+
}
|
288
|
+
]
|
289
|
+
}
|
290
|
+
]
|
291
|
+
end
|
292
|
+
|
293
|
+
it do
|
294
|
+
expect(resources_results.size).to eq(1)
|
295
|
+
expect(resources_results.first.type).to eq(2)
|
296
|
+
expect(resources_results.first.name).to eq(topic_name)
|
297
|
+
|
298
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
299
|
+
config.name == 'delete.retention.ms'
|
300
|
+
end
|
301
|
+
|
302
|
+
expect(ret_config.value).to eq(target_retention)
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
context 'when altering one topic with one valid config via delete' do
|
307
|
+
let(:target_retention) { (8640002 + rand(10_000)).to_s }
|
308
|
+
let(:resources_with_configs) do
|
309
|
+
[
|
310
|
+
{
|
311
|
+
resource_type: 2,
|
312
|
+
resource_name: topic_name,
|
313
|
+
configs: [
|
314
|
+
{
|
315
|
+
name: 'delete.retention.ms',
|
316
|
+
value: target_retention,
|
317
|
+
op_type: 1
|
318
|
+
}
|
319
|
+
]
|
320
|
+
}
|
321
|
+
]
|
322
|
+
end
|
323
|
+
|
324
|
+
it do
|
325
|
+
expect(resources_results.size).to eq(1)
|
326
|
+
expect(resources_results.first.type).to eq(2)
|
327
|
+
expect(resources_results.first.name).to eq(topic_name)
|
328
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
329
|
+
config.name == 'delete.retention.ms'
|
330
|
+
end
|
331
|
+
|
332
|
+
expect(ret_config.value).to eq('86400000')
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
context 'when altering one topic with one valid config via append' do
|
337
|
+
let(:target_policy) { 'compact' }
|
338
|
+
let(:resources_with_configs) do
|
339
|
+
[
|
340
|
+
{
|
341
|
+
resource_type: 2,
|
342
|
+
resource_name: topic_name,
|
343
|
+
configs: [
|
344
|
+
{
|
345
|
+
name: 'cleanup.policy',
|
346
|
+
value: target_policy,
|
347
|
+
op_type: 2
|
348
|
+
}
|
349
|
+
]
|
350
|
+
}
|
351
|
+
]
|
352
|
+
end
|
353
|
+
|
354
|
+
it do
|
355
|
+
expect(resources_results.size).to eq(1)
|
356
|
+
expect(resources_results.first.type).to eq(2)
|
357
|
+
expect(resources_results.first.name).to eq(topic_name)
|
358
|
+
|
359
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
360
|
+
config.name == 'cleanup.policy'
|
361
|
+
end
|
362
|
+
|
363
|
+
expect(ret_config.value).to eq("delete,#{target_policy}")
|
364
|
+
end
|
365
|
+
end
|
366
|
+
|
367
|
+
context 'when altering one topic with one valid config via subtrack' do
|
368
|
+
let(:target_policy) { 'delete' }
|
369
|
+
let(:resources_with_configs) do
|
370
|
+
[
|
371
|
+
{
|
372
|
+
resource_type: 2,
|
373
|
+
resource_name: topic_name,
|
374
|
+
configs: [
|
375
|
+
{
|
376
|
+
name: 'cleanup.policy',
|
377
|
+
value: target_policy,
|
378
|
+
op_type: 3
|
379
|
+
}
|
380
|
+
]
|
381
|
+
}
|
382
|
+
]
|
383
|
+
end
|
384
|
+
|
385
|
+
it do
|
386
|
+
expect(resources_results.size).to eq(1)
|
387
|
+
expect(resources_results.first.type).to eq(2)
|
388
|
+
expect(resources_results.first.name).to eq(topic_name)
|
389
|
+
|
390
|
+
ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
|
391
|
+
config.name == 'cleanup.policy'
|
392
|
+
end
|
393
|
+
|
394
|
+
expect(ret_config.value).to eq('')
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
context 'when altering one topic with invalid config' do
|
399
|
+
let(:target_retention) { '-10' }
|
400
|
+
let(:resources_with_configs) do
|
401
|
+
[
|
402
|
+
{
|
403
|
+
resource_type: 2,
|
404
|
+
resource_name: topic_name,
|
405
|
+
configs: [
|
406
|
+
{
|
407
|
+
name: 'delete.retention.ms',
|
408
|
+
value: target_retention,
|
409
|
+
op_type: 0
|
410
|
+
}
|
411
|
+
]
|
412
|
+
}
|
413
|
+
]
|
414
|
+
end
|
415
|
+
|
416
|
+
it 'expect to raise error' do
|
417
|
+
expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_config/)
|
418
|
+
end
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
145
422
|
describe "#delete_topic" do
|
146
423
|
describe "called with invalid input" do
|
147
424
|
# https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
|
@@ -412,7 +689,10 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
412
689
|
end
|
413
690
|
|
414
691
|
context 'when topic has less then desired number of partitions' do
|
415
|
-
before
|
692
|
+
before do
|
693
|
+
admin.create_topic(topic_name, 1, 1).wait
|
694
|
+
sleep(1)
|
695
|
+
end
|
416
696
|
|
417
697
|
it 'expect to change number of partitions' do
|
418
698
|
admin.create_partitions(topic_name, 10).wait
|
@@ -31,6 +31,48 @@ describe Rdkafka::Producer do
|
|
31
31
|
it { expect(producer.name).to include('rdkafka#producer-') }
|
32
32
|
end
|
33
33
|
|
34
|
+
describe '#produce with topic config alterations' do
|
35
|
+
context 'when config is not valid' do
|
36
|
+
it 'expect to raise error' do
|
37
|
+
expect do
|
38
|
+
producer.produce(topic: 'test', payload: '', topic_config: { 'invalid': 'invalid' })
|
39
|
+
end.to raise_error(Rdkafka::Config::ConfigError)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'when config is valid' do
|
44
|
+
it 'expect to raise error' do
|
45
|
+
expect do
|
46
|
+
producer.produce(topic: 'test', payload: '', topic_config: { 'acks': 1 }).wait
|
47
|
+
end.not_to raise_error
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'when alteration should change behavior' do
|
51
|
+
# This is set incorrectly for a reason
|
52
|
+
# If alteration would not work, this will hang the spec suite
|
53
|
+
let(:producer) do
|
54
|
+
rdkafka_producer_config(
|
55
|
+
'message.timeout.ms': 1_000_000,
|
56
|
+
:"bootstrap.servers" => "localhost:9094",
|
57
|
+
).producer
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'expect to give up on delivery fast based on alteration config' do
|
61
|
+
expect do
|
62
|
+
producer.produce(
|
63
|
+
topic: 'produce_config_test',
|
64
|
+
payload: 'test',
|
65
|
+
topic_config: {
|
66
|
+
'compression.type': 'gzip',
|
67
|
+
'message.timeout.ms': 1
|
68
|
+
}
|
69
|
+
).wait
|
70
|
+
end.to raise_error(Rdkafka::RdkafkaError, /msg_timed_out/)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
34
76
|
context "delivery callback" do
|
35
77
|
context "with a proc/lambda" do
|
36
78
|
it "should set the callback" do
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.0.
|
4
|
+
version: 0.16.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-05-27 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
@@ -193,6 +193,8 @@ files:
|
|
193
193
|
- lib/rdkafka/abstract_handle.rb
|
194
194
|
- lib/rdkafka/admin.rb
|
195
195
|
- lib/rdkafka/admin/acl_binding_result.rb
|
196
|
+
- lib/rdkafka/admin/config_binding_result.rb
|
197
|
+
- lib/rdkafka/admin/config_resource_binding_result.rb
|
196
198
|
- lib/rdkafka/admin/create_acl_handle.rb
|
197
199
|
- lib/rdkafka/admin/create_acl_report.rb
|
198
200
|
- lib/rdkafka/admin/create_partitions_handle.rb
|
@@ -207,6 +209,10 @@ files:
|
|
207
209
|
- lib/rdkafka/admin/delete_topic_report.rb
|
208
210
|
- lib/rdkafka/admin/describe_acl_handle.rb
|
209
211
|
- lib/rdkafka/admin/describe_acl_report.rb
|
212
|
+
- lib/rdkafka/admin/describe_configs_handle.rb
|
213
|
+
- lib/rdkafka/admin/describe_configs_report.rb
|
214
|
+
- lib/rdkafka/admin/incremental_alter_configs_handle.rb
|
215
|
+
- lib/rdkafka/admin/incremental_alter_configs_report.rb
|
210
216
|
- lib/rdkafka/bindings.rb
|
211
217
|
- lib/rdkafka/callbacks.rb
|
212
218
|
- lib/rdkafka/config.rb
|
@@ -272,7 +278,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
272
278
|
requirements:
|
273
279
|
- - ">="
|
274
280
|
- !ruby/object:Gem::Version
|
275
|
-
version: '
|
281
|
+
version: '3.0'
|
276
282
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
277
283
|
requirements:
|
278
284
|
- - ">="
|
metadata.gz.sig
CHANGED
Binary file
|