rdkafka 0.22.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +101 -3
- data/README.md +21 -14
- data/Rakefile +21 -21
- data/bin/verify_kafka_warnings +39 -0
- data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.2.tar.gz} +0 -0
- data/docker-compose-ssl.yml +35 -0
- data/docker-compose.yml +2 -2
- data/ext/Rakefile +27 -27
- data/lib/rdkafka/abstract_handle.rb +54 -7
- data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
- data/lib/rdkafka/admin/config_binding_result.rb +2 -2
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +10 -6
- data/lib/rdkafka/admin/create_acl_report.rb +3 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +8 -7
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +8 -7
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +9 -8
- data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +10 -7
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +8 -7
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +9 -8
- data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +7 -10
- data/lib/rdkafka/admin/describe_configs_report.rb +8 -6
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -10
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +8 -6
- data/lib/rdkafka/admin/list_offsets_handle.rb +31 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
- data/lib/rdkafka/admin.rb +357 -159
- data/lib/rdkafka/bindings.rb +205 -111
- data/lib/rdkafka/callbacks/base_handler.rb +62 -0
- data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
- data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
- data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
- data/lib/rdkafka/callbacks.rb +125 -216
- data/lib/rdkafka/config.rb +126 -66
- data/lib/rdkafka/consumer/headers.rb +22 -7
- data/lib/rdkafka/consumer/message.rb +12 -11
- data/lib/rdkafka/consumer/partition.rb +15 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +64 -38
- data/lib/rdkafka/consumer.rb +513 -59
- data/lib/rdkafka/defaults.rb +125 -0
- data/lib/rdkafka/error.rb +40 -14
- data/lib/rdkafka/helpers/metadata.rb +29 -0
- data/lib/rdkafka/helpers/oauth.rb +45 -13
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +128 -37
- data/lib/rdkafka/native_kafka.rb +91 -8
- data/lib/rdkafka/producer/delivery_handle.rb +6 -6
- data/lib/rdkafka/producer/delivery_report.rb +10 -6
- data/lib/rdkafka/producer/partitions_count_cache.rb +51 -55
- data/lib/rdkafka/producer.rb +193 -95
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +15 -0
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +58 -36
- data/renovate.json +39 -23
- metadata +39 -125
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
- data/.github/workflows/ci_macos_arm64.yml +0 -284
- data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
- data/.github/workflows/push_macos_arm64.yml +0 -54
- data/.github/workflows/push_ruby.yml +0 -37
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.gitignore +0 -14
- data/.rspec +0 -2
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.yardopts +0 -2
- data/Gemfile +0 -5
- data/ext/README.md +0 -19
- data/ext/build_common.sh +0 -361
- data/ext/build_linux_x86_64_gnu.sh +0 -306
- data/ext/build_linux_x86_64_musl.sh +0 -763
- data/ext/build_macos_arm64.sh +0 -550
- data/spec/rdkafka/abstract_handle_spec.rb +0 -117
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
- data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
- data/spec/rdkafka/admin_spec.rb +0 -971
- data/spec/rdkafka/bindings_spec.rb +0 -199
- data/spec/rdkafka/callbacks_spec.rb +0 -20
- data/spec/rdkafka/config_spec.rb +0 -258
- data/spec/rdkafka/consumer/headers_spec.rb +0 -73
- data/spec/rdkafka/consumer/message_spec.rb +0 -139
- data/spec/rdkafka/consumer/partition_spec.rb +0 -57
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
- data/spec/rdkafka/consumer_spec.rb +0 -1274
- data/spec/rdkafka/error_spec.rb +0 -89
- data/spec/rdkafka/metadata_spec.rb +0 -79
- data/spec/rdkafka/native_kafka_spec.rb +0 -130
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
- data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
- data/spec/rdkafka/producer_spec.rb +0 -1345
- data/spec/spec_helper.rb +0 -195
data/lib/rdkafka/callbacks.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
+
# Callback handlers for librdkafka events
|
|
5
|
+
# @private
|
|
4
6
|
module Callbacks
|
|
5
7
|
# Extracts attributes of a rd_kafka_topic_result_t
|
|
6
8
|
#
|
|
@@ -8,12 +10,16 @@ module Rdkafka
|
|
|
8
10
|
class TopicResult
|
|
9
11
|
attr_reader :result_error, :error_string, :result_name
|
|
10
12
|
|
|
13
|
+
# @param topic_result_pointer [FFI::Pointer] pointer to the topic result struct
|
|
11
14
|
def initialize(topic_result_pointer)
|
|
12
15
|
@result_error = Rdkafka::Bindings.rd_kafka_topic_result_error(topic_result_pointer)
|
|
13
16
|
@error_string = Rdkafka::Bindings.rd_kafka_topic_result_error_string(topic_result_pointer)
|
|
14
17
|
@result_name = Rdkafka::Bindings.rd_kafka_topic_result_name(topic_result_pointer)
|
|
15
18
|
end
|
|
16
19
|
|
|
20
|
+
# @param count [Integer] number of results
|
|
21
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
22
|
+
# @return [Array<TopicResult>] array of topic results
|
|
17
23
|
def self.create_topic_results_from_array(count, array_pointer)
|
|
18
24
|
(1..count).map do |index|
|
|
19
25
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -22,13 +28,18 @@ module Rdkafka
|
|
|
22
28
|
end
|
|
23
29
|
end
|
|
24
30
|
|
|
31
|
+
# Extracts attributes of rd_kafka_group_result_t
|
|
32
|
+
#
|
|
33
|
+
# @private
|
|
25
34
|
class GroupResult
|
|
26
35
|
attr_reader :result_error, :error_string, :result_name
|
|
36
|
+
|
|
37
|
+
# @param group_result_pointer [FFI::Pointer] pointer to the group result struct
|
|
27
38
|
def initialize(group_result_pointer)
|
|
28
39
|
native_error = Rdkafka::Bindings.rd_kafka_group_result_error(group_result_pointer)
|
|
29
40
|
|
|
30
41
|
if native_error.null?
|
|
31
|
-
@result_error =
|
|
42
|
+
@result_error = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
32
43
|
@error_string = FFI::Pointer::NULL
|
|
33
44
|
else
|
|
34
45
|
@result_error = native_error[:code]
|
|
@@ -37,6 +48,10 @@ module Rdkafka
|
|
|
37
48
|
|
|
38
49
|
@result_name = Rdkafka::Bindings.rd_kafka_group_result_name(group_result_pointer)
|
|
39
50
|
end
|
|
51
|
+
|
|
52
|
+
# @param count [Integer] number of results
|
|
53
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
54
|
+
# @return [Array<GroupResult>] array of group results
|
|
40
55
|
def self.create_group_results_from_array(count, array_pointer)
|
|
41
56
|
(1..count).map do |index|
|
|
42
57
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -51,12 +66,16 @@ module Rdkafka
|
|
|
51
66
|
class CreateAclResult
|
|
52
67
|
attr_reader :result_error, :error_string
|
|
53
68
|
|
|
69
|
+
# @param acl_result_pointer [FFI::Pointer] pointer to the ACL result struct
|
|
54
70
|
def initialize(acl_result_pointer)
|
|
55
71
|
rd_kafka_error_pointer = Bindings.rd_kafka_acl_result_error(acl_result_pointer)
|
|
56
72
|
@result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
|
|
57
73
|
@error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
|
|
58
74
|
end
|
|
59
75
|
|
|
76
|
+
# @param count [Integer] number of results
|
|
77
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
78
|
+
# @return [Array<CreateAclResult>] array of ACL results
|
|
60
79
|
def self.create_acl_results_from_array(count, array_pointer)
|
|
61
80
|
(1..count).map do |index|
|
|
62
81
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -71,19 +90,23 @@ module Rdkafka
|
|
|
71
90
|
class DeleteAclResult
|
|
72
91
|
attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
|
|
73
92
|
|
|
93
|
+
# @param acl_result_pointer [FFI::Pointer] pointer to the delete ACL result response struct
|
|
74
94
|
def initialize(acl_result_pointer)
|
|
75
|
-
@matching_acls=[]
|
|
95
|
+
@matching_acls = []
|
|
76
96
|
rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
|
|
77
97
|
@result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
|
|
78
98
|
@error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
|
|
79
|
-
if @result_error ==
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
100
|
+
# Get the number of matching acls
|
|
101
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
102
|
+
@matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
|
|
103
|
+
@matching_acls_count = pointer_to_size_t.read_int
|
|
84
104
|
end
|
|
85
105
|
end
|
|
86
106
|
|
|
107
|
+
# @param count [Integer] number of results
|
|
108
|
+
# @param array_pointer [FFI::Pointer] pointer to the results array
|
|
109
|
+
# @return [Array<DeleteAclResult>] array of delete ACL results
|
|
87
110
|
def self.delete_acl_results_from_array(count, array_pointer)
|
|
88
111
|
(1..count).map do |index|
|
|
89
112
|
result_pointer = (array_pointer + (index - 1)).read_pointer
|
|
@@ -98,251 +121,123 @@ module Rdkafka
|
|
|
98
121
|
class DescribeAclResult
|
|
99
122
|
attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
|
|
100
123
|
|
|
124
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
101
125
|
def initialize(event_ptr)
|
|
102
|
-
@matching_acls=[]
|
|
126
|
+
@matching_acls = []
|
|
103
127
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
104
128
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
105
|
-
if @result_error ==
|
|
129
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
106
130
|
acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
|
|
107
131
|
# Get the number of matching acls
|
|
108
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:
|
|
132
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
109
133
|
@matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
|
|
110
134
|
@matching_acls_count = pointer_to_size_t.read_int
|
|
111
135
|
end
|
|
112
136
|
end
|
|
113
137
|
end
|
|
114
138
|
|
|
139
|
+
# Extracts attributes of rd_kafka_DescribeConfigs_result_t
|
|
140
|
+
#
|
|
141
|
+
# @private
|
|
115
142
|
class DescribeConfigsResult
|
|
116
143
|
attr_reader :result_error, :error_string, :results, :results_count
|
|
117
144
|
|
|
145
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
118
146
|
def initialize(event_ptr)
|
|
119
|
-
@results=[]
|
|
147
|
+
@results = []
|
|
120
148
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
121
149
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
122
150
|
|
|
123
|
-
if @result_error ==
|
|
151
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
124
152
|
configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
|
|
125
153
|
# Get the number of matching acls
|
|
126
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:
|
|
154
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
127
155
|
@results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
|
|
128
156
|
@results_count = pointer_to_size_t.read_int
|
|
129
157
|
end
|
|
130
158
|
end
|
|
131
159
|
end
|
|
132
160
|
|
|
161
|
+
# Extracts attributes of rd_kafka_IncrementalAlterConfigs_result_t
|
|
162
|
+
#
|
|
163
|
+
# @private
|
|
133
164
|
class IncrementalAlterConfigsResult
|
|
134
165
|
attr_reader :result_error, :error_string, :results, :results_count
|
|
135
166
|
|
|
167
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
136
168
|
def initialize(event_ptr)
|
|
137
|
-
@results=[]
|
|
169
|
+
@results = []
|
|
138
170
|
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
139
171
|
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
140
172
|
|
|
141
|
-
if @result_error ==
|
|
173
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
142
174
|
incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
|
|
143
175
|
# Get the number of matching acls
|
|
144
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:
|
|
176
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
145
177
|
@results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
|
|
146
178
|
@results_count = pointer_to_size_t.read_int
|
|
147
179
|
end
|
|
148
180
|
end
|
|
149
181
|
end
|
|
150
182
|
|
|
183
|
+
# Extracts attributes of rd_kafka_ListOffsets_result_t
|
|
184
|
+
#
|
|
151
185
|
# @private
|
|
152
|
-
class
|
|
153
|
-
|
|
154
|
-
case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
|
155
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
|
|
156
|
-
process_create_topic(event_ptr)
|
|
157
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
|
|
158
|
-
process_describe_configs(event_ptr)
|
|
159
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
|
|
160
|
-
process_incremental_alter_configs(event_ptr)
|
|
161
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
|
|
162
|
-
process_delete_topic(event_ptr)
|
|
163
|
-
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
|
|
164
|
-
process_create_partitions(event_ptr)
|
|
165
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
|
|
166
|
-
process_create_acl(event_ptr)
|
|
167
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
|
|
168
|
-
process_delete_acl(event_ptr)
|
|
169
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
|
|
170
|
-
process_describe_acl(event_ptr)
|
|
171
|
-
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
|
|
172
|
-
process_delete_groups(event_ptr)
|
|
173
|
-
end
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
private
|
|
177
|
-
|
|
178
|
-
def self.process_create_topic(event_ptr)
|
|
179
|
-
create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
|
|
180
|
-
|
|
181
|
-
# Get the number of create topic results
|
|
182
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
183
|
-
create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
|
|
184
|
-
create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
|
|
185
|
-
create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
186
|
-
|
|
187
|
-
if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
|
|
188
|
-
create_topic_handle[:response] = create_topic_results[0].result_error
|
|
189
|
-
create_topic_handle[:error_string] = create_topic_results[0].error_string
|
|
190
|
-
create_topic_handle[:result_name] = create_topic_results[0].result_name
|
|
191
|
-
|
|
192
|
-
create_topic_handle.unlock
|
|
193
|
-
end
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
def self.process_describe_configs(event_ptr)
|
|
197
|
-
describe_configs = DescribeConfigsResult.new(event_ptr)
|
|
198
|
-
describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
199
|
-
|
|
200
|
-
if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
|
|
201
|
-
describe_configs_handle[:response] = describe_configs.result_error
|
|
202
|
-
describe_configs_handle[:response_string] = describe_configs.error_string
|
|
203
|
-
describe_configs_handle[:pending] = false
|
|
204
|
-
|
|
205
|
-
if describe_configs.result_error == 0
|
|
206
|
-
describe_configs_handle[:config_entries] = describe_configs.results
|
|
207
|
-
describe_configs_handle[:entry_count] = describe_configs.results_count
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
describe_configs_handle.unlock
|
|
211
|
-
end
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
def self.process_incremental_alter_configs(event_ptr)
|
|
215
|
-
incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
|
|
216
|
-
incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
217
|
-
|
|
218
|
-
if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
|
|
219
|
-
incremental_alter_handle[:response] = incremental_alter.result_error
|
|
220
|
-
incremental_alter_handle[:response_string] = incremental_alter.error_string
|
|
221
|
-
incremental_alter_handle[:pending] = false
|
|
222
|
-
|
|
223
|
-
if incremental_alter.result_error == 0
|
|
224
|
-
incremental_alter_handle[:config_entries] = incremental_alter.results
|
|
225
|
-
incremental_alter_handle[:entry_count] = incremental_alter.results_count
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
incremental_alter_handle.unlock
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
def self.process_delete_groups(event_ptr)
|
|
233
|
-
delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
|
|
234
|
-
|
|
235
|
-
# Get the number of delete group results
|
|
236
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
237
|
-
delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
|
|
238
|
-
delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
|
|
239
|
-
delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
240
|
-
|
|
241
|
-
if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
|
|
242
|
-
delete_group_handle[:response] = delete_group_results[0].result_error
|
|
243
|
-
delete_group_handle[:error_string] = delete_group_results[0].error_string
|
|
244
|
-
delete_group_handle[:result_name] = delete_group_results[0].result_name
|
|
186
|
+
class ListOffsetsResult
|
|
187
|
+
attr_reader :result_error, :error_string, :result_infos, :result_count
|
|
245
188
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
|
|
252
|
-
|
|
253
|
-
# Get the number of topic results
|
|
254
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
255
|
-
delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
|
|
256
|
-
delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
|
|
257
|
-
delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
258
|
-
|
|
259
|
-
if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
|
|
260
|
-
delete_topic_handle[:response] = delete_topic_results[0].result_error
|
|
261
|
-
delete_topic_handle[:error_string] = delete_topic_results[0].error_string
|
|
262
|
-
delete_topic_handle[:result_name] = delete_topic_results[0].result_name
|
|
263
|
-
|
|
264
|
-
delete_topic_handle.unlock
|
|
265
|
-
end
|
|
266
|
-
end
|
|
267
|
-
|
|
268
|
-
def self.process_create_partitions(event_ptr)
|
|
269
|
-
create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
|
|
270
|
-
|
|
271
|
-
# Get the number of create topic results
|
|
272
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
273
|
-
create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
|
|
274
|
-
create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
|
|
275
|
-
create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
276
|
-
|
|
277
|
-
if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
|
|
278
|
-
create_partitions_handle[:response] = create_partitions_results[0].result_error
|
|
279
|
-
create_partitions_handle[:error_string] = create_partitions_results[0].error_string
|
|
280
|
-
create_partitions_handle[:result_name] = create_partitions_results[0].result_name
|
|
281
|
-
|
|
282
|
-
create_partitions_handle.unlock
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
def self.process_create_acl(event_ptr)
|
|
287
|
-
create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
|
|
288
|
-
|
|
289
|
-
# Get the number of acl results
|
|
290
|
-
pointer_to_size_t = FFI::MemoryPointer.new(:int32)
|
|
291
|
-
create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
|
|
292
|
-
create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
|
|
293
|
-
create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
294
|
-
|
|
295
|
-
if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
|
|
296
|
-
create_acl_handle[:response] = create_acl_results[0].result_error
|
|
297
|
-
create_acl_handle[:response_string] = create_acl_results[0].error_string
|
|
189
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
190
|
+
def initialize(event_ptr)
|
|
191
|
+
@result_infos = FFI::Pointer::NULL
|
|
192
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
193
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
298
194
|
|
|
299
|
-
|
|
195
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
196
|
+
list_offsets_result = Rdkafka::Bindings.rd_kafka_event_ListOffsets_result(event_ptr)
|
|
197
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
198
|
+
@result_infos = Rdkafka::Bindings.rd_kafka_ListOffsets_result_infos(list_offsets_result, pointer_to_size_t)
|
|
199
|
+
@result_count = pointer_to_size_t.read(:size_t)
|
|
300
200
|
end
|
|
301
201
|
end
|
|
202
|
+
end
|
|
302
203
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
delete_acl_handle.unlock
|
|
204
|
+
# @private
|
|
205
|
+
class BackgroundEventCallback
|
|
206
|
+
# Handles background events from librdkafka
|
|
207
|
+
# @param _client_ptr [FFI::Pointer] unused client pointer
|
|
208
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
209
|
+
# @param _opaque_ptr [FFI::Pointer] unused opaque pointer
|
|
210
|
+
def self.call(_client_ptr, event_ptr, _opaque_ptr)
|
|
211
|
+
handler = case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
|
|
212
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT then CreateTopicHandler
|
|
213
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
|
|
214
|
+
when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
|
|
215
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
|
|
216
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
|
|
217
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
|
|
218
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT then DescribeAclHandler
|
|
219
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT then DescribeConfigsHandler
|
|
220
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
|
|
221
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
|
|
322
222
|
end
|
|
323
|
-
end
|
|
324
|
-
|
|
325
|
-
def self.process_describe_acl(event_ptr)
|
|
326
|
-
describe_acl = DescribeAclResult.new(event_ptr)
|
|
327
|
-
describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
328
|
-
|
|
329
|
-
if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
|
|
330
|
-
describe_acl_handle[:response] = describe_acl.result_error
|
|
331
|
-
describe_acl_handle[:response_string] = describe_acl.error_string
|
|
332
|
-
|
|
333
|
-
if describe_acl.result_error == 0
|
|
334
|
-
describe_acl_handle[:acls] = describe_acl.matching_acls
|
|
335
|
-
describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
|
|
336
|
-
end
|
|
337
223
|
|
|
338
|
-
|
|
339
|
-
|
|
224
|
+
handler&.call(event_ptr)
|
|
225
|
+
ensure
|
|
226
|
+
# Background queue events are owned by the application and must be destroyed once
|
|
227
|
+
# served (see rd_kafka_conf_set_background_event_cb). All event-owned memory (result
|
|
228
|
+
# arrays, strings) has to be copied into Ruby objects by the handlers above before this
|
|
229
|
+
# runs - nothing may retain pointers into the event past this point.
|
|
230
|
+
Rdkafka::Bindings.rd_kafka_event_destroy(event_ptr)
|
|
340
231
|
end
|
|
341
232
|
end
|
|
342
233
|
|
|
343
234
|
# @private
|
|
344
235
|
class DeliveryCallback
|
|
345
|
-
|
|
236
|
+
# Handles message delivery callbacks
|
|
237
|
+
# @param _client_ptr [FFI::Pointer] unused client pointer
|
|
238
|
+
# @param message_ptr [FFI::Pointer] pointer to the delivered message
|
|
239
|
+
# @param opaque_ptr [FFI::Pointer] pointer to the opaque object for callback context
|
|
240
|
+
def self.call(_client_ptr, message_ptr, opaque_ptr)
|
|
346
241
|
message = Rdkafka::Bindings::Message.new(message_ptr)
|
|
347
242
|
delivery_handle_ptr_address = message[:_private].address
|
|
348
243
|
if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
|
|
@@ -352,28 +247,42 @@ module Rdkafka
|
|
|
352
247
|
delivery_handle[:response] = message[:err]
|
|
353
248
|
delivery_handle[:partition] = message[:partition]
|
|
354
249
|
delivery_handle[:offset] = message[:offset]
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
#
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
250
|
+
# The topic is not stored on the handle here: it is already set as the handle's
|
|
251
|
+
# `topic` Ruby attribute during produce, which spares a native string copy that
|
|
252
|
+
# would otherwise be allocated and retained for every message.
|
|
253
|
+
|
|
254
|
+
begin
|
|
255
|
+
# Call delivery callback on opaque
|
|
256
|
+
if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
|
|
257
|
+
opaque.call_delivery_callback(
|
|
258
|
+
Rdkafka::Producer::DeliveryReport.new(
|
|
259
|
+
message[:partition],
|
|
260
|
+
message[:offset],
|
|
261
|
+
topic_name,
|
|
262
|
+
message[:err],
|
|
263
|
+
delivery_handle.label
|
|
264
|
+
),
|
|
265
|
+
delivery_handle
|
|
266
|
+
)
|
|
267
|
+
end
|
|
268
|
+
rescue Exception => err
|
|
269
|
+
# A user delivery callback that raises must not escape this FFI callback: it runs on
|
|
270
|
+
# librdkafka's polling thread (abort_on_exception = true), so an escaping exception
|
|
271
|
+
# would crash the whole process. We log and swallow it, matching the rebalance
|
|
272
|
+
# callback. The `ensure` below still unlocks the handle, so `wait` returns the
|
|
273
|
+
# delivery report instead of blocking until its timeout and raising
|
|
274
|
+
# WaitTimeoutError for a message that was in fact delivered.
|
|
275
|
+
Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
|
|
276
|
+
ensure
|
|
277
|
+
delivery_handle.unlock
|
|
369
278
|
end
|
|
370
|
-
|
|
371
|
-
delivery_handle.unlock
|
|
372
279
|
end
|
|
373
280
|
end
|
|
374
281
|
end
|
|
375
282
|
|
|
283
|
+
# @private
|
|
376
284
|
@@mutex = Mutex.new
|
|
285
|
+
# @private
|
|
377
286
|
@@current_pid = nil
|
|
378
287
|
|
|
379
288
|
class << self
|
|
@@ -392,14 +301,14 @@ module Rdkafka
|
|
|
392
301
|
|
|
393
302
|
# FFI Function used for Create Topic and Delete Topic callbacks
|
|
394
303
|
background_event_callback_function = FFI::Function.new(
|
|
395
|
-
|
|
304
|
+
:void, [:pointer, :pointer, :pointer]
|
|
396
305
|
) do |client_ptr, event_ptr, opaque_ptr|
|
|
397
306
|
BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
|
|
398
307
|
end
|
|
399
308
|
|
|
400
309
|
# FFI Function used for Message Delivery callbacks
|
|
401
310
|
delivery_callback_function = FFI::Function.new(
|
|
402
|
-
|
|
311
|
+
:void, [:pointer, :pointer, :pointer]
|
|
403
312
|
) do |client_ptr, message_ptr, opaque_ptr|
|
|
404
313
|
DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
|
|
405
314
|
end
|