rdkafka 0.22.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +63 -3
- data/Gemfile +8 -0
- data/Gemfile.lint +14 -0
- data/Gemfile.lint.lock +123 -0
- data/README.md +19 -14
- data/Rakefile +21 -21
- data/bin/verify_kafka_warnings +39 -0
- data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.0.tar.gz} +0 -0
- data/docker-compose-ssl.yml +35 -0
- data/docker-compose.yml +2 -2
- data/ext/Rakefile +27 -27
- data/lib/rdkafka/abstract_handle.rb +23 -5
- data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
- data/lib/rdkafka/admin/create_acl_report.rb +3 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
- data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
- data/lib/rdkafka/admin.rb +301 -135
- data/lib/rdkafka/bindings.rb +199 -110
- data/lib/rdkafka/callbacks.rb +124 -21
- data/lib/rdkafka/config.rb +81 -33
- data/lib/rdkafka/consumer/headers.rb +3 -2
- data/lib/rdkafka/consumer/message.rb +12 -11
- data/lib/rdkafka/consumer/partition.rb +8 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +21 -17
- data/lib/rdkafka/consumer.rb +397 -45
- data/lib/rdkafka/defaults.rb +106 -0
- data/lib/rdkafka/error.rb +40 -14
- data/lib/rdkafka/helpers/oauth.rb +45 -13
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +45 -21
- data/lib/rdkafka/native_kafka.rb +89 -4
- data/lib/rdkafka/producer/delivery_handle.rb +5 -5
- data/lib/rdkafka/producer/delivery_report.rb +10 -6
- data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
- data/lib/rdkafka/producer.rb +168 -82
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +3 -0
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +57 -36
- data/renovate.json +29 -24
- metadata +29 -124
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
- data/.github/workflows/ci_macos_arm64.yml +0 -284
- data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
- data/.github/workflows/push_macos_arm64.yml +0 -54
- data/.github/workflows/push_ruby.yml +0 -37
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.gitignore +0 -14
- data/.rspec +0 -2
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.yardopts +0 -2
- data/ext/README.md +0 -19
- data/ext/build_common.sh +0 -361
- data/ext/build_linux_x86_64_gnu.sh +0 -306
- data/ext/build_linux_x86_64_musl.sh +0 -763
- data/ext/build_macos_arm64.sh +0 -550
- data/spec/rdkafka/abstract_handle_spec.rb +0 -117
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
- data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
- data/spec/rdkafka/admin_spec.rb +0 -971
- data/spec/rdkafka/bindings_spec.rb +0 -199
- data/spec/rdkafka/callbacks_spec.rb +0 -20
- data/spec/rdkafka/config_spec.rb +0 -258
- data/spec/rdkafka/consumer/headers_spec.rb +0 -73
- data/spec/rdkafka/consumer/message_spec.rb +0 -139
- data/spec/rdkafka/consumer/partition_spec.rb +0 -57
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
- data/spec/rdkafka/consumer_spec.rb +0 -1274
- data/spec/rdkafka/error_spec.rb +0 -89
- data/spec/rdkafka/metadata_spec.rb +0 -79
- data/spec/rdkafka/native_kafka_spec.rb +0 -130
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
- data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
- data/spec/rdkafka/producer_spec.rb +0 -1345
- data/spec/spec_helper.rb +0 -195
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -2,18 +2,61 @@
|
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
4
|
# @private
|
|
5
|
+
#
|
|
6
|
+
# @note
|
|
7
|
+
# There are two types of responses related to errors:
|
|
8
|
+
# - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
|
|
9
|
+
# - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
|
|
10
|
+
#
|
|
11
|
+
# It is critical to ensure, that we handle them correctly. The result type should be:
|
|
12
|
+
# - rd_kafka_error_t - :pointer
|
|
13
|
+
# - rd_kafka_resp_err_t - :int
|
|
5
14
|
module Bindings
|
|
6
15
|
extend FFI::Library
|
|
7
16
|
|
|
17
|
+
# Returns the library extension based on the host OS
|
|
18
|
+
# @return [String] 'dylib' on macOS, 'so' on other systems
|
|
8
19
|
def self.lib_extension
|
|
9
|
-
if RbConfig::CONFIG[
|
|
10
|
-
|
|
20
|
+
if /darwin/.match?(RbConfig::CONFIG["host_os"])
|
|
21
|
+
"dylib"
|
|
11
22
|
else
|
|
12
|
-
|
|
23
|
+
"so"
|
|
13
24
|
end
|
|
14
25
|
end
|
|
15
26
|
|
|
16
|
-
ffi_lib
|
|
27
|
+
# Wrap ffi_lib to provide better error messages for glibc compatibility issues
|
|
28
|
+
begin
|
|
29
|
+
ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
|
|
30
|
+
rescue LoadError => e
|
|
31
|
+
error_message = e.message
|
|
32
|
+
|
|
33
|
+
# Check if this is a glibc version mismatch error
|
|
34
|
+
if /GLIBC_[\d.]+['"` ]?\s*not found/i.match?(error_message)
|
|
35
|
+
glibc_version = error_message[/GLIBC_([\d.]+)/, 1] || "unknown"
|
|
36
|
+
|
|
37
|
+
raise Rdkafka::LibraryLoadError, <<~ERROR_MSG.strip
|
|
38
|
+
Failed to load librdkafka due to glibc compatibility issue.
|
|
39
|
+
|
|
40
|
+
The precompiled librdkafka binary requires glibc version #{glibc_version} or higher,
|
|
41
|
+
but your system has an older version installed.
|
|
42
|
+
|
|
43
|
+
To resolve this issue, you have two options:
|
|
44
|
+
|
|
45
|
+
1. Upgrade your system to a supported platform (recommended)
|
|
46
|
+
|
|
47
|
+
2. Force compilation from source by reinstalling without the precompiled binary:
|
|
48
|
+
gem install rdkafka --platform=ruby
|
|
49
|
+
|
|
50
|
+
Or if using Bundler, add to your Gemfile:
|
|
51
|
+
gem 'rdkafka', force_ruby_platform: true
|
|
52
|
+
|
|
53
|
+
Original error: #{error_message}
|
|
54
|
+
ERROR_MSG
|
|
55
|
+
else
|
|
56
|
+
# Re-raise the original error if it's not a glibc issue
|
|
57
|
+
raise
|
|
58
|
+
end
|
|
59
|
+
end
|
|
17
60
|
|
|
18
61
|
RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
|
|
19
62
|
RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
|
|
@@ -21,13 +64,17 @@ module Rdkafka
|
|
|
21
64
|
RD_KAFKA_RESP_ERR__NOENT = -156
|
|
22
65
|
RD_KAFKA_RESP_ERR_NO_ERROR = 0
|
|
23
66
|
|
|
24
|
-
RD_KAFKA_OFFSET_END
|
|
67
|
+
RD_KAFKA_OFFSET_END = -1
|
|
25
68
|
RD_KAFKA_OFFSET_BEGINNING = -2
|
|
26
|
-
RD_KAFKA_OFFSET_STORED
|
|
27
|
-
RD_KAFKA_OFFSET_INVALID
|
|
69
|
+
RD_KAFKA_OFFSET_STORED = -1000
|
|
70
|
+
RD_KAFKA_OFFSET_INVALID = -1001
|
|
71
|
+
|
|
72
|
+
RD_KAFKA_PARTITION_UA = -1
|
|
73
|
+
RD_KAFKA_PARTITION_UA_STR = RD_KAFKA_PARTITION_UA.to_s.freeze
|
|
28
74
|
|
|
29
75
|
EMPTY_HASH = {}.freeze
|
|
30
76
|
|
|
77
|
+
# FFI struct for size_t pointer wrapper
|
|
31
78
|
class SizePtr < FFI::Struct
|
|
32
79
|
layout :value, :size_t
|
|
33
80
|
end
|
|
@@ -43,6 +90,12 @@ module Rdkafka
|
|
|
43
90
|
attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
|
|
44
91
|
attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
|
|
45
92
|
|
|
93
|
+
# Non-blocking poll variants (do not release GVL)
|
|
94
|
+
# These are more efficient for poll(0) calls in fiber schedulers where GVL
|
|
95
|
+
# release/reacquire overhead is wasteful since we don't expect to wait.
|
|
96
|
+
# Uses the same underlying C function but with blocking: false to skip GVL release.
|
|
97
|
+
attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
|
|
98
|
+
|
|
46
99
|
# Metadata
|
|
47
100
|
|
|
48
101
|
attach_function :rd_kafka_name, [:pointer], :string
|
|
@@ -51,18 +104,17 @@ module Rdkafka
|
|
|
51
104
|
attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
|
|
52
105
|
attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
|
|
53
106
|
|
|
54
|
-
#
|
|
55
|
-
|
|
107
|
+
# FFI struct representing a Kafka message (rd_kafka_message_t)
|
|
56
108
|
class Message < FFI::Struct
|
|
57
109
|
layout :err, :int,
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
110
|
+
:rkt, :pointer,
|
|
111
|
+
:partition, :int32,
|
|
112
|
+
:payload, :pointer,
|
|
113
|
+
:len, :size_t,
|
|
114
|
+
:key, :pointer,
|
|
115
|
+
:key_len, :size_t,
|
|
116
|
+
:offset, :int64,
|
|
117
|
+
:_private, :pointer
|
|
66
118
|
end
|
|
67
119
|
|
|
68
120
|
attach_function :rd_kafka_message_destroy, [:pointer], :void
|
|
@@ -71,40 +123,38 @@ module Rdkafka
|
|
|
71
123
|
attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
|
|
72
124
|
attach_function :rd_kafka_topic_name, [:pointer], :string
|
|
73
125
|
|
|
74
|
-
#
|
|
75
|
-
|
|
126
|
+
# FFI struct representing a topic partition (rd_kafka_topic_partition_t)
|
|
76
127
|
class TopicPartition < FFI::Struct
|
|
77
128
|
layout :topic, :string,
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
129
|
+
:partition, :int32,
|
|
130
|
+
:offset, :int64,
|
|
131
|
+
:metadata, :pointer,
|
|
132
|
+
:metadata_size, :size_t,
|
|
133
|
+
:opaque, :pointer,
|
|
134
|
+
:err, :int,
|
|
135
|
+
:_private, :pointer
|
|
85
136
|
end
|
|
86
137
|
|
|
138
|
+
# FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
|
|
87
139
|
class TopicPartitionList < FFI::Struct
|
|
88
140
|
layout :cnt, :int,
|
|
89
|
-
|
|
90
|
-
|
|
141
|
+
:size, :int,
|
|
142
|
+
:elems, :pointer
|
|
91
143
|
end
|
|
92
144
|
|
|
93
145
|
attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
|
|
94
|
-
attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :
|
|
146
|
+
attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
|
|
95
147
|
attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
|
|
96
148
|
attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
|
|
97
149
|
attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
|
|
98
150
|
|
|
99
|
-
#
|
|
100
|
-
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
# details. Each resource will also have separate errors results if obtaining configuration
|
|
104
|
-
# was not possible for any reason
|
|
151
|
+
# FFI struct representing a config resource (rd_kafka_ConfigResource_t)
|
|
152
|
+
# Structs for management of configurations. Each configuration is attached to a resource
|
|
153
|
+
# and one resource can have many configuration details. Each resource will also have
|
|
154
|
+
# separate errors results if obtaining configuration was not possible for any reason
|
|
105
155
|
class ConfigResource < FFI::Struct
|
|
106
156
|
layout :type, :int,
|
|
107
|
-
|
|
157
|
+
:name, :string
|
|
108
158
|
end
|
|
109
159
|
|
|
110
160
|
attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
|
@@ -133,20 +183,45 @@ module Rdkafka
|
|
|
133
183
|
RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
|
|
134
184
|
RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
|
|
135
185
|
|
|
136
|
-
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET
|
|
137
|
-
RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE
|
|
138
|
-
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND
|
|
186
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
|
|
187
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
|
|
188
|
+
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
|
139
189
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
|
140
190
|
|
|
141
|
-
#
|
|
191
|
+
# List Offsets
|
|
192
|
+
RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
|
|
193
|
+
RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
|
|
194
|
+
|
|
195
|
+
# rd_kafka_IsolationLevel_t
|
|
196
|
+
RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
|
|
197
|
+
RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
|
|
198
|
+
|
|
199
|
+
# rd_kafka_OffsetSpec_t
|
|
200
|
+
RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
|
|
201
|
+
RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
|
|
202
|
+
RD_KAFKA_OFFSET_SPEC_LATEST = -1
|
|
203
|
+
|
|
204
|
+
attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
|
|
205
|
+
attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
|
|
206
|
+
attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
|
|
207
|
+
attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
|
|
208
|
+
attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
|
|
209
|
+
attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
|
|
210
|
+
attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
|
|
211
|
+
|
|
212
|
+
# FFI struct for error description (rd_kafka_err_desc)
|
|
142
213
|
class NativeErrorDesc < FFI::Struct
|
|
143
214
|
layout :code, :int,
|
|
144
|
-
|
|
145
|
-
|
|
215
|
+
:name, :pointer,
|
|
216
|
+
:desc, :pointer
|
|
146
217
|
end
|
|
147
218
|
|
|
148
219
|
attach_function :rd_kafka_err2name, [:int], :string
|
|
149
220
|
attach_function :rd_kafka_err2str, [:int], :string
|
|
221
|
+
attach_function :rd_kafka_error_is_fatal, [:pointer], :int
|
|
222
|
+
attach_function :rd_kafka_error_is_retriable, [:pointer], :int
|
|
223
|
+
attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
|
|
224
|
+
attach_function :rd_kafka_error_destroy, [:pointer], :void
|
|
150
225
|
attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
|
|
151
226
|
|
|
152
227
|
# Configuration
|
|
@@ -159,6 +234,9 @@ module Rdkafka
|
|
|
159
234
|
|
|
160
235
|
attach_function :rd_kafka_conf_new, [], :pointer
|
|
161
236
|
attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
|
237
|
+
attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
|
|
238
|
+
attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
|
|
239
|
+
attach_function :rd_kafka_conf_destroy, [:pointer], :void
|
|
162
240
|
callback :log_cb, [:pointer, :int, :string, :string], :void
|
|
163
241
|
attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
|
|
164
242
|
attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
|
|
@@ -174,6 +252,15 @@ module Rdkafka
|
|
|
174
252
|
# Log queue
|
|
175
253
|
attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
|
|
176
254
|
attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
|
|
255
|
+
attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
|
|
256
|
+
|
|
257
|
+
# Queue IO Event Support - for fiber scheduler integration
|
|
258
|
+
# Enables notifications to a custom FD when queue transitions from empty to non-empty
|
|
259
|
+
# @param queue rd_kafka_queue_t* - the queue to monitor
|
|
260
|
+
# @param fd int - file descriptor to write to (provide your own pipe/eventfd)
|
|
261
|
+
# @param payload const void* - data to write to fd
|
|
262
|
+
# @param size size_t - size of payload
|
|
263
|
+
attach_function :rd_kafka_queue_io_event_enable, [:pointer, :int, :pointer, :size_t], :void
|
|
177
264
|
# Per topic configs
|
|
178
265
|
attach_function :rd_kafka_topic_conf_new, [], :pointer
|
|
179
266
|
attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
|
@@ -182,19 +269,19 @@ module Rdkafka
|
|
|
182
269
|
:void, [:pointer, :int, :string, :string]
|
|
183
270
|
) do |_client_ptr, level, _level_string, line|
|
|
184
271
|
severity = case level
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
272
|
+
when 0, 1, 2
|
|
273
|
+
Logger::FATAL
|
|
274
|
+
when 3
|
|
275
|
+
Logger::ERROR
|
|
276
|
+
when 4
|
|
277
|
+
Logger::WARN
|
|
278
|
+
when 5, 6
|
|
279
|
+
Logger::INFO
|
|
280
|
+
when 7
|
|
281
|
+
Logger::DEBUG
|
|
282
|
+
else
|
|
283
|
+
Logger::UNKNOWN
|
|
284
|
+
end
|
|
198
285
|
|
|
199
286
|
Rdkafka::Config.ensure_log_thread
|
|
200
287
|
Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
|
|
@@ -220,8 +307,8 @@ module Rdkafka
|
|
|
220
307
|
#
|
|
221
308
|
# Since this cache is shared, having few consumers and/or producers in one process will
|
|
222
309
|
# automatically improve the querying times even with low refresh times.
|
|
223
|
-
(stats[
|
|
224
|
-
partitions_count = details[
|
|
310
|
+
(stats["topics"] || EMPTY_HASH).each do |topic_name, details|
|
|
311
|
+
partitions_count = details["partitions"].keys.count { |k| !(k == RD_KAFKA_PARTITION_UA_STR) }
|
|
225
312
|
|
|
226
313
|
next unless partitions_count.positive?
|
|
227
314
|
|
|
@@ -232,14 +319,15 @@ module Rdkafka
|
|
|
232
319
|
end
|
|
233
320
|
|
|
234
321
|
# Return 0 so librdkafka frees the json string
|
|
235
|
-
|
|
322
|
+
RD_KAFKA_RESP_ERR_NO_ERROR
|
|
236
323
|
end
|
|
237
324
|
|
|
238
325
|
ErrorCallback = FFI::Function.new(
|
|
239
326
|
:void, [:pointer, :int, :string, :pointer]
|
|
240
|
-
) do |
|
|
327
|
+
) do |client_ptr, err_code, reason, _opaque|
|
|
241
328
|
if Rdkafka::Config.error_callback
|
|
242
|
-
|
|
329
|
+
instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
|
|
330
|
+
error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason, instance_name: instance_name)
|
|
243
331
|
error.set_backtrace(caller)
|
|
244
332
|
Rdkafka::Config.error_callback.call(error)
|
|
245
333
|
end
|
|
@@ -266,9 +354,7 @@ module Rdkafka
|
|
|
266
354
|
OAuthbearerTokenRefreshCallback = FFI::Function.new(
|
|
267
355
|
:void, [:pointer, :string, :pointer]
|
|
268
356
|
) do |client_ptr, config, _opaque|
|
|
269
|
-
|
|
270
|
-
Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
|
|
271
|
-
end
|
|
357
|
+
Rdkafka::Config.oauthbearer_token_refresh_callback&.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
|
|
272
358
|
end
|
|
273
359
|
|
|
274
360
|
# Handle
|
|
@@ -296,7 +382,13 @@ module Rdkafka
|
|
|
296
382
|
attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
|
|
297
383
|
attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
|
|
298
384
|
attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
|
|
385
|
+
# Non-blocking consumer poll variant (does not release GVL)
|
|
386
|
+
# More efficient for poll(0) calls in fiber schedulers.
|
|
387
|
+
attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
|
|
299
388
|
attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
|
|
389
|
+
attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
|
|
390
|
+
attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
|
|
391
|
+
attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
|
|
300
392
|
attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
|
|
301
393
|
attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
|
|
302
394
|
attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
|
|
@@ -355,7 +447,6 @@ module Rdkafka
|
|
|
355
447
|
attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
|
|
356
448
|
|
|
357
449
|
# Producer
|
|
358
|
-
|
|
359
450
|
RD_KAFKA_VTYPE_END = 0
|
|
360
451
|
RD_KAFKA_VTYPE_TOPIC = 1
|
|
361
452
|
RD_KAFKA_VTYPE_RKT = 2
|
|
@@ -377,16 +468,25 @@ module Rdkafka
|
|
|
377
468
|
callback :delivery_cb, [:pointer, :pointer, :pointer], :void
|
|
378
469
|
attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
|
|
379
470
|
|
|
380
|
-
#
|
|
381
|
-
|
|
382
|
-
|
|
471
|
+
# Hash mapping partitioner names to their FFI function symbols
|
|
472
|
+
# @return [Hash{String => Symbol}]
|
|
473
|
+
PARTITIONERS = %w[random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random].each_with_object({}) do |name, hsh|
|
|
474
|
+
method_name = :"rd_kafka_msg_partitioner_#{name}"
|
|
383
475
|
attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
|
|
384
476
|
hsh[name] = method_name
|
|
385
477
|
end
|
|
386
478
|
|
|
479
|
+
# Calculates the partition for a message based on the partitioner
|
|
480
|
+
#
|
|
481
|
+
# @param topic_ptr [FFI::Pointer] pointer to the topic handle
|
|
482
|
+
# @param str [String] the partition key string
|
|
483
|
+
# @param partition_count [Integer, nil] number of partitions
|
|
484
|
+
# @param partitioner [String] name of the partitioner to use
|
|
485
|
+
# @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
|
|
486
|
+
# @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
|
|
387
487
|
def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
|
|
388
488
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
|
389
|
-
return
|
|
489
|
+
return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
|
|
390
490
|
|
|
391
491
|
str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
|
|
392
492
|
method_name = PARTITIONERS.fetch(partitioner) do
|
|
@@ -397,8 +497,7 @@ module Rdkafka
|
|
|
397
497
|
end
|
|
398
498
|
|
|
399
499
|
# Create Topics
|
|
400
|
-
|
|
401
|
-
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
|
500
|
+
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
|
402
501
|
RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
|
|
403
502
|
|
|
404
503
|
attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
|
@@ -409,8 +508,7 @@ module Rdkafka
|
|
|
409
508
|
attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
|
|
410
509
|
|
|
411
510
|
# Delete Topics
|
|
412
|
-
|
|
413
|
-
RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
|
|
511
|
+
RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
|
|
414
512
|
RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
|
|
415
513
|
|
|
416
514
|
attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
|
|
@@ -430,7 +528,6 @@ module Rdkafka
|
|
|
430
528
|
attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
|
|
431
529
|
|
|
432
530
|
# Delete Group
|
|
433
|
-
|
|
434
531
|
RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
|
|
435
532
|
RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
|
|
436
533
|
|
|
@@ -442,7 +539,6 @@ module Rdkafka
|
|
|
442
539
|
|
|
443
540
|
# Background Queue and Callback
|
|
444
541
|
|
|
445
|
-
attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
|
|
446
542
|
attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
|
|
447
543
|
attach_function :rd_kafka_queue_destroy, [:pointer], :void
|
|
448
544
|
|
|
@@ -464,8 +560,7 @@ module Rdkafka
|
|
|
464
560
|
attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
|
|
465
561
|
|
|
466
562
|
# Create Acls
|
|
467
|
-
|
|
468
|
-
RD_KAFKA_ADMIN_OP_CREATEACLS = 9
|
|
563
|
+
RD_KAFKA_ADMIN_OP_CREATEACLS = 9
|
|
469
564
|
RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
|
|
470
565
|
|
|
471
566
|
attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
|
|
@@ -473,8 +568,7 @@ module Rdkafka
|
|
|
473
568
|
attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
|
|
474
569
|
|
|
475
570
|
# Delete Acls
|
|
476
|
-
|
|
477
|
-
RD_KAFKA_ADMIN_OP_DELETEACLS = 11
|
|
571
|
+
RD_KAFKA_ADMIN_OP_DELETEACLS = 11
|
|
478
572
|
RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
|
|
479
573
|
|
|
480
574
|
attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
|
|
@@ -484,8 +578,7 @@ module Rdkafka
|
|
|
484
578
|
attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
|
|
485
579
|
|
|
486
580
|
# Describe Acls
|
|
487
|
-
|
|
488
|
-
RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
|
|
581
|
+
RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
|
|
489
582
|
RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
|
|
490
583
|
|
|
491
584
|
attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
|
|
@@ -501,45 +594,42 @@ module Rdkafka
|
|
|
501
594
|
attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
|
|
502
595
|
attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
|
|
503
596
|
attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
|
|
504
|
-
attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t
|
|
505
|
-
attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t
|
|
597
|
+
attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
|
|
598
|
+
attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
|
|
506
599
|
attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
|
|
507
600
|
|
|
508
601
|
# rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
|
|
509
|
-
|
|
510
|
-
RD_KAFKA_RESOURCE_ANY = 1
|
|
602
|
+
RD_KAFKA_RESOURCE_ANY = 1
|
|
511
603
|
RD_KAFKA_RESOURCE_TOPIC = 2
|
|
512
604
|
RD_KAFKA_RESOURCE_GROUP = 3
|
|
513
605
|
RD_KAFKA_RESOURCE_BROKER = 4
|
|
514
606
|
RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
|
|
515
607
|
|
|
516
608
|
# rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
517
|
-
|
|
518
|
-
RD_KAFKA_RESOURCE_PATTERN_ANY
|
|
519
|
-
RD_KAFKA_RESOURCE_PATTERN_MATCH
|
|
520
|
-
RD_KAFKA_RESOURCE_PATTERN_LITERAL
|
|
609
|
+
RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
|
|
610
|
+
RD_KAFKA_RESOURCE_PATTERN_ANY = 1
|
|
611
|
+
RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
|
|
612
|
+
RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
|
|
521
613
|
RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
|
|
522
614
|
|
|
523
615
|
# rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
616
|
+
RD_KAFKA_ACL_OPERATION_ANY = 1
|
|
617
|
+
RD_KAFKA_ACL_OPERATION_ALL = 2
|
|
618
|
+
RD_KAFKA_ACL_OPERATION_READ = 3
|
|
619
|
+
RD_KAFKA_ACL_OPERATION_WRITE = 4
|
|
620
|
+
RD_KAFKA_ACL_OPERATION_CREATE = 5
|
|
621
|
+
RD_KAFKA_ACL_OPERATION_DELETE = 6
|
|
622
|
+
RD_KAFKA_ACL_OPERATION_ALTER = 7
|
|
623
|
+
RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
|
|
624
|
+
RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
|
|
534
625
|
RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
|
|
535
|
-
RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS
|
|
626
|
+
RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
|
|
536
627
|
RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
|
|
537
628
|
|
|
538
629
|
# rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
630
|
+
RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
|
|
631
|
+
RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
|
|
632
|
+
RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
|
|
543
633
|
|
|
544
634
|
# Extracting error details from Acl results
|
|
545
635
|
attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
|
|
@@ -549,14 +639,13 @@ module Rdkafka
|
|
|
549
639
|
attach_function :rd_kafka_event_error_string, [:pointer], :pointer
|
|
550
640
|
attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
|
|
551
641
|
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
class NativeError < FFI::Struct # rd_kafka_error_t
|
|
642
|
+
# FFI struct for native error (rd_kafka_error_t)
|
|
643
|
+
class NativeError < FFI::Struct
|
|
555
644
|
layout :code, :int32,
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
645
|
+
:errstr, :pointer,
|
|
646
|
+
:fatal, :u_int8_t,
|
|
647
|
+
:retriable, :u_int8_t,
|
|
648
|
+
:txn_requires_abort, :u_int8_t
|
|
560
649
|
end
|
|
561
650
|
|
|
562
651
|
attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
|