rdkafka 0.25.0-aarch64-linux-gnu → 0.25.1-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +5 -6
- data/Gemfile.lint +14 -0
- data/Gemfile.lint.lock +123 -0
- data/README.md +1 -1
- data/Rakefile +21 -21
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/admin/acl_binding_result.rb +4 -4
- data/lib/rdkafka/admin/create_acl_handle.rb +4 -4
- data/lib/rdkafka/admin/create_acl_report.rb +0 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +5 -5
- data/lib/rdkafka/admin/create_topic_handle.rb +5 -5
- data/lib/rdkafka/admin/delete_acl_handle.rb +6 -6
- data/lib/rdkafka/admin/delete_acl_report.rb +2 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +5 -5
- data/lib/rdkafka/admin/delete_topic_handle.rb +5 -5
- data/lib/rdkafka/admin/describe_acl_handle.rb +6 -6
- data/lib/rdkafka/admin/describe_acl_report.rb +2 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +4 -4
- data/lib/rdkafka/admin/describe_configs_report.rb +1 -1
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +4 -4
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +1 -1
- data/lib/rdkafka/admin.rb +86 -20
- data/lib/rdkafka/bindings.rb +97 -82
- data/lib/rdkafka/callbacks.rb +10 -10
- data/lib/rdkafka/config.rb +18 -18
- data/lib/rdkafka/consumer/message.rb +5 -8
- data/lib/rdkafka/consumer/partition.rb +2 -2
- data/lib/rdkafka/consumer/topic_partition_list.rb +10 -10
- data/lib/rdkafka/consumer.rb +207 -14
- data/lib/rdkafka/error.rb +13 -13
- data/lib/rdkafka/helpers/oauth.rb +0 -1
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +16 -16
- data/lib/rdkafka/native_kafka.rb +63 -2
- data/lib/rdkafka/producer/delivery_handle.rb +5 -5
- data/lib/rdkafka/producer/delivery_report.rb +1 -1
- data/lib/rdkafka/producer/partitions_count_cache.rb +6 -6
- data/lib/rdkafka/producer.rb +117 -57
- data/lib/rdkafka/version.rb +1 -1
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +39 -40
- data/renovate.json +21 -0
- metadata +5 -1
data/lib/rdkafka/config.rb
CHANGED
|
@@ -6,7 +6,7 @@ module Rdkafka
|
|
|
6
6
|
# configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
|
|
7
7
|
class Config
|
|
8
8
|
# @private
|
|
9
|
-
@@logger = Logger.new(
|
|
9
|
+
@@logger = Logger.new($stdout)
|
|
10
10
|
# @private
|
|
11
11
|
@@statistics_callback = nil
|
|
12
12
|
# @private
|
|
@@ -33,7 +33,7 @@ module Rdkafka
|
|
|
33
33
|
# Makes sure that there is a thread for consuming logs
|
|
34
34
|
# We do not spawn thread immediately and we need to check if it operates to support forking
|
|
35
35
|
def self.ensure_log_thread
|
|
36
|
-
return if @@log_thread
|
|
36
|
+
return if @@log_thread&.alive?
|
|
37
37
|
|
|
38
38
|
@@log_mutex.synchronize do
|
|
39
39
|
# Restart if dead (fork, crash)
|
|
@@ -74,7 +74,7 @@ module Rdkafka
|
|
|
74
74
|
# @param callback [Proc, #call, nil] callable object or nil to clear
|
|
75
75
|
# @return [nil]
|
|
76
76
|
def self.statistics_callback=(callback)
|
|
77
|
-
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback
|
|
77
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
|
|
78
78
|
@@statistics_callback = callback
|
|
79
79
|
end
|
|
80
80
|
|
|
@@ -109,7 +109,7 @@ module Rdkafka
|
|
|
109
109
|
# @param callback [Proc, #call, nil] callable object to handle token refresh or nil to clear
|
|
110
110
|
# @return [nil]
|
|
111
111
|
def self.oauthbearer_token_refresh_callback=(callback)
|
|
112
|
-
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback
|
|
112
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
|
|
113
113
|
@@oauthbearer_token_refresh_callback = callback
|
|
114
114
|
end
|
|
115
115
|
|
|
@@ -131,7 +131,7 @@ module Rdkafka
|
|
|
131
131
|
# Required config that cannot be overwritten.
|
|
132
132
|
REQUIRED_CONFIG = {
|
|
133
133
|
# Enable log queues so we get callbacks in our own Ruby threads
|
|
134
|
-
|
|
134
|
+
"log.queue": true
|
|
135
135
|
}.freeze
|
|
136
136
|
|
|
137
137
|
# Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
|
|
@@ -168,10 +168,8 @@ module Rdkafka
|
|
|
168
168
|
|
|
169
169
|
# Get notifications on partition assignment/revocation for the subscribed topics
|
|
170
170
|
#
|
|
171
|
-
# @
|
|
172
|
-
|
|
173
|
-
@consumer_rebalance_listener = listener
|
|
174
|
-
end
|
|
171
|
+
# @return [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
|
|
172
|
+
attr_writer :consumer_rebalance_listener
|
|
175
173
|
|
|
176
174
|
# Should we use a single queue for the underlying consumer and events.
|
|
177
175
|
#
|
|
@@ -184,10 +182,8 @@ module Rdkafka
|
|
|
184
182
|
# It is recommended to use the defaults and only set it to `false` in advance multi-threaded
|
|
185
183
|
# and complex cases where granular events handling control is needed.
|
|
186
184
|
#
|
|
187
|
-
# @
|
|
188
|
-
|
|
189
|
-
@consumer_poll_set = poll_set
|
|
190
|
-
end
|
|
185
|
+
# @return [Boolean]
|
|
186
|
+
attr_writer :consumer_poll_set
|
|
191
187
|
|
|
192
188
|
# Creates a consumer with this configuration.
|
|
193
189
|
#
|
|
@@ -228,11 +224,13 @@ module Rdkafka
|
|
|
228
224
|
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
|
229
225
|
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
|
230
226
|
# @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
|
|
227
|
+
# @param run_polling_thread [Boolean] should the background polling thread be started.
|
|
228
|
+
# Defaults to true. Set to false when using the FD API for fiber scheduler integration.
|
|
231
229
|
# @return [Producer] The created producer
|
|
232
230
|
#
|
|
233
231
|
# @raise [ConfigError] When the configuration contains invalid options
|
|
234
232
|
# @raise [ClientCreationError] When the native client cannot be created
|
|
235
|
-
def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
|
|
233
|
+
def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
|
|
236
234
|
# Create opaque
|
|
237
235
|
opaque = Opaque.new
|
|
238
236
|
# Create Kafka config
|
|
@@ -247,7 +245,7 @@ module Rdkafka
|
|
|
247
245
|
Rdkafka::Producer.new(
|
|
248
246
|
Rdkafka::NativeKafka.new(
|
|
249
247
|
kafka,
|
|
250
|
-
run_polling_thread:
|
|
248
|
+
run_polling_thread: run_polling_thread,
|
|
251
249
|
opaque: opaque,
|
|
252
250
|
auto_start: native_kafka_auto_start,
|
|
253
251
|
timeout_ms: native_kafka_poll_timeout_ms
|
|
@@ -263,11 +261,13 @@ module Rdkafka
|
|
|
263
261
|
# @param native_kafka_auto_start [Boolean] should the native kafka operations be started
|
|
264
262
|
# automatically. Defaults to true. Set to false only when doing complex initialization.
|
|
265
263
|
# @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
|
|
264
|
+
# @param run_polling_thread [Boolean] should the background polling thread be started.
|
|
265
|
+
# Defaults to true. Set to false when using the FD API for fiber scheduler integration.
|
|
266
266
|
# @return [Admin] The created admin instance
|
|
267
267
|
#
|
|
268
268
|
# @raise [ConfigError] When the configuration contains invalid options
|
|
269
269
|
# @raise [ClientCreationError] When the native client cannot be created
|
|
270
|
-
def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
|
|
270
|
+
def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
|
|
271
271
|
opaque = Opaque.new
|
|
272
272
|
config = native_config(opaque)
|
|
273
273
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
|
@@ -277,7 +277,7 @@ module Rdkafka
|
|
|
277
277
|
Rdkafka::Admin.new(
|
|
278
278
|
Rdkafka::NativeKafka.new(
|
|
279
279
|
kafka,
|
|
280
|
-
run_polling_thread:
|
|
280
|
+
run_polling_thread: run_polling_thread,
|
|
281
281
|
opaque: opaque,
|
|
282
282
|
auto_start: native_kafka_auto_start,
|
|
283
283
|
timeout_ms: native_kafka_poll_timeout_ms
|
|
@@ -383,7 +383,7 @@ module Rdkafka
|
|
|
383
383
|
# @param delivery_report [Rdkafka::Producer::DeliveryReport] the delivery report
|
|
384
384
|
# @param delivery_handle [Rdkafka::Producer::DeliveryHandle] the delivery handle
|
|
385
385
|
def call_delivery_callback(delivery_report, delivery_handle)
|
|
386
|
-
producer
|
|
386
|
+
producer&.call_delivery_callback(delivery_report, delivery_handle)
|
|
387
387
|
end
|
|
388
388
|
|
|
389
389
|
# Invokes the on_partitions_assigned callback on the rebalance listener if set
|
|
@@ -53,13 +53,11 @@ module Rdkafka
|
|
|
53
53
|
# Set timestamp
|
|
54
54
|
raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
|
|
55
55
|
@timestamp = if raw_timestamp && raw_timestamp > -1
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
nil
|
|
62
|
-
end
|
|
56
|
+
# Calculate seconds and microseconds
|
|
57
|
+
seconds = raw_timestamp / 1000
|
|
58
|
+
milliseconds = (raw_timestamp - seconds * 1000) * 1000
|
|
59
|
+
Time.at(seconds, milliseconds)
|
|
60
|
+
end
|
|
63
61
|
|
|
64
62
|
@headers = Headers.from_native(native_message)
|
|
65
63
|
end
|
|
@@ -84,7 +82,6 @@ module Rdkafka
|
|
|
84
82
|
string
|
|
85
83
|
end
|
|
86
84
|
end
|
|
87
|
-
|
|
88
85
|
end
|
|
89
86
|
end
|
|
90
87
|
end
|
|
@@ -9,7 +9,7 @@ module Rdkafka
|
|
|
9
9
|
# @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
|
|
10
10
|
#
|
|
11
11
|
# @return [TopicPartitionList]
|
|
12
|
-
def initialize(data=nil)
|
|
12
|
+
def initialize(data = nil)
|
|
13
13
|
@data = data || {}
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -18,10 +18,10 @@ module Rdkafka
|
|
|
18
18
|
def count
|
|
19
19
|
i = 0
|
|
20
20
|
@data.each do |_topic, partitions|
|
|
21
|
-
if partitions
|
|
22
|
-
|
|
21
|
+
i += if partitions
|
|
22
|
+
partitions.count
|
|
23
23
|
else
|
|
24
|
-
|
|
24
|
+
1
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
i
|
|
@@ -49,7 +49,7 @@ module Rdkafka
|
|
|
49
49
|
#
|
|
50
50
|
# @example Add a topic with all topics up to a count
|
|
51
51
|
# tpl.add_topic("topic", 9)
|
|
52
|
-
def add_topic(topic, partitions=nil)
|
|
52
|
+
def add_topic(topic, partitions = nil)
|
|
53
53
|
if partitions.nil?
|
|
54
54
|
@data[topic.to_s] = nil
|
|
55
55
|
else
|
|
@@ -90,7 +90,7 @@ module Rdkafka
|
|
|
90
90
|
# @param other [TopicPartitionList] object to compare with
|
|
91
91
|
# @return [Boolean]
|
|
92
92
|
def ==(other)
|
|
93
|
-
|
|
93
|
+
to_h == other.to_h
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
# Create a new topic partition list based of a native one.
|
|
@@ -114,10 +114,10 @@ module Rdkafka
|
|
|
114
114
|
else
|
|
115
115
|
partitions = data[elem[:topic]] || []
|
|
116
116
|
offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
117
|
+
nil
|
|
118
|
+
else
|
|
119
|
+
elem[:offset]
|
|
120
|
+
end
|
|
121
121
|
partition = Partition.new(elem[:partition], offset, elem[:err])
|
|
122
122
|
partitions.push(partition)
|
|
123
123
|
data[elem[:topic]] = partitions
|
data/lib/rdkafka/consumer.rb
CHANGED
|
@@ -34,6 +34,144 @@ module Rdkafka
|
|
|
34
34
|
end
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
# Enable IO event notifications for fiber scheduler integration
|
|
38
|
+
# When the consumer queue has messages, librdkafka will write to your FD
|
|
39
|
+
#
|
|
40
|
+
# @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
|
|
41
|
+
# @param payload [String] data to write to fd (default: "\x01")
|
|
42
|
+
# @return [nil]
|
|
43
|
+
# @raise [ClosedInnerError] when the consumer is closed
|
|
44
|
+
#
|
|
45
|
+
# @example Using with fiber scheduler
|
|
46
|
+
# consumer = config.consumer
|
|
47
|
+
# consumer.subscribe("topic")
|
|
48
|
+
#
|
|
49
|
+
# # Create notification FD
|
|
50
|
+
# signal_r, signal_w = IO.pipe
|
|
51
|
+
#
|
|
52
|
+
# # Enable librdkafka to signal when messages arrive
|
|
53
|
+
# consumer.enable_queue_io_events(signal_w.fileno)
|
|
54
|
+
#
|
|
55
|
+
# # Monitor with select/poll
|
|
56
|
+
# loop do
|
|
57
|
+
# readable, = IO.select([signal_r], nil, nil, timeout)
|
|
58
|
+
# if readable
|
|
59
|
+
# signal_r.read_nonblock(1024) rescue nil # Drain signal
|
|
60
|
+
# while msg = consumer.poll(0)
|
|
61
|
+
# process(msg)
|
|
62
|
+
# end
|
|
63
|
+
# end
|
|
64
|
+
# end
|
|
65
|
+
def enable_queue_io_events(fd, payload = "\x01")
|
|
66
|
+
@native_kafka.enable_main_queue_io_events(fd, payload)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Enable IO event notifications for background events
|
|
70
|
+
# @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
|
|
71
|
+
# @param payload [String] data to write to fd (default: "\x01")
|
|
72
|
+
# @return [nil]
|
|
73
|
+
# @raise [ClosedInnerError] when the consumer is closed
|
|
74
|
+
def enable_background_queue_io_events(fd, payload = "\x01")
|
|
75
|
+
@native_kafka.enable_background_queue_io_events(fd, payload)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Polls for events in a non-blocking loop, yielding the count after each iteration.
|
|
79
|
+
#
|
|
80
|
+
# This method processes events (stats, errors, etc.) in a single GVL/mutex session,
|
|
81
|
+
# which is more efficient than repeated individual polls. It uses non-blocking polls
|
|
82
|
+
# internally (no GVL release between polls).
|
|
83
|
+
#
|
|
84
|
+
# Yields the count of events processed after each poll iteration, allowing the caller
|
|
85
|
+
# to implement timeout or other termination logic by returning `:stop`.
|
|
86
|
+
#
|
|
87
|
+
# @yield [count] Called after each poll iteration
|
|
88
|
+
# @yieldparam count [Integer] Number of events processed in this iteration
|
|
89
|
+
# @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
|
|
90
|
+
# @return [nil]
|
|
91
|
+
# @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
|
|
92
|
+
#
|
|
93
|
+
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
94
|
+
# Other consumer operations will wait until this method returns.
|
|
95
|
+
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
96
|
+
# @note Do NOT use this if `consumer_poll_set` was set to `true`
|
|
97
|
+
#
|
|
98
|
+
# @example Drain all pending events
|
|
99
|
+
# consumer.events_poll_nb_each { |_count| }
|
|
100
|
+
#
|
|
101
|
+
# @example With timeout control
|
|
102
|
+
# deadline = monotonic_now + timeout_ms
|
|
103
|
+
# consumer.events_poll_nb_each do |_count|
|
|
104
|
+
# :stop if monotonic_now >= deadline
|
|
105
|
+
# end
|
|
106
|
+
def events_poll_nb_each
|
|
107
|
+
closed_consumer_check(__method__)
|
|
108
|
+
|
|
109
|
+
@native_kafka.with_inner do |inner|
|
|
110
|
+
loop do
|
|
111
|
+
count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
|
|
112
|
+
break if count.zero?
|
|
113
|
+
break if yield(count) == :stop
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Polls for messages in a non-blocking loop, yielding each message to the caller.
|
|
119
|
+
#
|
|
120
|
+
# This method processes messages in a single GVL/mutex session until the queue is empty
|
|
121
|
+
# or the caller returns `:stop`. It handles the message pointer lifecycle internally,
|
|
122
|
+
# ensuring proper cleanup via `rd_kafka_message_destroy`.
|
|
123
|
+
#
|
|
124
|
+
# @yield [message] Called for each message received
|
|
125
|
+
# @yieldparam message [Consumer::Message] The received message
|
|
126
|
+
# @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
|
|
127
|
+
# @return [nil]
|
|
128
|
+
# @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
|
|
129
|
+
# @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
|
|
130
|
+
#
|
|
131
|
+
# @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
|
|
132
|
+
# `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
|
|
133
|
+
# statistics, etc.). For consumers, use this method to receive messages and
|
|
134
|
+
# `events_poll_nb_each` for processing background events.
|
|
135
|
+
# @note This method holds the inner lock for the duration. Other consumer operations
|
|
136
|
+
# will wait until this method returns.
|
|
137
|
+
# @note Timeout/max_messages logic should be implemented by the caller
|
|
138
|
+
#
|
|
139
|
+
# @example Process messages until queue is empty
|
|
140
|
+
# consumer.poll_nb_each do |message|
|
|
141
|
+
# process(message)
|
|
142
|
+
# end
|
|
143
|
+
#
|
|
144
|
+
# @example Process with early termination
|
|
145
|
+
# count = 0
|
|
146
|
+
# consumer.poll_nb_each do |message|
|
|
147
|
+
# process(message)
|
|
148
|
+
# count += 1
|
|
149
|
+
# :stop if count >= 10
|
|
150
|
+
# end
|
|
151
|
+
def poll_nb_each
|
|
152
|
+
closed_consumer_check(__method__)
|
|
153
|
+
|
|
154
|
+
@native_kafka.with_inner do |inner|
|
|
155
|
+
loop do
|
|
156
|
+
message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
|
|
157
|
+
break if message_ptr.null?
|
|
158
|
+
|
|
159
|
+
begin
|
|
160
|
+
native_message = Rdkafka::Bindings::Message.new(message_ptr)
|
|
161
|
+
|
|
162
|
+
if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
163
|
+
raise Rdkafka::RdkafkaError.new(native_message[:err])
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
result = yield Consumer::Message.new(native_message)
|
|
167
|
+
break if result == :stop
|
|
168
|
+
ensure
|
|
169
|
+
Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
37
175
|
# @return [Proc] finalizer proc for closing the consumer
|
|
38
176
|
# @private
|
|
39
177
|
def finalizer
|
|
@@ -78,7 +216,7 @@ module Rdkafka
|
|
|
78
216
|
Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
|
|
79
217
|
end
|
|
80
218
|
if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
81
|
-
raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(
|
|
219
|
+
raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(", ")}'")
|
|
82
220
|
end
|
|
83
221
|
ensure
|
|
84
222
|
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
|
|
@@ -228,7 +366,7 @@ module Rdkafka
|
|
|
228
366
|
end
|
|
229
367
|
end
|
|
230
368
|
ensure
|
|
231
|
-
ptr
|
|
369
|
+
ptr&.free
|
|
232
370
|
end
|
|
233
371
|
|
|
234
372
|
# @return [Boolean] true if our current assignment has been lost involuntarily.
|
|
@@ -281,7 +419,7 @@ module Rdkafka
|
|
|
281
419
|
# @return [TopicPartitionList]
|
|
282
420
|
#
|
|
283
421
|
# @raise [RdkafkaError] When getting the positions fails.
|
|
284
|
-
def position(list=nil)
|
|
422
|
+
def position(list = nil)
|
|
285
423
|
if list.nil?
|
|
286
424
|
list = assignment
|
|
287
425
|
elsif !list.is_a?(TopicPartitionList)
|
|
@@ -321,17 +459,17 @@ module Rdkafka
|
|
|
321
459
|
partition,
|
|
322
460
|
low,
|
|
323
461
|
high,
|
|
324
|
-
timeout_ms
|
|
462
|
+
timeout_ms
|
|
325
463
|
)
|
|
326
464
|
end
|
|
327
465
|
if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
328
466
|
raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
|
|
329
467
|
end
|
|
330
468
|
|
|
331
|
-
|
|
469
|
+
[low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
|
|
332
470
|
ensure
|
|
333
|
-
low
|
|
334
|
-
high
|
|
471
|
+
low&.free
|
|
472
|
+
high&.free
|
|
335
473
|
end
|
|
336
474
|
|
|
337
475
|
# Calculate the consumer lag per partition for the provided topic partition list.
|
|
@@ -510,14 +648,14 @@ module Rdkafka
|
|
|
510
648
|
# @param async [Boolean] Whether to commit async or wait for the commit to finish
|
|
511
649
|
# @return [nil]
|
|
512
650
|
# @raise [RdkafkaError] When committing fails
|
|
513
|
-
def commit(list=nil, async=false)
|
|
651
|
+
def commit(list = nil, async = false)
|
|
514
652
|
closed_consumer_check(__method__)
|
|
515
653
|
|
|
516
654
|
if !list.nil? && !list.is_a?(TopicPartitionList)
|
|
517
655
|
raise TypeError.new("list has to be nil or a TopicPartitionList")
|
|
518
656
|
end
|
|
519
657
|
|
|
520
|
-
tpl = list
|
|
658
|
+
tpl = list&.to_native_tpl
|
|
521
659
|
|
|
522
660
|
begin
|
|
523
661
|
response = @native_kafka.with_inner do |inner|
|
|
@@ -561,6 +699,47 @@ module Rdkafka
|
|
|
561
699
|
end
|
|
562
700
|
end
|
|
563
701
|
|
|
702
|
+
# Poll for the next message without releasing the GVL (Global VM Lock).
|
|
703
|
+
#
|
|
704
|
+
# This is more efficient than regular polling for non-blocking poll(0) calls,
|
|
705
|
+
# particularly useful in fiber scheduler contexts where GVL release/reacquire
|
|
706
|
+
# overhead is wasteful since we don't expect to wait.
|
|
707
|
+
#
|
|
708
|
+
# @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
|
|
709
|
+
# @return [Message, nil] A message or nil if there was no new message within the timeout
|
|
710
|
+
# @raise [RdkafkaError] When polling fails
|
|
711
|
+
#
|
|
712
|
+
# @example Using with fiber scheduler
|
|
713
|
+
# # After receiving IO notification that messages are available
|
|
714
|
+
# while msg = consumer.poll_nb
|
|
715
|
+
# process(msg)
|
|
716
|
+
# end
|
|
717
|
+
def poll_nb(timeout_ms = 0)
|
|
718
|
+
closed_consumer_check(__method__)
|
|
719
|
+
|
|
720
|
+
message_ptr = @native_kafka.with_inner do |inner|
|
|
721
|
+
Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
if message_ptr.null?
|
|
725
|
+
nil
|
|
726
|
+
else
|
|
727
|
+
# Create struct wrapper
|
|
728
|
+
native_message = Rdkafka::Bindings::Message.new(message_ptr)
|
|
729
|
+
# Raise error if needed
|
|
730
|
+
if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
731
|
+
raise Rdkafka::RdkafkaError.new(native_message[:err])
|
|
732
|
+
end
|
|
733
|
+
# Create a message to pass out
|
|
734
|
+
Rdkafka::Consumer::Message.new(native_message)
|
|
735
|
+
end
|
|
736
|
+
ensure
|
|
737
|
+
# Clean up rdkafka message if there is one
|
|
738
|
+
if message_ptr && !message_ptr.null?
|
|
739
|
+
Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
|
|
740
|
+
end
|
|
741
|
+
end
|
|
742
|
+
|
|
564
743
|
# Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
|
|
565
744
|
# was set to `true`.
|
|
566
745
|
#
|
|
@@ -587,6 +766,22 @@ module Rdkafka
|
|
|
587
766
|
end
|
|
588
767
|
end
|
|
589
768
|
|
|
769
|
+
# Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
|
|
770
|
+
#
|
|
771
|
+
# This is more efficient than regular events_poll for non-blocking poll(0) calls,
|
|
772
|
+
# particularly useful in fiber scheduler contexts where GVL release/reacquire
|
|
773
|
+
# overhead is wasteful since we don't expect to wait.
|
|
774
|
+
#
|
|
775
|
+
# @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
|
|
776
|
+
# @return [Integer] the number of events served
|
|
777
|
+
#
|
|
778
|
+
# @see #events_poll for more details on when to use this method
|
|
779
|
+
def events_poll_nb(timeout_ms = 0)
|
|
780
|
+
@native_kafka.with_inner do |inner|
|
|
781
|
+
Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
|
|
782
|
+
end
|
|
783
|
+
end
|
|
784
|
+
|
|
590
785
|
# Poll for new messages and yield for each received one. Iteration
|
|
591
786
|
# will end when the consumer is closed.
|
|
592
787
|
#
|
|
@@ -602,12 +797,10 @@ module Rdkafka
|
|
|
602
797
|
message = poll(timeout_ms)
|
|
603
798
|
if message
|
|
604
799
|
yield(message)
|
|
800
|
+
elsif closed?
|
|
801
|
+
break
|
|
605
802
|
else
|
|
606
|
-
|
|
607
|
-
break
|
|
608
|
-
else
|
|
609
|
-
next
|
|
610
|
-
end
|
|
803
|
+
next
|
|
611
804
|
end
|
|
612
805
|
end
|
|
613
806
|
end
|
data/lib/rdkafka/error.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Rdkafka
|
|
|
22
22
|
# @param response [Integer] the raw error response code from librdkafka
|
|
23
23
|
# @param message_prefix [String, nil] optional prefix for error messages
|
|
24
24
|
# @param broker_message [String, nil] optional error message from the broker
|
|
25
|
-
def initialize(response, message_prefix=nil, broker_message: nil)
|
|
25
|
+
def initialize(response, message_prefix = nil, broker_message: nil)
|
|
26
26
|
raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
|
|
27
27
|
@rdkafka_response = response
|
|
28
28
|
@message_prefix = message_prefix
|
|
@@ -34,7 +34,7 @@ module Rdkafka
|
|
|
34
34
|
def code
|
|
35
35
|
code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
|
|
36
36
|
if code[0] == "_"
|
|
37
|
-
code[1
|
|
37
|
+
code[1..].to_sym
|
|
38
38
|
else
|
|
39
39
|
code.to_sym
|
|
40
40
|
end
|
|
@@ -44,10 +44,10 @@ module Rdkafka
|
|
|
44
44
|
# @return [String]
|
|
45
45
|
def to_s
|
|
46
46
|
message_prefix_part = if message_prefix
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
"#{message_prefix} - "
|
|
48
|
+
else
|
|
49
|
+
""
|
|
50
|
+
end
|
|
51
51
|
"#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
|
|
52
52
|
end
|
|
53
53
|
|
|
@@ -58,10 +58,10 @@ module Rdkafka
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
# Error comparison
|
|
61
|
-
# @param
|
|
61
|
+
# @param other [Object] object to compare with
|
|
62
62
|
# @return [Boolean]
|
|
63
|
-
def ==(
|
|
64
|
-
|
|
63
|
+
def ==(other)
|
|
64
|
+
other.is_a?(self.class) && (to_s == other.to_s)
|
|
65
65
|
end
|
|
66
66
|
end
|
|
67
67
|
|
|
@@ -74,7 +74,7 @@ module Rdkafka
|
|
|
74
74
|
# @param response [Integer] the raw error response code from librdkafka
|
|
75
75
|
# @param topic_partition_list [TopicPartitionList] the topic partition list with error info
|
|
76
76
|
# @param message_prefix [String, nil] optional prefix for error messages
|
|
77
|
-
def initialize(response, topic_partition_list, message_prefix=nil)
|
|
77
|
+
def initialize(response, topic_partition_list, message_prefix = nil)
|
|
78
78
|
super(response, message_prefix)
|
|
79
79
|
@topic_partition_list = topic_partition_list
|
|
80
80
|
end
|
|
@@ -84,7 +84,7 @@ module Rdkafka
|
|
|
84
84
|
class ClosedConsumerError < BaseError
|
|
85
85
|
# @param method [Symbol] the method that was called
|
|
86
86
|
def initialize(method)
|
|
87
|
-
super("Illegal call to #{method
|
|
87
|
+
super("Illegal call to #{method} on a closed consumer")
|
|
88
88
|
end
|
|
89
89
|
end
|
|
90
90
|
|
|
@@ -92,7 +92,7 @@ module Rdkafka
|
|
|
92
92
|
class ClosedProducerError < BaseError
|
|
93
93
|
# @param method [Symbol] the method that was called
|
|
94
94
|
def initialize(method)
|
|
95
|
-
super("Illegal call to #{method
|
|
95
|
+
super("Illegal call to #{method} on a closed producer")
|
|
96
96
|
end
|
|
97
97
|
end
|
|
98
98
|
|
|
@@ -100,7 +100,7 @@ module Rdkafka
|
|
|
100
100
|
class ClosedAdminError < BaseError
|
|
101
101
|
# @param method [Symbol] the method that was called
|
|
102
102
|
def initialize(method)
|
|
103
|
-
super("Illegal call to #{method
|
|
103
|
+
super("Illegal call to #{method} on a closed admin")
|
|
104
104
|
end
|
|
105
105
|
end
|
|
106
106
|
|
|
@@ -2,7 +2,6 @@ module Rdkafka
|
|
|
2
2
|
module Helpers
|
|
3
3
|
# OAuth helper methods for setting and refreshing SASL/OAUTHBEARER tokens
|
|
4
4
|
module OAuth
|
|
5
|
-
|
|
6
5
|
# Set the OAuthBearer token
|
|
7
6
|
#
|
|
8
7
|
# @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
|
data/lib/rdkafka/helpers/time.rb
CHANGED
|
@@ -9,6 +9,11 @@ module Rdkafka
|
|
|
9
9
|
def monotonic_now
|
|
10
10
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
11
11
|
end
|
|
12
|
+
|
|
13
|
+
# @return [Integer] current monotonic time in milliseconds
|
|
14
|
+
def monotonic_now_ms
|
|
15
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
|
16
|
+
end
|
|
12
17
|
end
|
|
13
18
|
end
|
|
14
19
|
end
|
data/lib/rdkafka/metadata.rb
CHANGED
|
@@ -100,40 +100,40 @@ module Rdkafka
|
|
|
100
100
|
# FFI struct for rd_kafka_metadata_t
|
|
101
101
|
class Metadata < CustomFFIStruct
|
|
102
102
|
layout :brokers_count, :int,
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
103
|
+
:brokers_metadata, :pointer,
|
|
104
|
+
:topics_count, :int,
|
|
105
|
+
:topics_metadata, :pointer,
|
|
106
|
+
:broker_id, :int32,
|
|
107
|
+
:broker_name, :string
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
# @private
|
|
111
111
|
# FFI struct for rd_kafka_metadata_broker_t
|
|
112
112
|
class BrokerMetadata < CustomFFIStruct
|
|
113
113
|
layout :broker_id, :int32,
|
|
114
|
-
|
|
115
|
-
|
|
114
|
+
:broker_name, :string,
|
|
115
|
+
:broker_port, :int
|
|
116
116
|
end
|
|
117
117
|
|
|
118
118
|
# @private
|
|
119
119
|
# FFI struct for rd_kafka_metadata_topic_t
|
|
120
120
|
class TopicMetadata < CustomFFIStruct
|
|
121
121
|
layout :topic_name, :string,
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
:partition_count, :int,
|
|
123
|
+
:partitions_metadata, :pointer,
|
|
124
|
+
:rd_kafka_resp_err, :int
|
|
125
125
|
end
|
|
126
126
|
|
|
127
127
|
# @private
|
|
128
128
|
# FFI struct for rd_kafka_metadata_partition_t
|
|
129
129
|
class PartitionMetadata < CustomFFIStruct
|
|
130
130
|
layout :partition_id, :int32,
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
131
|
+
:rd_kafka_resp_err, :int,
|
|
132
|
+
:leader, :int32,
|
|
133
|
+
:replica_count, :int,
|
|
134
|
+
:replicas, :pointer,
|
|
135
|
+
:in_sync_replica_brokers, :int,
|
|
136
|
+
:isrs, :pointer
|
|
137
137
|
end
|
|
138
138
|
end
|
|
139
139
|
end
|