karafka-rdkafka 0.12.2 → 0.13.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +1 -1
- data/CHANGELOG.md +26 -0
- data/Gemfile +2 -0
- data/README.md +26 -0
- data/Rakefile +2 -0
- data/ext/Rakefile +2 -0
- data/karafka-rdkafka.gemspec +2 -0
- data/lib/rdkafka/abstract_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin.rb +95 -73
- data/lib/rdkafka/bindings.rb +53 -37
- data/lib/rdkafka/callbacks.rb +7 -1
- data/lib/rdkafka/config.rb +13 -10
- data/lib/rdkafka/consumer/headers.rb +24 -7
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +2 -0
- data/lib/rdkafka/consumer.rb +100 -44
- data/lib/rdkafka/error.rb +9 -0
- data/lib/rdkafka/metadata.rb +25 -2
- data/lib/rdkafka/native_kafka.rb +83 -0
- data/lib/rdkafka/producer/delivery_handle.rb +5 -2
- data/lib/rdkafka/producer/delivery_report.rb +9 -2
- data/lib/rdkafka/producer.rb +75 -12
- data/lib/rdkafka/version.rb +3 -1
- data/lib/rdkafka.rb +3 -1
- data/spec/rdkafka/abstract_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin_spec.rb +4 -3
- data/spec/rdkafka/bindings_spec.rb +2 -0
- data/spec/rdkafka/callbacks_spec.rb +2 -0
- data/spec/rdkafka/config_spec.rb +17 -2
- data/spec/rdkafka/consumer/headers_spec.rb +62 -0
- data/spec/rdkafka/consumer/message_spec.rb +2 -0
- data/spec/rdkafka/consumer/partition_spec.rb +2 -0
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +2 -0
- data/spec/rdkafka/consumer_spec.rb +124 -22
- data/spec/rdkafka/error_spec.rb +2 -0
- data/spec/rdkafka/metadata_spec.rb +2 -0
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -34
- data/spec/rdkafka/producer/delivery_handle_spec.rb +5 -0
- data/spec/rdkafka/producer/delivery_report_spec.rb +8 -2
- data/spec/rdkafka/producer_spec.rb +124 -19
- data/spec/spec_helper.rb +17 -1
- data.tar.gz.sig +0 -0
- metadata +10 -10
- metadata.gz.sig +0 -0
- data/bin/console +0 -11
- data/lib/rdkafka/producer/client.rb +0 -47
data/lib/rdkafka/config.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "logger"
|
2
4
|
|
3
5
|
module Rdkafka
|
@@ -30,7 +32,6 @@ module Rdkafka
|
|
30
32
|
@@logger
|
31
33
|
end
|
32
34
|
|
33
|
-
|
34
35
|
# Returns a queue whose contents will be passed to the configured logger. Each entry
|
35
36
|
# should follow the format [Logger::Severity, String]. The benefit over calling the
|
36
37
|
# logger directly is that this is safe to use from trap contexts.
|
@@ -47,7 +48,7 @@ module Rdkafka
|
|
47
48
|
# @return [nil]
|
48
49
|
def self.logger=(logger)
|
49
50
|
raise NoLoggerError if logger.nil?
|
50
|
-
@@logger=logger
|
51
|
+
@@logger = logger
|
51
52
|
end
|
52
53
|
|
53
54
|
# Set a callback that will be called every time the underlying client emits statistics.
|
@@ -156,13 +157,14 @@ module Rdkafka
|
|
156
157
|
Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
|
157
158
|
end
|
158
159
|
|
160
|
+
# Create native client
|
159
161
|
kafka = native_kafka(config, :rd_kafka_consumer)
|
160
162
|
|
161
163
|
# Redirect the main queue to the consumer
|
162
164
|
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
|
163
165
|
|
164
166
|
# Return consumer with Kafka client
|
165
|
-
Rdkafka::Consumer.new(kafka)
|
167
|
+
Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
|
166
168
|
end
|
167
169
|
|
168
170
|
# Create a producer with this configuration.
|
@@ -179,7 +181,8 @@ module Rdkafka
|
|
179
181
|
# Set callback to receive delivery reports on config
|
180
182
|
Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
|
181
183
|
# Return producer with Kafka client
|
182
|
-
|
184
|
+
partitioner_name = self[:partitioner] || self["partitioner"]
|
185
|
+
Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
|
183
186
|
opaque.producer = producer
|
184
187
|
end
|
185
188
|
end
|
@@ -194,7 +197,7 @@ module Rdkafka
|
|
194
197
|
opaque = Opaque.new
|
195
198
|
config = native_config(opaque)
|
196
199
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
197
|
-
Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
|
200
|
+
Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
|
198
201
|
end
|
199
202
|
|
200
203
|
# Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
|
@@ -210,7 +213,7 @@ module Rdkafka
|
|
210
213
|
|
211
214
|
# This method is only intended to be used to create a client,
|
212
215
|
# using it in another way will leak memory.
|
213
|
-
def native_config(opaque=nil)
|
216
|
+
def native_config(opaque = nil)
|
214
217
|
Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
|
215
218
|
# Create config
|
216
219
|
@config_hash.merge(REQUIRED_CONFIG).each do |key, value|
|
@@ -282,18 +285,18 @@ module Rdkafka
|
|
282
285
|
producer.call_delivery_callback(delivery_report, delivery_handle) if producer
|
283
286
|
end
|
284
287
|
|
285
|
-
def call_on_partitions_assigned(
|
288
|
+
def call_on_partitions_assigned(list)
|
286
289
|
return unless consumer_rebalance_listener
|
287
290
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
|
288
291
|
|
289
|
-
consumer_rebalance_listener.on_partitions_assigned(
|
292
|
+
consumer_rebalance_listener.on_partitions_assigned(list)
|
290
293
|
end
|
291
294
|
|
292
|
-
def call_on_partitions_revoked(
|
295
|
+
def call_on_partitions_revoked(list)
|
293
296
|
return unless consumer_rebalance_listener
|
294
297
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
|
295
298
|
|
296
|
-
consumer_rebalance_listener.on_partitions_revoked(
|
299
|
+
consumer_rebalance_listener.on_partitions_revoked(list)
|
297
300
|
end
|
298
301
|
end
|
299
302
|
end
|
@@ -1,10 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Consumer
|
3
|
-
#
|
4
|
-
|
5
|
-
|
5
|
+
# Interface to return headers for a consumer message
|
6
|
+
module Headers
|
7
|
+
class HashWithSymbolKeysTreatedLikeStrings < Hash
|
8
|
+
def [](key)
|
9
|
+
if key.is_a?(Symbol)
|
10
|
+
Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
|
11
|
+
"Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
|
12
|
+
super(key.to_s)
|
13
|
+
else
|
14
|
+
super
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Reads a librdkafka native message's headers and returns them as a Ruby Hash
|
20
|
+
#
|
21
|
+
# @param [librdkakfa message] native_message
|
6
22
|
#
|
7
|
-
# @return [Hash<String, String>]
|
23
|
+
# @return [Hash<String, String>] headers Hash for the native_message
|
8
24
|
#
|
9
25
|
# @raise [Rdkafka::RdkafkaError] when fail to read headers
|
10
26
|
#
|
@@ -24,7 +40,8 @@ module Rdkafka
|
|
24
40
|
name_ptrptr = FFI::MemoryPointer.new(:pointer)
|
25
41
|
value_ptrptr = FFI::MemoryPointer.new(:pointer)
|
26
42
|
size_ptr = Rdkafka::Bindings::SizePtr.new
|
27
|
-
|
43
|
+
|
44
|
+
headers = HashWithSymbolKeysTreatedLikeStrings.new
|
28
45
|
|
29
46
|
idx = 0
|
30
47
|
loop do
|
@@ -51,12 +68,12 @@ module Rdkafka
|
|
51
68
|
|
52
69
|
value = value_ptr.read_string(size)
|
53
70
|
|
54
|
-
headers[name
|
71
|
+
headers[name] = value
|
55
72
|
|
56
73
|
idx += 1
|
57
74
|
end
|
58
75
|
|
59
|
-
headers
|
76
|
+
headers.freeze
|
60
77
|
end
|
61
78
|
end
|
62
79
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Consumer
|
3
5
|
# A message that was consumed from a topic.
|
@@ -18,7 +20,7 @@ module Rdkafka
|
|
18
20
|
# @return [String, nil]
|
19
21
|
attr_reader :key
|
20
22
|
|
21
|
-
# This message's offset in
|
23
|
+
# This message's offset in its partition
|
22
24
|
# @return [Integer]
|
23
25
|
attr_reader :offset
|
24
26
|
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
|
3
5
|
# brokers automatically assign partitions and load balance partitions over consumers that
|
@@ -14,18 +16,33 @@ module Rdkafka
|
|
14
16
|
# @private
|
15
17
|
def initialize(native_kafka)
|
16
18
|
@native_kafka = native_kafka
|
17
|
-
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] consumer name
|
22
|
+
def name
|
23
|
+
@name ||= @native_kafka.with_inner do |inner|
|
24
|
+
::Rdkafka::Bindings.rd_kafka_name(inner)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def finalizer
|
29
|
+
->(_) { close }
|
18
30
|
end
|
19
31
|
|
20
32
|
# Close this consumer
|
21
33
|
# @return [nil]
|
22
34
|
def close
|
23
|
-
return
|
35
|
+
return if closed?
|
36
|
+
ObjectSpace.undefine_finalizer(self)
|
37
|
+
@native_kafka.with_inner do |inner|
|
38
|
+
Rdkafka::Bindings.rd_kafka_consumer_close(inner)
|
39
|
+
end
|
40
|
+
@native_kafka.close
|
41
|
+
end
|
24
42
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@native_kafka = nil
|
43
|
+
# Whether this consumer has closed
|
44
|
+
def closed?
|
45
|
+
@native_kafka.closed?
|
29
46
|
end
|
30
47
|
|
31
48
|
# Subscribe to one or more topics letting Kafka handle partition assignments.
|
@@ -46,7 +63,9 @@ module Rdkafka
|
|
46
63
|
end
|
47
64
|
|
48
65
|
# Subscribe to topic partition list and check this was successful
|
49
|
-
response =
|
66
|
+
response = @native_kafka.with_inner do |inner|
|
67
|
+
Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
|
68
|
+
end
|
50
69
|
if response != 0
|
51
70
|
raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
|
52
71
|
end
|
@@ -62,7 +81,9 @@ module Rdkafka
|
|
62
81
|
def unsubscribe
|
63
82
|
closed_consumer_check(__method__)
|
64
83
|
|
65
|
-
response =
|
84
|
+
response = @native_kafka.with_inner do |inner|
|
85
|
+
Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
|
86
|
+
end
|
66
87
|
if response != 0
|
67
88
|
raise Rdkafka::RdkafkaError.new(response)
|
68
89
|
end
|
@@ -85,7 +106,9 @@ module Rdkafka
|
|
85
106
|
tpl = list.to_native_tpl
|
86
107
|
|
87
108
|
begin
|
88
|
-
response =
|
109
|
+
response = @native_kafka.with_inner do |inner|
|
110
|
+
Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
|
111
|
+
end
|
89
112
|
|
90
113
|
if response != 0
|
91
114
|
list = TopicPartitionList.from_native_tpl(tpl)
|
@@ -113,7 +136,9 @@ module Rdkafka
|
|
113
136
|
tpl = list.to_native_tpl
|
114
137
|
|
115
138
|
begin
|
116
|
-
response =
|
139
|
+
response = @native_kafka.with_inner do |inner|
|
140
|
+
Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
|
141
|
+
end
|
117
142
|
if response != 0
|
118
143
|
raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
|
119
144
|
end
|
@@ -131,7 +156,9 @@ module Rdkafka
|
|
131
156
|
closed_consumer_check(__method__)
|
132
157
|
|
133
158
|
ptr = FFI::MemoryPointer.new(:pointer)
|
134
|
-
response =
|
159
|
+
response = @native_kafka.with_inner do |inner|
|
160
|
+
Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
|
161
|
+
end
|
135
162
|
|
136
163
|
if response != 0
|
137
164
|
raise Rdkafka::RdkafkaError.new(response)
|
@@ -161,7 +188,9 @@ module Rdkafka
|
|
161
188
|
tpl = list.to_native_tpl
|
162
189
|
|
163
190
|
begin
|
164
|
-
response =
|
191
|
+
response = @native_kafka.with_inner do |inner|
|
192
|
+
Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
|
193
|
+
end
|
165
194
|
if response != 0
|
166
195
|
raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
|
167
196
|
end
|
@@ -179,7 +208,9 @@ module Rdkafka
|
|
179
208
|
closed_consumer_check(__method__)
|
180
209
|
|
181
210
|
ptr = FFI::MemoryPointer.new(:pointer)
|
182
|
-
response =
|
211
|
+
response = @native_kafka.with_inner do |inner|
|
212
|
+
Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
|
213
|
+
end
|
183
214
|
if response != 0
|
184
215
|
raise Rdkafka::RdkafkaError.new(response)
|
185
216
|
end
|
@@ -197,6 +228,15 @@ module Rdkafka
|
|
197
228
|
ptr.free unless ptr.nil?
|
198
229
|
end
|
199
230
|
|
231
|
+
# @return [Boolean] true if our current assignment has been lost involuntarily.
|
232
|
+
def assignment_lost?
|
233
|
+
closed_consumer_check(__method__)
|
234
|
+
|
235
|
+
@native_kafka.with_inner do |inner|
|
236
|
+
!Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
200
240
|
# Return the current committed offset per partition for this consumer group.
|
201
241
|
# The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
|
202
242
|
#
|
@@ -218,7 +258,9 @@ module Rdkafka
|
|
218
258
|
tpl = list.to_native_tpl
|
219
259
|
|
220
260
|
begin
|
221
|
-
response =
|
261
|
+
response = @native_kafka.with_inner do |inner|
|
262
|
+
Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
|
263
|
+
end
|
222
264
|
if response != 0
|
223
265
|
raise Rdkafka::RdkafkaError.new(response)
|
224
266
|
end
|
@@ -243,14 +285,16 @@ module Rdkafka
|
|
243
285
|
low = FFI::MemoryPointer.new(:int64, 1)
|
244
286
|
high = FFI::MemoryPointer.new(:int64, 1)
|
245
287
|
|
246
|
-
response =
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
288
|
+
response = @native_kafka.with_inner do |inner|
|
289
|
+
Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
|
290
|
+
inner,
|
291
|
+
topic,
|
292
|
+
partition,
|
293
|
+
low,
|
294
|
+
high,
|
295
|
+
timeout_ms,
|
296
|
+
)
|
297
|
+
end
|
254
298
|
if response != 0
|
255
299
|
raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
|
256
300
|
end
|
@@ -298,7 +342,9 @@ module Rdkafka
|
|
298
342
|
# @return [String, nil]
|
299
343
|
def cluster_id
|
300
344
|
closed_consumer_check(__method__)
|
301
|
-
|
345
|
+
@native_kafka.with_inner do |inner|
|
346
|
+
Rdkafka::Bindings.rd_kafka_clusterid(inner)
|
347
|
+
end
|
302
348
|
end
|
303
349
|
|
304
350
|
# Returns this client's broker-assigned group member id
|
@@ -308,7 +354,9 @@ module Rdkafka
|
|
308
354
|
# @return [String, nil]
|
309
355
|
def member_id
|
310
356
|
closed_consumer_check(__method__)
|
311
|
-
|
357
|
+
@native_kafka.with_inner do |inner|
|
358
|
+
Rdkafka::Bindings.rd_kafka_memberid(inner)
|
359
|
+
end
|
312
360
|
end
|
313
361
|
|
314
362
|
# Store offset of a message to be used in the next commit of this consumer
|
@@ -325,11 +373,13 @@ module Rdkafka
|
|
325
373
|
|
326
374
|
# rd_kafka_offset_store is one of the few calls that does not support
|
327
375
|
# a string as the topic, so create a native topic for it.
|
328
|
-
native_topic =
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
376
|
+
native_topic = @native_kafka.with_inner do |inner|
|
377
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
378
|
+
inner,
|
379
|
+
message.topic,
|
380
|
+
nil
|
381
|
+
)
|
382
|
+
end
|
333
383
|
response = Rdkafka::Bindings.rd_kafka_offset_store(
|
334
384
|
native_topic,
|
335
385
|
message.partition,
|
@@ -357,11 +407,13 @@ module Rdkafka
|
|
357
407
|
|
358
408
|
# rd_kafka_offset_store is one of the few calls that does not support
|
359
409
|
# a string as the topic, so create a native topic for it.
|
360
|
-
native_topic =
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
410
|
+
native_topic = @native_kafka.with_inner do |inner|
|
411
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
412
|
+
inner,
|
413
|
+
message.topic,
|
414
|
+
nil
|
415
|
+
)
|
416
|
+
end
|
365
417
|
response = Rdkafka::Bindings.rd_kafka_seek(
|
366
418
|
native_topic,
|
367
419
|
message.partition,
|
@@ -402,7 +454,9 @@ module Rdkafka
|
|
402
454
|
tpl = list ? list.to_native_tpl : nil
|
403
455
|
|
404
456
|
begin
|
405
|
-
response =
|
457
|
+
response = @native_kafka.with_inner do |inner|
|
458
|
+
Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
|
459
|
+
end
|
406
460
|
if response != 0
|
407
461
|
raise Rdkafka::RdkafkaError.new(response)
|
408
462
|
end
|
@@ -421,7 +475,9 @@ module Rdkafka
|
|
421
475
|
def poll(timeout_ms)
|
422
476
|
closed_consumer_check(__method__)
|
423
477
|
|
424
|
-
message_ptr =
|
478
|
+
message_ptr = @native_kafka.with_inner do |inner|
|
479
|
+
Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
|
480
|
+
end
|
425
481
|
if message_ptr.null?
|
426
482
|
nil
|
427
483
|
else
|
@@ -436,7 +492,7 @@ module Rdkafka
|
|
436
492
|
end
|
437
493
|
ensure
|
438
494
|
# Clean up rdkafka message if there is one
|
439
|
-
if
|
495
|
+
if message_ptr && !message_ptr.null?
|
440
496
|
Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
|
441
497
|
end
|
442
498
|
end
|
@@ -459,7 +515,7 @@ module Rdkafka
|
|
459
515
|
if message
|
460
516
|
yield(message)
|
461
517
|
else
|
462
|
-
if
|
518
|
+
if closed?
|
463
519
|
break
|
464
520
|
else
|
465
521
|
next
|
@@ -468,10 +524,6 @@ module Rdkafka
|
|
468
524
|
end
|
469
525
|
end
|
470
526
|
|
471
|
-
def closed_consumer_check(method)
|
472
|
-
raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
|
473
|
-
end
|
474
|
-
|
475
527
|
# Poll for new messages and yield them in batches that may contain
|
476
528
|
# messages from more than one partition.
|
477
529
|
#
|
@@ -527,7 +579,7 @@ module Rdkafka
|
|
527
579
|
bytes = 0
|
528
580
|
end_time = monotonic_now + timeout_ms / 1000.0
|
529
581
|
loop do
|
530
|
-
break if
|
582
|
+
break if closed?
|
531
583
|
max_wait = end_time - monotonic_now
|
532
584
|
max_wait_ms = if max_wait <= 0
|
533
585
|
0 # should not block, but may retrieve a message
|
@@ -545,7 +597,7 @@ module Rdkafka
|
|
545
597
|
end
|
546
598
|
if message
|
547
599
|
slice << message
|
548
|
-
bytes += message.payload.bytesize
|
600
|
+
bytes += message.payload.bytesize if message.payload
|
549
601
|
end
|
550
602
|
if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
|
551
603
|
yield slice.dup, nil
|
@@ -561,5 +613,9 @@ module Rdkafka
|
|
561
613
|
# needed because Time.now can go backwards
|
562
614
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
563
615
|
end
|
616
|
+
|
617
|
+
def closed_consumer_check(method)
|
618
|
+
raise Rdkafka::ClosedConsumerError.new(method) if closed?
|
619
|
+
end
|
564
620
|
end
|
565
621
|
end
|
data/lib/rdkafka/error.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# Base error class.
|
3
5
|
class BaseError < RuntimeError; end
|
@@ -83,4 +85,11 @@ module Rdkafka
|
|
83
85
|
super("Illegal call to #{method.to_s} on a closed producer")
|
84
86
|
end
|
85
87
|
end
|
88
|
+
|
89
|
+
# Error class for public consumer method calls on a closed admin.
|
90
|
+
class ClosedAdminError < BaseError
|
91
|
+
def initialize(method)
|
92
|
+
super("Illegal call to #{method.to_s} on a closed admin")
|
93
|
+
end
|
94
|
+
end
|
86
95
|
end
|
data/lib/rdkafka/metadata.rb
CHANGED
@@ -1,8 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Metadata
|
3
5
|
attr_reader :brokers, :topics
|
4
6
|
|
5
|
-
|
7
|
+
# Errors upon which we retry the metadata fetch
|
8
|
+
RETRIED_ERRORS = %i[
|
9
|
+
timed_out
|
10
|
+
leader_not_available
|
11
|
+
].freeze
|
12
|
+
|
13
|
+
private_constant :RETRIED_ERRORS
|
14
|
+
|
15
|
+
def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
|
16
|
+
attempt ||= 0
|
17
|
+
attempt += 1
|
18
|
+
|
6
19
|
native_topic = if topic_name
|
7
20
|
Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
|
8
21
|
end
|
@@ -14,12 +27,22 @@ module Rdkafka
|
|
14
27
|
topic_flag = topic_name.nil? ? 1 : 0
|
15
28
|
|
16
29
|
# Retrieve the Metadata
|
17
|
-
result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr,
|
30
|
+
result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
|
18
31
|
|
19
32
|
# Error Handling
|
20
33
|
raise Rdkafka::RdkafkaError.new(result) unless result.zero?
|
21
34
|
|
22
35
|
metadata_from_native(ptr.read_pointer)
|
36
|
+
rescue ::Rdkafka::RdkafkaError => e
|
37
|
+
raise unless RETRIED_ERRORS.include?(e.code)
|
38
|
+
raise if attempt > 10
|
39
|
+
|
40
|
+
backoff_factor = 2**attempt
|
41
|
+
timeout = backoff_factor * 0.1
|
42
|
+
|
43
|
+
sleep(timeout)
|
44
|
+
|
45
|
+
retry
|
23
46
|
ensure
|
24
47
|
Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
|
25
48
|
Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
# @private
|
5
|
+
# A wrapper around a native kafka that polls and cleanly exits
|
6
|
+
class NativeKafka
|
7
|
+
def initialize(inner, run_polling_thread:)
|
8
|
+
@inner = inner
|
9
|
+
# Lock around external access
|
10
|
+
@access_mutex = Mutex.new
|
11
|
+
# Lock around internal polling
|
12
|
+
@poll_mutex = Mutex.new
|
13
|
+
|
14
|
+
if run_polling_thread
|
15
|
+
# Start thread to poll client for delivery callbacks,
|
16
|
+
# not used in consumer.
|
17
|
+
@polling_thread = Thread.new do
|
18
|
+
loop do
|
19
|
+
@poll_mutex.synchronize do
|
20
|
+
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Exit thread if closing and the poll queue is empty
|
24
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
|
25
|
+
break
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
@polling_thread.abort_on_exception = true
|
31
|
+
@polling_thread[:closing] = false
|
32
|
+
end
|
33
|
+
|
34
|
+
@closing = false
|
35
|
+
end
|
36
|
+
|
37
|
+
def with_inner
|
38
|
+
return if @inner.nil?
|
39
|
+
|
40
|
+
@access_mutex.synchronize do
|
41
|
+
yield @inner
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def finalizer
|
46
|
+
->(_) { close }
|
47
|
+
end
|
48
|
+
|
49
|
+
def closed?
|
50
|
+
@closing || @inner.nil?
|
51
|
+
end
|
52
|
+
|
53
|
+
def close(object_id=nil)
|
54
|
+
return if closed?
|
55
|
+
|
56
|
+
@access_mutex.lock
|
57
|
+
|
58
|
+
# Indicate to the outside world that we are closing
|
59
|
+
@closing = true
|
60
|
+
|
61
|
+
if @polling_thread
|
62
|
+
# Indicate to polling thread that we're closing
|
63
|
+
@polling_thread[:closing] = true
|
64
|
+
|
65
|
+
# Wait for the polling thread to finish up,
|
66
|
+
# this can be aborted in practice if this
|
67
|
+
# code runs from a finalizer.
|
68
|
+
@polling_thread.join
|
69
|
+
end
|
70
|
+
|
71
|
+
# Destroy the client after locking both mutexes
|
72
|
+
@poll_mutex.lock
|
73
|
+
|
74
|
+
# This check prevents a race condition, where we would enter the close in two threads
|
75
|
+
# and after unlocking the primary one that hold the lock but finished, ours would be unlocked
|
76
|
+
# and would continue to run, trying to destroy inner twice
|
77
|
+
return unless @inner
|
78
|
+
|
79
|
+
Rdkafka::Bindings.rd_kafka_destroy(@inner)
|
80
|
+
@inner = nil
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Producer
|
3
5
|
# Handle to wait for a delivery report which is returned when
|
@@ -6,7 +8,8 @@ module Rdkafka
|
|
6
8
|
layout :pending, :bool,
|
7
9
|
:response, :int,
|
8
10
|
:partition, :int,
|
9
|
-
:offset, :int64
|
11
|
+
:offset, :int64,
|
12
|
+
:topic_name, :pointer
|
10
13
|
|
11
14
|
# @return [String] the name of the operation (e.g. "delivery")
|
12
15
|
def operation_name
|
@@ -15,7 +18,7 @@ module Rdkafka
|
|
15
18
|
|
16
19
|
# @return [DeliveryReport] a report on the delivery of the message
|
17
20
|
def create_result
|
18
|
-
DeliveryReport.new(self[:partition], self[:offset])
|
21
|
+
DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
|
19
22
|
end
|
20
23
|
end
|
21
24
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Producer
|
3
5
|
# Delivery report for a successfully produced message.
|
@@ -10,15 +12,20 @@ module Rdkafka
|
|
10
12
|
# @return [Integer]
|
11
13
|
attr_reader :offset
|
12
14
|
|
13
|
-
#
|
15
|
+
# The name of the topic this message was produced to.
|
14
16
|
# @return [String]
|
17
|
+
attr_reader :topic_name
|
18
|
+
|
19
|
+
# Error in case happen during produce.
|
20
|
+
# @return [Integer]
|
15
21
|
attr_reader :error
|
16
22
|
|
17
23
|
private
|
18
24
|
|
19
|
-
def initialize(partition, offset, error = nil)
|
25
|
+
def initialize(partition, offset, topic_name = nil, error = nil)
|
20
26
|
@partition = partition
|
21
27
|
@offset = offset
|
28
|
+
@topic_name = topic_name
|
22
29
|
@error = error
|
23
30
|
end
|
24
31
|
end
|