ruby-kafka 0.6.0.beta3 → 0.6.0.beta4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -0
- data/lib/kafka/consumer.rb +46 -22
- data/lib/kafka/datadog.rb +13 -0
- data/lib/kafka/pause.rb +90 -0
- data/lib/kafka/statsd.rb +11 -0
- data/lib/kafka/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 58773abdf15d690fb38890feb0788e3a47cb26312aa8022f869691dc6b56c3c6
|
4
|
+
data.tar.gz: e63116cf01fd6a140976179d63b9368b0a78855a3cbbb580ad84da74750b31a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 421cb2a9813a72cb8b6821323427e2c2dd86467b9c5658b0a095650c87578dccc69bd48aa3c094a8c9827ea5bb18e752c24fe846c92540522c3bf2710ac41f07
|
7
|
+
data.tar.gz: cbef389f1c4b6336fcc957d27f18d0e37e76149a814fbfee39aeb7b8ec21c3aa39d5f4f3945ad953a70c67aea06273e096937da7be9e9daa3978ec7c84994fde
|
data/CHANGELOG.md
CHANGED
data/lib/kafka/consumer.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require "kafka/consumer_group"
|
2
2
|
require "kafka/offset_manager"
|
3
3
|
require "kafka/fetcher"
|
4
|
+
require "kafka/pause"
|
4
5
|
|
5
6
|
module Kafka
|
6
7
|
|
@@ -50,8 +51,11 @@ module Kafka
|
|
50
51
|
@fetcher = fetcher
|
51
52
|
@heartbeat = heartbeat
|
52
53
|
|
53
|
-
|
54
|
-
|
54
|
+
@pauses = Hash.new {|h, k|
|
55
|
+
h[k] = Hash.new {|h2, k2|
|
56
|
+
h2[k2] = Pause.new
|
57
|
+
}
|
58
|
+
}
|
55
59
|
|
56
60
|
# Whether or not the consumer is currently consuming messages.
|
57
61
|
@running = false
|
@@ -113,16 +117,28 @@ module Kafka
|
|
113
117
|
# the rest of the partitions to continue being processed.
|
114
118
|
#
|
115
119
|
# If the `timeout` argument is passed, the partition will automatically be
|
116
|
-
# resumed when the timeout expires.
|
120
|
+
# resumed when the timeout expires. If `exponential_backoff` is enabled, each
|
121
|
+
# subsequent pause will cause the timeout to double until a message from the
|
122
|
+
# partition has been successfully processed.
|
117
123
|
#
|
118
124
|
# @param topic [String]
|
119
125
|
# @param partition [Integer]
|
120
|
-
# @param timeout [Integer] the number of seconds to pause the partition for,
|
126
|
+
# @param timeout [nil, Integer] the number of seconds to pause the partition for,
|
121
127
|
# or `nil` if the partition should not be automatically resumed.
|
128
|
+
# @param max_timeout [nil, Integer] the maximum number of seconds to pause for,
|
129
|
+
# or `nil` if no maximum should be enforced.
|
130
|
+
# @param exponential_backoff [Boolean] whether to enable exponential backoff.
|
122
131
|
# @return [nil]
|
123
|
-
def pause(topic, partition, timeout: nil)
|
124
|
-
|
125
|
-
|
132
|
+
def pause(topic, partition, timeout: nil, max_timeout: nil, exponential_backoff: false)
|
133
|
+
if max_timeout && !exponential_backoff
|
134
|
+
raise ArgumentError, "`max_timeout` only makes sense when `exponential_backoff` is enabled"
|
135
|
+
end
|
136
|
+
|
137
|
+
pause_for(topic, partition).pause!(
|
138
|
+
timeout: timeout,
|
139
|
+
max_timeout: max_timeout,
|
140
|
+
exponential_backoff: exponential_backoff,
|
141
|
+
)
|
126
142
|
end
|
127
143
|
|
128
144
|
# Resume processing of a topic partition.
|
@@ -132,8 +148,7 @@ module Kafka
|
|
132
148
|
# @param partition [Integer]
|
133
149
|
# @return [nil]
|
134
150
|
def resume(topic, partition)
|
135
|
-
|
136
|
-
paused_partitions.delete(partition)
|
151
|
+
pause_for(topic, partition).resume!
|
137
152
|
|
138
153
|
seek_to_next(topic, partition)
|
139
154
|
end
|
@@ -145,16 +160,7 @@ module Kafka
|
|
145
160
|
# @param partition [Integer]
|
146
161
|
# @return [Boolean] true if the partition is paused, false otherwise.
|
147
162
|
def paused?(topic, partition)
|
148
|
-
|
149
|
-
|
150
|
-
if partitions.key?(partition)
|
151
|
-
# Users can set an optional timeout, after which the partition is
|
152
|
-
# automatically resumed. When pausing, the timeout is translated to an
|
153
|
-
# absolute point in time.
|
154
|
-
timeout = partitions.fetch(partition)
|
155
|
-
|
156
|
-
timeout.nil? || Time.now < timeout
|
157
|
-
end
|
163
|
+
pause_for(topic, partition).paused?
|
158
164
|
end
|
159
165
|
|
160
166
|
# Fetches and enumerates the messages in the topics that the consumer group
|
@@ -230,6 +236,10 @@ module Kafka
|
|
230
236
|
|
231
237
|
return if !@running
|
232
238
|
end
|
239
|
+
|
240
|
+
# We've successfully processed a batch from the partition, so we can clear
|
241
|
+
# the pause.
|
242
|
+
pause_for(batch.topic, batch.partition).reset!
|
233
243
|
end
|
234
244
|
|
235
245
|
# We may not have received any messages, but it's still a good idea to
|
@@ -305,6 +315,10 @@ module Kafka
|
|
305
315
|
end
|
306
316
|
|
307
317
|
mark_message_as_processed(batch.messages.last) if automatically_mark_as_processed
|
318
|
+
|
319
|
+
# We've successfully processed a batch from the partition, so we can clear
|
320
|
+
# the pause.
|
321
|
+
pause_for(batch.topic, batch.partition).reset!
|
308
322
|
end
|
309
323
|
|
310
324
|
@offset_manager.commit_offsets_if_necessary
|
@@ -449,9 +463,15 @@ module Kafka
|
|
449
463
|
end
|
450
464
|
|
451
465
|
def resume_paused_partitions!
|
452
|
-
@
|
453
|
-
partitions.
|
454
|
-
|
466
|
+
@pauses.each do |topic, partitions|
|
467
|
+
partitions.each do |partition, pause|
|
468
|
+
@instrumenter.instrument("pause_status.consumer", {
|
469
|
+
topic: topic,
|
470
|
+
partition: partition,
|
471
|
+
duration: pause.pause_duration,
|
472
|
+
})
|
473
|
+
|
474
|
+
if pause.paused? && pause.expired?
|
455
475
|
@logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
|
456
476
|
resume(topic, partition)
|
457
477
|
end
|
@@ -494,5 +514,9 @@ module Kafka
|
|
494
514
|
|
495
515
|
raise FetchError, e
|
496
516
|
end
|
517
|
+
|
518
|
+
def pause_for(topic, partition)
|
519
|
+
@pauses[topic][partition]
|
520
|
+
end
|
497
521
|
end
|
498
522
|
end
|
data/lib/kafka/datadog.rb
CHANGED
@@ -217,6 +217,19 @@ module Kafka
|
|
217
217
|
end
|
218
218
|
end
|
219
219
|
|
220
|
+
def pause_status(event)
|
221
|
+
tags = {
|
222
|
+
client: event.payload.fetch(:client_id),
|
223
|
+
group_id: event.payload.fetch(:group_id),
|
224
|
+
topic: event.payload.fetch(:topic),
|
225
|
+
partition: event.payload.fetch(:partition),
|
226
|
+
}
|
227
|
+
|
228
|
+
duration = event.payload.fetch(:duration)
|
229
|
+
|
230
|
+
gauge("consumer.pause.duration", duration, tags: tags)
|
231
|
+
end
|
232
|
+
|
220
233
|
attach_to "consumer.kafka"
|
221
234
|
end
|
222
235
|
|
data/lib/kafka/pause.rb
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
module Kafka
|
2
|
+
# Manages the pause state of a partition.
|
3
|
+
#
|
4
|
+
# The processing of messages in a partition can be paused, e.g. if there was
|
5
|
+
# an exception during processing. This could be caused by a downstream service
|
6
|
+
# not being available. A typical way of solving such an issue is to back off
|
7
|
+
# for a little while and then try again. In order to do that, _pause_ the
|
8
|
+
# partition.
|
9
|
+
class Pause
|
10
|
+
def initialize(clock: Time)
|
11
|
+
@clock = clock
|
12
|
+
@started_at = nil
|
13
|
+
@pauses = 0
|
14
|
+
@timeout = nil
|
15
|
+
@max_timeout = nil
|
16
|
+
@exponential_backoff = false
|
17
|
+
end
|
18
|
+
|
19
|
+
# Mark the partition as paused.
|
20
|
+
#
|
21
|
+
# If exponential backoff is enabled, each subsequent pause of a partition will
|
22
|
+
# cause a doubling of the actual timeout, i.e. for pause number _n_, the actual
|
23
|
+
# timeout will be _2^n * timeout_.
|
24
|
+
#
|
25
|
+
# Only when {#reset!} is called is this state cleared.
|
26
|
+
#
|
27
|
+
# @param timeout [nil, Integer] if specified, the partition will automatically
|
28
|
+
# resume after this many seconds.
|
29
|
+
# @param exponential_backoff [Boolean] whether to enable exponential timeouts.
|
30
|
+
def pause!(timeout: nil, max_timeout: nil, exponential_backoff: false)
|
31
|
+
@started_at = @clock.now
|
32
|
+
@timeout = timeout
|
33
|
+
@max_timeout = max_timeout
|
34
|
+
@exponential_backoff = exponential_backoff
|
35
|
+
@pauses += 1
|
36
|
+
end
|
37
|
+
|
38
|
+
# Resumes the partition.
|
39
|
+
#
|
40
|
+
# The number of pauses is still retained, and if the partition is paused again
|
41
|
+
# it may be with an exponential backoff.
|
42
|
+
def resume!
|
43
|
+
@started_at = nil
|
44
|
+
@timeout = nil
|
45
|
+
@max_timeout = nil
|
46
|
+
end
|
47
|
+
|
48
|
+
# Whether the partition is currently paused.
|
49
|
+
def paused?
|
50
|
+
# This is nil if we're not currently paused.
|
51
|
+
return false if @started_at.nil?
|
52
|
+
|
53
|
+
# If no timeout is set we pause forever.
|
54
|
+
return true if @timeout.nil?
|
55
|
+
|
56
|
+
!expired?
|
57
|
+
end
|
58
|
+
|
59
|
+
def pause_duration
|
60
|
+
if paused?
|
61
|
+
Time.now - @started_at
|
62
|
+
else
|
63
|
+
0
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# Whether the pause has expired.
|
68
|
+
def expired?
|
69
|
+
!@timeout.nil? && @clock.now >= ends_at
|
70
|
+
end
|
71
|
+
|
72
|
+
# Resets the pause state, ensuring that the next pause is not exponential.
|
73
|
+
def reset!
|
74
|
+
@pauses = 0
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
def ends_at
|
80
|
+
# Apply an exponential backoff to the timeout.
|
81
|
+
backoff_factor = @exponential_backoff ? 2**(@pauses - 1) : 1
|
82
|
+
timeout = backoff_factor * @timeout
|
83
|
+
|
84
|
+
# If set, don't allow a timeout longer than max_timeout.
|
85
|
+
timeout = @max_timeout if @max_timeout && timeout > @max_timeout
|
86
|
+
|
87
|
+
@started_at + timeout
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
data/lib/kafka/statsd.rb
CHANGED
@@ -155,6 +155,17 @@ module Kafka
|
|
155
155
|
end
|
156
156
|
end
|
157
157
|
|
158
|
+
def pause_status(event)
|
159
|
+
client = event.payload.fetch(:client_id)
|
160
|
+
group_id = event.payload.fetch(:group_id)
|
161
|
+
topic = event.payload.fetch(:topic)
|
162
|
+
partition = event.payload.fetch(:partition)
|
163
|
+
|
164
|
+
duration = event.payload.fetch(:duration)
|
165
|
+
|
166
|
+
gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.pause.duration", duration)
|
167
|
+
end
|
168
|
+
|
158
169
|
attach_to "consumer.kafka"
|
159
170
|
end
|
160
171
|
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.0.
|
4
|
+
version: 0.6.0.beta4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -331,6 +331,7 @@ files:
|
|
331
331
|
- lib/kafka/message_buffer.rb
|
332
332
|
- lib/kafka/offset_manager.rb
|
333
333
|
- lib/kafka/partitioner.rb
|
334
|
+
- lib/kafka/pause.rb
|
334
335
|
- lib/kafka/pending_message.rb
|
335
336
|
- lib/kafka/pending_message_queue.rb
|
336
337
|
- lib/kafka/produce_operation.rb
|