karafka 2.2.7 → 2.2.8.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile.lock +3 -5
- data/bin/karafka +2 -3
- data/docker-compose.yml +3 -1
- data/karafka.gemspec +1 -2
- data/lib/karafka/base_consumer.rb +1 -0
- data/lib/karafka/cli/base.rb +45 -34
- data/lib/karafka/cli/console.rb +5 -4
- data/lib/karafka/cli/help.rb +24 -0
- data/lib/karafka/cli/info.rb +2 -2
- data/lib/karafka/cli/install.rb +4 -4
- data/lib/karafka/cli/server.rb +68 -33
- data/lib/karafka/cli/topics.rb +1 -1
- data/lib/karafka/cli.rb +23 -19
- data/lib/karafka/connection/client.rb +9 -4
- data/lib/karafka/connection/rebalance_manager.rb +36 -21
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +64 -0
- data/lib/karafka/instrumentation/notifications.rb +5 -1
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +30 -0
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +122 -0
- data/lib/karafka/instrumentation/vendors/appsignal/dashboard.json +222 -0
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +30 -0
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +331 -0
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +2 -2
- data/lib/karafka/patches/rdkafka/bindings.rb +22 -39
- data/lib/karafka/patches/rdkafka/opaque.rb +36 -0
- data/lib/karafka/pro/processing/coordinator.rb +6 -7
- data/lib/karafka/pro/processing/strategies/vp/default.rb +20 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +14 -20
- metadata.gz.sig +0 -0
@@ -0,0 +1,331 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
|
5
|
+
module Karafka
|
6
|
+
module Instrumentation
|
7
|
+
module Vendors
|
8
|
+
# Namespace for Appsignal instrumentation
|
9
|
+
module Appsignal
|
10
|
+
# Listener that ships metrics to Appsignal
|
11
|
+
class MetricsListener < Base
|
12
|
+
def_delegators :config, :client, :rd_kafka_metrics, :namespace
|
13
|
+
|
14
|
+
# Value object for storing a single rdkafka metric publishing details
|
15
|
+
RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
|
16
|
+
|
17
|
+
setting :namespace, default: 'karafka'
|
18
|
+
|
19
|
+
setting :client, default: Client.new
|
20
|
+
|
21
|
+
setting :rd_kafka_metrics, default: [
|
22
|
+
# Broker metrics
|
23
|
+
RdKafkaMetric.new(:count, :brokers, 'requests_retries', 'txretries_d'),
|
24
|
+
RdKafkaMetric.new(:count, :brokers, 'transmission_errors', 'txerrs_d'),
|
25
|
+
RdKafkaMetric.new(:count, :brokers, 'receive_errors', 'rxerrs_d'),
|
26
|
+
RdKafkaMetric.new(:count, :brokers, 'connection_connects', 'connects_d'),
|
27
|
+
RdKafkaMetric.new(:count, :brokers, 'connection_disconnects', 'disconnects_d'),
|
28
|
+
RdKafkaMetric.new(:gauge, :brokers, 'network_latency_avg', %w[rtt avg]),
|
29
|
+
RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p95', %w[rtt p95]),
|
30
|
+
RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p99', %w[rtt p99]),
|
31
|
+
|
32
|
+
# Topics partitions metrics
|
33
|
+
RdKafkaMetric.new(:gauge, :topics, 'consumer_lag', 'consumer_lag_stored'),
|
34
|
+
RdKafkaMetric.new(:gauge, :topics, 'consumer_lag_delta', 'consumer_lag_stored_d')
|
35
|
+
].freeze
|
36
|
+
|
37
|
+
# Metrics that sum values on topics levels and not on partition levels
|
38
|
+
setting :aggregated_rd_kafka_metrics, default: [
|
39
|
+
# Topic aggregated metrics
|
40
|
+
RdKafkaMetric.new(:gauge, :topics, 'consumer_aggregated_lag', 'consumer_lag_stored')
|
41
|
+
].freeze
|
42
|
+
|
43
|
+
configure
|
44
|
+
|
45
|
+
# Before each consumption process, lets start a transaction associated with it
|
46
|
+
# We also set some basic metadata about the given consumption that can be useful for
|
47
|
+
# debugging
|
48
|
+
#
|
49
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
50
|
+
def on_consumer_consume(event)
|
51
|
+
consumer = event.payload[:caller]
|
52
|
+
|
53
|
+
start_transaction(consumer, 'consume')
|
54
|
+
|
55
|
+
client.metadata = {
|
56
|
+
batch_size: consumer.messages.size,
|
57
|
+
first_offset: consumer.messages.metadata.first_offset,
|
58
|
+
last_offset: consumer.messages.metadata.last_offset,
|
59
|
+
consumer_group: consumer.topic.consumer_group.id,
|
60
|
+
topic: consumer.topic.name,
|
61
|
+
partition: consumer.partition,
|
62
|
+
attempt: consumer.coordinator.pause_tracker.attempt
|
63
|
+
}
|
64
|
+
end
|
65
|
+
|
66
|
+
# Once we're done with consumption, we bump counters about that
|
67
|
+
#
|
68
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
69
|
+
def on_consumer_consumed(event)
|
70
|
+
consumer = event.payload[:caller]
|
71
|
+
messages = consumer.messages
|
72
|
+
metadata = messages.metadata
|
73
|
+
|
74
|
+
with_multiple_resolutions(consumer) do |tags|
|
75
|
+
count('consumer_messages', messages.size, tags)
|
76
|
+
count('consumer_batches', 1, tags)
|
77
|
+
gauge('consumer_offsets', metadata.last_offset, tags)
|
78
|
+
end
|
79
|
+
|
80
|
+
stop_transaction
|
81
|
+
end
|
82
|
+
|
83
|
+
# Register minute based probe only on app running. Otherwise if we would always register
|
84
|
+
# minute probe, it would report on processes using Karafka but not running the
|
85
|
+
# consumption process
|
86
|
+
#
|
87
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
88
|
+
def on_app_running(_event)
|
89
|
+
return if @probe_registered
|
90
|
+
|
91
|
+
@probe_registered = true
|
92
|
+
|
93
|
+
# Registers the minutely probe for one-every-minute metrics
|
94
|
+
client.register_probe(:karafka, -> { minute_probe })
|
95
|
+
end
|
96
|
+
|
97
|
+
# Keeps track of revocation user code execution
|
98
|
+
#
|
99
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
100
|
+
def on_consumer_revoke(event)
|
101
|
+
consumer = event.payload[:caller]
|
102
|
+
start_transaction(consumer, 'revoked')
|
103
|
+
end
|
104
|
+
|
105
|
+
# Finishes the revocation transaction
|
106
|
+
#
|
107
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
108
|
+
def on_consumer_revoked(_event)
|
109
|
+
stop_transaction
|
110
|
+
end
|
111
|
+
|
112
|
+
# Keeps track of revocation user code execution
|
113
|
+
#
|
114
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
115
|
+
def on_consumer_shutting_down(event)
|
116
|
+
consumer = event.payload[:caller]
|
117
|
+
start_transaction(consumer, 'shutdown')
|
118
|
+
end
|
119
|
+
|
120
|
+
# Finishes the shutdown transaction
|
121
|
+
#
|
122
|
+
# @param _event [Karafka::Core::Monitoring::Event]
|
123
|
+
def on_consumer_shutdown(_event)
|
124
|
+
stop_transaction
|
125
|
+
end
|
126
|
+
|
127
|
+
# Counts DLQ dispatches
|
128
|
+
#
|
129
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
130
|
+
def on_dead_letter_queue_dispatched(event)
|
131
|
+
consumer = event.payload[:caller]
|
132
|
+
|
133
|
+
with_multiple_resolutions(consumer) do |tags|
|
134
|
+
count('consumer_dead', 1, tags)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
# Reports on **any** error that occurs. This also includes non-user related errors
|
139
|
+
# originating from the framework.
|
140
|
+
#
|
141
|
+
# @param event [Karafka::Core::Monitoring::Event] error event details
|
142
|
+
def on_error_occurred(event)
|
143
|
+
# If this is a user consumption related error, we bump the counters for metrics
|
144
|
+
if event[:type] == 'consumer.consume.error'
|
145
|
+
consumer = event.payload[:caller]
|
146
|
+
|
147
|
+
with_multiple_resolutions(consumer) do |tags|
|
148
|
+
count('consumer_errors', 1, tags)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
stop_transaction
|
153
|
+
end
|
154
|
+
|
155
|
+
# Hooks up to Karafka instrumentation for emitted statistics
|
156
|
+
#
|
157
|
+
# @param event [Karafka::Core::Monitoring::Event]
|
158
|
+
def on_statistics_emitted(event)
|
159
|
+
statistics = event[:statistics]
|
160
|
+
consumer_group_id = event[:consumer_group_id]
|
161
|
+
|
162
|
+
rd_kafka_metrics.each do |metric|
|
163
|
+
report_metric(metric, statistics, consumer_group_id)
|
164
|
+
end
|
165
|
+
|
166
|
+
report_aggregated_topics_metrics(statistics, consumer_group_id)
|
167
|
+
end
|
168
|
+
|
169
|
+
# Reports a given metric statistics to Appsignal
|
170
|
+
# @param metric [RdKafkaMetric] metric value object
|
171
|
+
# @param statistics [Hash] hash with all the statistics emitted
|
172
|
+
# @param consumer_group_id [String] cg in context which we operate
|
173
|
+
def report_metric(metric, statistics, consumer_group_id)
|
174
|
+
case metric.scope
|
175
|
+
when :root
|
176
|
+
# Do nothing on the root metrics as the same metrics are reported in a granular
|
177
|
+
# way from other places
|
178
|
+
nil
|
179
|
+
when :brokers
|
180
|
+
statistics.fetch('brokers').each_value do |broker_statistics|
|
181
|
+
# Skip bootstrap nodes
|
182
|
+
# Bootstrap nodes have nodeid -1, other nodes have positive
|
183
|
+
# node ids
|
184
|
+
next if broker_statistics['nodeid'] == -1
|
185
|
+
|
186
|
+
public_send(
|
187
|
+
metric.type,
|
188
|
+
metric.name,
|
189
|
+
broker_statistics.dig(*metric.key_location),
|
190
|
+
{
|
191
|
+
broker: broker_statistics['nodename']
|
192
|
+
}
|
193
|
+
)
|
194
|
+
end
|
195
|
+
when :topics
|
196
|
+
statistics.fetch('topics').each do |topic_name, topic_values|
|
197
|
+
topic_values['partitions'].each do |partition_name, partition_statistics|
|
198
|
+
next if partition_name == '-1'
|
199
|
+
# Skip until lag info is available
|
200
|
+
next if partition_statistics['consumer_lag'] == -1
|
201
|
+
next if partition_statistics['consumer_lag_stored'] == -1
|
202
|
+
|
203
|
+
# Skip if we do not own the fetch assignment
|
204
|
+
next if partition_statistics['fetch_state'] == 'stopped'
|
205
|
+
next if partition_statistics['fetch_state'] == 'none'
|
206
|
+
|
207
|
+
public_send(
|
208
|
+
metric.type,
|
209
|
+
metric.name,
|
210
|
+
partition_statistics.dig(*metric.key_location),
|
211
|
+
{
|
212
|
+
consumer_group: consumer_group_id,
|
213
|
+
topic: topic_name,
|
214
|
+
partition: partition_name
|
215
|
+
}
|
216
|
+
)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
else
|
220
|
+
raise ArgumentError, metric.scope
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
# Publishes aggregated topic-level metrics that are sum of per partition metrics
|
225
|
+
#
|
226
|
+
# @param statistics [Hash] hash with all the statistics emitted
|
227
|
+
# @param consumer_group_id [String] cg in context which we operate
|
228
|
+
def report_aggregated_topics_metrics(statistics, consumer_group_id)
|
229
|
+
config.aggregated_rd_kafka_metrics.each do |metric|
|
230
|
+
statistics.fetch('topics').each do |topic_name, topic_values|
|
231
|
+
sum = 0
|
232
|
+
|
233
|
+
topic_values['partitions'].each do |partition_name, partition_statistics|
|
234
|
+
next if partition_name == '-1'
|
235
|
+
# Skip until lag info is available
|
236
|
+
next if partition_statistics['consumer_lag'] == -1
|
237
|
+
next if partition_statistics['consumer_lag_stored'] == -1
|
238
|
+
|
239
|
+
sum += partition_statistics.dig(*metric.key_location)
|
240
|
+
end
|
241
|
+
|
242
|
+
public_send(
|
243
|
+
metric.type,
|
244
|
+
metric.name,
|
245
|
+
sum,
|
246
|
+
{
|
247
|
+
consumer_group: consumer_group_id,
|
248
|
+
topic: topic_name
|
249
|
+
}
|
250
|
+
)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
# Increments a counter with a namespace key, value and tags
|
256
|
+
#
|
257
|
+
# @param key [String] key we want to use (without the namespace)
|
258
|
+
# @param value [Integer] count value
|
259
|
+
# @param tags [Hash] additional extra tags
|
260
|
+
def count(key, value, tags)
|
261
|
+
client.count(
|
262
|
+
namespaced_metric(key),
|
263
|
+
value,
|
264
|
+
tags
|
265
|
+
)
|
266
|
+
end
|
267
|
+
|
268
|
+
# Sets the gauge value
|
269
|
+
#
|
270
|
+
# @param key [String] key we want to use (without the namespace)
|
271
|
+
# @param value [Integer] gauge value
|
272
|
+
# @param tags [Hash] additional extra tags
|
273
|
+
def gauge(key, value, tags)
|
274
|
+
client.gauge(
|
275
|
+
namespaced_metric(key),
|
276
|
+
value,
|
277
|
+
tags
|
278
|
+
)
|
279
|
+
end
|
280
|
+
|
281
|
+
private
|
282
|
+
|
283
|
+
# Wraps metric name in listener's namespace
|
284
|
+
# @param metric_name [String] RdKafkaMetric name
|
285
|
+
# @return [String]
|
286
|
+
def namespaced_metric(metric_name)
|
287
|
+
"#{namespace}_#{metric_name}"
|
288
|
+
end
|
289
|
+
|
290
|
+
# Starts the transaction for monitoring user code
|
291
|
+
#
|
292
|
+
# @param consumer [Karafka::BaseConsumer] karafka consumer instance
|
293
|
+
# @param action_name [String] lifecycle user method name
|
294
|
+
def start_transaction(consumer, action_name)
|
295
|
+
client.start_transaction(
|
296
|
+
"#{consumer.class}##{action_name}"
|
297
|
+
)
|
298
|
+
end
|
299
|
+
|
300
|
+
# Stops the transaction wrapping user code
|
301
|
+
def stop_transaction
|
302
|
+
client.stop_transaction
|
303
|
+
end
|
304
|
+
|
305
|
+
# @param consumer [Karafka::BaseConsumer] Karafka consumer instance
|
306
|
+
def with_multiple_resolutions(consumer)
|
307
|
+
topic_name = consumer.topic.name
|
308
|
+
consumer_group_id = consumer.topic.consumer_group.id
|
309
|
+
partition = consumer.partition
|
310
|
+
|
311
|
+
tags = {
|
312
|
+
consumer_group: consumer_group_id,
|
313
|
+
topic: topic_name
|
314
|
+
}
|
315
|
+
|
316
|
+
yield(tags)
|
317
|
+
yield(tags.merge(partition: partition))
|
318
|
+
end
|
319
|
+
|
320
|
+
# Sends minute based probing metrics
|
321
|
+
def minute_probe
|
322
|
+
concurrency = Karafka::App.config.concurrency
|
323
|
+
|
324
|
+
count('processes_count', 1, {})
|
325
|
+
count('threads_count', concurrency, {})
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
@@ -67,7 +67,7 @@ module Karafka
|
|
67
67
|
configure(&block)
|
68
68
|
end
|
69
69
|
|
70
|
-
# Hooks up to
|
70
|
+
# Hooks up to Karafka instrumentation for emitted statistics
|
71
71
|
#
|
72
72
|
# @param event [Karafka::Core::Monitoring::Event]
|
73
73
|
def on_statistics_emitted(event)
|
@@ -101,7 +101,7 @@ module Karafka
|
|
101
101
|
time_taken = event[:time]
|
102
102
|
messages_count = event[:messages_buffer].size
|
103
103
|
|
104
|
-
consumer_group_id = event[:subscription_group].
|
104
|
+
consumer_group_id = event[:subscription_group].consumer_group.id
|
105
105
|
|
106
106
|
extra_tags = ["consumer_group:#{consumer_group_id}"]
|
107
107
|
|
@@ -18,15 +18,23 @@ module Karafka
|
|
18
18
|
# @param client_ptr [FFI::Pointer]
|
19
19
|
# @param code [Integer]
|
20
20
|
# @param partitions_ptr [FFI::Pointer]
|
21
|
-
|
21
|
+
# @param tpl [Rdkafka::Consumer::TopicPartitionList]
|
22
|
+
# @param opaque [Rdkafka::Opaque]
|
23
|
+
def on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
22
24
|
case code
|
23
25
|
when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
|
26
|
+
opaque&.call_on_partitions_assign(tpl)
|
24
27
|
RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
|
28
|
+
opaque&.call_on_partitions_assigned(tpl)
|
25
29
|
when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
|
30
|
+
opaque&.call_on_partitions_revoke(tpl)
|
26
31
|
RB.rd_kafka_commit(client_ptr, nil, false)
|
27
32
|
RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
|
33
|
+
opaque&.call_on_partitions_revoked(tpl)
|
28
34
|
else
|
35
|
+
opaque&.call_on_partitions_assign(tpl)
|
29
36
|
RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
|
37
|
+
opaque&.call_on_partitions_assigned(tpl)
|
30
38
|
end
|
31
39
|
end
|
32
40
|
|
@@ -35,45 +43,25 @@ module Karafka
|
|
35
43
|
# @param client_ptr [FFI::Pointer]
|
36
44
|
# @param code [Integer]
|
37
45
|
# @param partitions_ptr [FFI::Pointer]
|
38
|
-
|
46
|
+
# @param tpl [Rdkafka::Consumer::TopicPartitionList]
|
47
|
+
# @param opaque [Rdkafka::Opaque]
|
48
|
+
def on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
39
49
|
case code
|
40
50
|
when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
|
51
|
+
opaque&.call_on_partitions_assign(tpl)
|
41
52
|
RB.rd_kafka_assign(client_ptr, partitions_ptr)
|
53
|
+
opaque&.call_on_partitions_assigned(tpl)
|
42
54
|
when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
|
55
|
+
opaque&.call_on_partitions_revoke(tpl)
|
43
56
|
RB.rd_kafka_commit(client_ptr, nil, false)
|
44
57
|
RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
|
58
|
+
opaque&.call_on_partitions_revoked(tpl)
|
45
59
|
else
|
60
|
+
opaque&.call_on_partitions_assign(tpl)
|
46
61
|
RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
|
62
|
+
opaque&.call_on_partitions_assigned(tpl)
|
47
63
|
end
|
48
64
|
end
|
49
|
-
|
50
|
-
# Trigger Karafka callbacks
|
51
|
-
#
|
52
|
-
# @param code [Integer]
|
53
|
-
# @param opaque [Rdkafka::Opaque]
|
54
|
-
# @param tpl [Rdkafka::Consumer::TopicPartitionList]
|
55
|
-
def trigger_callbacks(code, opaque, tpl)
|
56
|
-
Karafka.monitor.instrument(
|
57
|
-
'connection.client.rebalance_callback',
|
58
|
-
caller: self,
|
59
|
-
code: code,
|
60
|
-
tpl: tpl
|
61
|
-
) do
|
62
|
-
case code
|
63
|
-
when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
|
64
|
-
opaque.call_on_partitions_assigned(tpl)
|
65
|
-
when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
|
66
|
-
opaque.call_on_partitions_revoked(tpl)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
rescue StandardError => e
|
70
|
-
Karafka.monitor.instrument(
|
71
|
-
'error.occurred',
|
72
|
-
caller: self,
|
73
|
-
error: e,
|
74
|
-
type: 'connection.client.rebalance_callback.error'
|
75
|
-
)
|
76
|
-
end
|
77
65
|
end
|
78
66
|
|
79
67
|
# This patch changes few things:
|
@@ -89,19 +77,14 @@ module Karafka
|
|
89
77
|
) do |client_ptr, code, partitions_ptr, opaque_ptr|
|
90
78
|
# Patch reference
|
91
79
|
pr = ::Karafka::Patches::Rdkafka::Bindings
|
80
|
+
tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
|
81
|
+
opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
|
92
82
|
|
93
83
|
if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
|
94
|
-
pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr)
|
84
|
+
pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
95
85
|
else
|
96
|
-
pr.on_eager_rebalance(client_ptr, code, partitions_ptr)
|
86
|
+
pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
97
87
|
end
|
98
|
-
|
99
|
-
opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
|
100
|
-
return unless opaque
|
101
|
-
|
102
|
-
tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
|
103
|
-
|
104
|
-
pr.trigger_callbacks(code, opaque, tpl)
|
105
88
|
end
|
106
89
|
end
|
107
90
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Patches
|
5
|
+
module Rdkafka
|
6
|
+
# Patches allowing us to run events on both pre and post rebalance events.
|
7
|
+
# Thanks to that, we can easily connect to the whole flow despite of the moment when things
|
8
|
+
# are happening
|
9
|
+
module Opaque
|
10
|
+
# Handles pre-assign phase of rebalance
|
11
|
+
#
|
12
|
+
# @param tpl [Rdkafka::Consumer::TopicPartitionList]
|
13
|
+
def call_on_partitions_assign(tpl)
|
14
|
+
return unless consumer_rebalance_listener
|
15
|
+
return unless consumer_rebalance_listener.respond_to?(:on_partitions_assign)
|
16
|
+
|
17
|
+
consumer_rebalance_listener.on_partitions_assign(tpl)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Handles pre-revoke phase of rebalance
|
21
|
+
#
|
22
|
+
# @param tpl [Rdkafka::Consumer::TopicPartitionList]
|
23
|
+
def call_on_partitions_revoke(tpl)
|
24
|
+
return unless consumer_rebalance_listener
|
25
|
+
return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoke)
|
26
|
+
|
27
|
+
consumer_rebalance_listener.on_partitions_revoke(tpl)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
::Rdkafka::Opaque.include(
|
35
|
+
Karafka::Patches::Rdkafka::Opaque
|
36
|
+
)
|
@@ -17,6 +17,10 @@ module Karafka
|
|
17
17
|
# Pro coordinator that provides extra orchestration methods useful for parallel processing
|
18
18
|
# within the same partition
|
19
19
|
class Coordinator < ::Karafka::Processing::Coordinator
|
20
|
+
extend Forwardable
|
21
|
+
|
22
|
+
def_delegators :@collapser, :collapsed?, :collapse_until!
|
23
|
+
|
20
24
|
attr_reader :filter, :virtual_offset_manager
|
21
25
|
|
22
26
|
# @param args [Object] anything the base coordinator accepts
|
@@ -57,7 +61,7 @@ module Karafka
|
|
57
61
|
|
58
62
|
# We keep the old processed offsets until the collapsing is done and regular processing
|
59
63
|
# with virtualization is restored
|
60
|
-
@virtual_offset_manager.clear if topic.virtual_partitions? &&
|
64
|
+
@virtual_offset_manager.clear if topic.virtual_partitions? && !collapsed?
|
61
65
|
|
62
66
|
@last_message = messages.last
|
63
67
|
end
|
@@ -68,12 +72,7 @@ module Karafka
|
|
68
72
|
# @param error [StandardError] error from the failure
|
69
73
|
def failure!(consumer, error)
|
70
74
|
super
|
71
|
-
|
72
|
-
end
|
73
|
-
|
74
|
-
# @return [Boolean] are we in a collapsed state at the moment
|
75
|
-
def collapsed?
|
76
|
-
@collapser.collapsed?
|
75
|
+
collapse_until!(@last_message.offset + 1)
|
77
76
|
end
|
78
77
|
|
79
78
|
# @return [Boolean] did any of the filters apply any logic that would cause use to run
|
@@ -72,6 +72,15 @@ module Karafka
|
|
72
72
|
coordinator.collapsed?
|
73
73
|
end
|
74
74
|
|
75
|
+
# @param offset [Integer] first offset from which we should not operate in a collapsed
|
76
|
+
# mode.
|
77
|
+
# @note Keep in mind, that if a batch contains this but also messages earlier messages
|
78
|
+
# that should be collapsed, all will continue to operate in a collapsed mode until
|
79
|
+
# first full batch with only messages that should not be collapsed.
|
80
|
+
def collapse_until!(offset)
|
81
|
+
coordinator.collapse_until!(offset)
|
82
|
+
end
|
83
|
+
|
75
84
|
# @return [Boolean] true if any of virtual partition we're operating in the entangled
|
76
85
|
# mode has already failed and we know we are failing collectively.
|
77
86
|
# Useful for early stop to minimize number of things processed twice.
|
@@ -84,6 +93,17 @@ module Karafka
|
|
84
93
|
coordinator.failure?
|
85
94
|
end
|
86
95
|
|
96
|
+
# Allows for cross-virtual-partition consumers locks
|
97
|
+
#
|
98
|
+
# This is not needed in the non-VP flows because there is always only one consumer
|
99
|
+
# per partition at the same time, so no coordination is needed directly for the
|
100
|
+
# end users
|
101
|
+
#
|
102
|
+
# @param block [Proc] block we want to run in a mutex to prevent race-conditions
|
103
|
+
def synchronize(&block)
|
104
|
+
coordinator.synchronize(&block)
|
105
|
+
end
|
106
|
+
|
87
107
|
private
|
88
108
|
|
89
109
|
# Prior to adding work to the queue, registers all the messages offsets into the
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.2.
|
4
|
+
version: 2.2.8.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-10-
|
38
|
+
date: 2023-10-20 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -57,20 +57,6 @@ dependencies:
|
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 2.3.0
|
60
|
-
- !ruby/object:Gem::Dependency
|
61
|
-
name: thor
|
62
|
-
requirement: !ruby/object:Gem::Requirement
|
63
|
-
requirements:
|
64
|
-
- - ">="
|
65
|
-
- !ruby/object:Gem::Version
|
66
|
-
version: '0.20'
|
67
|
-
type: :runtime
|
68
|
-
prerelease: false
|
69
|
-
version_requirements: !ruby/object:Gem::Requirement
|
70
|
-
requirements:
|
71
|
-
- - ">="
|
72
|
-
- !ruby/object:Gem::Version
|
73
|
-
version: '0.20'
|
74
60
|
- !ruby/object:Gem::Dependency
|
75
61
|
name: waterdrop
|
76
62
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,6 +156,7 @@ files:
|
|
170
156
|
- lib/karafka/cli.rb
|
171
157
|
- lib/karafka/cli/base.rb
|
172
158
|
- lib/karafka/cli/console.rb
|
159
|
+
- lib/karafka/cli/help.rb
|
173
160
|
- lib/karafka/cli/info.rb
|
174
161
|
- lib/karafka/cli/install.rb
|
175
162
|
- lib/karafka/cli/server.rb
|
@@ -196,12 +183,18 @@ files:
|
|
196
183
|
- lib/karafka/helpers/colorize.rb
|
197
184
|
- lib/karafka/helpers/multi_delegator.rb
|
198
185
|
- lib/karafka/instrumentation/callbacks/error.rb
|
186
|
+
- lib/karafka/instrumentation/callbacks/rebalance.rb
|
199
187
|
- lib/karafka/instrumentation/callbacks/statistics.rb
|
200
188
|
- lib/karafka/instrumentation/logger.rb
|
201
189
|
- lib/karafka/instrumentation/logger_listener.rb
|
202
190
|
- lib/karafka/instrumentation/monitor.rb
|
203
191
|
- lib/karafka/instrumentation/notifications.rb
|
204
192
|
- lib/karafka/instrumentation/proctitle_listener.rb
|
193
|
+
- lib/karafka/instrumentation/vendors/appsignal/base.rb
|
194
|
+
- lib/karafka/instrumentation/vendors/appsignal/client.rb
|
195
|
+
- lib/karafka/instrumentation/vendors/appsignal/dashboard.json
|
196
|
+
- lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb
|
197
|
+
- lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb
|
205
198
|
- lib/karafka/instrumentation/vendors/datadog/dashboard.json
|
206
199
|
- lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
|
207
200
|
- lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
|
@@ -217,6 +210,7 @@ files:
|
|
217
210
|
- lib/karafka/messages/parser.rb
|
218
211
|
- lib/karafka/messages/seek.rb
|
219
212
|
- lib/karafka/patches/rdkafka/bindings.rb
|
213
|
+
- lib/karafka/patches/rdkafka/opaque.rb
|
220
214
|
- lib/karafka/pro.rb
|
221
215
|
- lib/karafka/pro/active_job/consumer.rb
|
222
216
|
- lib/karafka/pro/active_job/dispatcher.rb
|
@@ -440,7 +434,7 @@ licenses:
|
|
440
434
|
metadata:
|
441
435
|
funding_uri: https://karafka.io/#become-pro
|
442
436
|
homepage_uri: https://karafka.io
|
443
|
-
changelog_uri: https://
|
437
|
+
changelog_uri: https://karafka.io/docs/Changelog-Karafka
|
444
438
|
bug_tracker_uri: https://github.com/karafka/karafka/issues
|
445
439
|
source_code_uri: https://github.com/karafka/karafka
|
446
440
|
documentation_uri: https://karafka.io/docs
|
@@ -456,11 +450,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
456
450
|
version: '0'
|
457
451
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
458
452
|
requirements:
|
459
|
-
- - "
|
453
|
+
- - ">"
|
460
454
|
- !ruby/object:Gem::Version
|
461
|
-
version:
|
455
|
+
version: 1.3.1
|
462
456
|
requirements: []
|
463
|
-
rubygems_version: 3.4.
|
457
|
+
rubygems_version: 3.4.19
|
464
458
|
signing_key:
|
465
459
|
specification_version: 4
|
466
460
|
summary: Karafka is Ruby and Rails efficient Kafka processing framework.
|
metadata.gz.sig
CHANGED
Binary file
|