karafka 2.1.7 → 2.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +7 -0
- data/Gemfile.lock +4 -4
- data/lib/karafka/base_consumer.rb +10 -0
- data/lib/karafka/connection/client.rb +23 -1
- data/lib/karafka/connection/rebalance_manager.rb +11 -0
- data/lib/karafka/messages/messages.rb +4 -2
- data/lib/karafka/processing/executor.rb +1 -1
- data/lib/karafka/routing/topic.rb +2 -1
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +2 -2
- metadata.gz.sig +1 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 042f365fb134a24ae360d678590ce798014751c8a23fb267001c920a42aa5324
|
4
|
+
data.tar.gz: ba2950de557a5f6c775577ce392d60ee839184f50b7d9225969c684625c9ecd0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 30b5fcd92c348c50482cb84542380ad28b317d47e01efad2d2049cd3ba7872c5f66a7a4fde93d8bfa262d7fcb3745dedbdb89a4fd5e6cdf2288a43606ea2361d
|
7
|
+
data.tar.gz: d9c8ba95b2b71f46a3d35e2f5be634473a7aa9f45d9b5924e1019dcb53c7cea6aca8c594f9f4d3bf85a14176c3cb98a7167b4eb6e1f6f2192059d8040440e4a9
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 2.1.8 (2023-07-29)
|
4
|
+
- [Improvement] Introduce `Karafka::BaseConsumer#used?` method to indicate, that at least one invocation of `#consume` took or will take place. This can be used as a replacement to the non-direct `messages.count` check for shutdown and revocation to ensure, that the consumption took place or is taking place (in case of running LRJ).
|
5
|
+
- [Improvement] Make `messages#to_a` return copy of the underlying array to prevent scenarios, where the mutation impacts offset management.
|
6
|
+
- [Improvement] Mitigate a librdkafka `cooperative-sticky` rebalance crash issue.
|
7
|
+
- [Improvement] Provide ability to overwrite `consumer_persistence` per subscribed topic. This is mostly useful for plugins and extensions developers.
|
8
|
+
- [Fix] Fix a case where the performance tracker would crash in case of mutation of messages to an empty state.
|
9
|
+
|
3
10
|
## 2.1.7 (2023-07-22)
|
4
11
|
- [Improvement] Always query for watermarks in the Iterator to improve the initial response time.
|
5
12
|
- [Improvement] Add `max_wait_time` option to the Iterator.
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.1.
|
4
|
+
karafka (2.1.8)
|
5
5
|
karafka-core (>= 2.1.1, < 2.2.0)
|
6
6
|
thor (>= 0.20)
|
7
7
|
waterdrop (>= 2.6.2, < 3.0.0)
|
@@ -37,13 +37,13 @@ GEM
|
|
37
37
|
ffi (~> 1.15)
|
38
38
|
mini_portile2 (~> 2.6)
|
39
39
|
rake (> 12)
|
40
|
-
karafka-web (0.6.
|
40
|
+
karafka-web (0.6.3)
|
41
41
|
erubi (~> 1.4)
|
42
42
|
karafka (>= 2.1.4, < 3.0.0)
|
43
43
|
karafka-core (>= 2.0.13, < 3.0.0)
|
44
44
|
roda (~> 3.68, >= 3.68)
|
45
45
|
tilt (~> 2.0)
|
46
|
-
mini_portile2 (2.8.
|
46
|
+
mini_portile2 (2.8.4)
|
47
47
|
minitest (5.18.1)
|
48
48
|
rack (3.0.8)
|
49
49
|
rake (13.0.6)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
tilt (2.2.0)
|
73
73
|
tzinfo (2.0.6)
|
74
74
|
concurrent-ruby (~> 1.0)
|
75
|
-
waterdrop (2.6.
|
75
|
+
waterdrop (2.6.5)
|
76
76
|
karafka-core (>= 2.1.1, < 3.0.0)
|
77
77
|
zeitwerk (~> 2.3)
|
78
78
|
zeitwerk (2.6.8)
|
@@ -25,6 +25,7 @@ module Karafka
|
|
25
25
|
# Creates new consumer and assigns it an id
|
26
26
|
def initialize
|
27
27
|
@id = SecureRandom.hex(6)
|
28
|
+
@used = false
|
28
29
|
end
|
29
30
|
|
30
31
|
# Can be used to run preparation code prior to the job being enqueued
|
@@ -34,6 +35,7 @@ module Karafka
|
|
34
35
|
# not as a part of the public api. This should not perform any extensive operations as it is
|
35
36
|
# blocking and running in the listener thread.
|
36
37
|
def on_before_enqueue
|
38
|
+
@used = true
|
37
39
|
handle_before_enqueue
|
38
40
|
rescue StandardError => e
|
39
41
|
Karafka.monitor.instrument(
|
@@ -160,6 +162,14 @@ module Karafka
|
|
160
162
|
# some teardown procedures (closing file handler, etc).
|
161
163
|
def shutdown; end
|
162
164
|
|
165
|
+
# @return [Boolean] was this consumer in active use. Active use means running `#consume` at
|
166
|
+
# least once. Consumer may have to run `#revoked` or `#shutdown` despite not running
|
167
|
+
# `#consume` previously in delayed job cases and other cases that potentially involve running
|
168
|
+
# the `Jobs::Idle` for house-keeping
|
169
|
+
def used?
|
170
|
+
@used
|
171
|
+
end
|
172
|
+
|
163
173
|
# Pauses processing on a given offset for the current topic partition
|
164
174
|
#
|
165
175
|
# After given partition is resumed, it will continue processing from the given offset
|
@@ -23,11 +23,17 @@ module Karafka
|
|
23
23
|
# Max time for a TPL request. We increase it to compensate for remote clusters latency
|
24
24
|
TPL_REQUEST_TIMEOUT = 2_000
|
25
25
|
|
26
|
+
# 1 minute of max wait for the first rebalance before a forceful attempt
|
27
|
+
# This applies only to a case when a short-lived Karafka instance with a client would be
|
28
|
+
# closed before first rebalance. Mitigates a librdkafka bug.
|
29
|
+
COOPERATIVE_STICKY_MAX_WAIT = 60_000
|
30
|
+
|
26
31
|
# We want to make sure we never close several clients in the same moment to prevent
|
27
32
|
# potential race conditions and other issues
|
28
33
|
SHUTDOWN_MUTEX = Mutex.new
|
29
34
|
|
30
|
-
private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT
|
35
|
+
private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT,
|
36
|
+
:COOPERATIVE_STICKY_MAX_WAIT
|
31
37
|
|
32
38
|
# Creates a new consumer instance.
|
33
39
|
#
|
@@ -226,6 +232,22 @@ module Karafka
|
|
226
232
|
# as until all the consumers are stopped, the server will keep running serving only
|
227
233
|
# part of the messages
|
228
234
|
def stop
|
235
|
+
# This ensures, that we do not stop the underlying client until it passes the first
|
236
|
+
# rebalance for cooperative-sticky. Otherwise librdkafka may crash
|
237
|
+
#
|
238
|
+
# We set a timeout just in case the rebalance would never happen or would last for an
|
239
|
+
# extensive time period.
|
240
|
+
#
|
241
|
+
# @see https://github.com/confluentinc/librdkafka/issues/4312
|
242
|
+
if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
|
243
|
+
(COOPERATIVE_STICKY_MAX_WAIT / 100).times do
|
244
|
+
# If we're past the first rebalance, no need to wait
|
245
|
+
break if @rebalance_manager.active?
|
246
|
+
|
247
|
+
sleep(0.1)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
229
251
|
close
|
230
252
|
end
|
231
253
|
|
@@ -30,6 +30,7 @@ module Karafka
|
|
30
30
|
@assigned_partitions = {}
|
31
31
|
@revoked_partitions = {}
|
32
32
|
@changed = false
|
33
|
+
@active = false
|
33
34
|
end
|
34
35
|
|
35
36
|
# Resets the rebalance manager state
|
@@ -46,11 +47,20 @@ module Karafka
|
|
46
47
|
@changed
|
47
48
|
end
|
48
49
|
|
50
|
+
# @return [Boolean] true if there was at least one rebalance
|
51
|
+
# @note This method is needed to make sure that when using cooperative-sticky, we do not
|
52
|
+
# close until first rebalance. Otherwise librdkafka may crash.
|
53
|
+
# @see https://github.com/confluentinc/librdkafka/issues/4312
|
54
|
+
def active?
|
55
|
+
@active
|
56
|
+
end
|
57
|
+
|
49
58
|
# Callback that kicks in inside of rdkafka, when new partitions are assigned.
|
50
59
|
#
|
51
60
|
# @private
|
52
61
|
# @param partitions [Rdkafka::Consumer::TopicPartitionList]
|
53
62
|
def on_partitions_assigned(partitions)
|
63
|
+
@active = true
|
54
64
|
@assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
|
55
65
|
@changed = true
|
56
66
|
end
|
@@ -60,6 +70,7 @@ module Karafka
|
|
60
70
|
# @private
|
61
71
|
# @param partitions [Rdkafka::Consumer::TopicPartitionList]
|
62
72
|
def on_partitions_revoked(partitions)
|
73
|
+
@active = true
|
63
74
|
@revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
|
64
75
|
@changed = true
|
65
76
|
end
|
@@ -60,10 +60,12 @@ module Karafka
|
|
60
60
|
@messages_array.size
|
61
61
|
end
|
62
62
|
|
63
|
-
# @return [Array<Karafka::Messages::Message>] pure array with messages
|
63
|
+
# @return [Array<Karafka::Messages::Message>] copy of the pure array with messages
|
64
64
|
def to_a
|
65
|
-
@messages_array
|
65
|
+
@messages_array.dup
|
66
66
|
end
|
67
|
+
|
68
|
+
alias count size
|
67
69
|
end
|
68
70
|
end
|
69
71
|
end
|
@@ -48,7 +48,7 @@ module Karafka
|
|
48
48
|
# We reload the consumers with each batch instead of relying on some external signals
|
49
49
|
# when needed for consistency. That way devs may have it on or off and not in this
|
50
50
|
# middle state, where re-creation of a consumer instance would occur only sometimes
|
51
|
-
@consumer = nil unless
|
51
|
+
@consumer = nil unless topic.consumer_persistence
|
52
52
|
|
53
53
|
# First we build messages batch...
|
54
54
|
consumer.messages = Messages::Builders::Messages.call(
|
@@ -17,6 +17,7 @@ module Karafka
|
|
17
17
|
max_messages
|
18
18
|
max_wait_time
|
19
19
|
initial_offset
|
20
|
+
consumer_persistence
|
20
21
|
].freeze
|
21
22
|
|
22
23
|
private_constant :INHERITABLE_ATTRIBUTES
|
@@ -50,7 +51,7 @@ module Karafka
|
|
50
51
|
|
51
52
|
# @return [Class] consumer class that we should use
|
52
53
|
def consumer
|
53
|
-
if
|
54
|
+
if consumer_persistence
|
54
55
|
# When persistence of consumers is on, no need to reload them
|
55
56
|
@consumer
|
56
57
|
else
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-07-
|
38
|
+
date: 2023-07-29 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
metadata.gz.sig
CHANGED
@@ -1,6 +1 @@
|
|
1
|
-
|
2
|
-
V���� & �')��O��9zނ{<
|
3
|
-
[�:��ZA�嵘P@'�t��G�fc�ҍ���麍��B�e-Ծ�x�F}�Wn������q�?�
|
4
|
-
�&�/���|'*�������Vc� �/) �� ���h�R�
|
5
|
-
���\xl�C?H��9m��� ��g|���>)c�+�z���i7���*��Ҟ�":͍�O7P��e'�������V��
|
6
|
-
� ��Lhav���
|
1
|
+
�$tJt�po�n��K����>���9mK�*��=��pc�3��;����;zP���{ȁ½������Iԋ���ɹX*�����:`�� �>]�I��GA���wN�^E���oZO��M�͔�+�`jlX����Z^ֿ��S�S��\ꡛ����-�f�HU@����1� WSn�H.�KfN��[������v+��Xt����A:��lO0hy��x3�]?�|(���k��C+� Xf���B�١����L!��}�(@tl��G����>)ߙ������Rxܦ�)5E�>�&-�Z���l����ç���<��@cػM;�?ۗ��郆p)6��N��\�6ÄD�C��G?��~c��%
|