karafka 2.4.14 → 2.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +6 -6
- data/docker-compose.yml +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +7 -2
- data/lib/karafka/pro/processing/strategies/default.rb +24 -0
- data/lib/karafka/routing/builder.rb +15 -1
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +2 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b49de31147b2a64d8927c91e25e6b607fdf1a8a7280b7cc3ba8c3663cf96b6f
|
4
|
+
data.tar.gz: 4a3d9e439e21b79b9ac6a2c8bf4e76219f970be1f6bb7316feb5c7f2f792271e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 527a1c169ddc5a0f978f69dc3dbae6d30288997d2cfc95104ad6343d7a468c67afbde4c55051c1e7902d2c161fc1e615f8d703b9dfba123f734b9fbd7ccec5d1
|
7
|
+
data.tar.gz: a895657a8dece0c9bdb65afeeb00bc8df2e03e1957d345bc71ceab9d6cce74e5d0b4afcc251e26dfe230778a05b8742e40505a779695325844c6be630307ad82
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Karafka Framework Changelog
|
2
2
|
|
3
|
+
## 2.4.15 (2024-12-04)
|
4
|
+
- [Fix] Assignment tracker current state fetch during a rebalance loop can cause an error on multi CG setup.
|
5
|
+
- [Fix] Prevent double post-transaction offset dispatch to Kafka.
|
6
|
+
|
3
7
|
## 2.4.14 (2024-11-25)
|
4
8
|
- [Enhancement] Improve low-level critical error reporting.
|
5
9
|
- [Enhancement] Expand Kubernetes Liveness state reporting with critical errors detection.
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.4.
|
4
|
+
karafka (2.4.15)
|
5
5
|
base64 (~> 0.2)
|
6
6
|
karafka-core (>= 2.4.4, < 2.5.0)
|
7
7
|
karafka-rdkafka (>= 0.17.2)
|
@@ -49,8 +49,8 @@ GEM
|
|
49
49
|
activesupport (>= 6.1)
|
50
50
|
i18n (1.14.6)
|
51
51
|
concurrent-ruby (~> 1.0)
|
52
|
-
karafka-core (2.4.
|
53
|
-
karafka-rdkafka (>= 0.
|
52
|
+
karafka-core (2.4.5)
|
53
|
+
karafka-rdkafka (>= 0.17.6, < 0.18.0)
|
54
54
|
karafka-rdkafka (0.17.6)
|
55
55
|
ffi (~> 1.15)
|
56
56
|
mini_portile2 (~> 2.6)
|
@@ -58,18 +58,18 @@ GEM
|
|
58
58
|
karafka-testing (2.4.6)
|
59
59
|
karafka (>= 2.4.0, < 2.5.0)
|
60
60
|
waterdrop (>= 2.7.0)
|
61
|
-
karafka-web (0.10.
|
61
|
+
karafka-web (0.10.4)
|
62
62
|
erubi (~> 1.4)
|
63
63
|
karafka (>= 2.4.10, < 2.5.0)
|
64
64
|
karafka-core (>= 2.4.0, < 2.5.0)
|
65
65
|
roda (~> 3.68, >= 3.69)
|
66
66
|
tilt (~> 2.0)
|
67
67
|
logger (1.6.1)
|
68
|
-
mini_portile2 (2.8.
|
68
|
+
mini_portile2 (2.8.8)
|
69
69
|
minitest (5.25.1)
|
70
70
|
ostruct (0.6.1)
|
71
71
|
raabro (1.4.0)
|
72
|
-
rack (3.1.
|
72
|
+
rack (3.1.8)
|
73
73
|
rake (13.2.1)
|
74
74
|
roda (3.84.0)
|
75
75
|
rack
|
data/docker-compose.yml
CHANGED
@@ -31,8 +31,13 @@ module Karafka
|
|
31
31
|
def current
|
32
32
|
assignments = {}
|
33
33
|
|
34
|
-
|
35
|
-
|
34
|
+
# Since the `@assignments` state can change during a rebalance, if we would iterate over
|
35
|
+
# it exactly during state change, we would end up with the following error:
|
36
|
+
# RuntimeError: can't add a new key into hash during iteration
|
37
|
+
@mutex.synchronize do
|
38
|
+
@assignments.each do |topic, partitions|
|
39
|
+
assignments[topic] = partitions.dup.freeze
|
40
|
+
end
|
36
41
|
end
|
37
42
|
|
38
43
|
assignments.freeze
|
@@ -63,6 +63,16 @@ module Karafka
|
|
63
63
|
# Ignore earlier offsets than the one we already committed
|
64
64
|
return true if coordinator.seek_offset > message.offset
|
65
65
|
return false if revoked?
|
66
|
+
|
67
|
+
# If we have already marked this successfully in a transaction that was running
|
68
|
+
# we should not mark it again with the client offset delegation but instead we should
|
69
|
+
# just align the in-memory state
|
70
|
+
if @_in_transaction_marked
|
71
|
+
coordinator.seek_offset = message.offset + 1
|
72
|
+
|
73
|
+
return true
|
74
|
+
end
|
75
|
+
|
66
76
|
return revoked? unless client.mark_as_consumed(message, offset_metadata)
|
67
77
|
|
68
78
|
coordinator.seek_offset = message.offset + 1
|
@@ -90,6 +100,12 @@ module Karafka
|
|
90
100
|
return true if coordinator.seek_offset > message.offset
|
91
101
|
return false if revoked?
|
92
102
|
|
103
|
+
if @_in_transaction_marked
|
104
|
+
coordinator.seek_offset = message.offset + 1
|
105
|
+
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
|
93
109
|
return revoked? unless client.mark_as_consumed!(message, offset_metadata)
|
94
110
|
|
95
111
|
coordinator.seek_offset = message.offset + 1
|
@@ -132,6 +148,7 @@ module Karafka
|
|
132
148
|
transaction_started = true
|
133
149
|
@_transaction_marked = []
|
134
150
|
@_in_transaction = true
|
151
|
+
@_in_transaction_marked = false
|
135
152
|
|
136
153
|
producer.transaction(&block)
|
137
154
|
|
@@ -143,6 +160,11 @@ module Karafka
|
|
143
160
|
# @note We never need to use the blocking `#mark_as_consumed!` here because the offset
|
144
161
|
# anyhow was already stored during the transaction
|
145
162
|
#
|
163
|
+
# @note Since the offset could have been already stored in Kafka (could have because
|
164
|
+
# you can have transactions without marking), we use the `@_in_transaction_marked`
|
165
|
+
# state to decide if we need to dispatch the offset via client at all
|
166
|
+
# (if post transaction, then we do not have to)
|
167
|
+
#
|
146
168
|
# @note In theory we could only keep reference to the most recent marking and reject
|
147
169
|
# others. We however do not do it for two reasons:
|
148
170
|
# - User may have non standard flow relying on some alternative order and we want to
|
@@ -158,6 +180,7 @@ module Karafka
|
|
158
180
|
if transaction_started
|
159
181
|
@_transaction_marked.clear
|
160
182
|
@_in_transaction = false
|
183
|
+
@_in_transaction_marked = false
|
161
184
|
end
|
162
185
|
end
|
163
186
|
|
@@ -178,6 +201,7 @@ module Karafka
|
|
178
201
|
offset_metadata
|
179
202
|
)
|
180
203
|
|
204
|
+
@_in_transaction_marked = true
|
181
205
|
@_transaction_marked ||= []
|
182
206
|
@_transaction_marked << [message, offset_metadata, async]
|
183
207
|
end
|
@@ -67,6 +67,20 @@ module Karafka
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
+
# Clear out the drawn routes.
|
71
|
+
alias array_clear clear
|
72
|
+
private :array_clear
|
73
|
+
|
74
|
+
# Clear routes and draw them again with the given block. Helpful for testing purposes.
|
75
|
+
# @param block [Proc] block we will evaluate within the builder context
|
76
|
+
def redraw(&block)
|
77
|
+
@mutex.synchronize do
|
78
|
+
@draws.clear
|
79
|
+
array_clear
|
80
|
+
end
|
81
|
+
draw(&block)
|
82
|
+
end
|
83
|
+
|
70
84
|
# @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
|
71
85
|
# we want to use. Since Karafka supports multi-process setup, we need to be able
|
72
86
|
# to pick only those consumer groups that should be active in our given process context
|
@@ -79,7 +93,7 @@ module Karafka
|
|
79
93
|
@mutex.synchronize do
|
80
94
|
@defaults = EMPTY_DEFAULTS
|
81
95
|
@draws.clear
|
82
|
-
|
96
|
+
array_clear
|
83
97
|
end
|
84
98
|
end
|
85
99
|
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.4.
|
4
|
+
version: 2.4.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
|
36
36
|
ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-
|
38
|
+
date: 2024-12-04 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: base64
|
metadata.gz.sig
CHANGED
Binary file
|