karafka 2.3.0.rc1 → 2.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +9 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +23 -23
- data/README.md +2 -2
- data/karafka.gemspec +1 -1
- data/lib/karafka/admin.rb +31 -24
- data/lib/karafka/constraints.rb +49 -0
- data/lib/karafka/errors.rb +7 -0
- data/lib/karafka/helpers/async.rb +13 -3
- data/lib/karafka/runner.rb +4 -0
- data/lib/karafka/server.rb +2 -3
- data/lib/karafka/templates/karafka.rb.erb +0 -2
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +5 -4
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d28565a68d29a1f8f14be1878bf3a1f41611028069586fb4ece1572b8eb72eb
|
4
|
+
data.tar.gz: 8c7f6ea1ec67e07ed107c7347d594a26ba923cecc7282c346eb24863e0324412
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 73d43079b8e83babe2f722acdd11a67d6a5c361998ffada9b46b0f28b3ebc9c430d7b28cf56cb2aba808e25c234fffc7b2ee945e9e265160fb7cba6f821fcc70
|
7
|
+
data.tar.gz: 465b1088763ddc3ea63e53ef92765ae5719d6d1a83522288976321d1a64d24616b06d6571ce11425fd8a0dbf36d8c595871b92abfc8de8bcecc444181afe5123
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
-
## 2.3.
|
3
|
+
## 2.3.1 (2024-02-08)
|
4
|
+
- [Refactor] Ensure that `Karafka::Helpers::Async#async_call` can run from multiple threads.
|
5
|
+
|
6
|
+
## 2.3.0 (2024-01-26)
|
4
7
|
- **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
|
5
8
|
- **[Feature]** Provide ability to multiplex subscription groups (Pro)
|
6
9
|
- **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
|
@@ -23,8 +26,13 @@
|
|
23
26
|
- [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
|
24
27
|
- [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
|
25
28
|
- [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
|
29
|
+
- [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
|
26
30
|
- [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
|
27
31
|
|
32
|
+
### Upgrade Notes
|
33
|
+
|
34
|
+
Available [here](https://karafka.io/docs/Upgrades-2.3/).
|
35
|
+
|
28
36
|
## 2.2.14 (2023-12-07)
|
29
37
|
- **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
|
30
38
|
- **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.3.
|
5
|
-
karafka-core (>= 2.3.0
|
4
|
+
karafka (2.3.1)
|
5
|
+
karafka-core (>= 2.3.0, < 2.4.0)
|
6
6
|
waterdrop (>= 2.6.12, < 3.0.0)
|
7
7
|
zeitwerk (~> 2.3)
|
8
8
|
|
@@ -27,7 +27,7 @@ GEM
|
|
27
27
|
byebug (11.1.3)
|
28
28
|
concurrent-ruby (1.2.3)
|
29
29
|
connection_pool (2.4.1)
|
30
|
-
diff-lcs (1.5.
|
30
|
+
diff-lcs (1.5.1)
|
31
31
|
docile (1.4.0)
|
32
32
|
drb (2.2.0)
|
33
33
|
ruby2_keywords
|
@@ -39,38 +39,38 @@ GEM
|
|
39
39
|
activesupport (>= 6.1)
|
40
40
|
i18n (1.14.1)
|
41
41
|
concurrent-ruby (~> 1.0)
|
42
|
-
karafka-core (2.3.0
|
43
|
-
karafka-rdkafka (>= 0.14.
|
44
|
-
karafka-rdkafka (0.14.
|
42
|
+
karafka-core (2.3.0)
|
43
|
+
karafka-rdkafka (>= 0.14.8, < 0.15.0)
|
44
|
+
karafka-rdkafka (0.14.9)
|
45
45
|
ffi (~> 1.15)
|
46
46
|
mini_portile2 (~> 2.6)
|
47
47
|
rake (> 12)
|
48
|
-
karafka-web (0.
|
48
|
+
karafka-web (0.8.1)
|
49
49
|
erubi (~> 1.4)
|
50
|
-
karafka (>= 2.
|
51
|
-
karafka-core (>= 2.
|
50
|
+
karafka (>= 2.3.0, < 2.4.0)
|
51
|
+
karafka-core (>= 2.3.0, < 2.4.0)
|
52
52
|
roda (~> 3.68, >= 3.69)
|
53
53
|
tilt (~> 2.0)
|
54
54
|
mini_portile2 (2.8.5)
|
55
55
|
minitest (5.21.2)
|
56
56
|
mutex_m (0.2.0)
|
57
|
-
rack (3.0.
|
57
|
+
rack (3.0.9)
|
58
58
|
rake (13.1.0)
|
59
59
|
roda (3.76.0)
|
60
60
|
rack
|
61
|
-
rspec (3.
|
62
|
-
rspec-core (~> 3.
|
63
|
-
rspec-expectations (~> 3.
|
64
|
-
rspec-mocks (~> 3.
|
65
|
-
rspec-core (3.
|
66
|
-
rspec-support (~> 3.
|
67
|
-
rspec-expectations (3.
|
61
|
+
rspec (3.13.0)
|
62
|
+
rspec-core (~> 3.13.0)
|
63
|
+
rspec-expectations (~> 3.13.0)
|
64
|
+
rspec-mocks (~> 3.13.0)
|
65
|
+
rspec-core (3.13.0)
|
66
|
+
rspec-support (~> 3.13.0)
|
67
|
+
rspec-expectations (3.13.0)
|
68
68
|
diff-lcs (>= 1.2.0, < 2.0)
|
69
|
-
rspec-support (~> 3.
|
70
|
-
rspec-mocks (3.
|
69
|
+
rspec-support (~> 3.13.0)
|
70
|
+
rspec-mocks (3.13.0)
|
71
71
|
diff-lcs (>= 1.2.0, < 2.0)
|
72
|
-
rspec-support (~> 3.
|
73
|
-
rspec-support (3.
|
72
|
+
rspec-support (~> 3.13.0)
|
73
|
+
rspec-support (3.13.0)
|
74
74
|
ruby2_keywords (0.0.5)
|
75
75
|
simplecov (0.22.0)
|
76
76
|
docile (~> 1.1)
|
@@ -81,7 +81,7 @@ GEM
|
|
81
81
|
tilt (2.3.0)
|
82
82
|
tzinfo (2.0.6)
|
83
83
|
concurrent-ruby (~> 1.0)
|
84
|
-
waterdrop (2.6.
|
84
|
+
waterdrop (2.6.13)
|
85
85
|
karafka-core (>= 2.2.3, < 3.0.0)
|
86
86
|
zeitwerk (~> 2.3)
|
87
87
|
zeitwerk (2.6.12)
|
@@ -95,7 +95,7 @@ DEPENDENCIES
|
|
95
95
|
byebug
|
96
96
|
factory_bot
|
97
97
|
karafka!
|
98
|
-
karafka-web
|
98
|
+
karafka-web (>= 0.8.0)
|
99
99
|
rspec
|
100
100
|
simplecov
|
101
101
|
|
data/README.md
CHANGED
@@ -57,8 +57,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
|
|
57
57
|
1. Add and install Karafka:
|
58
58
|
|
59
59
|
```bash
|
60
|
-
# Make sure to install Karafka 2.
|
61
|
-
bundle add karafka --version ">= 2.
|
60
|
+
# Make sure to install Karafka 2.3
|
61
|
+
bundle add karafka --version ">= 2.3.0"
|
62
62
|
|
63
63
|
bundle exec karafka install
|
64
64
|
```
|
data/karafka.gemspec
CHANGED
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
without having to focus on things that are not your business domain.
|
22
22
|
DESC
|
23
23
|
|
24
|
-
spec.add_dependency 'karafka-core', '>= 2.3.0
|
24
|
+
spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
|
25
25
|
spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
|
26
26
|
spec.add_dependency 'zeitwerk', '~> 2.3'
|
27
27
|
|
data/lib/karafka/admin.rb
CHANGED
@@ -191,34 +191,41 @@ module Karafka
|
|
191
191
|
mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
|
192
192
|
settings = { 'group.id': mapped_consumer_group_id }
|
193
193
|
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
#
|
212
|
-
offset
|
194
|
+
# This error can occur when we query a broker that is not a coordinator because something
|
195
|
+
# was changing in the cluster. We should be able to safely restart our seeking request
|
196
|
+
# when this happens without any issues
|
197
|
+
#
|
198
|
+
# We wrap the consumer creation, so we retry with a new consumer instance
|
199
|
+
with_rdkafka_retry(codes: %i[not_coordinator]) do
|
200
|
+
with_consumer(settings) do |consumer|
|
201
|
+
# If we have any time based stuff to resolve, we need to do it prior to commits
|
202
|
+
unless time_tpl.empty?
|
203
|
+
real_offsets = consumer.offsets_for_times(time_tpl)
|
204
|
+
|
205
|
+
real_offsets.to_h.each do |name, results|
|
206
|
+
results.each do |result|
|
207
|
+
raise(Errors::InvalidTimeBasedOffsetError) unless result
|
208
|
+
|
209
|
+
partition = result.partition
|
210
|
+
|
211
|
+
# Negative offset means we're beyond last message and we need to query for the
|
212
|
+
# high watermark offset to get the most recent offset and move there
|
213
|
+
if result.offset.negative?
|
214
|
+
_, offset = consumer.query_watermark_offsets(name, result.partition)
|
215
|
+
else
|
216
|
+
# If we get an offset, it means there existed a message close to this time
|
217
|
+
# location
|
218
|
+
offset = result.offset
|
219
|
+
end
|
220
|
+
|
221
|
+
# Since now we have proper offsets, we can add this to the final tpl for commit
|
222
|
+
tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
|
213
223
|
end
|
214
|
-
|
215
|
-
# Since now we have proper offsets, we can add this to the final tpl for commit
|
216
|
-
tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
|
217
224
|
end
|
218
225
|
end
|
219
|
-
end
|
220
226
|
|
221
|
-
|
227
|
+
consumer.commit(tpl, false)
|
228
|
+
end
|
222
229
|
end
|
223
230
|
end
|
224
231
|
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Module used to check some constraints that cannot be easily defined by Bundler
|
5
|
+
# At the moment we use it to ensure, that if Karafka is used, it operates with the expected
|
6
|
+
# web ui version and that older versions of Web UI that would not be compatible with the API
|
7
|
+
# changes in karafka are not used.
|
8
|
+
#
|
9
|
+
# We can make Web UI require certain karafka version range, but at the moment we do not have a
|
10
|
+
# strict 1:1 release pattern matching those two.
|
11
|
+
module Constraints
|
12
|
+
class << self
|
13
|
+
# Verifies that optional requirements are met.
|
14
|
+
def verify!
|
15
|
+
# Skip verification if web is not used at all
|
16
|
+
return unless require_version('karafka/web')
|
17
|
+
|
18
|
+
# All good if version higher than 0.7.x because we expect 0.8.0 or higher
|
19
|
+
return if version(Karafka::Web::VERSION) >= version('0.7.100')
|
20
|
+
|
21
|
+
# If older web-ui used, we cannot allow it
|
22
|
+
raise(
|
23
|
+
Errors::DependencyConstraintsError,
|
24
|
+
'karafka-web < 0.8.0 is not compatible with this karafka version'
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# Requires given version file from a gem location
|
31
|
+
# @param version_location [String]
|
32
|
+
# @return [Boolean] true if it was required or false if not reachable
|
33
|
+
def require_version(version_location)
|
34
|
+
require "#{version_location}/version"
|
35
|
+
|
36
|
+
true
|
37
|
+
rescue LoadError
|
38
|
+
false
|
39
|
+
end
|
40
|
+
|
41
|
+
# Builds a version object for comparing
|
42
|
+
# @param string [String]
|
43
|
+
# @return [::Gem::Version]
|
44
|
+
def version(string)
|
45
|
+
::Gem::Version.new(string)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/karafka/errors.rb
CHANGED
@@ -80,5 +80,12 @@ module Karafka
|
|
80
80
|
# Raised in transactions when we attempt to store offset for a partition that we have lost
|
81
81
|
# This does not affect producer only transactions, hence we raise it only on offset storage
|
82
82
|
AssignmentLostError = Class.new(BaseError)
|
83
|
+
|
84
|
+
# Raised if optional dependencies like karafka-web are required in a version that is not
|
85
|
+
# supported by the current framework version.
|
86
|
+
#
|
87
|
+
# Because we do not want to require web out of the box and we do not want to lock web with
|
88
|
+
# karafka 1:1, we do such a sanity check
|
89
|
+
DependencyConstraintsError = Class.new(BaseError)
|
83
90
|
end
|
84
91
|
end
|
@@ -8,6 +8,12 @@ module Karafka
|
|
8
8
|
# @note Thread running code needs to manage it's own exceptions. If they leak out, they will
|
9
9
|
# abort thread on exception.
|
10
10
|
module Async
|
11
|
+
# Mutex used to ensure we do not create multiple threads if we decide to run this
|
12
|
+
# in parallel on multiple threads
|
13
|
+
MUTEX = Mutex.new
|
14
|
+
|
15
|
+
private_constant :MUTEX
|
16
|
+
|
11
17
|
class << self
|
12
18
|
# Adds forwardable to redirect thread-based control methods to the underlying thread that
|
13
19
|
# runs the async operations
|
@@ -22,10 +28,14 @@ module Karafka
|
|
22
28
|
|
23
29
|
# Runs the `#call` method in a new thread
|
24
30
|
def async_call
|
25
|
-
|
26
|
-
|
31
|
+
MUTEX.synchronize do
|
32
|
+
return if @thread&.alive?
|
33
|
+
|
34
|
+
@thread = Thread.new do
|
35
|
+
Thread.current.abort_on_exception = true
|
27
36
|
|
28
|
-
|
37
|
+
call
|
38
|
+
end
|
29
39
|
end
|
30
40
|
end
|
31
41
|
end
|
data/lib/karafka/runner.rb
CHANGED
@@ -18,6 +18,10 @@ module Karafka
|
|
18
18
|
workers = Processing::WorkersBatch.new(jobs_queue)
|
19
19
|
listeners = Connection::ListenersBatch.new(jobs_queue)
|
20
20
|
|
21
|
+
# We mark it prior to delegating to the manager as manager will have to start at least one
|
22
|
+
# connection to Kafka, hence running
|
23
|
+
Karafka::App.run!
|
24
|
+
|
21
25
|
# Register all the listeners so they can be started and managed
|
22
26
|
@manager.register(listeners)
|
23
27
|
|
data/lib/karafka/server.rb
CHANGED
@@ -61,10 +61,9 @@ module Karafka
|
|
61
61
|
end
|
62
62
|
|
63
63
|
# Starts Karafka with a supervision
|
64
|
-
# @note We don't need to sleep because Karafka::
|
65
|
-
#
|
64
|
+
# @note We don't need to sleep because Karafka::Runner is locking and waiting to finish loop
|
65
|
+
# (and it won't happen until we explicitly want to stop)
|
66
66
|
def start
|
67
|
-
Karafka::App.run!
|
68
67
|
Karafka::Runner.new.call
|
69
68
|
end
|
70
69
|
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.3.
|
4
|
+
version: 2.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-
|
38
|
+
date: 2024-02-09 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.3.0
|
46
|
+
version: 2.3.0
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: 2.4.0
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.3.0
|
56
|
+
version: 2.3.0
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 2.4.0
|
@@ -174,6 +174,7 @@ files:
|
|
174
174
|
- lib/karafka/connection/raw_messages_buffer.rb
|
175
175
|
- lib/karafka/connection/rebalance_manager.rb
|
176
176
|
- lib/karafka/connection/status.rb
|
177
|
+
- lib/karafka/constraints.rb
|
177
178
|
- lib/karafka/contracts.rb
|
178
179
|
- lib/karafka/contracts/base.rb
|
179
180
|
- lib/karafka/contracts/config.rb
|
metadata.gz.sig
CHANGED
Binary file
|