karafka 2.3.0.rc1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +6 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +9 -9
- data/karafka.gemspec +1 -1
- data/lib/karafka/admin.rb +31 -24
- data/lib/karafka/constraints.rb +49 -0
- data/lib/karafka/errors.rb +7 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +5 -4
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bceb8995575bd7c51d4fa6df9905619082080c932b5295749d9562fb53eba2a7
|
4
|
+
data.tar.gz: d16247c466fe08f6a671e48e8b25af7edcf2c32d769a497a46245773a66794de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3367c6bace7ab7a1ed7c6ef04e41eefa40c6fef7baf41d0e8e32bb891e1f2799e1564a6ee36b38735b5d514272c88c62e90bcbb54ffa45f175e80b888837d1e5
|
7
|
+
data.tar.gz: 2d31b87c97ce9deb3d47c0332782ad116dbcf4661a7ea55b35ac04a0e87003a8eb73a5b50f7fc9a21eb1ad288eaafff0528b95133fe58f97ada832526959b628
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
-
## 2.3.0 (
|
3
|
+
## 2.3.0 (2024-01-26)
|
4
4
|
- **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
|
5
5
|
- **[Feature]** Provide ability to multiplex subscription groups (Pro)
|
6
6
|
- **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
|
@@ -23,8 +23,13 @@
|
|
23
23
|
- [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
|
24
24
|
- [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
|
25
25
|
- [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
|
26
|
+
- [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
|
26
27
|
- [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
|
27
28
|
|
29
|
+
### Upgrade Notes
|
30
|
+
|
31
|
+
Available [here](https://karafka.io/docs/Upgrades-2.3/).
|
32
|
+
|
28
33
|
## 2.2.14 (2023-12-07)
|
29
34
|
- **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
|
30
35
|
- **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.3.0
|
5
|
-
karafka-core (>= 2.3.0
|
4
|
+
karafka (2.3.0)
|
5
|
+
karafka-core (>= 2.3.0, < 2.4.0)
|
6
6
|
waterdrop (>= 2.6.12, < 3.0.0)
|
7
7
|
zeitwerk (~> 2.3)
|
8
8
|
|
@@ -39,16 +39,16 @@ GEM
|
|
39
39
|
activesupport (>= 6.1)
|
40
40
|
i18n (1.14.1)
|
41
41
|
concurrent-ruby (~> 1.0)
|
42
|
-
karafka-core (2.3.0
|
43
|
-
karafka-rdkafka (>= 0.14.
|
44
|
-
karafka-rdkafka (0.14.
|
42
|
+
karafka-core (2.3.0)
|
43
|
+
karafka-rdkafka (>= 0.14.8, < 0.15.0)
|
44
|
+
karafka-rdkafka (0.14.8)
|
45
45
|
ffi (~> 1.15)
|
46
46
|
mini_portile2 (~> 2.6)
|
47
47
|
rake (> 12)
|
48
|
-
karafka-web (0.
|
48
|
+
karafka-web (0.8.0.rc1)
|
49
49
|
erubi (~> 1.4)
|
50
|
-
karafka (>= 2.
|
51
|
-
karafka-core (>= 2.
|
50
|
+
karafka (>= 2.3.0.rc1, < 2.4.0)
|
51
|
+
karafka-core (>= 2.3.0.rc1, < 2.4.0)
|
52
52
|
roda (~> 3.68, >= 3.69)
|
53
53
|
tilt (~> 2.0)
|
54
54
|
mini_portile2 (2.8.5)
|
@@ -95,7 +95,7 @@ DEPENDENCIES
|
|
95
95
|
byebug
|
96
96
|
factory_bot
|
97
97
|
karafka!
|
98
|
-
karafka-web
|
98
|
+
karafka-web (>= 0.8.0.rc1)
|
99
99
|
rspec
|
100
100
|
simplecov
|
101
101
|
|
data/karafka.gemspec
CHANGED
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
without having to focus on things that are not your business domain.
|
22
22
|
DESC
|
23
23
|
|
24
|
-
spec.add_dependency 'karafka-core', '>= 2.3.0
|
24
|
+
spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
|
25
25
|
spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
|
26
26
|
spec.add_dependency 'zeitwerk', '~> 2.3'
|
27
27
|
|
data/lib/karafka/admin.rb
CHANGED
@@ -191,34 +191,41 @@ module Karafka
|
|
191
191
|
mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
|
192
192
|
settings = { 'group.id': mapped_consumer_group_id }
|
193
193
|
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
#
|
212
|
-
offset
|
194
|
+
# This error can occur when we query a broker that is not a coordinator because something
|
195
|
+
# was changing in the cluster. We should be able to safely restart our seeking request
|
196
|
+
# when this happens without any issues
|
197
|
+
#
|
198
|
+
# We wrap the consumer creation, so we retry with a new consumer instance
|
199
|
+
with_rdkafka_retry(codes: %i[not_coordinator]) do
|
200
|
+
with_consumer(settings) do |consumer|
|
201
|
+
# If we have any time based stuff to resolve, we need to do it prior to commits
|
202
|
+
unless time_tpl.empty?
|
203
|
+
real_offsets = consumer.offsets_for_times(time_tpl)
|
204
|
+
|
205
|
+
real_offsets.to_h.each do |name, results|
|
206
|
+
results.each do |result|
|
207
|
+
raise(Errors::InvalidTimeBasedOffsetError) unless result
|
208
|
+
|
209
|
+
partition = result.partition
|
210
|
+
|
211
|
+
# Negative offset means we're beyond last message and we need to query for the
|
212
|
+
# high watermark offset to get the most recent offset and move there
|
213
|
+
if result.offset.negative?
|
214
|
+
_, offset = consumer.query_watermark_offsets(name, result.partition)
|
215
|
+
else
|
216
|
+
# If we get an offset, it means there existed a message close to this time
|
217
|
+
# location
|
218
|
+
offset = result.offset
|
219
|
+
end
|
220
|
+
|
221
|
+
# Since now we have proper offsets, we can add this to the final tpl for commit
|
222
|
+
tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
|
213
223
|
end
|
214
|
-
|
215
|
-
# Since now we have proper offsets, we can add this to the final tpl for commit
|
216
|
-
tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
|
217
224
|
end
|
218
225
|
end
|
219
|
-
end
|
220
226
|
|
221
|
-
|
227
|
+
consumer.commit(tpl, false)
|
228
|
+
end
|
222
229
|
end
|
223
230
|
end
|
224
231
|
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Module used to check some constraints that cannot be easily defined by Bundler
|
5
|
+
# At the moment we use it to ensure, that if Karafka is used, it operates with the expected
|
6
|
+
# web ui version and that older versions of Web UI that would not be compatible with the API
|
7
|
+
# changes in karafka are not used.
|
8
|
+
#
|
9
|
+
# We can make Web UI require certain karafka version range, but at the moment we do not have a
|
10
|
+
# strict 1:1 release pattern matching those two.
|
11
|
+
module Constraints
|
12
|
+
class << self
|
13
|
+
# Verifies that optional requirements are met.
|
14
|
+
def verify!
|
15
|
+
# Skip verification if web is not used at all
|
16
|
+
return unless require_version('karafka/web')
|
17
|
+
|
18
|
+
# All good if version higher than 0.7.x because we expect 0.8.0 or higher
|
19
|
+
return if version(Karafka::Web::VERSION) >= version('0.7.100')
|
20
|
+
|
21
|
+
# If older web-ui used, we cannot allow it
|
22
|
+
raise(
|
23
|
+
Errors::DependencyConstraintsError,
|
24
|
+
'karafka-web < 0.8.0 is not compatible with this karafka version'
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# Requires given version file from a gem location
|
31
|
+
# @param version_location [String]
|
32
|
+
# @return [Boolean] true if it was required or false if not reachable
|
33
|
+
def require_version(version_location)
|
34
|
+
require "#{version_location}/version"
|
35
|
+
|
36
|
+
true
|
37
|
+
rescue LoadError
|
38
|
+
false
|
39
|
+
end
|
40
|
+
|
41
|
+
# Builds a version object for comparing
|
42
|
+
# @param string [String]
|
43
|
+
# @return [::Gem::Version]
|
44
|
+
def version(string)
|
45
|
+
::Gem::Version.new(string)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/karafka/errors.rb
CHANGED
@@ -80,5 +80,12 @@ module Karafka
|
|
80
80
|
# Raised in transactions when we attempt to store offset for a partition that we have lost
|
81
81
|
# This does not affect producer only transactions, hence we raise it only on offset storage
|
82
82
|
AssignmentLostError = Class.new(BaseError)
|
83
|
+
|
84
|
+
# Raised if optional dependencies like karafka-web are required in a version that is not
|
85
|
+
# supported by the current framework version.
|
86
|
+
#
|
87
|
+
# Because we do not want to require web out of the box and we do not want to lock web with
|
88
|
+
# karafka 1:1, we do such a sanity check
|
89
|
+
DependencyConstraintsError = Class.new(BaseError)
|
83
90
|
end
|
84
91
|
end
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.3.0
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-01-
|
38
|
+
date: 2024-01-26 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.3.0
|
46
|
+
version: 2.3.0
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: 2.4.0
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.3.0
|
56
|
+
version: 2.3.0
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 2.4.0
|
@@ -174,6 +174,7 @@ files:
|
|
174
174
|
- lib/karafka/connection/raw_messages_buffer.rb
|
175
175
|
- lib/karafka/connection/rebalance_manager.rb
|
176
176
|
- lib/karafka/connection/status.rb
|
177
|
+
- lib/karafka/constraints.rb
|
177
178
|
- lib/karafka/contracts.rb
|
178
179
|
- lib/karafka/contracts/base.rb
|
179
180
|
- lib/karafka/contracts/config.rb
|
metadata.gz.sig
CHANGED
Binary file
|