karafka 2.3.0.rc1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b07678937b0dc9c4154c90ccea2e9bbbc25aa2669461d937e859b77e134ddc48
4
- data.tar.gz: a2e3eee8a9351a789dc33a1f78a48899a007da29b81205b4383e75a9fa0cc13c
3
+ metadata.gz: bceb8995575bd7c51d4fa6df9905619082080c932b5295749d9562fb53eba2a7
4
+ data.tar.gz: d16247c466fe08f6a671e48e8b25af7edcf2c32d769a497a46245773a66794de
5
5
  SHA512:
6
- metadata.gz: d2331d328849f4ffb65aa5deb921bf8d12d732f06b30acc5b17ef1fb844d5da23166dbcd27a6b53a6d57d76c6a7d508e9c5479e064a73aa6e0c2eaed69597c8a
7
- data.tar.gz: da8afa92e4b5fc552319b15322b3b3e883c2ec5c2b46ec887d098d34f7a0fbd7a7408558a89d781699eb521309c8fe69d2df7e6a1f46de19259451fb3050e391
6
+ metadata.gz: 3367c6bace7ab7a1ed7c6ef04e41eefa40c6fef7baf41d0e8e32bb891e1f2799e1564a6ee36b38735b5d514272c88c62e90bcbb54ffa45f175e80b888837d1e5
7
+ data.tar.gz: 2d31b87c97ce9deb3d47c0332782ad116dbcf4661a7ea55b35ac04a0e87003a8eb73a5b50f7fc9a21eb1ad288eaafff0528b95133fe58f97ada832526959b628
checksums.yaml.gz.sig CHANGED
Binary file
data/CHANGELOG.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Karafka framework changelog
2
2
 
3
- ## 2.3.0 (Unreleased)
3
+ ## 2.3.0 (2024-01-26)
4
4
  - **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
5
5
  - **[Feature]** Provide ability to multiplex subscription groups (Pro)
6
6
  - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
@@ -23,8 +23,13 @@
23
23
  - [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
24
24
  - [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
25
25
  - [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
26
+ - [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
26
27
  - [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
27
28
 
29
+ ### Upgrade Notes
30
+
31
+ Available [here](https://karafka.io/docs/Upgrades-2.3/).
32
+
28
33
  ## 2.2.14 (2023-12-07)
29
34
  - **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
30
35
  - **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.
data/Gemfile CHANGED
@@ -10,7 +10,7 @@ gemspec
10
10
  # They are added here because they are part of the integration suite
11
11
  group :integrations do
12
12
  gem 'activejob'
13
- gem 'karafka-web'
13
+ gem 'karafka-web', '>= 0.8.0.rc1'
14
14
  end
15
15
 
16
16
  group :test do
data/Gemfile.lock CHANGED
@@ -1,8 +1,8 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka (2.3.0.rc1)
5
- karafka-core (>= 2.3.0.rc1, < 2.4.0)
4
+ karafka (2.3.0)
5
+ karafka-core (>= 2.3.0, < 2.4.0)
6
6
  waterdrop (>= 2.6.12, < 3.0.0)
7
7
  zeitwerk (~> 2.3)
8
8
 
@@ -39,16 +39,16 @@ GEM
39
39
  activesupport (>= 6.1)
40
40
  i18n (1.14.1)
41
41
  concurrent-ruby (~> 1.0)
42
- karafka-core (2.3.0.rc1)
43
- karafka-rdkafka (>= 0.14.7, < 0.15.0)
44
- karafka-rdkafka (0.14.7)
42
+ karafka-core (2.3.0)
43
+ karafka-rdkafka (>= 0.14.8, < 0.15.0)
44
+ karafka-rdkafka (0.14.8)
45
45
  ffi (~> 1.15)
46
46
  mini_portile2 (~> 2.6)
47
47
  rake (> 12)
48
- karafka-web (0.7.10)
48
+ karafka-web (0.8.0.rc1)
49
49
  erubi (~> 1.4)
50
- karafka (>= 2.2.9, < 3.0.0)
51
- karafka-core (>= 2.2.4, < 3.0.0)
50
+ karafka (>= 2.3.0.rc1, < 2.4.0)
51
+ karafka-core (>= 2.3.0.rc1, < 2.4.0)
52
52
  roda (~> 3.68, >= 3.69)
53
53
  tilt (~> 2.0)
54
54
  mini_portile2 (2.8.5)
@@ -95,7 +95,7 @@ DEPENDENCIES
95
95
  byebug
96
96
  factory_bot
97
97
  karafka!
98
- karafka-web
98
+ karafka-web (>= 0.8.0.rc1)
99
99
  rspec
100
100
  simplecov
101
101
 
data/karafka.gemspec CHANGED
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
21
21
  without having to focus on things that are not your business domain.
22
22
  DESC
23
23
 
24
- spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
24
+ spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
25
25
  spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
26
26
  spec.add_dependency 'zeitwerk', '~> 2.3'
27
27
 
data/lib/karafka/admin.rb CHANGED
@@ -191,34 +191,41 @@ module Karafka
191
191
  mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
192
192
  settings = { 'group.id': mapped_consumer_group_id }
193
193
 
194
- with_consumer(settings) do |consumer|
195
- # If we have any time based stuff to resolve, we need to do it prior to commits
196
- unless time_tpl.empty?
197
- real_offsets = consumer.offsets_for_times(time_tpl)
198
-
199
- real_offsets.to_h.each do |name, results|
200
- results.each do |result|
201
- raise(Errors::InvalidTimeBasedOffsetError) unless result
202
-
203
- partition = result.partition
204
-
205
- # Negative offset means we're beyond last message and we need to query for the
206
- # high watermark offset to get the most recent offset and move there
207
- if result.offset.negative?
208
- _, offset = consumer.query_watermark_offsets(name, result.partition)
209
- else
210
- # If we get an offset, it means there existed a message close to this time
211
- # location
212
- offset = result.offset
194
+ # This error can occur when we query a broker that is not a coordinator because something
195
+ # was changing in the cluster. We should be able to safely restart our seeking request
196
+ # when this happens without any issues
197
+ #
198
+ # We wrap the consumer creation, so we retry with a new consumer instance
199
+ with_rdkafka_retry(codes: %i[not_coordinator]) do
200
+ with_consumer(settings) do |consumer|
201
+ # If we have any time based stuff to resolve, we need to do it prior to commits
202
+ unless time_tpl.empty?
203
+ real_offsets = consumer.offsets_for_times(time_tpl)
204
+
205
+ real_offsets.to_h.each do |name, results|
206
+ results.each do |result|
207
+ raise(Errors::InvalidTimeBasedOffsetError) unless result
208
+
209
+ partition = result.partition
210
+
211
+ # Negative offset means we're beyond last message and we need to query for the
212
+ # high watermark offset to get the most recent offset and move there
213
+ if result.offset.negative?
214
+ _, offset = consumer.query_watermark_offsets(name, result.partition)
215
+ else
216
+ # If we get an offset, it means there existed a message close to this time
217
+ # location
218
+ offset = result.offset
219
+ end
220
+
221
+ # Since now we have proper offsets, we can add this to the final tpl for commit
222
+ tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
213
223
  end
214
-
215
- # Since now we have proper offsets, we can add this to the final tpl for commit
216
- tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
217
224
  end
218
225
  end
219
- end
220
226
 
221
- consumer.commit(tpl, false)
227
+ consumer.commit(tpl, false)
228
+ end
222
229
  end
223
230
  end
224
231
 
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Module used to check some constraints that cannot be easily defined by Bundler
5
+ # At the moment we use it to ensure, that if Karafka is used, it operates with the expected
6
+ # web ui version and that older versions of Web UI that would not be compatible with the API
7
+ # changes in karafka are not used.
8
+ #
9
+ # We can make Web UI require certain karafka version range, but at the moment we do not have a
10
+ # strict 1:1 release pattern matching those two.
11
+ module Constraints
12
+ class << self
13
+ # Verifies that optional requirements are met.
14
+ def verify!
15
+ # Skip verification if web is not used at all
16
+ return unless require_version('karafka/web')
17
+
18
+ # All good if version higher than 0.7.x because we expect 0.8.0 or higher
19
+ return if version(Karafka::Web::VERSION) >= version('0.7.100')
20
+
21
+ # If older web-ui used, we cannot allow it
22
+ raise(
23
+ Errors::DependencyConstraintsError,
24
+ 'karafka-web < 0.8.0 is not compatible with this karafka version'
25
+ )
26
+ end
27
+
28
+ private
29
+
30
+ # Requires given version file from a gem location
31
+ # @param version_location [String]
32
+ # @return [Boolean] true if it was required or false if not reachable
33
+ def require_version(version_location)
34
+ require "#{version_location}/version"
35
+
36
+ true
37
+ rescue LoadError
38
+ false
39
+ end
40
+
41
+ # Builds a version object for comparing
42
+ # @param string [String]
43
+ # @return [::Gem::Version]
44
+ def version(string)
45
+ ::Gem::Version.new(string)
46
+ end
47
+ end
48
+ end
49
+ end
@@ -80,5 +80,12 @@ module Karafka
80
80
  # Raised in transactions when we attempt to store offset for a partition that we have lost
81
81
  # This does not affect producer only transactions, hence we raise it only on offset storage
82
82
  AssignmentLostError = Class.new(BaseError)
83
+
84
+ # Raised if optional dependencies like karafka-web are required in a version that is not
85
+ # supported by the current framework version.
86
+ #
87
+ # Because we do not want to require web out of the box and we do not want to lock web with
88
+ # karafka 1:1, we do such a sanity check
89
+ DependencyConstraintsError = Class.new(BaseError)
83
90
  end
84
91
  end
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.3.0.rc1'
6
+ VERSION = '2.3.0'
7
7
  end
data/lib/karafka.rb CHANGED
@@ -143,3 +143,5 @@ end
143
143
 
144
144
  # Load railtie after everything else is ready so we know we can rely on it.
145
145
  require 'karafka/railtie'
146
+
147
+ Karafka::Constraints.verify!
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.0.rc1
4
+ version: 2.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
36
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
37
  -----END CERTIFICATE-----
38
- date: 2024-01-21 00:00:00.000000000 Z
38
+ date: 2024-01-26 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
43
43
  requirements:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: 2.3.0.rc1
46
+ version: 2.3.0
47
47
  - - "<"
48
48
  - !ruby/object:Gem::Version
49
49
  version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
53
53
  requirements:
54
54
  - - ">="
55
55
  - !ruby/object:Gem::Version
56
- version: 2.3.0.rc1
56
+ version: 2.3.0
57
57
  - - "<"
58
58
  - !ruby/object:Gem::Version
59
59
  version: 2.4.0
@@ -174,6 +174,7 @@ files:
174
174
  - lib/karafka/connection/raw_messages_buffer.rb
175
175
  - lib/karafka/connection/rebalance_manager.rb
176
176
  - lib/karafka/connection/status.rb
177
+ - lib/karafka/constraints.rb
177
178
  - lib/karafka/contracts.rb
178
179
  - lib/karafka/contracts/base.rb
179
180
  - lib/karafka/contracts/config.rb
metadata.gz.sig CHANGED
Binary file