karafka 2.4.8 → 2.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +0 -1
- data/CHANGELOG.md +8 -0
- data/Gemfile +8 -5
- data/Gemfile.lock +23 -14
- data/bin/integrations +5 -0
- data/certs/cert.pem +26 -0
- data/config/locales/errors.yml +4 -0
- data/config/locales/pro_errors.yml +17 -0
- data/karafka.gemspec +1 -1
- data/lib/karafka/admin.rb +42 -0
- data/lib/karafka/contracts/config.rb +2 -0
- data/lib/karafka/errors.rb +3 -2
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +16 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +5 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +17 -1
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +17 -1
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +22 -6
- data/lib/karafka/pro/recurring_tasks/consumer.rb +105 -0
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +53 -0
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +41 -0
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +35 -0
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +87 -0
- data/lib/karafka/pro/recurring_tasks/errors.rb +34 -0
- data/lib/karafka/pro/recurring_tasks/executor.rb +152 -0
- data/lib/karafka/pro/recurring_tasks/listener.rb +38 -0
- data/lib/karafka/pro/recurring_tasks/matcher.rb +38 -0
- data/lib/karafka/pro/recurring_tasks/schedule.rb +63 -0
- data/lib/karafka/pro/recurring_tasks/serializer.rb +113 -0
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +52 -0
- data/lib/karafka/pro/recurring_tasks/task.rb +151 -0
- data/lib/karafka/pro/recurring_tasks.rb +87 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +130 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +28 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +40 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +27 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +25 -0
- data/lib/karafka/processing/strategies/dlq.rb +16 -2
- data/lib/karafka/processing/strategies/dlq_mom.rb +25 -6
- data/lib/karafka/processing/worker.rb +11 -1
- data/lib/karafka/railtie.rb +11 -22
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +3 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
- data/lib/karafka/routing/features/dead_letter_queue/topic.rb +7 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +12 -0
- data/lib/karafka/routing/topic.rb +14 -0
- data/lib/karafka/setup/config.rb +3 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +44 -24
- metadata.gz.sig +0 -0
- data/certs/cert_chain.pem +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 69bfb8dac259cdb89eadfa86b8a13b7e901fbf82f7562e0869c7bd88f6111f30
|
4
|
+
data.tar.gz: 68959cc13a83db2611a41556ce99db5fed6f98272626dc12f64e4c18415bc633
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 54d7bb2ad1d65df9b2e92f28a8ae2255dac95c37fcbc3b1bde554a7d6be0c48c79ee164f6d7b8ebd57d9b09a18ff3f955cc9a193e5ca0207403d503224fc0f61
|
7
|
+
data.tar.gz: 94b08e703378d7b074d32f220dbb24a02a0e3039f948780bb7c434be721df53a52b03ca349d3e26fd8330ff6d7049d389fe27965b8120ef9d4047b1acc3984d1
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Karafka Framework Changelog
|
2
2
|
|
3
|
+
## 2.4.9 (2024-08-23)
|
4
|
+
- **[Feature]** Provide Kafka based Recurring (Cron) Tasks.
|
5
|
+
- [Enhancement] Wrap worker work with Rails Reloader/Executor (fusion2004)
|
6
|
+
- [Enhancement] Allow for partial topic level kafka scope settings reconfiguration via `inherit` flag.
|
7
|
+
- [Enhancement] Validate `eof` kafka scope flag when `eofed` in routing enabled.
|
8
|
+
- [Enhancement] Provide `mark_after_dispatch` setting for granular DLQ marking control.
|
9
|
+
- [Enhancement] Provide `Karafka::Admin.rename_consumer_group`.
|
10
|
+
|
3
11
|
## 2.4.8 (2024-08-09)
|
4
12
|
- **[Feature]** Introduce ability to react to `#eof` either from `#consume` or from `#eofed` when EOF without new messages.
|
5
13
|
- [Enhancement] Provide `Consumer#eofed?` to indicate reaching EOF.
|
data/Gemfile
CHANGED
@@ -6,20 +6,23 @@ plugin 'diffend'
|
|
6
6
|
|
7
7
|
gemspec
|
8
8
|
|
9
|
-
# Karafka gem does not require activejob
|
9
|
+
# Karafka gem does not require activejob, karafka-web or fugit to work
|
10
10
|
# They are added here because they are part of the integration suite
|
11
11
|
# Since some of those are only needed for some specs, they should never be required automatically
|
12
|
+
group :integrations, :test do
|
13
|
+
gem 'fugit', require: false
|
14
|
+
gem 'rspec', require: false
|
15
|
+
end
|
16
|
+
|
12
17
|
group :integrations do
|
13
18
|
gem 'activejob', require: false
|
14
|
-
gem 'karafka-testing', '>= 2.4.
|
15
|
-
gem 'karafka-web', '>= 0.10.0.
|
16
|
-
gem 'rspec', require: false
|
19
|
+
gem 'karafka-testing', '>= 2.4.6', require: false
|
20
|
+
gem 'karafka-web', '>= 0.10.0.rc2', require: false
|
17
21
|
end
|
18
22
|
|
19
23
|
group :test do
|
20
24
|
gem 'byebug'
|
21
25
|
gem 'factory_bot'
|
22
26
|
gem 'ostruct'
|
23
|
-
gem 'rspec'
|
24
27
|
gem 'simplecov'
|
25
28
|
end
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (2.4.
|
4
|
+
karafka (2.4.9)
|
5
5
|
base64 (~> 0.2)
|
6
6
|
karafka-core (>= 2.4.3, < 2.5.0)
|
7
7
|
karafka-rdkafka (>= 0.17.2)
|
@@ -11,31 +11,37 @@ PATH
|
|
11
11
|
GEM
|
12
12
|
remote: https://rubygems.org/
|
13
13
|
specs:
|
14
|
-
activejob (7.1
|
15
|
-
activesupport (= 7.1
|
14
|
+
activejob (7.2.1)
|
15
|
+
activesupport (= 7.2.1)
|
16
16
|
globalid (>= 0.3.6)
|
17
|
-
activesupport (7.1
|
17
|
+
activesupport (7.2.1)
|
18
18
|
base64
|
19
19
|
bigdecimal
|
20
|
-
concurrent-ruby (~> 1.0, >= 1.
|
20
|
+
concurrent-ruby (~> 1.0, >= 1.3.1)
|
21
21
|
connection_pool (>= 2.2.5)
|
22
22
|
drb
|
23
23
|
i18n (>= 1.6, < 2)
|
24
|
+
logger (>= 1.4.2)
|
24
25
|
minitest (>= 5.1)
|
25
|
-
|
26
|
-
tzinfo (~> 2.0)
|
26
|
+
securerandom (>= 0.3)
|
27
|
+
tzinfo (~> 2.0, >= 2.0.5)
|
27
28
|
base64 (0.2.0)
|
28
29
|
bigdecimal (3.1.8)
|
29
30
|
byebug (11.1.3)
|
30
|
-
concurrent-ruby (1.3.
|
31
|
+
concurrent-ruby (1.3.4)
|
31
32
|
connection_pool (2.4.1)
|
32
33
|
diff-lcs (1.5.1)
|
33
34
|
docile (1.4.1)
|
34
35
|
drb (2.2.1)
|
35
36
|
erubi (1.13.0)
|
37
|
+
et-orbi (1.2.11)
|
38
|
+
tzinfo
|
36
39
|
factory_bot (6.4.6)
|
37
40
|
activesupport (>= 5.0.0)
|
38
41
|
ffi (1.17.0)
|
42
|
+
fugit (1.11.1)
|
43
|
+
et-orbi (~> 1, >= 1.2.11)
|
44
|
+
raabro (~> 1.4)
|
39
45
|
globalid (1.2.1)
|
40
46
|
activesupport (>= 6.1)
|
41
47
|
i18n (1.14.5)
|
@@ -49,19 +55,20 @@ GEM
|
|
49
55
|
karafka-testing (2.4.6)
|
50
56
|
karafka (>= 2.4.0, < 2.5.0)
|
51
57
|
waterdrop (>= 2.7.0)
|
52
|
-
karafka-web (0.10.0
|
58
|
+
karafka-web (0.10.0)
|
53
59
|
erubi (~> 1.4)
|
54
60
|
karafka (>= 2.4.7, < 2.5.0)
|
55
61
|
karafka-core (>= 2.4.0, < 2.5.0)
|
56
62
|
roda (~> 3.68, >= 3.69)
|
57
63
|
tilt (~> 2.0)
|
64
|
+
logger (1.6.0)
|
58
65
|
mini_portile2 (2.8.7)
|
59
|
-
minitest (5.
|
60
|
-
mutex_m (0.2.0)
|
66
|
+
minitest (5.25.1)
|
61
67
|
ostruct (0.6.0)
|
68
|
+
raabro (1.4.0)
|
62
69
|
rack (3.1.7)
|
63
70
|
rake (13.2.1)
|
64
|
-
roda (3.
|
71
|
+
roda (3.83.0)
|
65
72
|
rack
|
66
73
|
rspec (3.13.0)
|
67
74
|
rspec-core (~> 3.13.0)
|
@@ -76,6 +83,7 @@ GEM
|
|
76
83
|
diff-lcs (>= 1.2.0, < 2.0)
|
77
84
|
rspec-support (~> 3.13.0)
|
78
85
|
rspec-support (3.13.1)
|
86
|
+
securerandom (0.3.1)
|
79
87
|
simplecov (0.22.0)
|
80
88
|
docile (~> 1.1)
|
81
89
|
simplecov-html (~> 0.11)
|
@@ -99,9 +107,10 @@ DEPENDENCIES
|
|
99
107
|
activejob
|
100
108
|
byebug
|
101
109
|
factory_bot
|
110
|
+
fugit
|
102
111
|
karafka!
|
103
|
-
karafka-testing (>= 2.4.
|
104
|
-
karafka-web (>= 0.10.0.
|
112
|
+
karafka-testing (>= 2.4.6)
|
113
|
+
karafka-web (>= 0.10.0.rc2)
|
105
114
|
ostruct
|
106
115
|
rspec
|
107
116
|
simplecov
|
data/bin/integrations
CHANGED
@@ -240,6 +240,11 @@ ARGV.each do |filter|
|
|
240
240
|
end
|
241
241
|
end
|
242
242
|
|
243
|
+
# Remove Rails 7.2 specs from Ruby 3.0 because it requires 3.1
|
244
|
+
specs.delete_if do |spec|
|
245
|
+
RUBY_VERSION < '3.1' && spec.include?('rails72')
|
246
|
+
end
|
247
|
+
|
243
248
|
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
244
249
|
|
245
250
|
# Randomize order
|
data/certs/cert.pem
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
3
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
4
|
+
MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
|
5
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
6
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
|
7
|
+
PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
|
8
|
+
ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
|
9
|
+
Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
|
10
|
+
JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
|
11
|
+
T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
|
12
|
+
flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
|
13
|
+
vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
|
14
|
+
iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
|
15
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
|
16
|
+
sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
17
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
|
18
|
+
eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
|
19
|
+
Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
|
20
|
+
agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
|
21
|
+
TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
|
22
|
+
Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
|
23
|
+
oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
|
24
|
+
i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
|
25
|
+
ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
|
26
|
+
-----END CERTIFICATE-----
|
data/config/locales/errors.yml
CHANGED
@@ -33,6 +33,8 @@ en:
|
|
33
33
|
internal.processing.partitioner_class_format: cannot be nil
|
34
34
|
internal.processing.strategy_selector_format: cannot be nil
|
35
35
|
internal.processing.expansions_selector_format: cannot be nil
|
36
|
+
internal.processing.executor_class_format: cannot be nil
|
37
|
+
internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
|
36
38
|
|
37
39
|
internal.active_job.dispatcher_format: cannot be nil
|
38
40
|
internal.active_job.job_options_contract_format: cannot be nil
|
@@ -113,10 +115,12 @@ en:
|
|
113
115
|
dead_letter_queue.transactional_format: needs to be either true or false
|
114
116
|
dead_letter_queue.dispatch_method_format: 'needs to be either #produce_sync or #produce_async'
|
115
117
|
dead_letter_queue.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
|
118
|
+
dead_letter_queue.mark_after_dispatch_format: 'needs to be true, false or nil'
|
116
119
|
|
117
120
|
active_format: needs to be either true or false
|
118
121
|
|
119
122
|
eofed.active_format: needs to be either true or false
|
123
|
+
eofed.kafka_enable: 'cannot be enabled without enable.partition.eof set to true'
|
120
124
|
|
121
125
|
declaratives.partitions_format: needs to be more or equal to 1
|
122
126
|
declaratives.active_format: needs to be true
|
@@ -63,6 +63,8 @@ en:
|
|
63
63
|
swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
|
64
64
|
swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
|
65
65
|
|
66
|
+
recurring_tasks.active_format: 'needs to be boolean'
|
67
|
+
|
66
68
|
direct_assignments.active_missing: needs to be present
|
67
69
|
direct_assignments.active_format: 'needs to be boolean'
|
68
70
|
direct_assignments.partitions_missing: 'needs to be present'
|
@@ -99,5 +101,20 @@ en:
|
|
99
101
|
patterns.ttl_format: needs to be an integer bigger than 0
|
100
102
|
patterns.ttl_missing: needs to be present
|
101
103
|
|
104
|
+
recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
|
105
|
+
recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
|
106
|
+
recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
|
107
|
+
recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
|
108
|
+
recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
|
109
|
+
recurring_tasks.deserializer_format: 'needs to be configured'
|
110
|
+
recurring_tasks.logging_format: needs to be a boolean
|
111
|
+
|
102
112
|
routing:
|
103
113
|
swarm_nodes_not_used: 'At least one of the nodes has no assignments'
|
114
|
+
|
115
|
+
recurring_tasks:
|
116
|
+
id_format: 'can include only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_)'
|
117
|
+
cron_format: must be a non-empty string
|
118
|
+
enabled_format: needs to be a boolean
|
119
|
+
changed_format: needs to be a boolean
|
120
|
+
previous_time_format: needs to be a numerical or time
|
data/karafka.gemspec
CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
34
34
|
end
|
35
35
|
|
36
|
-
spec.cert_chain = %w[certs/
|
36
|
+
spec.cert_chain = %w[certs/cert.pem]
|
37
37
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
38
38
|
spec.executables = %w[karafka]
|
39
39
|
spec.require_paths = %w[lib]
|
data/lib/karafka/admin.rb
CHANGED
@@ -274,6 +274,48 @@ module Karafka
|
|
274
274
|
end
|
275
275
|
end
|
276
276
|
|
277
|
+
# Takes consumer group and its topics and migrates all the offsets to a new named group
|
278
|
+
#
|
279
|
+
# @param previous_name [String] old consumer group name
|
280
|
+
# @param new_name [String] new consumer group name
|
281
|
+
# @param topics [Array<String>] topics for which we want to migrate offsets during rename
|
282
|
+
# @param delete_previous [Boolean] should we delete previous consumer group after rename.
|
283
|
+
# Defaults to true.
|
284
|
+
#
|
285
|
+
# @note This method should **not** be executed on a running consumer group as it creates a
|
286
|
+
# "fake" consumer and uses it to move offsets.
|
287
|
+
#
|
288
|
+
# @note After migration unless `delete_previous` is set to `false`, old group will be
|
289
|
+
# removed.
|
290
|
+
#
|
291
|
+
# @note If new consumer group exists, old offsets will be added to it.
|
292
|
+
def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
|
293
|
+
remap = Hash.new { |h, k| h[k] = {} }
|
294
|
+
|
295
|
+
old_lags = read_lags_with_offsets({ previous_name => topics })
|
296
|
+
|
297
|
+
return if old_lags.empty?
|
298
|
+
|
299
|
+
read_lags_with_offsets({ previous_name => topics })
|
300
|
+
.fetch(previous_name)
|
301
|
+
.each do |topic, partitions|
|
302
|
+
partitions.each do |partition_id, details|
|
303
|
+
offset = details[:offset]
|
304
|
+
|
305
|
+
# No offset on this partition
|
306
|
+
next if offset.negative?
|
307
|
+
|
308
|
+
remap[topic][partition_id] = offset
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
312
|
+
seek_consumer_group(new_name, remap)
|
313
|
+
|
314
|
+
return unless delete_previous
|
315
|
+
|
316
|
+
delete_consumer_group(previous_name)
|
317
|
+
end
|
318
|
+
|
277
319
|
# Removes given consumer group (if exists)
|
278
320
|
#
|
279
321
|
# @param consumer_group_id [String] consumer group name
|
@@ -116,6 +116,8 @@ module Karafka
|
|
116
116
|
required(:partitioner_class) { |val| !val.nil? }
|
117
117
|
required(:strategy_selector) { |val| !val.nil? }
|
118
118
|
required(:expansions_selector) { |val| !val.nil? }
|
119
|
+
required(:executor_class) { |val| !val.nil? }
|
120
|
+
required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
|
119
121
|
end
|
120
122
|
|
121
123
|
nested(:active_job) do
|
data/lib/karafka/errors.rb
CHANGED
@@ -82,10 +82,11 @@ module Karafka
|
|
82
82
|
AssignmentLostError = Class.new(BaseError)
|
83
83
|
|
84
84
|
# Raised if optional dependencies like karafka-web are required in a version that is not
|
85
|
-
# supported by the current framework version.
|
85
|
+
# supported by the current framework version or when an optional dependency is missing.
|
86
86
|
#
|
87
87
|
# Because we do not want to require web out of the box and we do not want to lock web with
|
88
|
-
# karafka 1:1, we do such a sanity check
|
88
|
+
# karafka 1:1, we do such a sanity check. This also applies to cases where some external
|
89
|
+
# optional dependencies are needed but not available.
|
89
90
|
DependencyConstraintsError = Class.new(BaseError)
|
90
91
|
|
91
92
|
# Raised when we were not able to open pidfd for given pid
|
data/lib/karafka/pro/loader.rb
CHANGED
@@ -135,7 +135,12 @@ module Karafka
|
|
135
135
|
|
136
136
|
dispatch = lambda do
|
137
137
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
138
|
-
|
138
|
+
|
139
|
+
if mark_after_dispatch?
|
140
|
+
mark_dispatched_to_dlq(skippable_message)
|
141
|
+
else
|
142
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
143
|
+
end
|
139
144
|
end
|
140
145
|
|
141
146
|
if dispatch_in_a_transaction?
|
@@ -193,6 +198,16 @@ module Karafka
|
|
193
198
|
producer.transactional? && topic.dead_letter_queue.transactional?
|
194
199
|
end
|
195
200
|
|
201
|
+
# @return [Boolean] should we mark given message as consumed after dispatch.
|
202
|
+
# For default non MOM strategies if user did not explicitly tell us not to, we mark
|
203
|
+
# it. Default is `nil`, which means `true` in this case. If user provided alternative
|
204
|
+
# value, we go with it.
|
205
|
+
def mark_after_dispatch?
|
206
|
+
return true if topic.dead_letter_queue.mark_after_dispatch.nil?
|
207
|
+
|
208
|
+
topic.dead_letter_queue.mark_after_dispatch
|
209
|
+
end
|
210
|
+
|
196
211
|
# Runs the DLQ strategy and based on it it performs certain operations
|
197
212
|
#
|
198
213
|
# In case of `:skip` and `:dispatch` will run the exact flow provided in a block
|
@@ -55,7 +55,11 @@ module Karafka
|
|
55
55
|
skippable_message, _marked = find_skippable_message
|
56
56
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
57
57
|
|
58
|
-
|
58
|
+
if mark_after_dispatch?
|
59
|
+
mark_dispatched_to_dlq(skippable_message)
|
60
|
+
else
|
61
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
62
|
+
end
|
59
63
|
end
|
60
64
|
end
|
61
65
|
end
|
@@ -46,11 +46,27 @@ module Karafka
|
|
46
46
|
skippable_message, _marked = find_skippable_message
|
47
47
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
48
48
|
|
49
|
-
|
49
|
+
if mark_after_dispatch?
|
50
|
+
mark_dispatched_to_dlq(skippable_message)
|
51
|
+
else
|
52
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
53
|
+
end
|
50
54
|
end
|
51
55
|
end
|
52
56
|
end
|
53
57
|
end
|
58
|
+
|
59
|
+
# @return [Boolean] should we mark given message as consumed after dispatch. For
|
60
|
+
# MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
|
61
|
+
# is `nil`, which means `false` in this case. If user provided alternative value, we
|
62
|
+
# go with it.
|
63
|
+
#
|
64
|
+
# @note Please note, this is the opposite behavior than in case of AOM strategies.
|
65
|
+
def mark_after_dispatch?
|
66
|
+
return false if topic.dead_letter_queue.mark_after_dispatch.nil?
|
67
|
+
|
68
|
+
topic.dead_letter_queue.mark_after_dispatch
|
69
|
+
end
|
54
70
|
end
|
55
71
|
end
|
56
72
|
end
|
@@ -49,11 +49,27 @@ module Karafka
|
|
49
49
|
skippable_message, _marked = find_skippable_message
|
50
50
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
51
51
|
|
52
|
-
|
52
|
+
if mark_after_dispatch?
|
53
|
+
mark_dispatched_to_dlq(skippable_message)
|
54
|
+
else
|
55
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
56
|
+
end
|
53
57
|
end
|
54
58
|
end
|
55
59
|
end
|
56
60
|
end
|
61
|
+
|
62
|
+
# @return [Boolean] should we mark given message as consumed after dispatch. For
|
63
|
+
# MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
|
64
|
+
# is `nil`, which means `false` in this case. If user provided alternative value, we
|
65
|
+
# go with it.
|
66
|
+
#
|
67
|
+
# @note Please note, this is the opposite behavior than in case of AOM strategies.
|
68
|
+
def mark_after_dispatch?
|
69
|
+
return false if topic.dead_letter_queue.mark_after_dispatch.nil?
|
70
|
+
|
71
|
+
topic.dead_letter_queue.mark_after_dispatch
|
72
|
+
end
|
57
73
|
end
|
58
74
|
end
|
59
75
|
end
|
@@ -40,16 +40,32 @@ module Karafka
|
|
40
40
|
skippable_message, = find_skippable_message
|
41
41
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
42
42
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
43
|
+
if mark_after_dispatch?
|
44
|
+
mark_dispatched_to_dlq(skippable_message)
|
45
|
+
else
|
46
|
+
# Save the next offset we want to go with after moving given message to DLQ
|
47
|
+
# Without this, we would not be able to move forward and we would end up
|
48
|
+
# in an infinite loop trying to un-pause from the message we've already
|
49
|
+
# processed. Of course, since it's a MoM a rebalance or kill, will move it
|
50
|
+
# back as no offsets are being committed
|
51
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
52
|
+
end
|
49
53
|
end
|
50
54
|
end
|
51
55
|
end
|
52
56
|
end
|
57
|
+
|
58
|
+
# @return [Boolean] should we mark given message as consumed after dispatch. For
|
59
|
+
# MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
|
60
|
+
# is `nil`, which means `false` in this case. If user provided alternative value, we
|
61
|
+
# go with it.
|
62
|
+
#
|
63
|
+
# @note Please note, this is the opposite behavior than in case of AOM strategies.
|
64
|
+
def mark_after_dispatch?
|
65
|
+
return false if topic.dead_letter_queue.mark_after_dispatch.nil?
|
66
|
+
|
67
|
+
topic.dead_letter_queue.mark_after_dispatch
|
68
|
+
end
|
53
69
|
end
|
54
70
|
end
|
55
71
|
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module RecurringTasks
|
17
|
+
# Consumer responsible for management of the recurring tasks and their execution
|
18
|
+
# There are some assumptions made here that always need to be satisfied:
|
19
|
+
# - we only run schedules that are of same or newer version
|
20
|
+
# - we always mark as consumed in such a way, that the first message received after
|
21
|
+
# assignment (if any) is a state
|
22
|
+
class Consumer < ::Karafka::BaseConsumer
|
23
|
+
# @param args [Array] all arguments accepted by the consumer
|
24
|
+
def initialize(*args)
|
25
|
+
super
|
26
|
+
@executor = Executor.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def consume
|
30
|
+
# There is nothing we can do if we operate on a newer schedule. In such cases we should
|
31
|
+
# just wait and re-raise error hoping someone will notice or that this will be
|
32
|
+
# reassigned to a process with newer schedule
|
33
|
+
raise(Errors::IncompatibleScheduleError) if @executor.incompatible?
|
34
|
+
|
35
|
+
messages.each do |message|
|
36
|
+
payload = message.payload
|
37
|
+
type = payload[:type]
|
38
|
+
|
39
|
+
case type
|
40
|
+
when 'schedule'
|
41
|
+
# If we're replaying data, we need to record the most recent stored state, so we
|
42
|
+
# can use this data to fully initialize the scheduler
|
43
|
+
@executor.update_state(payload) if @executor.replaying?
|
44
|
+
|
45
|
+
# If this is first message we cannot mark it on the previous offset
|
46
|
+
next if message.offset.zero?
|
47
|
+
|
48
|
+
# We always mark as consumed in such a way, that when replaying, we start from a
|
49
|
+
# schedule state message. This makes it easier to recover.
|
50
|
+
mark_as_consumed Karafka::Messages::Seek.new(
|
51
|
+
topic.name,
|
52
|
+
partition,
|
53
|
+
message.offset - 1
|
54
|
+
)
|
55
|
+
when 'command'
|
56
|
+
@executor.apply_command(payload)
|
57
|
+
|
58
|
+
next if @executor.replaying?
|
59
|
+
|
60
|
+
# Execute on each incoming command to have nice latency but only after replaying
|
61
|
+
# During replaying we should not execute because there may be more state changes
|
62
|
+
# that collectively have a different outcome
|
63
|
+
@executor.call
|
64
|
+
else
|
65
|
+
raise ::Karafka::Errors::UnsupportedCaseError, type
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
eofed if eofed?
|
70
|
+
end
|
71
|
+
|
72
|
+
# Starts the final replay process if we reached eof during replaying
|
73
|
+
def eofed
|
74
|
+
# We only mark as replayed if we were replaying in the first place
|
75
|
+
# If already replayed, nothing to do
|
76
|
+
return unless @executor.replaying?
|
77
|
+
|
78
|
+
@executor.replay
|
79
|
+
end
|
80
|
+
|
81
|
+
# Runs the cron execution if all good.
|
82
|
+
def tick
|
83
|
+
# Do nothing until we fully recover the correct state
|
84
|
+
return if @executor.replaying?
|
85
|
+
|
86
|
+
# If the state is incompatible, we can only raise an error.
|
87
|
+
# We do it here and in the `#consume` so the pause-retry kicks in basically reporting
|
88
|
+
# on this issue once every minute. That way user should not miss this.
|
89
|
+
# We use seek to move so we can achieve a pause of 60 seconds in between consecutive
|
90
|
+
# errors instead of on each tick because it is much more frequent.
|
91
|
+
if @executor.incompatible?
|
92
|
+
if messages.empty?
|
93
|
+
raise Errors::IncompatibleScheduleError
|
94
|
+
else
|
95
|
+
return seek(messages.last.offset - 1)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# If all good and compatible we can execute the recurring tasks
|
100
|
+
@executor.call
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module RecurringTasks
|
17
|
+
# Recurring Tasks related contracts
|
18
|
+
module Contracts
|
19
|
+
# Makes sure, all the expected config is defined as it should be
|
20
|
+
class Config < ::Karafka::Contracts::Base
|
21
|
+
configure do |config|
|
22
|
+
config.error_messages = YAML.safe_load(
|
23
|
+
File.read(
|
24
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
+
)
|
26
|
+
).fetch('en').fetch('validations').fetch('config')
|
27
|
+
end
|
28
|
+
|
29
|
+
nested(:recurring_tasks) do
|
30
|
+
required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
|
31
|
+
required(:deserializer) { |val| !val.nil? }
|
32
|
+
required(:logging) { |val| [true, false].include?(val) }
|
33
|
+
# Do not allow to run more often than every 5 seconds
|
34
|
+
required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
35
|
+
required(:group_id) do |val|
|
36
|
+
val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
|
37
|
+
end
|
38
|
+
|
39
|
+
nested(:topics) do
|
40
|
+
required(:schedules) do |val|
|
41
|
+
val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
|
42
|
+
end
|
43
|
+
|
44
|
+
required(:logs) do |val|
|
45
|
+
val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|