karafka 2.4.8 → 2.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +0 -1
  4. data/CHANGELOG.md +8 -0
  5. data/Gemfile +8 -5
  6. data/Gemfile.lock +23 -14
  7. data/bin/integrations +5 -0
  8. data/certs/cert.pem +26 -0
  9. data/config/locales/errors.yml +4 -0
  10. data/config/locales/pro_errors.yml +17 -0
  11. data/karafka.gemspec +1 -1
  12. data/lib/karafka/admin.rb +42 -0
  13. data/lib/karafka/contracts/config.rb +2 -0
  14. data/lib/karafka/errors.rb +3 -2
  15. data/lib/karafka/pro/loader.rb +2 -1
  16. data/lib/karafka/pro/processing/strategies/dlq/default.rb +16 -1
  17. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +5 -1
  18. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +17 -1
  19. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +17 -1
  20. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +22 -6
  21. data/lib/karafka/pro/recurring_tasks/consumer.rb +105 -0
  22. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +53 -0
  23. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +41 -0
  24. data/lib/karafka/pro/recurring_tasks/deserializer.rb +35 -0
  25. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +87 -0
  26. data/lib/karafka/pro/recurring_tasks/errors.rb +34 -0
  27. data/lib/karafka/pro/recurring_tasks/executor.rb +152 -0
  28. data/lib/karafka/pro/recurring_tasks/listener.rb +38 -0
  29. data/lib/karafka/pro/recurring_tasks/matcher.rb +38 -0
  30. data/lib/karafka/pro/recurring_tasks/schedule.rb +63 -0
  31. data/lib/karafka/pro/recurring_tasks/serializer.rb +113 -0
  32. data/lib/karafka/pro/recurring_tasks/setup/config.rb +52 -0
  33. data/lib/karafka/pro/recurring_tasks/task.rb +151 -0
  34. data/lib/karafka/pro/recurring_tasks.rb +87 -0
  35. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +130 -0
  36. data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +28 -0
  37. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +40 -0
  38. data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +27 -0
  39. data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +44 -0
  40. data/lib/karafka/pro/routing/features/recurring_tasks.rb +25 -0
  41. data/lib/karafka/processing/strategies/dlq.rb +16 -2
  42. data/lib/karafka/processing/strategies/dlq_mom.rb +25 -6
  43. data/lib/karafka/processing/worker.rb +11 -1
  44. data/lib/karafka/railtie.rb +11 -22
  45. data/lib/karafka/routing/features/dead_letter_queue/config.rb +3 -0
  46. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
  47. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +7 -2
  48. data/lib/karafka/routing/features/eofed/contracts/topic.rb +12 -0
  49. data/lib/karafka/routing/topic.rb +14 -0
  50. data/lib/karafka/setup/config.rb +3 -0
  51. data/lib/karafka/version.rb +1 -1
  52. data.tar.gz.sig +0 -0
  53. metadata +44 -24
  54. metadata.gz.sig +0 -0
  55. data/certs/cert_chain.pem +0 -26
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c7e8e51a5c0c4ded0d074965cee2090e64ca77e43443f4f36ab03cc3a21ddfd6
4
- data.tar.gz: c6e912c518d301f55974a9e4deb491ebe4c3e073e6748fe62ce1003eaea7bed7
3
+ metadata.gz: 69bfb8dac259cdb89eadfa86b8a13b7e901fbf82f7562e0869c7bd88f6111f30
4
+ data.tar.gz: 68959cc13a83db2611a41556ce99db5fed6f98272626dc12f64e4c18415bc633
5
5
  SHA512:
6
- metadata.gz: 8769a192c7ebb852250afd96611e115807172bf320d4e9d513bffbcc66f5570ca637aee1b8f3b68e46350b37054935ea905893139bc264a02949f975b39f2041
7
- data.tar.gz: 13dc8e118850aace7127bae5fe667b6f73118d220b61b1d2b590cc18e71243b3c1467d43df9ae134cd7b26c0274a100e34d8a07db0c6091bc02fd9c95ffb24f0
6
+ metadata.gz: 54d7bb2ad1d65df9b2e92f28a8ae2255dac95c37fcbc3b1bde554a7d6be0c48c79ee164f6d7b8ebd57d9b09a18ff3f955cc9a193e5ca0207403d503224fc0f61
7
+ data.tar.gz: 94b08e703378d7b074d32f220dbb24a02a0e3039f948780bb7c434be721df53a52b03ca349d3e26fd8330ff6d7049d389fe27965b8120ef9d4047b1acc3984d1
checksums.yaml.gz.sig CHANGED
Binary file
@@ -77,7 +77,6 @@ jobs:
77
77
  - '3.3'
78
78
  - '3.2'
79
79
  - '3.1'
80
- - '3.0'
81
80
  include:
82
81
  - ruby: '3.3'
83
82
  coverage: 'true'
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # Karafka Framework Changelog
2
2
 
3
+ ## 2.4.9 (2024-08-23)
4
+ - **[Feature]** Provide Kafka based Recurring (Cron) Tasks.
5
+ - [Enhancement] Wrap worker work with Rails Reloader/Executor (fusion2004)
6
+ - [Enhancement] Allow for partial topic level kafka scope settings reconfiguration via `inherit` flag.
7
+ - [Enhancement] Validate `eof` kafka scope flag when `eofed` in routing enabled.
8
+ - [Enhancement] Provide `mark_after_dispatch` setting for granular DLQ marking control.
9
+ - [Enhancement] Provide `Karafka::Admin.rename_consumer_group`.
10
+
3
11
  ## 2.4.8 (2024-08-09)
4
12
  - **[Feature]** Introduce ability to react to `#eof` either from `#consume` or from `#eofed` when EOF without new messages.
5
13
  - [Enhancement] Provide `Consumer#eofed?` to indicate reaching EOF.
data/Gemfile CHANGED
@@ -6,20 +6,23 @@ plugin 'diffend'
6
6
 
7
7
  gemspec
8
8
 
9
- # Karafka gem does not require activejob nor karafka-web to work
9
+ # Karafka gem does not require activejob, karafka-web or fugit to work
10
10
  # They are added here because they are part of the integration suite
11
11
  # Since some of those are only needed for some specs, they should never be required automatically
12
+ group :integrations, :test do
13
+ gem 'fugit', require: false
14
+ gem 'rspec', require: false
15
+ end
16
+
12
17
  group :integrations do
13
18
  gem 'activejob', require: false
14
- gem 'karafka-testing', '>= 2.4.0', require: false
15
- gem 'karafka-web', '>= 0.10.0.beta1', require: false
16
- gem 'rspec', require: false
19
+ gem 'karafka-testing', '>= 2.4.6', require: false
20
+ gem 'karafka-web', '>= 0.10.0.rc2', require: false
17
21
  end
18
22
 
19
23
  group :test do
20
24
  gem 'byebug'
21
25
  gem 'factory_bot'
22
26
  gem 'ostruct'
23
- gem 'rspec'
24
27
  gem 'simplecov'
25
28
  end
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka (2.4.8)
4
+ karafka (2.4.9)
5
5
  base64 (~> 0.2)
6
6
  karafka-core (>= 2.4.3, < 2.5.0)
7
7
  karafka-rdkafka (>= 0.17.2)
@@ -11,31 +11,37 @@ PATH
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activejob (7.1.3.4)
15
- activesupport (= 7.1.3.4)
14
+ activejob (7.2.1)
15
+ activesupport (= 7.2.1)
16
16
  globalid (>= 0.3.6)
17
- activesupport (7.1.3.4)
17
+ activesupport (7.2.1)
18
18
  base64
19
19
  bigdecimal
20
- concurrent-ruby (~> 1.0, >= 1.0.2)
20
+ concurrent-ruby (~> 1.0, >= 1.3.1)
21
21
  connection_pool (>= 2.2.5)
22
22
  drb
23
23
  i18n (>= 1.6, < 2)
24
+ logger (>= 1.4.2)
24
25
  minitest (>= 5.1)
25
- mutex_m
26
- tzinfo (~> 2.0)
26
+ securerandom (>= 0.3)
27
+ tzinfo (~> 2.0, >= 2.0.5)
27
28
  base64 (0.2.0)
28
29
  bigdecimal (3.1.8)
29
30
  byebug (11.1.3)
30
- concurrent-ruby (1.3.3)
31
+ concurrent-ruby (1.3.4)
31
32
  connection_pool (2.4.1)
32
33
  diff-lcs (1.5.1)
33
34
  docile (1.4.1)
34
35
  drb (2.2.1)
35
36
  erubi (1.13.0)
37
+ et-orbi (1.2.11)
38
+ tzinfo
36
39
  factory_bot (6.4.6)
37
40
  activesupport (>= 5.0.0)
38
41
  ffi (1.17.0)
42
+ fugit (1.11.1)
43
+ et-orbi (~> 1, >= 1.2.11)
44
+ raabro (~> 1.4)
39
45
  globalid (1.2.1)
40
46
  activesupport (>= 6.1)
41
47
  i18n (1.14.5)
@@ -49,19 +55,20 @@ GEM
49
55
  karafka-testing (2.4.6)
50
56
  karafka (>= 2.4.0, < 2.5.0)
51
57
  waterdrop (>= 2.7.0)
52
- karafka-web (0.10.0.rc1)
58
+ karafka-web (0.10.0)
53
59
  erubi (~> 1.4)
54
60
  karafka (>= 2.4.7, < 2.5.0)
55
61
  karafka-core (>= 2.4.0, < 2.5.0)
56
62
  roda (~> 3.68, >= 3.69)
57
63
  tilt (~> 2.0)
64
+ logger (1.6.0)
58
65
  mini_portile2 (2.8.7)
59
- minitest (5.24.1)
60
- mutex_m (0.2.0)
66
+ minitest (5.25.1)
61
67
  ostruct (0.6.0)
68
+ raabro (1.4.0)
62
69
  rack (3.1.7)
63
70
  rake (13.2.1)
64
- roda (3.82.0)
71
+ roda (3.83.0)
65
72
  rack
66
73
  rspec (3.13.0)
67
74
  rspec-core (~> 3.13.0)
@@ -76,6 +83,7 @@ GEM
76
83
  diff-lcs (>= 1.2.0, < 2.0)
77
84
  rspec-support (~> 3.13.0)
78
85
  rspec-support (3.13.1)
86
+ securerandom (0.3.1)
79
87
  simplecov (0.22.0)
80
88
  docile (~> 1.1)
81
89
  simplecov-html (~> 0.11)
@@ -99,9 +107,10 @@ DEPENDENCIES
99
107
  activejob
100
108
  byebug
101
109
  factory_bot
110
+ fugit
102
111
  karafka!
103
- karafka-testing (>= 2.4.0)
104
- karafka-web (>= 0.10.0.beta1)
112
+ karafka-testing (>= 2.4.6)
113
+ karafka-web (>= 0.10.0.rc2)
105
114
  ostruct
106
115
  rspec
107
116
  simplecov
data/bin/integrations CHANGED
@@ -240,6 +240,11 @@ ARGV.each do |filter|
240
240
  end
241
241
  end
242
242
 
243
+ # Remove Rails 7.2 specs from Ruby 3.0 because it requires 3.1
244
+ specs.delete_if do |spec|
245
+ RUBY_VERSION < '3.1' && spec.include?('rails72')
246
+ end
247
+
243
248
  raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
244
249
 
245
250
  # Randomize order
data/certs/cert.pem ADDED
@@ -0,0 +1,26 @@
1
+ -----BEGIN CERTIFICATE-----
2
+ MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
3
+ YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
4
+ MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
5
+ dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
6
+ bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
7
+ PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
8
+ ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
9
+ Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
10
+ JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
11
+ T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
12
+ flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
13
+ vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
14
+ iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
15
+ HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
16
+ sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
17
+ bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
18
+ eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
19
+ Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
20
+ agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
21
+ TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
22
+ Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
23
+ oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
24
+ i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
25
+ ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
26
+ -----END CERTIFICATE-----
@@ -33,6 +33,8 @@ en:
33
33
  internal.processing.partitioner_class_format: cannot be nil
34
34
  internal.processing.strategy_selector_format: cannot be nil
35
35
  internal.processing.expansions_selector_format: cannot be nil
36
+ internal.processing.executor_class_format: cannot be nil
37
+ internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
36
38
 
37
39
  internal.active_job.dispatcher_format: cannot be nil
38
40
  internal.active_job.job_options_contract_format: cannot be nil
@@ -113,10 +115,12 @@ en:
113
115
  dead_letter_queue.transactional_format: needs to be either true or false
114
116
  dead_letter_queue.dispatch_method_format: 'needs to be either #produce_sync or #produce_async'
115
117
  dead_letter_queue.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
118
+ dead_letter_queue.mark_after_dispatch_format: 'needs to be true, false or nil'
116
119
 
117
120
  active_format: needs to be either true or false
118
121
 
119
122
  eofed.active_format: needs to be either true or false
123
+ eofed.kafka_enable: 'cannot be enabled without enable.partition.eof set to true'
120
124
 
121
125
  declaratives.partitions_format: needs to be more or equal to 1
122
126
  declaratives.active_format: needs to be true
@@ -63,6 +63,8 @@ en:
63
63
  swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
64
64
  swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
65
65
 
66
+ recurring_tasks.active_format: 'needs to be boolean'
67
+
66
68
  direct_assignments.active_missing: needs to be present
67
69
  direct_assignments.active_format: 'needs to be boolean'
68
70
  direct_assignments.partitions_missing: 'needs to be present'
@@ -99,5 +101,20 @@ en:
99
101
  patterns.ttl_format: needs to be an integer bigger than 0
100
102
  patterns.ttl_missing: needs to be present
101
103
 
104
+ recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
105
+ recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
106
+ recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
107
+ recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
108
+ recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
109
+ recurring_tasks.deserializer_format: 'needs to be configured'
110
+ recurring_tasks.logging_format: needs to be a boolean
111
+
102
112
  routing:
103
113
  swarm_nodes_not_used: 'At least one of the nodes has no assignments'
114
+
115
+ recurring_tasks:
116
+ id_format: 'can include only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_)'
117
+ cron_format: must be a non-empty string
118
+ enabled_format: needs to be a boolean
119
+ changed_format: needs to be a boolean
120
+ previous_time_format: needs to be a numerical or time
data/karafka.gemspec CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
33
33
  spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
34
34
  end
35
35
 
36
- spec.cert_chain = %w[certs/cert_chain.pem]
36
+ spec.cert_chain = %w[certs/cert.pem]
37
37
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
38
38
  spec.executables = %w[karafka]
39
39
  spec.require_paths = %w[lib]
data/lib/karafka/admin.rb CHANGED
@@ -274,6 +274,48 @@ module Karafka
274
274
  end
275
275
  end
276
276
 
277
+ # Takes consumer group and its topics and migrates all the offsets to a new named group
278
+ #
279
+ # @param previous_name [String] old consumer group name
280
+ # @param new_name [String] new consumer group name
281
+ # @param topics [Array<String>] topics for which we want to migrate offsets during rename
282
+ # @param delete_previous [Boolean] should we delete previous consumer group after rename.
283
+ # Defaults to true.
284
+ #
285
+ # @note This method should **not** be executed on a running consumer group as it creates a
286
+ # "fake" consumer and uses it to move offsets.
287
+ #
288
+ # @note After migration unless `delete_previous` is set to `false`, old group will be
289
+ # removed.
290
+ #
291
+ # @note If new consumer group exists, old offsets will be added to it.
292
+ def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
293
+ remap = Hash.new { |h, k| h[k] = {} }
294
+
295
+ old_lags = read_lags_with_offsets({ previous_name => topics })
296
+
297
+ return if old_lags.empty?
298
+
299
+ read_lags_with_offsets({ previous_name => topics })
300
+ .fetch(previous_name)
301
+ .each do |topic, partitions|
302
+ partitions.each do |partition_id, details|
303
+ offset = details[:offset]
304
+
305
+ # No offset on this partition
306
+ next if offset.negative?
307
+
308
+ remap[topic][partition_id] = offset
309
+ end
310
+ end
311
+
312
+ seek_consumer_group(new_name, remap)
313
+
314
+ return unless delete_previous
315
+
316
+ delete_consumer_group(previous_name)
317
+ end
318
+
277
319
  # Removes given consumer group (if exists)
278
320
  #
279
321
  # @param consumer_group_id [String] consumer group name
@@ -116,6 +116,8 @@ module Karafka
116
116
  required(:partitioner_class) { |val| !val.nil? }
117
117
  required(:strategy_selector) { |val| !val.nil? }
118
118
  required(:expansions_selector) { |val| !val.nil? }
119
+ required(:executor_class) { |val| !val.nil? }
120
+ required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
119
121
  end
120
122
 
121
123
  nested(:active_job) do
@@ -82,10 +82,11 @@ module Karafka
82
82
  AssignmentLostError = Class.new(BaseError)
83
83
 
84
84
  # Raised if optional dependencies like karafka-web are required in a version that is not
85
- # supported by the current framework version.
85
+ # supported by the current framework version or when an optional dependency is missing.
86
86
  #
87
87
  # Because we do not want to require web out of the box and we do not want to lock web with
88
- # karafka 1:1, we do such a sanity check
88
+ # karafka 1:1, we do such a sanity check. This also applies to cases where some external
89
+ # optional dependencies are needed but not available.
89
90
  DependencyConstraintsError = Class.new(BaseError)
90
91
 
91
92
  # Raised when we were not able to open pidfd for given pid
@@ -75,7 +75,8 @@ module Karafka
75
75
  def features
76
76
  [
77
77
  Encryption,
78
- Cleaner
78
+ Cleaner,
79
+ RecurringTasks
79
80
  ]
80
81
  end
81
82
 
@@ -135,7 +135,12 @@ module Karafka
135
135
 
136
136
  dispatch = lambda do
137
137
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
138
- mark_dispatched_to_dlq(skippable_message)
138
+
139
+ if mark_after_dispatch?
140
+ mark_dispatched_to_dlq(skippable_message)
141
+ else
142
+ coordinator.seek_offset = skippable_message.offset + 1
143
+ end
139
144
  end
140
145
 
141
146
  if dispatch_in_a_transaction?
@@ -193,6 +198,16 @@ module Karafka
193
198
  producer.transactional? && topic.dead_letter_queue.transactional?
194
199
  end
195
200
 
201
+ # @return [Boolean] should we mark given message as consumed after dispatch.
202
+ # For default non MOM strategies if user did not explicitly tell us not to, we mark
203
+ # it. Default is `nil`, which means `true` in this case. If user provided alternative
204
+ # value, we go with it.
205
+ def mark_after_dispatch?
206
+ return true if topic.dead_letter_queue.mark_after_dispatch.nil?
207
+
208
+ topic.dead_letter_queue.mark_after_dispatch
209
+ end
210
+
196
211
  # Runs the DLQ strategy and based on it it performs certain operations
197
212
  #
198
213
  # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
@@ -55,7 +55,11 @@ module Karafka
55
55
  skippable_message, _marked = find_skippable_message
56
56
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
57
57
 
58
- coordinator.seek_offset = skippable_message.offset + 1
58
+ if mark_after_dispatch?
59
+ mark_dispatched_to_dlq(skippable_message)
60
+ else
61
+ coordinator.seek_offset = skippable_message.offset + 1
62
+ end
59
63
  end
60
64
  end
61
65
  end
@@ -46,11 +46,27 @@ module Karafka
46
46
  skippable_message, _marked = find_skippable_message
47
47
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
48
48
 
49
- coordinator.seek_offset = skippable_message.offset + 1
49
+ if mark_after_dispatch?
50
+ mark_dispatched_to_dlq(skippable_message)
51
+ else
52
+ coordinator.seek_offset = skippable_message.offset + 1
53
+ end
50
54
  end
51
55
  end
52
56
  end
53
57
  end
58
+
59
+ # @return [Boolean] should we mark given message as consumed after dispatch. For
60
+ # MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
61
+ # is `nil`, which means `false` in this case. If user provided alternative value, we
62
+ # go with it.
63
+ #
64
+ # @note Please note, this is the opposite behavior than in case of AOM strategies.
65
+ def mark_after_dispatch?
66
+ return false if topic.dead_letter_queue.mark_after_dispatch.nil?
67
+
68
+ topic.dead_letter_queue.mark_after_dispatch
69
+ end
54
70
  end
55
71
  end
56
72
  end
@@ -49,11 +49,27 @@ module Karafka
49
49
  skippable_message, _marked = find_skippable_message
50
50
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
51
51
 
52
- coordinator.seek_offset = skippable_message.offset + 1
52
+ if mark_after_dispatch?
53
+ mark_dispatched_to_dlq(skippable_message)
54
+ else
55
+ coordinator.seek_offset = skippable_message.offset + 1
56
+ end
53
57
  end
54
58
  end
55
59
  end
56
60
  end
61
+
62
+ # @return [Boolean] should we mark given message as consumed after dispatch. For
63
+ # MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
64
+ # is `nil`, which means `false` in this case. If user provided alternative value, we
65
+ # go with it.
66
+ #
67
+ # @note Please note, this is the opposite behavior than in case of AOM strategies.
68
+ def mark_after_dispatch?
69
+ return false if topic.dead_letter_queue.mark_after_dispatch.nil?
70
+
71
+ topic.dead_letter_queue.mark_after_dispatch
72
+ end
57
73
  end
58
74
  end
59
75
  end
@@ -40,16 +40,32 @@ module Karafka
40
40
  skippable_message, = find_skippable_message
41
41
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
42
42
 
43
- # Save the next offset we want to go with after moving given message to DLQ
44
- # Without this, we would not be able to move forward and we would end up
45
- # in an infinite loop trying to un-pause from the message we've already
46
- # processed. Of course, since it's a MoM a rebalance or kill, will move it back
47
- # as no offsets are being committed
48
- coordinator.seek_offset = skippable_message.offset + 1
43
+ if mark_after_dispatch?
44
+ mark_dispatched_to_dlq(skippable_message)
45
+ else
46
+ # Save the next offset we want to go with after moving given message to DLQ
47
+ # Without this, we would not be able to move forward and we would end up
48
+ # in an infinite loop trying to un-pause from the message we've already
49
+ # processed. Of course, since it's a MoM a rebalance or kill, will move it
50
+ # back as no offsets are being committed
51
+ coordinator.seek_offset = skippable_message.offset + 1
52
+ end
49
53
  end
50
54
  end
51
55
  end
52
56
  end
57
+
58
+ # @return [Boolean] should we mark given message as consumed after dispatch. For
59
+ # MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
60
+ # is `nil`, which means `false` in this case. If user provided alternative value, we
61
+ # go with it.
62
+ #
63
+ # @note Please note, this is the opposite behavior than in case of AOM strategies.
64
+ def mark_after_dispatch?
65
+ return false if topic.dead_letter_queue.mark_after_dispatch.nil?
66
+
67
+ topic.dead_letter_queue.mark_after_dispatch
68
+ end
53
69
  end
54
70
  end
55
71
  end
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module RecurringTasks
17
+ # Consumer responsible for management of the recurring tasks and their execution
18
+ # There are some assumptions made here that always need to be satisfied:
19
+ # - we only run schedules that are of same or newer version
20
+ # - we always mark as consumed in such a way, that the first message received after
21
+ # assignment (if any) is a state
22
+ class Consumer < ::Karafka::BaseConsumer
23
+ # @param args [Array] all arguments accepted by the consumer
24
+ def initialize(*args)
25
+ super
26
+ @executor = Executor.new
27
+ end
28
+
29
+ def consume
30
+ # There is nothing we can do if we operate on a newer schedule. In such cases we should
31
+ # just wait and re-raise error hoping someone will notice or that this will be
32
+ # reassigned to a process with newer schedule
33
+ raise(Errors::IncompatibleScheduleError) if @executor.incompatible?
34
+
35
+ messages.each do |message|
36
+ payload = message.payload
37
+ type = payload[:type]
38
+
39
+ case type
40
+ when 'schedule'
41
+ # If we're replaying data, we need to record the most recent stored state, so we
42
+ # can use this data to fully initialize the scheduler
43
+ @executor.update_state(payload) if @executor.replaying?
44
+
45
+ # If this is first message we cannot mark it on the previous offset
46
+ next if message.offset.zero?
47
+
48
+ # We always mark as consumed in such a way, that when replaying, we start from a
49
+ # schedule state message. This makes it easier to recover.
50
+ mark_as_consumed Karafka::Messages::Seek.new(
51
+ topic.name,
52
+ partition,
53
+ message.offset - 1
54
+ )
55
+ when 'command'
56
+ @executor.apply_command(payload)
57
+
58
+ next if @executor.replaying?
59
+
60
+ # Execute on each incoming command to have nice latency but only after replaying
61
+ # During replaying we should not execute because there may be more state changes
62
+ # that collectively have a different outcome
63
+ @executor.call
64
+ else
65
+ raise ::Karafka::Errors::UnsupportedCaseError, type
66
+ end
67
+ end
68
+
69
+ eofed if eofed?
70
+ end
71
+
72
+ # Starts the final replay process if we reached eof during replaying
73
+ def eofed
74
+ # We only mark as replayed if we were replaying in the first place
75
+ # If already replayed, nothing to do
76
+ return unless @executor.replaying?
77
+
78
+ @executor.replay
79
+ end
80
+
81
+ # Runs the cron execution if all good.
82
+ def tick
83
+ # Do nothing until we fully recover the correct state
84
+ return if @executor.replaying?
85
+
86
+ # If the state is incompatible, we can only raise an error.
87
+ # We do it here and in the `#consume` so the pause-retry kicks in basically reporting
88
+ # on this issue once every minute. That way user should not miss this.
89
+ # We use seek to move so we can achieve a pause of 60 seconds in between consecutive
90
+ # errors instead of on each tick because it is much more frequent.
91
+ if @executor.incompatible?
92
+ if messages.empty?
93
+ raise Errors::IncompatibleScheduleError
94
+ else
95
+ return seek(messages.last.offset - 1)
96
+ end
97
+ end
98
+
99
+ # If all good and compatible we can execute the recurring tasks
100
+ @executor.call
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module RecurringTasks
17
+ # Recurring Tasks related contracts
18
+ module Contracts
19
+ # Makes sure, all the expected config is defined as it should be
20
+ class Config < ::Karafka::Contracts::Base
21
+ configure do |config|
22
+ config.error_messages = YAML.safe_load(
23
+ File.read(
24
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
+ )
26
+ ).fetch('en').fetch('validations').fetch('config')
27
+ end
28
+
29
+ nested(:recurring_tasks) do
30
+ required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
31
+ required(:deserializer) { |val| !val.nil? }
32
+ required(:logging) { |val| [true, false].include?(val) }
33
+ # Do not allow to run more often than every 5 seconds
34
+ required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
35
+ required(:group_id) do |val|
36
+ val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
37
+ end
38
+
39
+ nested(:topics) do
40
+ required(:schedules) do |val|
41
+ val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
42
+ end
43
+
44
+ required(:logs) do |val|
45
+ val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end