karafka 2.0.38 → 2.0.40

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +1 -1
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +34 -0
  6. data/Gemfile.lock +4 -4
  7. data/bin/integrations +1 -1
  8. data/config/locales/errors.yml +0 -7
  9. data/config/locales/pro_errors.yml +18 -0
  10. data/lib/karafka/base_consumer.rb +35 -55
  11. data/lib/karafka/connection/listener.rb +15 -10
  12. data/lib/karafka/errors.rb +0 -3
  13. data/lib/karafka/instrumentation/logger_listener.rb +44 -3
  14. data/lib/karafka/instrumentation/notifications.rb +4 -0
  15. data/lib/karafka/messages/builders/batch_metadata.rb +6 -5
  16. data/lib/karafka/messages/builders/messages.rb +3 -1
  17. data/lib/karafka/messages/messages.rb +5 -0
  18. data/lib/karafka/pro/active_job/consumer.rb +10 -1
  19. data/lib/karafka/pro/processing/coordinator.rb +13 -4
  20. data/lib/karafka/pro/processing/filters/base.rb +61 -0
  21. data/lib/karafka/pro/processing/filters/delayer.rb +70 -0
  22. data/lib/karafka/pro/processing/filters/expirer.rb +51 -0
  23. data/lib/karafka/pro/processing/filters/throttler.rb +84 -0
  24. data/lib/karafka/pro/processing/filters_applier.rb +100 -0
  25. data/lib/karafka/pro/processing/jobs_builder.rb +7 -3
  26. data/lib/karafka/pro/processing/scheduler.rb +24 -7
  27. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +68 -0
  28. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +74 -0
  29. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +72 -0
  30. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +76 -0
  31. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +62 -0
  32. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +68 -0
  33. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +64 -0
  34. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +69 -0
  35. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +38 -0
  36. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +64 -0
  37. data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +38 -0
  38. data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +58 -0
  39. data/lib/karafka/pro/processing/strategies/{dlq_lrj_vp.rb → aj/lrj_mom.rb} +14 -13
  40. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +77 -0
  41. data/lib/karafka/pro/processing/strategies/aj/mom.rb +36 -0
  42. data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +52 -0
  43. data/lib/karafka/pro/processing/strategies/dlq/default.rb +131 -0
  44. data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +61 -0
  45. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +75 -0
  46. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +74 -0
  47. data/lib/karafka/pro/processing/strategies/{mom.rb → dlq/ftr_lrj_vp.rb} +16 -19
  48. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +73 -0
  49. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +39 -0
  50. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +63 -0
  51. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +66 -0
  52. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +38 -0
  53. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +67 -0
  54. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +40 -0
  55. data/lib/karafka/pro/processing/strategies/ftr/default.rb +104 -0
  56. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +40 -0
  57. data/lib/karafka/pro/processing/strategies/lrj/default.rb +85 -0
  58. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +69 -0
  59. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +67 -0
  60. data/lib/karafka/pro/processing/strategies/{vp.rb → lrj/ftr_vp.rb} +15 -13
  61. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +78 -0
  62. data/lib/karafka/pro/processing/strategies/{aj_lrj_mom.rb → lrj/vp.rb} +13 -12
  63. data/lib/karafka/pro/processing/strategies/mom/default.rb +46 -0
  64. data/lib/karafka/pro/processing/strategies/mom/ftr.rb +53 -0
  65. data/lib/karafka/pro/processing/strategies/vp/default.rb +53 -0
  66. data/lib/karafka/pro/processing/{strategies/lrj_vp.rb → strategies.rb} +1 -13
  67. data/lib/karafka/pro/processing/strategy_selector.rb +44 -18
  68. data/lib/karafka/pro/{processing/strategies/aj_mom.rb → routing/features/delaying/config.rb} +7 -13
  69. data/lib/karafka/pro/routing/features/delaying/contract.rb +38 -0
  70. data/lib/karafka/pro/routing/features/delaying/topic.rb +59 -0
  71. data/lib/karafka/pro/routing/features/delaying.rb +29 -0
  72. data/lib/karafka/pro/routing/features/expiring/config.rb +27 -0
  73. data/lib/karafka/pro/routing/features/expiring/contract.rb +38 -0
  74. data/lib/karafka/pro/routing/features/expiring/topic.rb +59 -0
  75. data/lib/karafka/pro/routing/features/expiring.rb +27 -0
  76. data/lib/karafka/pro/routing/features/filtering/config.rb +40 -0
  77. data/lib/karafka/pro/routing/features/filtering/contract.rb +41 -0
  78. data/lib/karafka/pro/routing/features/filtering/topic.rb +51 -0
  79. data/lib/karafka/pro/routing/features/filtering.rb +27 -0
  80. data/lib/karafka/pro/routing/features/long_running_job/contract.rb +1 -1
  81. data/lib/karafka/pro/routing/features/throttling/config.rb +32 -0
  82. data/lib/karafka/pro/routing/features/throttling/contract.rb +41 -0
  83. data/lib/karafka/pro/routing/features/throttling/topic.rb +69 -0
  84. data/lib/karafka/pro/routing/features/throttling.rb +30 -0
  85. data/lib/karafka/processing/coordinator.rb +60 -30
  86. data/lib/karafka/processing/coordinators_buffer.rb +5 -1
  87. data/lib/karafka/processing/executor.rb +37 -21
  88. data/lib/karafka/processing/executors_buffer.rb +10 -26
  89. data/lib/karafka/processing/jobs/consume.rb +2 -4
  90. data/lib/karafka/processing/jobs/idle.rb +24 -0
  91. data/lib/karafka/processing/jobs_builder.rb +2 -3
  92. data/lib/karafka/processing/result.rb +5 -0
  93. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  94. data/lib/karafka/processing/strategies/base.rb +5 -0
  95. data/lib/karafka/processing/strategies/default.rb +50 -0
  96. data/lib/karafka/processing/strategies/dlq.rb +13 -4
  97. data/lib/karafka/processing/strategies/dlq_mom.rb +8 -3
  98. data/lib/karafka/processing/strategy_selector.rb +27 -10
  99. data/lib/karafka/version.rb +1 -1
  100. data/renovate.json +6 -0
  101. data.tar.gz.sig +0 -0
  102. metadata +66 -22
  103. metadata.gz.sig +0 -0
  104. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb +0 -42
  105. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +0 -70
  106. data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +0 -62
  107. data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +0 -68
  108. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +0 -75
  109. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +0 -62
  110. data/lib/karafka/pro/processing/strategies/dlq.rb +0 -120
  111. data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +0 -65
  112. data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +0 -62
  113. data/lib/karafka/pro/processing/strategies/dlq_mom.rb +0 -62
  114. data/lib/karafka/pro/processing/strategies/dlq_vp.rb +0 -37
  115. data/lib/karafka/pro/processing/strategies/lrj.rb +0 -83
  116. data/lib/karafka/pro/processing/strategies/lrj_mom.rb +0 -73
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ # Namespace for all the strategies starting with DLQ
19
+ module Dlq
20
+ # Only dead letter queue enabled
21
+ module Default
22
+ include Strategies::Default
23
+
24
+ # Features for this strategy
25
+ FEATURES = %i[
26
+ dead_letter_queue
27
+ ].freeze
28
+
29
+ # When we encounter non-recoverable message, we skip it and go on with our lives
30
+ def handle_after_consume
31
+ coordinator.on_finished do
32
+ return if revoked?
33
+
34
+ if coordinator.success?
35
+ coordinator.pause_tracker.reset
36
+
37
+ return if coordinator.manual_pause?
38
+
39
+ mark_as_consumed(messages.last)
40
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
41
+ retry_after_pause
42
+ # If we've reached number of retries that we could, we need to skip the first
43
+ # message that was not marked as consumed, pause and continue, while also moving
44
+ # this message to the dead topic
45
+ else
46
+ # We reset the pause to indicate we will now consider it as "ok".
47
+ coordinator.pause_tracker.reset
48
+ skippable_message, = find_skippable_message
49
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
50
+ mark_as_consumed(skippable_message)
51
+ pause(coordinator.seek_offset, nil, false)
52
+ end
53
+ end
54
+ end
55
+
56
+ # Finds the message may want to skip (all, starting from first)
57
+ # @private
58
+ # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
59
+ # information if this message was from marked offset or figured out via mom flow
60
+ def find_skippable_message
61
+ skippable_message = messages.find do |msg|
62
+ coordinator.marked? && msg.offset == coordinator.seek_offset
63
+ end
64
+
65
+ # If we don't have the message matching the last comitted offset, it means that
66
+ # user operates with manual offsets and we're beyond the batch in which things
67
+ # broke for the first time. Then we skip the first (as no markings) and we
68
+ # move on one by one.
69
+ skippable_message ? [skippable_message, true] : [messages.first, false]
70
+ end
71
+
72
+ # Moves the broken message into a separate queue defined via the settings
73
+ #
74
+ # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
75
+ # dispatch to DLQ
76
+ def dispatch_to_dlq(skippable_message)
77
+ producer.produce_async(
78
+ build_dlq_message(
79
+ skippable_message
80
+ )
81
+ )
82
+
83
+ # Notify about dispatch on the events bus
84
+ Karafka.monitor.instrument(
85
+ 'dead_letter_queue.dispatched',
86
+ caller: self,
87
+ message: skippable_message
88
+ )
89
+ end
90
+
91
+ # @param skippable_message [Array<Karafka::Messages::Message>]
92
+ # @return [Hash] dispatch DLQ message
93
+ def build_dlq_message(skippable_message)
94
+ original_partition = skippable_message.partition.to_s
95
+
96
+ dlq_message = {
97
+ topic: topic.dead_letter_queue.topic,
98
+ key: original_partition,
99
+ payload: skippable_message.raw_payload,
100
+ headers: skippable_message.headers.merge(
101
+ 'original_topic' => topic.name,
102
+ 'original_partition' => original_partition,
103
+ 'original_offset' => skippable_message.offset.to_s,
104
+ 'original_consumer_group' => topic.consumer_group.id
105
+ )
106
+ }
107
+
108
+ # Optional method user can define in consumer to enhance the dlq message hash with
109
+ # some extra details if needed or to replace payload, etc
110
+ if respond_to?(:enhance_dlq_message, true)
111
+ enhance_dlq_message(
112
+ dlq_message,
113
+ skippable_message
114
+ )
115
+ end
116
+
117
+ dlq_message
118
+ end
119
+
120
+ # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch
121
+ # topic is set to false, we will skip the dispatch, effectively ignoring the broken
122
+ # message without taking any action.
123
+ def dispatch_to_dlq?
124
+ topic.dead_letter_queue.topic
125
+ end
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # - DLQ
20
+ # - Ftr
21
+ module Ftr
22
+ include Strategies::Ftr::Default
23
+ include Strategies::Dlq::Vp
24
+
25
+ # Features for this strategy
26
+ FEATURES = %i[
27
+ dead_letter_queue
28
+ filtering
29
+ ].freeze
30
+
31
+ # DLQ flow is standard here, what is not, is the success component where we need to
32
+ # take into consideration the filtering
33
+ def handle_after_consume
34
+ coordinator.on_finished do
35
+ return if revoked?
36
+
37
+ if coordinator.success?
38
+ coordinator.pause_tracker.reset
39
+
40
+ return if coordinator.manual_pause?
41
+
42
+ mark_as_consumed(messages.last)
43
+
44
+ handle_post_filtering
45
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
46
+ retry_after_pause
47
+ else
48
+ coordinator.pause_tracker.reset
49
+ skippable_message, = find_skippable_message
50
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
51
+ mark_as_consumed(skippable_message)
52
+ pause(coordinator.seek_offset, nil, false)
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # Dead-Letter Queue enabled
20
+ # Filtering enabled
21
+ # Long-Running Job enabled
22
+ module FtrLrj
23
+ include Strategies::Dlq::Lrj
24
+ include Strategies::Lrj::Ftr
25
+
26
+ # Features for this strategy
27
+ FEATURES = %i[
28
+ dead_letter_queue
29
+ filtering
30
+ long_running_job
31
+ ].freeze
32
+
33
+ # This is one of more complex cases.
34
+ # We need to ensure, that we always resume (inline or via paused backoff) and we need
35
+ # to make sure we dispatch to DLQ when needed. Because revocation on LRJ can happen
36
+ # any time, we need to make sure we do not dispatch to DLQ when error happens but we
37
+ # no longer own the assignment. Throttling is another factor that has to be taken
38
+ # into consideration on the successful path
39
+ def handle_after_consume
40
+ coordinator.on_finished do |last_group_message|
41
+ if coordinator.success?
42
+ coordinator.pause_tracker.reset
43
+
44
+ return if coordinator.manual_pause?
45
+
46
+ mark_as_consumed(last_group_message) unless revoked?
47
+
48
+ if coordinator.filtered? && !revoked?
49
+ handle_post_filtering
50
+ elsif !revoked?
51
+ seek(coordinator.seek_offset)
52
+ resume
53
+ else
54
+ resume
55
+ end
56
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
57
+ retry_after_pause
58
+ else
59
+ coordinator.pause_tracker.reset
60
+
61
+ return resume if revoked?
62
+
63
+ skippable_message, = find_skippable_message
64
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
65
+ mark_as_consumed(skippable_message)
66
+ pause(coordinator.seek_offset, nil, false)
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # DLQ enabled
20
+ # Ftr enabled
21
+ # LRJ enabled
22
+ # MoM enabled
23
+ module FtrLrjMom
24
+ include Strategies::Ftr::Default
25
+ include Strategies::Dlq::LrjMom
26
+
27
+ # Features for this strategy
28
+ FEATURES = %i[
29
+ dead_letter_queue
30
+ filtering
31
+ long_running_job
32
+ manual_offset_management
33
+ ].freeze
34
+
35
+ # Post execution flow of this strategy
36
+ def handle_after_consume
37
+ coordinator.on_finished do |last_group_message|
38
+ if coordinator.success?
39
+ coordinator.pause_tracker.reset
40
+
41
+ return if coordinator.manual_pause?
42
+
43
+ if coordinator.filtered? && !revoked?
44
+ handle_post_filtering
45
+ elsif !revoked?
46
+ seek(last_group_message.offset + 1)
47
+ resume
48
+ else
49
+ resume
50
+ end
51
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
52
+ retry_after_pause
53
+ else
54
+ coordinator.pause_tracker.reset
55
+
56
+ return resume if revoked?
57
+
58
+ skippable_message, marked = find_skippable_message
59
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
60
+
61
+ if marked
62
+ pause(coordinator.seek_offset, nil, false)
63
+ else
64
+ pause(skippable_message.offset + 1, nil, false)
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -15,26 +15,23 @@ module Karafka
15
15
  module Pro
16
16
  module Processing
17
17
  module Strategies
18
- # Manual offset management enabled
19
- module Mom
20
- include Default
18
+ module Dlq
19
+ # Dead-Letter Queue enabled
20
+ # Filtering enabled
21
+ # Long-Running Job enabled
22
+ # Virtual Partitions
23
+ module FtrLrjVp
24
+ include Strategies::Vp::Default
25
+ # Same as non VP because of the coordinator post-execution lock
26
+ include Strategies::Dlq::FtrLrj
21
27
 
22
- # Features for this strategy
23
- FEATURES = %i[
24
- manual_offset_management
25
- ].freeze
26
-
27
- # When mom is enabled, we do not mark messages as consumed after processing
28
- def handle_after_consume
29
- coordinator.on_finished do
30
- return if revoked?
31
-
32
- if coordinator.success?
33
- coordinator.pause_tracker.reset
34
- else
35
- retry_after_pause
36
- end
37
- end
28
+ # Features for this strategy
29
+ FEATURES = %i[
30
+ dead_letter_queue
31
+ filtering
32
+ long_running_job
33
+ virtual_partitions
34
+ ].freeze
38
35
  end
39
36
  end
40
37
  end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # - DLQ
20
+ # - Ftr
21
+ # - Mom
22
+ module FtrMom
23
+ include Strategies::Ftr::Default
24
+ include Strategies::Dlq::Default
25
+
26
+ # Features for this strategy
27
+ FEATURES = %i[
28
+ dead_letter_queue
29
+ filtering
30
+ manual_offset_management
31
+ ].freeze
32
+
33
+ # On mom we do not mark, throttling and seeking as in other strategies
34
+ def handle_after_consume
35
+ coordinator.on_finished do
36
+ return if revoked?
37
+
38
+ if coordinator.success?
39
+ coordinator.pause_tracker.reset
40
+
41
+ return if coordinator.manual_pause?
42
+
43
+ handle_post_filtering
44
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
45
+ retry_after_pause
46
+ # If we've reached number of retries that we could, we need to skip the first
47
+ # message that was not marked as consumed, pause and continue, while also moving
48
+ # this message to the dead topic.
49
+ #
50
+ # For a Mom setup, this means, that user has to manage the checkpointing by
51
+ # himself. If no checkpointing is ever done, we end up with an endless loop.
52
+ else
53
+ # We reset the pause to indicate we will now consider it as "ok".
54
+ coordinator.pause_tracker.reset
55
+
56
+ skippable_message, marked = find_skippable_message
57
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
58
+
59
+ # Move beyond the broken message
60
+ if marked
61
+ pause(coordinator.seek_offset, nil, false)
62
+ else
63
+ pause(skippable_message.offset + 1, nil, false)
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # - DLQ
20
+ # - Ftr
21
+ # - VPs
22
+ #
23
+ # Behaves same as non-VP due to coordinator lock
24
+ module FtrVp
25
+ include Strategies::Vp::Default
26
+ include Strategies::Dlq::Ftr
27
+
28
+ # Features for this strategy
29
+ FEATURES = %i[
30
+ dead_letter_queue
31
+ filtering
32
+ virtual_partitions
33
+ ].freeze
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # DLQ enabled
20
+ # Long-Running Job enabled
21
+ module Lrj
22
+ # Order here matters, lrj needs to be second
23
+ include Strategies::Dlq::Default
24
+ include Strategies::Lrj::Default
25
+
26
+ # Features for this strategy
27
+ FEATURES = %i[
28
+ dead_letter_queue
29
+ long_running_job
30
+ ].freeze
31
+
32
+ # LRJ standard flow after consumption with DLQ dispatch
33
+ def handle_after_consume
34
+ coordinator.on_finished do |last_group_message|
35
+ if coordinator.success?
36
+ coordinator.pause_tracker.reset
37
+
38
+ return if coordinator.manual_pause?
39
+
40
+ mark_as_consumed(last_group_message) unless revoked?
41
+ seek(coordinator.seek_offset) unless revoked?
42
+
43
+ resume
44
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
45
+ retry_after_pause
46
+ else
47
+ coordinator.pause_tracker.reset
48
+
49
+ return resume if revoked?
50
+
51
+ skippable_message, = find_skippable_message
52
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
53
+ mark_as_consumed(skippable_message)
54
+ pause(coordinator.seek_offset, nil, false)
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # Strategy for supporting DLQ with Mom and LRJ enabled
20
+ module LrjMom
21
+ # This strategy needs to pause and revoke same way as DlqLrj but without the offset
22
+ # management
23
+ include Strategies::Dlq::Lrj
24
+
25
+ # Features for this strategy
26
+ FEATURES = %i[
27
+ dead_letter_queue
28
+ long_running_job
29
+ manual_offset_management
30
+ ].freeze
31
+
32
+ # LRJ standard flow after consumption with DLQ dispatch and no offset management
33
+ def handle_after_consume
34
+ coordinator.on_finished do |last_group_message|
35
+ if coordinator.success?
36
+ coordinator.pause_tracker.reset
37
+
38
+ return if coordinator.manual_pause?
39
+
40
+ seek(last_group_message.offset + 1) unless revoked?
41
+
42
+ resume
43
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
44
+ retry_after_pause
45
+ else
46
+ coordinator.pause_tracker.reset
47
+
48
+ return resume if revoked?
49
+
50
+ skippable_message, marked = find_skippable_message
51
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
52
+
53
+ if marked
54
+ pause(coordinator.seek_offset, nil, false)
55
+ else
56
+ pause(skippable_message.offset + 1, nil, false)
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ module Dlq
19
+ # Dead-Letter Queue enabled
20
+ # Long-Running Job enabled
21
+ # Virtual Partitions enabled
22
+ module LrjVp
23
+ # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
24
+ include Strategies::Vp::Default
25
+ include Strategies::Dlq::Lrj
26
+
27
+ # Features for this strategy
28
+ FEATURES = %i[
29
+ dead_letter_queue
30
+ long_running_job
31
+ virtual_partitions
32
+ ].freeze
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end