karafka 2.0.38 → 2.0.40

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +1 -1
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +34 -0
  6. data/Gemfile.lock +4 -4
  7. data/bin/integrations +1 -1
  8. data/config/locales/errors.yml +0 -7
  9. data/config/locales/pro_errors.yml +18 -0
  10. data/lib/karafka/base_consumer.rb +35 -55
  11. data/lib/karafka/connection/listener.rb +15 -10
  12. data/lib/karafka/errors.rb +0 -3
  13. data/lib/karafka/instrumentation/logger_listener.rb +44 -3
  14. data/lib/karafka/instrumentation/notifications.rb +4 -0
  15. data/lib/karafka/messages/builders/batch_metadata.rb +6 -5
  16. data/lib/karafka/messages/builders/messages.rb +3 -1
  17. data/lib/karafka/messages/messages.rb +5 -0
  18. data/lib/karafka/pro/active_job/consumer.rb +10 -1
  19. data/lib/karafka/pro/processing/coordinator.rb +13 -4
  20. data/lib/karafka/pro/processing/filters/base.rb +61 -0
  21. data/lib/karafka/pro/processing/filters/delayer.rb +70 -0
  22. data/lib/karafka/pro/processing/filters/expirer.rb +51 -0
  23. data/lib/karafka/pro/processing/filters/throttler.rb +84 -0
  24. data/lib/karafka/pro/processing/filters_applier.rb +100 -0
  25. data/lib/karafka/pro/processing/jobs_builder.rb +7 -3
  26. data/lib/karafka/pro/processing/scheduler.rb +24 -7
  27. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +68 -0
  28. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +74 -0
  29. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +72 -0
  30. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +76 -0
  31. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +62 -0
  32. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +68 -0
  33. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +64 -0
  34. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +69 -0
  35. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +38 -0
  36. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +64 -0
  37. data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +38 -0
  38. data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +58 -0
  39. data/lib/karafka/pro/processing/strategies/{dlq_lrj_vp.rb → aj/lrj_mom.rb} +14 -13
  40. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +77 -0
  41. data/lib/karafka/pro/processing/strategies/aj/mom.rb +36 -0
  42. data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +52 -0
  43. data/lib/karafka/pro/processing/strategies/dlq/default.rb +131 -0
  44. data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +61 -0
  45. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +75 -0
  46. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +74 -0
  47. data/lib/karafka/pro/processing/strategies/{mom.rb → dlq/ftr_lrj_vp.rb} +16 -19
  48. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +73 -0
  49. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +39 -0
  50. data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +63 -0
  51. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +66 -0
  52. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +38 -0
  53. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +67 -0
  54. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +40 -0
  55. data/lib/karafka/pro/processing/strategies/ftr/default.rb +104 -0
  56. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +40 -0
  57. data/lib/karafka/pro/processing/strategies/lrj/default.rb +85 -0
  58. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +69 -0
  59. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +67 -0
  60. data/lib/karafka/pro/processing/strategies/{vp.rb → lrj/ftr_vp.rb} +15 -13
  61. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +78 -0
  62. data/lib/karafka/pro/processing/strategies/{aj_lrj_mom.rb → lrj/vp.rb} +13 -12
  63. data/lib/karafka/pro/processing/strategies/mom/default.rb +46 -0
  64. data/lib/karafka/pro/processing/strategies/mom/ftr.rb +53 -0
  65. data/lib/karafka/pro/processing/strategies/vp/default.rb +53 -0
  66. data/lib/karafka/pro/processing/{strategies/lrj_vp.rb → strategies.rb} +1 -13
  67. data/lib/karafka/pro/processing/strategy_selector.rb +44 -18
  68. data/lib/karafka/pro/{processing/strategies/aj_mom.rb → routing/features/delaying/config.rb} +7 -13
  69. data/lib/karafka/pro/routing/features/delaying/contract.rb +38 -0
  70. data/lib/karafka/pro/routing/features/delaying/topic.rb +59 -0
  71. data/lib/karafka/pro/routing/features/delaying.rb +29 -0
  72. data/lib/karafka/pro/routing/features/expiring/config.rb +27 -0
  73. data/lib/karafka/pro/routing/features/expiring/contract.rb +38 -0
  74. data/lib/karafka/pro/routing/features/expiring/topic.rb +59 -0
  75. data/lib/karafka/pro/routing/features/expiring.rb +27 -0
  76. data/lib/karafka/pro/routing/features/filtering/config.rb +40 -0
  77. data/lib/karafka/pro/routing/features/filtering/contract.rb +41 -0
  78. data/lib/karafka/pro/routing/features/filtering/topic.rb +51 -0
  79. data/lib/karafka/pro/routing/features/filtering.rb +27 -0
  80. data/lib/karafka/pro/routing/features/long_running_job/contract.rb +1 -1
  81. data/lib/karafka/pro/routing/features/throttling/config.rb +32 -0
  82. data/lib/karafka/pro/routing/features/throttling/contract.rb +41 -0
  83. data/lib/karafka/pro/routing/features/throttling/topic.rb +69 -0
  84. data/lib/karafka/pro/routing/features/throttling.rb +30 -0
  85. data/lib/karafka/processing/coordinator.rb +60 -30
  86. data/lib/karafka/processing/coordinators_buffer.rb +5 -1
  87. data/lib/karafka/processing/executor.rb +37 -21
  88. data/lib/karafka/processing/executors_buffer.rb +10 -26
  89. data/lib/karafka/processing/jobs/consume.rb +2 -4
  90. data/lib/karafka/processing/jobs/idle.rb +24 -0
  91. data/lib/karafka/processing/jobs_builder.rb +2 -3
  92. data/lib/karafka/processing/result.rb +5 -0
  93. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  94. data/lib/karafka/processing/strategies/base.rb +5 -0
  95. data/lib/karafka/processing/strategies/default.rb +50 -0
  96. data/lib/karafka/processing/strategies/dlq.rb +13 -4
  97. data/lib/karafka/processing/strategies/dlq_mom.rb +8 -3
  98. data/lib/karafka/processing/strategy_selector.rb +27 -10
  99. data/lib/karafka/version.rb +1 -1
  100. data/renovate.json +6 -0
  101. data.tar.gz.sig +0 -0
  102. metadata +66 -22
  103. metadata.gz.sig +0 -0
  104. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb +0 -42
  105. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +0 -70
  106. data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +0 -62
  107. data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +0 -68
  108. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +0 -75
  109. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +0 -62
  110. data/lib/karafka/pro/processing/strategies/dlq.rb +0 -120
  111. data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +0 -65
  112. data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +0 -62
  113. data/lib/karafka/pro/processing/strategies/dlq_mom.rb +0 -62
  114. data/lib/karafka/pro/processing/strategies/dlq_vp.rb +0 -37
  115. data/lib/karafka/pro/processing/strategies/lrj.rb +0 -83
  116. data/lib/karafka/pro/processing/strategies/lrj_mom.rb +0 -73
@@ -1,62 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # ActiveJob enabled
19
- # Manual offset management enabled
20
- # Virtual Partitions enabled
21
- module AjMomVp
22
- include Vp
23
- include Default
24
-
25
- # Features for this strategy
26
- FEATURES = %i[
27
- active_job
28
- manual_offset_management
29
- virtual_partitions
30
- ].freeze
31
-
32
- # Standard flow without any features
33
- def handle_after_consume
34
- coordinator.on_finished do |last_group_message|
35
- if coordinator.success?
36
- coordinator.pause_tracker.reset
37
-
38
- # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
39
- # intermediate jobs as processed not to mess up with the ordering.
40
- # Only when all the jobs are processed and we did not loose the partition
41
- # assignment and we are not stopping (Pro ActiveJob has an early break) we can
42
- # commit offsets on this as only then we can be sure, that all the jobs were
43
- # processed.
44
- # For a non virtual partitions case, the flow is regular and state is marked after
45
- # each successfully processed job
46
- #
47
- # We can mark and we do mark intermediate jobs in the collapsed mode when running
48
- # VPs
49
- return if revoked?
50
- return if Karafka::App.stopping?
51
-
52
- mark_as_consumed(last_group_message)
53
- else
54
- retry_after_pause
55
- end
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
62
- end
@@ -1,120 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Only dead letter queue enabled
19
- module Dlq
20
- include Default
21
-
22
- # Features for this strategy
23
- FEATURES = %i[
24
- dead_letter_queue
25
- ].freeze
26
-
27
- # When we encounter non-recoverable message, we skip it and go on with our lives
28
- def handle_after_consume
29
- coordinator.on_finished do
30
- return if revoked?
31
-
32
- if coordinator.success?
33
- coordinator.pause_tracker.reset
34
-
35
- return if coordinator.manual_pause?
36
-
37
- mark_as_consumed(messages.last)
38
- elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
39
- retry_after_pause
40
- # If we've reached number of retries that we could, we need to skip the first message
41
- # that was not marked as consumed, pause and continue, while also moving this message
42
- # to the dead topic
43
- else
44
- # We reset the pause to indicate we will now consider it as "ok".
45
- coordinator.pause_tracker.reset
46
- skippable_message = find_skippable_message
47
- dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
48
- mark_as_consumed(skippable_message)
49
- pause(coordinator.seek_offset, nil, false)
50
- end
51
- end
52
- end
53
-
54
- # Finds the message may want to skip (all, starting from first)
55
- # @private
56
- # @return [Karafka::Messages::Message] message we may want to skip
57
- def find_skippable_message
58
- skippable_message = messages.find { |msg| msg.offset == coordinator.seek_offset }
59
- skippable_message || raise(Errors::SkipMessageNotFoundError, topic.name)
60
- end
61
-
62
- # Moves the broken message into a separate queue defined via the settings
63
- #
64
- # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
65
- # dispatch to DLQ
66
- def dispatch_to_dlq(skippable_message)
67
- producer.produce_async(
68
- build_dlq_message(
69
- skippable_message
70
- )
71
- )
72
-
73
- # Notify about dispatch on the events bus
74
- Karafka.monitor.instrument(
75
- 'dead_letter_queue.dispatched',
76
- caller: self,
77
- message: skippable_message
78
- )
79
- end
80
-
81
- # @param skippable_message [Array<Karafka::Messages::Message>]
82
- # @return [Hash] dispatch DLQ message
83
- def build_dlq_message(skippable_message)
84
- original_partition = skippable_message.partition.to_s
85
-
86
- dlq_message = {
87
- topic: topic.dead_letter_queue.topic,
88
- key: original_partition,
89
- payload: skippable_message.raw_payload,
90
- headers: skippable_message.headers.merge(
91
- 'original_topic' => topic.name,
92
- 'original_partition' => original_partition,
93
- 'original_offset' => skippable_message.offset.to_s,
94
- 'original_consumer_group' => topic.consumer_group.id
95
- )
96
- }
97
-
98
- # Optional method user can define in consumer to enhance the dlq message hash with
99
- # some extra details if needed or to replace payload, etc
100
- if respond_to?(:enhance_dlq_message, true)
101
- enhance_dlq_message(
102
- dlq_message,
103
- skippable_message
104
- )
105
- end
106
-
107
- dlq_message
108
- end
109
-
110
- # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
111
- # is set to false, we will skip the dispatch, effectively ignoring the broken message
112
- # without taking any action.
113
- def dispatch_to_dlq?
114
- topic.dead_letter_queue.topic
115
- end
116
- end
117
- end
118
- end
119
- end
120
- end
@@ -1,65 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # DLQ enabled
19
- # Long-Running Job enabled
20
- module DlqLrj
21
- # Order here matters, lrj needs to be second
22
- include Dlq
23
- include Lrj
24
-
25
- # Features for this strategy
26
- FEATURES = %i[
27
- dead_letter_queue
28
- long_running_job
29
- ].freeze
30
-
31
- # LRJ standard flow after consumption with DLQ dispatch
32
- def handle_after_consume
33
- coordinator.on_finished do |last_group_message|
34
- if coordinator.success?
35
- coordinator.pause_tracker.reset
36
-
37
- return if coordinator.manual_pause?
38
-
39
- mark_as_consumed(last_group_message) unless revoked?
40
- seek(coordinator.seek_offset) unless revoked?
41
-
42
- resume
43
- elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
44
- retry_after_pause
45
- else
46
- coordinator.pause_tracker.reset
47
-
48
- unless revoked?
49
- skippable_message = find_skippable_message
50
- dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
51
- mark_as_consumed(skippable_message)
52
- end
53
-
54
- # This revoke might have changed state due to marking, hence checked again
55
- seek(coordinator.seek_offset) unless revoked?
56
-
57
- resume
58
- end
59
- end
60
- end
61
- end
62
- end
63
- end
64
- end
65
- end
@@ -1,62 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Strategy for supporting DLQ with Mom and LRJ enabled
19
- module DlqLrjMom
20
- # This strategy needs to pause and revoke same way as DlqLrj but without the offset
21
- # management
22
- include DlqLrj
23
-
24
- # Features for this strategy
25
- FEATURES = %i[
26
- dead_letter_queue
27
- long_running_job
28
- manual_offset_management
29
- ].freeze
30
-
31
- # LRJ standard flow after consumption with DLQ dispatch and no offset management
32
- def handle_after_consume
33
- coordinator.on_finished do
34
- if coordinator.success?
35
- coordinator.pause_tracker.reset
36
-
37
- seek(coordinator.seek_offset) unless revoked?
38
-
39
- resume
40
- elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
41
- retry_after_pause
42
- else
43
- coordinator.pause_tracker.reset
44
-
45
- unless revoked?
46
- if dispatch_to_dlq?
47
- skippable_message = find_skippable_message
48
- dispatch_to_dlq(skippable_message)
49
- end
50
-
51
- seek(coordinator.seek_offset)
52
- end
53
-
54
- resume
55
- end
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
62
- end
@@ -1,62 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Strategy for supporting DLQ with Mom enabled
19
- module DlqMom
20
- # The broken message lookup is the same in this scenario
21
- include Dlq
22
-
23
- # Features for this strategy
24
- FEATURES = %i[
25
- dead_letter_queue
26
- manual_offset_management
27
- ].freeze
28
-
29
- # When manual offset management is on, we do not mark anything as consumed automatically
30
- # and we rely on the user to figure things out
31
- def handle_after_consume
32
- coordinator.on_finished do
33
- return if revoked?
34
-
35
- if coordinator.success?
36
- coordinator.pause_tracker.reset
37
- elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
38
- retry_after_pause
39
- # If we've reached number of retries that we could, we need to skip the first message
40
- # that was not marked as consumed, pause and continue, while also moving this message
41
- # to the dead topic.
42
- #
43
- # For a Mom setup, this means, that user has to manage the checkpointing by himself.
44
- # If no checkpointing is ever done, we end up with an endless loop.
45
- else
46
- # We reset the pause to indicate we will now consider it as "ok".
47
- coordinator.pause_tracker.reset
48
-
49
- if dispatch_to_dlq?
50
- skippable_message = find_skippable_message
51
- dispatch_to_dlq(skippable_message)
52
- end
53
-
54
- pause(coordinator.seek_offset, nil, false)
55
- end
56
- end
57
- end
58
- end
59
- end
60
- end
61
- end
62
- end
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Dead Letter Queue enabled
19
- # Virtual Partitions enabled
20
- #
21
- # In general because we collapse processing in virtual partitions to one on errors, there
22
- # is no special action that needs to be taken because we warranty that even with VPs
23
- # on errors a retry collapses into a single state.
24
- module DlqVp
25
- # Features for this strategy
26
- FEATURES = %i[
27
- dead_letter_queue
28
- virtual_partitions
29
- ].freeze
30
-
31
- include Dlq
32
- include Vp
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,83 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Long-Running Job enabled
19
- module Lrj
20
- include Default
21
-
22
- # Pause for tops 31 years
23
- MAX_PAUSE_TIME = 1_000_000_000_000
24
-
25
- # Features for this strategy
26
- FEATURES = %i[
27
- long_running_job
28
- ].freeze
29
-
30
- # We always need to pause prior to doing any jobs for LRJ
31
- def handle_before_enqueue
32
- # This ensures that when running LRJ with VP, things operate as expected run only once
33
- # for all the virtual partitions collectively
34
- coordinator.on_enqueued do
35
- # Pause at the first message in a batch. That way in case of a crash, we will not
36
- # loose any messages.
37
- #
38
- # For VP it applies the same way and since VP cannot be used with MOM we should not
39
- # have any edge cases here.
40
- pause(coordinator.seek_offset, MAX_PAUSE_TIME, false)
41
- end
42
- end
43
-
44
- # LRJ standard flow after consumption
45
- def handle_after_consume
46
- coordinator.on_finished do |last_group_message|
47
- if coordinator.success?
48
- coordinator.pause_tracker.reset
49
-
50
- return if coordinator.manual_pause?
51
-
52
- mark_as_consumed(last_group_message) unless revoked?
53
- seek(coordinator.seek_offset) unless revoked?
54
-
55
- resume
56
- else
57
- # If processing failed, we need to pause
58
- # For long running job this will overwrite the default never-ending pause and will
59
- # cause the processing to keep going after the error backoff
60
- retry_after_pause
61
- end
62
- end
63
- end
64
-
65
- # We do not un-pause on revokations for LRJ
66
- def handle_revoked
67
- coordinator.on_revoked do
68
- # We do not want to resume on revocation in case of a LRJ.
69
- # For LRJ we resume after the successful processing or do a backoff pause in case of
70
- # a failure. Double non-blocking resume could cause problems in coordination.
71
- coordinator.revoke
72
- end
73
-
74
- Karafka.monitor.instrument('consumer.revoke', caller: self)
75
- Karafka.monitor.instrument('consumer.revoked', caller: self) do
76
- revoked
77
- end
78
- end
79
- end
80
- end
81
- end
82
- end
83
- end
@@ -1,73 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Processing
17
- module Strategies
18
- # Long-Running Job enabled
19
- # Manual offset management enabled
20
- module LrjMom
21
- include Default
22
-
23
- # Features for this strategy
24
- FEATURES = %i[
25
- long_running_job
26
- manual_offset_management
27
- ].freeze
28
-
29
- # We always need to pause prior to doing any jobs for LRJ
30
- def handle_before_enqueue
31
- # This ensures that when running LRJ with VP, things operate as expected run only once
32
- # for all the virtual partitions collectively
33
- coordinator.on_enqueued do
34
- # Pause at the first message in a batch. That way in case of a crash, we will not
35
- # loose any messages.
36
- #
37
- # For VP it applies the same way and since VP cannot be used with MOM we should not
38
- # have any edge cases here.
39
- pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME, false)
40
- end
41
- end
42
-
43
- # No offset management, aside from that typical LRJ
44
- def handle_after_consume
45
- coordinator.on_finished do
46
- if coordinator.success?
47
- coordinator.pause_tracker.reset
48
-
49
- seek(coordinator.seek_offset) unless revoked?
50
-
51
- resume
52
- else
53
- retry_after_pause
54
- end
55
- end
56
- end
57
-
58
- # We do not un-pause on revokations for LRJ
59
- def handle_revoked
60
- coordinator.on_revoked do
61
- coordinator.revoke
62
- end
63
-
64
- Karafka.monitor.instrument('consumer.revoke', caller: self)
65
- Karafka.monitor.instrument('consumer.revoked', caller: self) do
66
- revoked
67
- end
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end