karafka 2.0.27 → 2.0.29

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +34 -795
  4. data/Gemfile.lock +11 -10
  5. data/README.md +3 -2
  6. data/bin/karafka +1 -19
  7. data/bin/verify_license_integrity +2 -0
  8. data/config/locales/pro_errors.yml +1 -1
  9. data/karafka.gemspec +1 -1
  10. data/lib/karafka/cli/base.rb +20 -0
  11. data/lib/karafka/cli/console.rb +13 -8
  12. data/lib/karafka/connection/listener.rb +1 -1
  13. data/lib/karafka/instrumentation/callbacks/error.rb +1 -0
  14. data/lib/karafka/pro/active_job/consumer.rb +3 -2
  15. data/lib/karafka/pro/processing/collapser.rb +62 -0
  16. data/lib/karafka/pro/processing/coordinator.rb +17 -0
  17. data/lib/karafka/pro/processing/partitioner.rb +19 -5
  18. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +70 -0
  19. data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +68 -0
  20. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -0
  21. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +4 -0
  22. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  23. data/lib/karafka/pro/processing/strategies/dlq.rb +31 -7
  24. data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb +36 -0
  25. data/lib/karafka/pro/processing/strategies/dlq_vp.rb +37 -0
  26. data/lib/karafka/pro/processing/strategies/lrj_vp.rb +1 -0
  27. data/lib/karafka/pro/processing/strategies/vp.rb +5 -0
  28. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +6 -5
  29. data/lib/karafka/processing/coordinator.rb +27 -8
  30. data/lib/karafka/processing/partitioner.rb +3 -1
  31. data/lib/karafka/processing/strategies/default.rb +2 -3
  32. data/lib/karafka/railtie.rb +2 -9
  33. data/lib/karafka/setup/config.rb +3 -1
  34. data/lib/karafka/templates/karafka.rb.erb +7 -1
  35. data/lib/karafka/version.rb +1 -1
  36. data/lib/karafka.rb +13 -0
  37. data.tar.gz.sig +0 -0
  38. metadata +9 -4
  39. metadata.gz.sig +0 -0
data/Gemfile.lock CHANGED
@@ -1,31 +1,31 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka (2.0.27)
4
+ karafka (2.0.29)
5
5
  karafka-core (>= 2.0.9, < 3.0.0)
6
6
  thor (>= 0.20)
7
- waterdrop (>= 2.4.9, < 3.0.0)
7
+ waterdrop (>= 2.4.10, < 3.0.0)
8
8
  zeitwerk (~> 2.3)
9
9
 
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- activejob (7.0.4)
14
- activesupport (= 7.0.4)
13
+ activejob (7.0.4.2)
14
+ activesupport (= 7.0.4.2)
15
15
  globalid (>= 0.3.6)
16
- activesupport (7.0.4)
16
+ activesupport (7.0.4.2)
17
17
  concurrent-ruby (~> 1.0, >= 1.0.2)
18
18
  i18n (>= 1.6, < 2)
19
19
  minitest (>= 5.1)
20
20
  tzinfo (~> 2.0)
21
21
  byebug (11.1.3)
22
- concurrent-ruby (1.1.10)
22
+ concurrent-ruby (1.2.0)
23
23
  diff-lcs (1.5.0)
24
24
  docile (1.4.0)
25
25
  factory_bot (6.2.1)
26
26
  activesupport (>= 5.0.0)
27
27
  ffi (1.15.5)
28
- globalid (1.0.0)
28
+ globalid (1.0.1)
29
29
  activesupport (>= 5.0)
30
30
  i18n (1.12.0)
31
31
  concurrent-ruby (~> 1.0)
@@ -48,7 +48,7 @@ GEM
48
48
  rspec-expectations (3.12.2)
49
49
  diff-lcs (>= 1.2.0, < 2.0)
50
50
  rspec-support (~> 3.12.0)
51
- rspec-mocks (3.12.2)
51
+ rspec-mocks (3.12.3)
52
52
  diff-lcs (>= 1.2.0, < 2.0)
53
53
  rspec-support (~> 3.12.0)
54
54
  rspec-support (3.12.0)
@@ -61,12 +61,13 @@ GEM
61
61
  thor (1.2.1)
62
62
  tzinfo (2.0.5)
63
63
  concurrent-ruby (~> 1.0)
64
- waterdrop (2.4.9)
64
+ waterdrop (2.4.10)
65
65
  karafka-core (>= 2.0.9, < 3.0.0)
66
66
  zeitwerk (~> 2.3)
67
67
  zeitwerk (2.6.6)
68
68
 
69
69
  PLATFORMS
70
+ arm64-darwin-21
70
71
  x86_64-linux
71
72
 
72
73
  DEPENDENCIES
@@ -78,4 +79,4 @@ DEPENDENCIES
78
79
  simplecov
79
80
 
80
81
  BUNDLED WITH
81
- 2.4.2
82
+ 2.4.5
data/README.md CHANGED
@@ -58,7 +58,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
58
58
  1. Add and install Karafka:
59
59
 
60
60
  ```bash
61
- bundle add karafka
61
+ # Make sure to install Karafka 2.0
62
+ bundle add karafka --version ">= 2.0.28"
62
63
 
63
64
  bundle exec karafka install
64
65
  ```
@@ -86,7 +87,7 @@ I also sell Karafka Pro subscriptions. It includes a commercial-friendly license
86
87
 
87
88
  **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
88
89
 
89
- Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io) for more details.
90
+ Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io/#become-pro) for more details.
90
91
 
91
92
  ## Support
92
93
 
data/bin/karafka CHANGED
@@ -6,24 +6,6 @@ require 'karafka'
6
6
  # our bin/karafka cli
7
7
  ENV['KARAFKA_CLI'] = 'true'
8
8
 
9
- # If there is a boot file, we need to require it as we expect it to contain
10
- # Karafka app setup, routes, etc
11
- if File.exist?(Karafka.boot_file)
12
- rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
13
-
14
- # Load Rails environment file that starts Rails, so we can reference consumers and other things
15
- # from `karafka.rb` file. This will work only for Rails, for non-rails a manual setup is needed
16
- require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
17
-
18
- require Karafka.boot_file.to_s
19
- else
20
- # However when it is unavailable, we still want to be able to run help command
21
- # and install command as they don't require configured app itself to run
22
- raise(
23
- Karafka::Errors::MissingBootFileError,
24
- Karafka.boot_file
25
- ) unless %w[-h install].include?(ARGV[0])
26
- end
27
-
9
+ Karafka::Cli::Base.load
28
10
  Karafka::Cli.prepare
29
11
  Karafka::Cli.start
@@ -11,6 +11,8 @@ set -e
11
11
  if [ "$MODE" != "after" ]; then
12
12
  # Check the remote license prior to bundle installing
13
13
  curl \
14
+ --retry 5 \
15
+ --retry-delay 1 \
14
16
  --fail \
15
17
  -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
16
18
  https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \
@@ -5,7 +5,7 @@ en:
5
5
  virtual_partitions.max_partitions_format: needs to be equal or more than 1
6
6
  manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
7
7
  long_running_job.active_format: needs to be either true or false
8
- dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions
8
+ dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
9
9
 
10
10
  config:
11
11
  encryption.active_format: 'needs to be either true or false'
data/karafka.gemspec CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
 
24
24
  spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
25
25
  spec.add_dependency 'thor', '>= 0.20'
26
- spec.add_dependency 'waterdrop', '>= 2.4.9', '< 3.0.0'
26
+ spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
27
27
  spec.add_dependency 'zeitwerk', '~> 2.3'
28
28
 
29
29
  if $PROGRAM_NAME.end_with?('gem')
@@ -34,6 +34,26 @@ module Karafka
34
34
  end
35
35
 
36
36
  class << self
37
+ # Loads proper environment with what is needed to run the CLI
38
+ def load
39
+ # If there is a boot file, we need to require it as we expect it to contain
40
+ # Karafka app setup, routes, etc
41
+ if File.exist?(::Karafka.boot_file)
42
+ rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
43
+
44
+ # Load Rails environment file that starts Rails, so we can reference consumers and
45
+ # other things from `karafka.rb` file. This will work only for Rails, for non-rails
46
+ # a manual setup is needed
47
+ require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
48
+
49
+ require Karafka.boot_file.to_s
50
+ # However when it is unavailable, we still want to be able to run help command
51
+ # and install command as they don't require configured app itself to run
52
+ elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
53
+ raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
54
+ end
55
+ end
56
+
37
57
  # Allows to set options for Thor cli
38
58
  # @see https://github.com/erikhuda/thor
39
59
  # @param option Single option details
@@ -9,22 +9,27 @@ module Karafka
9
9
  option aliases: 'c'
10
10
 
11
11
  class << self
12
- # @return [String] Console executing command
12
+ # @return [String] Console executing command for non-Rails setup
13
13
  # @example
14
14
  # Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
15
- def command
16
- envs = [
17
- "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
18
- 'KARAFKA_CONSOLE=true'
19
- ]
20
- "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
15
+ def console
16
+ "IRBRC='#{Karafka.gem_root}/.console_irbrc' bundle exec irb -r #{Karafka.boot_file}"
17
+ end
18
+
19
+ # @return [String] Console executing command for Rails setup
20
+ # @note In case of Rails, it has its own console, hence we can just defer to it
21
+ def rails_console
22
+ 'bundle exec rails console'
21
23
  end
22
24
  end
23
25
 
24
26
  # Start the Karafka console
25
27
  def call
26
28
  cli.info
27
- exec self.class.command
29
+
30
+ command = ::Karafka.rails? ? self.class.rails_console : self.class.console
31
+
32
+ exec "KARAFKA_CONSOLE=true #{command}"
28
33
  end
29
34
  end
30
35
  end
@@ -267,7 +267,7 @@ module Karafka
267
267
  # Start work coordination for this topic partition
268
268
  coordinator.start(messages)
269
269
 
270
- @partitioner.call(topic, messages) do |group_id, partition_messages|
270
+ @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
271
271
  # Count the job we're going to create here
272
272
  coordinator.increment
273
273
  executor = @executors.find_or_create(topic, partition, group_id)
@@ -28,6 +28,7 @@ module Karafka
28
28
 
29
29
  @monitor.instrument(
30
30
  'error.occurred',
31
+ caller: self,
31
32
  subscription_group_id: @subscription_group_id,
32
33
  consumer_group_id: @consumer_group_id,
33
34
  type: 'librdkafka.error',
@@ -36,8 +36,9 @@ module Karafka
36
36
  )
37
37
 
38
38
  # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
39
- # this could create random markings
40
- next if topic.virtual_partitions?
39
+ # this could create random markings.
40
+ # The exception here is the collapsed state where we can move one after another
41
+ next if topic.virtual_partitions? && !collapsed?
41
42
 
42
43
  mark_as_consumed(message)
43
44
  end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ # Manages the collapse of virtual partitions
18
+ # Since any non-virtual partition is actually a virtual partition of size one, we can use
19
+ # it in a generic manner without having to distinguish between those cases.
20
+ #
21
+ # We need to have notion of the offset until we want to collapse because upon pause and retry
22
+ # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
23
+ # (different) dataset and without tracking the end of collapse, there would be a chance for
24
+ # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
25
+ # messages from the corrupted batch are processed.
26
+ class Collapser
27
+ # When initialized, nothing is collapsed
28
+ def initialize
29
+ @collapsed = false
30
+ @until_offset = -1
31
+ @mutex = Mutex.new
32
+ end
33
+
34
+ # @return [Boolean] Should we collapse into a single consumer
35
+ def collapsed?
36
+ @collapsed
37
+ end
38
+
39
+ # Collapse until given offset. Until given offset is encountered or offset bigger than that
40
+ # we keep collapsing.
41
+ # @param offset [Integer] offset until which we keep the collapse
42
+ def collapse_until!(offset)
43
+ @mutex.synchronize do
44
+ # We check it here in case after a pause and re-fetch we would get less messages and
45
+ # one of them would cause an error. We do not want to overwrite the offset here unless
46
+ # it is bigger.
47
+ @until_offset = offset if offset > @until_offset
48
+ end
49
+ end
50
+
51
+ # Sets the collapse state based on the first collective offset that we are going to process
52
+ # and makes the decision whether or not we need to still keep the collapse.
53
+ # @param first_offset [Integer] first offset from a collective batch
54
+ def refresh!(first_offset)
55
+ @mutex.synchronize do
56
+ @collapsed = first_offset < @until_offset
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -23,6 +23,7 @@ module Karafka
23
23
 
24
24
  @executed = []
25
25
  @flow_lock = Mutex.new
26
+ @collapser = Collapser.new
26
27
  end
27
28
 
28
29
  # Starts the coordination process
@@ -31,12 +32,28 @@ module Karafka
31
32
  def start(messages)
32
33
  super
33
34
 
35
+ @collapser.refresh!(messages.first.offset)
36
+
34
37
  @mutex.synchronize do
35
38
  @executed.clear
36
39
  @last_message = messages.last
37
40
  end
38
41
  end
39
42
 
43
+ # Sets the consumer failure status and additionally starts the collapse until
44
+ #
45
+ # @param consumer [Karafka::BaseConsumer] consumer that failed
46
+ # @param error [StandardError] error from the failure
47
+ def failure!(consumer, error)
48
+ super
49
+ @collapser.collapse_until!(@last_message.offset + 1)
50
+ end
51
+
52
+ # @return [Boolean] are we in a collapsed state at the moment
53
+ def collapsed?
54
+ @collapser.collapsed?
55
+ end
56
+
40
57
  # @return [Boolean] is the coordinated work finished or not
41
58
  def finished?
42
59
  @running_jobs.zero?
@@ -18,15 +18,29 @@ module Karafka
18
18
  class Partitioner < ::Karafka::Processing::Partitioner
19
19
  # @param topic [String] topic name
20
20
  # @param messages [Array<Karafka::Messages::Message>] karafka messages
21
+ # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
22
+ # will be used with those messages
21
23
  # @yieldparam [Integer] group id
22
24
  # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
23
- def call(topic, messages)
25
+ def call(topic, messages, coordinator)
24
26
  ktopic = @subscription_group.topics.find(topic)
25
27
 
26
- # We only partition work if we have a virtual partitioner and more than one thread to
27
- # process the data. With one thread it is not worth partitioning the work as the work
28
- # itself will be assigned to one thread (pointless work)
29
- if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
28
+ # We only partition work if we have:
29
+ # - a virtual partitioner
30
+ # - more than one thread to process the data
31
+ # - collective is not collapsed via coordinator
32
+ #
33
+ # With one thread it is not worth partitioning the work as the work itself will be
34
+ # assigned to one thread (pointless work)
35
+ #
36
+ # We collapse the partitioning on errors because we "regain" full ordering on a batch
37
+ # that potentially contains the data that caused the error.
38
+ #
39
+ # This is great because it allows us to run things without the parallelization that adds
40
+ # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
41
+ if ktopic.virtual_partitions? &&
42
+ ktopic.virtual_partitions.max_partitions > 1 &&
43
+ !coordinator.collapsed?
30
44
  # We need to reduce it to the max concurrency, so the group_id is not a direct effect
31
45
  # of the end user action. Otherwise the persistence layer for consumers would cache
32
46
  # it forever and it would cause memory leaks
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ # ActiveJob enabled
19
+ # DLQ enabled
20
+ # Long-Running Job enabled
21
+ # Manual offset management enabled
22
+ # Virtual Partitions enabled
23
+ #
24
+ # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
25
+ # offset management works in this case.
26
+ module AjDlqLrjMomVp
27
+ include AjDlqMomVp
28
+ include AjLrjMom
29
+
30
+ # Features for this strategy
31
+ FEATURES = %i[
32
+ active_job
33
+ long_running_job
34
+ manual_offset_management
35
+ dead_letter_queue
36
+ virtual_partitions
37
+ ].freeze
38
+
39
+ # This strategy is pretty much as non VP one because of the collapse
40
+ def handle_after_consume
41
+ coordinator.on_finished do |last_group_message|
42
+ if coordinator.success?
43
+ coordinator.pause_tracker.reset
44
+
45
+ return if revoked?
46
+ return if Karafka::App.stopping?
47
+
48
+ # Since we have VP here we do not commit intermediate offsets and need to commit
49
+ # them here. We do commit in collapsed mode but this is generalized.
50
+ mark_as_consumed(last_group_message)
51
+
52
+ seek(coordinator.seek_offset) unless revoked?
53
+
54
+ resume
55
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
56
+ retry_after_pause
57
+ else
58
+ coordinator.pause_tracker.reset
59
+ skippable_message = find_skippable_message
60
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
61
+ mark_as_consumed(skippable_message)
62
+ pause(coordinator.seek_offset, nil, false)
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ # ActiveJob enabled
19
+ # Manual offset management enabled
20
+ # Virtual Partitions enabled
21
+ module AjDlqMomVp
22
+ include Dlq
23
+ include Vp
24
+ include Default
25
+
26
+ # Features for this strategy
27
+ FEATURES = %i[
28
+ active_job
29
+ dead_letter_queue
30
+ manual_offset_management
31
+ virtual_partitions
32
+ ].freeze
33
+
34
+ # Flow including moving to DLQ in the collapsed mode
35
+ def handle_after_consume
36
+ coordinator.on_finished do |last_group_message|
37
+ if coordinator.success?
38
+ coordinator.pause_tracker.reset
39
+
40
+ # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
41
+ # intermediate jobs as processed not to mess up with the ordering.
42
+ # Only when all the jobs are processed and we did not loose the partition
43
+ # assignment and we are not stopping (Pro ActiveJob has an early break) we can
44
+ # commit offsets on this as only then we can be sure, that all the jobs were
45
+ # processed.
46
+ # For a non virtual partitions case, the flow is regular and state is marked after
47
+ # each successfully processed job
48
+ return if revoked?
49
+ return if Karafka::App.stopping?
50
+
51
+ mark_as_consumed(last_group_message)
52
+ elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
53
+ retry_after_pause
54
+ else
55
+ # Here we are in a collapsed state, hence we can apply the same logic as AjDlqMom
56
+ coordinator.pause_tracker.reset
57
+ skippable_message = find_skippable_message
58
+ dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
59
+ mark_as_consumed(skippable_message)
60
+ pause(coordinator.seek_offset, nil, false)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -21,6 +21,7 @@ module Karafka
21
21
  # Virtual Partitions enabled
22
22
  module AjLrjMomVp
23
23
  include Default
24
+ include Vp
24
25
 
25
26
  # Features for this strategy
26
27
  FEATURES = %i[
@@ -19,6 +19,7 @@ module Karafka
19
19
  # Manual offset management enabled
20
20
  # Virtual Partitions enabled
21
21
  module AjMomVp
22
+ include Vp
22
23
  include Default
23
24
 
24
25
  # Features for this strategy
@@ -42,6 +43,9 @@ module Karafka
42
43
  # processed.
43
44
  # For a non virtual partitions case, the flow is regular and state is marked after
44
45
  # each successfully processed job
46
+ #
47
+ # We can mark and we do mark intermediate jobs in the collapsed mode when running
48
+ # VPs
45
49
  return if revoked?
46
50
  return if Karafka::App.stopping?
47
51
 
@@ -53,10 +53,10 @@ module Karafka
53
53
  end
54
54
 
55
55
  # Mark job as successful
56
- coordinator.consumption(self).success!
56
+ coordinator.success!(self)
57
57
  rescue StandardError => e
58
58
  # If failed, mark as failed
59
- coordinator.consumption(self).failure!(e)
59
+ coordinator.failure!(self, e)
60
60
 
61
61
  # Re-raise so reported in the consumer
62
62
  raise e
@@ -65,13 +65,8 @@ module Karafka
65
65
  # dispatch to DLQ
66
66
  def dispatch_to_dlq(skippable_message)
67
67
  producer.produce_async(
68
- topic: topic.dead_letter_queue.topic,
69
- payload: skippable_message.raw_payload,
70
- key: skippable_message.partition.to_s,
71
- headers: skippable_message.headers.merge(
72
- 'original_topic' => topic.name,
73
- 'original_partition' => skippable_message.partition.to_s,
74
- 'original_offset' => skippable_message.offset.to_s
68
+ build_dlq_message(
69
+ skippable_message
75
70
  )
76
71
  )
77
72
 
@@ -83,6 +78,35 @@ module Karafka
83
78
  )
84
79
  end
85
80
 
81
+ # @param skippable_message [Array<Karafka::Messages::Message>]
82
+ # @return [Hash] dispatch DLQ message
83
+ def build_dlq_message(skippable_message)
84
+ original_partition = skippable_message.partition.to_s
85
+
86
+ dlq_message = {
87
+ topic: topic.dead_letter_queue.topic,
88
+ key: original_partition,
89
+ payload: skippable_message.raw_payload,
90
+ headers: skippable_message.headers.merge(
91
+ 'original_topic' => topic.name,
92
+ 'original_partition' => original_partition,
93
+ 'original_offset' => skippable_message.offset.to_s,
94
+ 'original_consumer_group' => topic.consumer_group.id
95
+ )
96
+ }
97
+
98
+ # Optional method user can define in consumer to enhance the dlq message hash with
99
+ # some extra details if needed or to replace payload, etc
100
+ if respond_to?(:enhance_dlq_message, true)
101
+ enhance_dlq_message(
102
+ dlq_message,
103
+ skippable_message
104
+ )
105
+ end
106
+
107
+ dlq_message
108
+ end
109
+
86
110
  # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
87
111
  # is set to false, we will skip the dispatch, effectively ignoring the broken message
88
112
  # without taking any action.
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ # Dead-Letter Queue enabled
19
+ # Long-Running Job enabled
20
+ # Virtual Partitions enabled
21
+ module DlqLrjVp
22
+ # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
23
+ include Vp
24
+ include DlqLrj
25
+
26
+ # Features for this strategy
27
+ FEATURES = %i[
28
+ dead_letter_queue
29
+ long_running_job
30
+ virtual_partitions
31
+ ].freeze
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end