karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/processing/strategies/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro processing strategies namespace
+      module Strategies
+        # Base strategy for Pro
+        module Base
+          include ::Karafka::Processing::Strategies::Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # No features enabled.
+        # No manual offset management
+        # No long running jobs
+        # No virtual partitions
+        # Nothing. Just standard, automatic flow
+        module Default
+          include Base
+          include ::Karafka::Processing::Strategies::Default
+
+          # Apply strategy for a non-feature based flow
+          FEATURES = %i[].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            nil
+          end
+
+          # Increment number of attempts per one "full" job. For all VP on a single topic partition
+          # this also should run once.
+          def handle_before_consume
+            coordinator.on_started do
+              coordinator.pause_tracker.increment
+            end
+          end
+
+          # Run the user consumption code
+          def handle_consume
+            # We should not run the work at all on a partition that was revoked
+            # This can happen primarily when an LRJ job gets to the internal worker queue and
+            # this partition is revoked prior processing.
+            unless revoked?
+              Karafka.monitor.instrument('consumer.consume', caller: self)
+              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+                consume
+              end
+            end
+
+            # Mark job as successful
+            coordinator.success!(self)
+          rescue StandardError => e
+            # If failed, mark as failed
+            coordinator.failure!(self, e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has
+            # finished
+            coordinator.decrement
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # Do not mark last message if pause happened. This prevents a scenario where pause
+                # is overridden upon rebalance by marking
+                return if coordinator.manual_pause?
+
+                mark_as_consumed(last_group_message)
+              else
+                retry_after_pause
+              end
+            end
+          end
+
+          # Standard
+          def handle_revoked
+            coordinator.on_revoked do
+              resume
+
+              coordinator.revoke
+            end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the strategies starting with DLQ
+        module Dlq
+          # Only dead letter queue enabled
+          module Default
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+            ].freeze
+
+            # When we encounter non-recoverable message, we skip it and go on with our lives
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+
+            # Finds the message may want to skip (all, starting from first)
+            # @private
+            # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+            #   information if this message was from marked offset or figured out via mom flow
+            def find_skippable_message
+              skippable_message = messages.find do |msg|
+                coordinator.marked? && msg.offset == coordinator.seek_offset
+              end
+
+              # If we don't have the message matching the last comitted offset, it means that
+              # user operates with manual offsets and we're beyond the batch in which things
+              # broke for the first time. Then we skip the first (as no markings) and we
+              # move on one by one.
+              skippable_message ? [skippable_message, true] : [messages.first, false]
+            end
+
+            # Moves the broken message into a separate queue defined via the settings
+            #
+            # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
+            #   dispatch to DLQ
+            def dispatch_to_dlq(skippable_message)
+              # DLQ should never try to dispatch a message that was cleaned. It message was
+              # cleaned, we will not have all the needed data. If you see this error, it means
+              # that your processing flow is not as expected and you have cleaned message that
+              # should not be cleaned as it should go to the DLQ
+              raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
+
+              producer.produce_async(
+                build_dlq_message(
+                  skippable_message
+                )
+              )
+
+              # Notify about dispatch on the events bus
+              Karafka.monitor.instrument(
+                'dead_letter_queue.dispatched',
+                caller: self,
+                message: skippable_message
+              )
+            end
+
+            # @param skippable_message [Array<Karafka::Messages::Message>]
+            # @return [Hash] dispatch DLQ message
+            def build_dlq_message(skippable_message)
+              original_partition = skippable_message.partition.to_s
+
+              dlq_message = {
+                topic: topic.dead_letter_queue.topic,
+                key: original_partition,
+                payload: skippable_message.raw_payload,
+                headers: skippable_message.headers.merge(
+                  'original_topic' => topic.name,
+                  'original_partition' => original_partition,
+                  'original_offset' => skippable_message.offset.to_s,
+                  'original_consumer_group' => topic.consumer_group.id
+                )
+              }
+
+              # Optional method user can define in consumer to enhance the dlq message hash with
+              # some extra details if needed or to replace payload, etc
+              if respond_to?(:enhance_dlq_message, true)
+                enhance_dlq_message(
+                  dlq_message,
+                  skippable_message
+                )
+              end
+
+              dlq_message
+            end
+
+            # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch
+            #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
+            #   message without taking any action.
+            def dispatch_to_dlq?
+              topic.dead_letter_queue.topic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+            ].freeze
+
+            # DLQ flow is standard here, what is not, is the success component where we need to
+            # take into consideration the filtering
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          module FtrLrj
+            include Strategies::Dlq::Lrj
+            include Strategies::Lrj::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+            ].freeze
+
+            # This is one of more complex cases.
+            # We need to ensure, that we always resume (inline or via paused backoff) and we need
+            # to make sure we dispatch to DLQ when needed. Because revocation on LRJ can happen
+            # any time, we need to make sure we do not dispatch to DLQ when error happens but we
+            # no longer own the assignment. Throttling is another factor that has to be taken
+            # into consideration on the successful path
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked? && !coordinator.manual_seek?
+                    seek(coordinator.seek_offset, false)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          module FtrLrjMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # Post execution flow of this strategy
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked? && !coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, _marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          # VP enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Virtual Partitions
+          module FtrLrjVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          module FtrMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # On mom we do not mark, throttling and seeking as in other strategies
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+
+                  skippable_message, _marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          # - VP
+          # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
+          module FtrMomVp
+            include Strategies::Mom::Vp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end