karafka 2.0.15 → 2.0.17

data/lib/karafka/pro/performance_tracker.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -21,6 +23,7 @@ module Karafka
           @on_enqueued_invoked = false
           @on_started_invoked = false
           @on_finished_invoked = false
+          @on_revoked_invoked = false
           @flow_lock = Mutex.new
         end
 
@@ -79,6 +82,18 @@ module Karafka
             yield(@last_message)
           end
         end
+
+        # Runs once when a partition is revoked
+        def on_revoked
+          @flow_lock.synchronize do
+            return unless finished?
+            return if @on_revoked_invoked
+
+            @on_revoked_invoked = true
+
+            yield(@last_message)
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/partitioner.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/scheduler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        #
+        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+        # offset management works in this case.
+        module AjDlqLrjMom
+          include AjLrjMom
+          include AjDlqMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            dead_letter_queue
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Manual offset management enabled
+        #
+        # AJ has manual offset management on by default and the offset management is delegated to
+        # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
+        # consumed when we skip given job upon errors. In all the other scenarions marking as
+        # consumed needs to happen in the AJ consumer on a per job basis.
+        module AjDlqMom
+          include DlqMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            dead_letter_queue
+            manual_offset_management
+          ].freeze
+
+          # How should we post-finalize consumption.
+          def handle_after_consume
+            coordinator.on_finished do
+              return if revoked?
+
+              if coordinator.success?
+                # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+                coordinator.pause_tracker.reset
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              else
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message)
+                # We can commit the offset here because we know that we skip it "forever" and
+                # since AJ consumer commits the offset after each job, we also know that the
+                # previous job was successful
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        module AjLrjMom
+          # Same behaviour as LrjMom
+          include LrjMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjLrjMomVp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            coordinator.on_enqueued do
+              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME)
+            end
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message) unless revoked? || Karafka::App.stopping?
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              else
+                # If processing failed, we need to pause
+                # For long running job this will overwrite the default never-ending pause and will
+                # cause the processing to keep going after the error backoff
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # LRJ cannot resume here. Only in handling the after consumption
+          def handle_revoked
+            coordinator.on_revoked do
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_mom.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual Offset management enabled
+        module AjMom
+          # Standard ActiveJob strategy is the same one we use for Mom
+          include Mom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            manual_offset_management
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjMomVp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
+                # intermediate jobs as processed not to mess up with the ordering.
+                # Only when all the jobs are processed and we did not loose the partition
+                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                # commit offsets on this as only then we can be sure, that all the jobs were
+                # processed.
+                # For a non virtual partitions case, the flow is regular and state is marked after
+                # each successfully processed job
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                mark_as_consumed(last_group_message)
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro processing strategies namespace
+      module Strategies
+        # Base strategy for Pro
+        module Base
+          include ::Karafka::Processing::Strategies::Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # No features enabled.
+        # No manual offset management
+        # No long running jobs
+        # No virtual partitions
+        # Nothing. Just standard, automatic flow
+        module Default
+          include Base
+
+          # Apply strategy for a non-feature based flow
+          FEATURES = %i[].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            nil
+          end
+
+          # Increment number of attempts per one "full" job. For all VP on a single topic partition
+          # this also should run once.
+          def handle_before_consume
+            coordinator.on_started do
+              coordinator.pause_tracker.increment
+            end
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message)
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # Standard
+          def handle_revoked
+            coordinator.on_revoked do
+              resume
+
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Only dead letter queue enabled
+        module Dlq
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+          ].freeze
+
+          # When we encounter non-recoverable message, we skip it and go on with our lives
+          def handle_after_consume
+            coordinator.on_finished do
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(messages.last)
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              # If we've reached number of retries that we could, we need to skip the first message
+              # that was not marked as consumed, pause and continue, while also moving this message
+              # to the dead topic
+              else
+                # We reset the pause to indicate we will now consider it as "ok".
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message)
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # Finds the message may want to skip (all, starting from first)
+          # @private
+          # @return [Karafka::Messages::Message] message we may want to skip
+          def find_skippable_message
+            skippable_message = messages.find { |msg| msg.offset == coordinator.seek_offset }
+            skippable_message || raise(Errors::SkipMessageNotFoundError, topic.name)
+          end
+
+          # Moves the broken message into a separate queue defined via the settings
+          #
+          # @private
+          # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
+          #   dispatch to DLQ
+          def dispatch_to_dlq(skippable_message)
+            producer.produce_async(
+              topic: topic.dead_letter_queue.topic,
+              payload: skippable_message.raw_payload,
+              key: skippable_message.partition.to_s,
+              headers: skippable_message.headers.merge(
+                'original_topic' => topic.name,
+                'original_partition' => skippable_message.partition.to_s,
+                'original_offset' => skippable_message.offset.to_s
+              )
+            )
+
+            # Notify about dispatch on the events bus
+            Karafka.monitor.instrument(
+              'dead_letter_queue.dispatched',
+              caller: self,
+              message: skippable_message
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # DLQ enabled
+        # Long-Running Job enabled
+        module DlqLrj
+          # Order here matters, lrj needs to be second
+          include Dlq
+          include Lrj
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            long_running_job
+          ].freeze
+
+          # LRJ standard flow after consumption with DLQ dispatch
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message) unless revoked?
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              else
+                coordinator.pause_tracker.reset
+
+                skippable_message = find_skippable_message
+
+                unless revoked?
+                  dispatch_to_dlq(skippable_message)
+                  mark_as_consumed(skippable_message)
+                end
+
+                # This revoke might have changed state due to marking, hence checked again
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end