karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Filtering enabled
+          # Long-Running Job enabled
+          #
+          # In general aside from throttling this one will behave the same way as the Lrj
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Lrj::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+            ].freeze
+
+            # LRJ standard flow after consumption with potential throttling on success
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Manual pausing has the highest priority
+                  return if coordinator.manual_pause?
+
+                  # It's not a MoM, so for successful we need to mark as consumed
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  # If still not revoked and was throttled, we need to apply throttling logic
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked? && !coordinator.manual_seek?
+                    # If not revoked and not throttled, we move to where we were suppose to and
+                    # resume
+                    seek(coordinator.seek_offset, false)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Filtering enabled
+          # Manual offset management enabled
+          #
+          # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
+          module FtrMom
+            include Strategies::Lrj::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # LRJ standard flow after consumption with potential filtering on success
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Manual pausing has the highest priority
+                  return if coordinator.manual_pause?
+
+                  # If still not revoked and was throttled, we need to apply filtering logic
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked? && !coordinator.manual_seek?
+                    # If not revoked and not throttled, we move to where we were suppose to and
+                    # resume
+                    seek(last_group_message.offset + 1, false)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Filtering related init strategies
+        module Lrj
+          # Filtering enabled
+          # LRJ enabled
+          # MoM enabled
+          # VPs enabled
+          module FtrMomVp
+            # Filtering + LRJ + Mom + VPs
+            FEATURES = %i[
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Lrj::MomVp
+            include Strategies::Lrj::FtrMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Filtering enabled
+          # Virtual Partitions enabled
+          #
+          # Behaves same as non-VP because of the aggregated flow in the coordinator.
+          module FtrVp
+            include Strategies::Vp::Default
+            include Strategies::Lrj::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the LRJ starting strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module Mom
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # We always need to pause prior to doing any jobs for LRJ
+            def handle_before_enqueue
+              super
+
+              # This ensures that when running LRJ with VP, things operate as expected run only
+              # once for all the virtual partitions collectively
+              coordinator.on_enqueued do
+                pause(:consecutive, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
+              end
+            end
+
+            # No offset management, aside from that typical LRJ
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  unless revoked? || coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                  end
+
+                  resume
+                else
+                  retry_after_pause
+                end
+              end
+            end
+
+            # We do not un-pause on revokations for LRJ
+            def handle_revoked
+              coordinator.on_revoked do
+                coordinator.revoke
+              end
+
+              Karafka.monitor.instrument('consumer.revoke', caller: self)
+              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+                revoked
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the LRJ starting strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Mom::Vp
+            include Strategies::Lrj::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Virtual Partitions enabled
+          module Vp
+            # Same flow as the standard Lrj
+            include Strategies::Vp::Default
+            include Strategies::Lrj::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/default.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for Mom starting strategies
+        module Mom
+          # Manual offset management enabled
+          module Default
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              manual_offset_management
+            ].freeze
+
+            # When mom is enabled, we do not mark messages as consumed after processing
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/ftr.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # Filtering support for MoM
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Mom::Default
+
+            # MoM + Ftr
+            FEATURES = %i[
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # When mom is enabled, we do not mark messages as consumed after processing
+            # but we also need to keep in mind throttling here
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Do not throttle if paused
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Ftr enabled
+          # - Vp enabled
+          module FtrVp
+            include Strategies::Mom::Vp
+            include Strategies::Mom::Ftr
+
+            # Features of this strategy
+            FEATURES = %i[
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/vp.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Vp enabled
+          module Vp
+            include Strategies::Vp::Default
+            include Strategies::Mom::Default
+
+            # Features of this strategy
+            FEATURES = %i[
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # VP starting strategies
+        module Vp
+          # Just Virtual Partitions enabled
+          module Default
+            # This flow is exactly the same as the default one because the default one is wrapper
+            # with `coordinator#on_finished`
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              virtual_partitions
+            ].freeze
+
+            # @param message [Karafka::Messages::Message] marks message as consumed
+            # @note This virtual offset management uses a regular default marking API underneath.
+            #   We do not alter the "real" marking API, as VPs are just one of many cases we want
+            #   to support and we do not want to impact them with collective offsets management
+            def mark_as_consumed(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                # If this is last marking on a finished flow, we can use the original
+                # last message and in order to do so, we need to mark all previous messages as
+                # consumed as otherwise the computed offset could be different
+                # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                # want to mark all but until the expected location
+                manager.mark_until(message) if coordinator.finished?
+
+                return revoked? unless manager.markable?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
+            # @param message [Karafka::Messages::Message] blocking marks message as consumed
+            def mark_as_consumed!(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                manager.mark_until(message) if coordinator.finished?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
+            # @return [Boolean] is the virtual processing collapsed in the context of given
+            #   consumer.
+            def collapsed?
+              coordinator.collapsed?
+            end
+
+            # @param offset [Integer] first offset from which we should not operate in a collapsed
+            #   mode.
+            # @note Keep in mind, that if a batch contains this but also messages earlier messages
+            #   that should be collapsed, all will continue to operate in a collapsed mode until
+            #   first full batch with only messages that should not be collapsed.
+            def collapse_until!(offset)
+              coordinator.collapse_until!(offset)
+            end
+
+            # @return [Boolean] true if any of virtual partition we're operating in the entangled
+            #   mode has already failed and we know we are failing collectively.
+            #   Useful for early stop to minimize number of things processed twice.
+            #
+            # @note We've named it `#failing?` instead of `#failure?`  because it aims to be used
+            #   from within virtual partitions where we want to have notion of collective failing
+            #   not just "local" to our processing. We "are" failing with other virtual partitions
+            #   raising an error, but locally we are still processing.
+            def failing?
+              coordinator.failure?
+            end
+
+            # Allows for cross-virtual-partition consumers locks
+            #
+            # This is not needed in the non-VP flows because there is always only one consumer
+            # per partition at the same time, so no coordination is needed directly for the
+            # end users
+            #
+            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
+            def synchronize(&block)
+              coordinator.synchronize(&block)
+            end
+
+            private
+
+            # Prior to adding work to the queue, registers all the messages offsets into the
+            # virtual offset group.
+            #
+            # @note This can be done without the mutex, because it happens from the same thread
+            #   for all the work (listener thread)
+            def handle_before_enqueue
+              coordinator.virtual_offset_manager.register(
+                messages.map(&:offset)
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro processing strategies namespace
+      module Strategies
+      end
+    end
+  end
+end