karafka 2.0.41 → 2.1.4

data/lib/karafka/pro/active_job/consumer.rb

@@ -35,18 +35,9 @@ module Karafka
             # double-processing
             break if Karafka::App.stopping? && !topic.virtual_partitions?
 
-            # Break if we already know, that one of virtual partitions has failed and we will
-            # be restarting processing all together after all VPs are done. This will minimize
-            # number of jobs that will be re-processed
-            break if topic.virtual_partitions? && failing?
-
             consume_job(message)
 
-            # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
-            # this could create random markings.
-            # The exception here is the collapsed state where we can move one after another
-            next if topic.virtual_partitions? && !collapsed?
-
+            # We can always mark because of the virtual offset management that we have in VPs
             mark_as_consumed(message)
           end
         end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -39,7 +39,7 @@ module Karafka
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(job.serialize)
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           )
         end
@@ -54,7 +54,7 @@ module Karafka
 
             dispatches[d_method] << dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(job.serialize)
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           end
 

data/lib/karafka/pro/processing/coordinator.rb

@@ -17,7 +17,7 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
-        attr_reader :filter
+        attr_reader :filter, :virtual_offset_manager
 
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
@@ -27,6 +27,20 @@ module Karafka
           @flow_lock = Mutex.new
           @collapser = Collapser.new
           @filter = FiltersApplier.new(self)
+
+          return unless topic.virtual_partitions?
+
+          @virtual_offset_manager = VirtualOffsetManager.new(
+            topic.name,
+            partition
+          )
+
+          # We register our own "internal" filter to support filtering of messages that were marked
+          # as consumed virtually
+          @filter.filters << Filters::VirtualLimiter.new(
+            @virtual_offset_manager,
+            @collapser
+          )
         end
 
         # Starts the coordination process
@@ -40,6 +54,11 @@ module Karafka
           @filter.apply!(messages)
 
           @executed.clear
+
+          # We keep the old processed offsets until the collapsing is done and regular processing
+          # with virtualization is restored
+          @virtual_offset_manager.clear if topic.virtual_partitions? && !@collapser.collapsed?
+
           @last_message = messages.last
         end
 

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Removes messages that are already marked as consumed in the virtual offset manager
+        # This should operate only when using virtual partitions.
+        #
+        # This cleaner prevents us from duplicated processing of messages that were virtually
+        # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
+        # to limit reprocessing when errors occur drastically when operating with virtual
+        # partitions
+        #
+        # @note It should be registered only when VPs are used
+        class VirtualLimiter < Base
+          # @param manager [Processing::VirtualOffsetManager]
+          # @param collapser [Processing::Collapser]
+          def initialize(manager, collapser)
+            @manager = manager
+            @collapser = collapser
+
+            super()
+          end
+
+          # Remove messages that we already marked as virtually consumed. Does nothing if not in
+          # the collapsed mode.
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            return unless @collapser.collapsed?
+
+            marked = @manager.marked
+
+            messages.delete_if { |message| marked.include?(message.offset) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -21,6 +21,10 @@ module Karafka
       # This means that this is the API we expose as a single filter, allowing us to control
       # the filtering via many filters easily.
       class FiltersApplier
+        # @return [Array] registered filters array. Useful if we want to inject internal context
+        #   aware filters.
+        attr_reader :filters
+
         # @param coordinator [Pro::Coordinator] pro coordinator
         def initialize(coordinator)
           # Builds filters out of their factories

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -22,7 +22,7 @@ module Karafka
           # - Mom
           # - VP
           module DlqFtrMomVp
-            include Strategies::Vp::Default
+            include Strategies::Aj::DlqMomVp
             include Strategies::Aj::DlqFtrMom
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -24,7 +24,9 @@ module Karafka
           # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
           # offset management works in this case.
           module DlqLrjMom
-            include Strategies::Aj::DlqLrjMomVp
+            include Strategies::Default
+            include Strategies::Dlq::Default
+            include Strategies::Aj::LrjMom
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -20,9 +20,9 @@ module Karafka
           # Manual offset management enabled
           # Virtual Partitions enabled
           module DlqMomVp
-            include Strategies::Dlq::Default
-            include Strategies::Vp::Default
             include Strategies::Default
+            include Strategies::Dlq::Vp
+            include Strategies::Vp::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -34,6 +34,8 @@ module Karafka
 
             # No actions needed for the standard flow here
             def handle_before_enqueue
+              super
+
               coordinator.on_enqueued do
                 pause(coordinator.seek_offset, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
               end

data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb

@@ -20,8 +20,8 @@ module Karafka
           # Manual offset management enabled
           # Virtual Partitions enabled
           module MomVp
-            include Strategies::Vp::Default
             include Strategies::Default
+            include Strategies::Vp::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -20,7 +20,7 @@ module Karafka
           # - Ftr
           module Ftr
             include Strategies::Ftr::Default
-            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -55,14 +55,11 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          # VP enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -22,6 +22,7 @@ module Karafka
           # Virtual Partitions
           module FtrLrjVp
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             # Same as non VP because of the coordinator post-execution lock
             include Strategies::Dlq::FtrLrj
 

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -53,15 +53,11 @@ module Karafka
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Move beyond the broken message
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          # - VP
+          # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
+          module FtrMomVp
+            include Strategies::Mom::Vp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -23,6 +23,7 @@ module Karafka
           # Behaves same as non-VP due to coordinator lock
           module FtrVp
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             include Strategies::Dlq::Ftr
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -47,14 +47,11 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom, LRJ and VP enabled
+          module LrjMomVp
+            # Feature set for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::LrjMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb

@@ -22,6 +22,7 @@ module Karafka
           module LrjVp
             # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             include Strategies::Dlq::Lrj
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -47,15 +47,16 @@ module Karafka
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Move beyond the broken message
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  # Save the next offset we want to go with after moving given message to DLQ
+                  # Without this, we would not be able to move forward and we would end up
+                  # in an infinite loop trying to un-pause from the message we've already processed
+                  # Of course, since it's a MoM a rebalance or kill, will move it back as no
+                  # offsets are being committed
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # MoM enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -30,6 +30,8 @@ module Karafka
 
             # We always need to pause prior to doing any jobs for LRJ
             def handle_before_enqueue
+              super
+
               # This ensures that when running LRJ with VP, things operate as expected run only
               # once for all the virtual partitions collectively
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Filtering related init strategies
+        module Lrj
+          # Filtering enabled
+          # LRJ enabled
+          # MoM enabled
+          # VPs enabled
+          module FtrMomVp
+            # Filtering + LRJ + Mom + VPs
+            FEATURES = %i[
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Lrj::MomVp
+            include Strategies::Lrj::FtrMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -30,6 +30,8 @@ module Karafka
 
             # We always need to pause prior to doing any jobs for LRJ
             def handle_before_enqueue
+              super
+
               # This ensures that when running LRJ with VP, things operate as expected run only
               # once for all the virtual partitions collectively
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the LRJ starting strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Mom::Vp
+            include Strategies::Lrj::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Ftr enabled
+          # - Vp enabled
+          module FtrVp
+            include Strategies::Mom::Vp
+            include Strategies::Mom::Ftr
+
+            # Features of this strategy
+            FEATURES = %i[
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/{base_consumer.rb → processing/strategies/mom/vp.rb}

@@ -13,13 +13,23 @@
 
 module Karafka
   module Pro
-    # Karafka PRO consumer.
-    #
-    # If you use PRO, all your consumers should inherit (indirectly) from it.
-    #
-    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
-    #   after each batch is processed.
-    class BaseConsumer < Karafka::BaseConsumer
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Vp enabled
+          module Vp
+            include Strategies::Vp::Default
+            include Strategies::Mom::Default
+
+            # Features of this strategy
+            FEATURES = %i[
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
     end
   end
 end