karafka 2.0.40 → 2.1.0

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          # VP enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -22,6 +22,7 @@ module Karafka
           # Virtual Partitions
           module FtrLrjVp
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             # Same as non VP because of the coordinator post-execution lock
             include Strategies::Dlq::FtrLrj
 

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -53,15 +53,11 @@ module Karafka
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Move beyond the broken message
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          # - VP
+          # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
+          module FtrMomVp
+            include Strategies::Mom::Vp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -23,6 +23,7 @@ module Karafka
           # Behaves same as non-VP due to coordinator lock
           module FtrVp
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             include Strategies::Dlq::Ftr
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -47,14 +47,11 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom, LRJ and VP enabled
+          module LrjMomVp
+            # Feature set for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::LrjMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb

@@ -22,6 +22,7 @@ module Karafka
           module LrjVp
             # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
             include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
             include Strategies::Dlq::Lrj
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -47,15 +47,16 @@ module Karafka
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Move beyond the broken message
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  # Save the next offset we want to go with after moving given message to DLQ
+                  # Without this, we would not be able to move forward and we would end up
+                  # in an infinite loop trying to un-pause from the message we've already processed
+                  # Of course, since it's a MoM a rebalance or kill, will move it back as no
+                  # offsets are being committed
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # MoM enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -30,6 +30,8 @@ module Karafka
 
             # We always need to pause prior to doing any jobs for LRJ
             def handle_before_enqueue
+              super
+
               # This ensures that when running LRJ with VP, things operate as expected run only
               # once for all the virtual partitions collectively
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Filtering related init strategies
+        module Lrj
+          # Filtering enabled
+          # LRJ enabled
+          # MoM enabled
+          # VPs enabled
+          module FtrMomVp
+            # Filtering + LRJ + Mom + VPs
+            FEATURES = %i[
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Lrj::MomVp
+            include Strategies::Lrj::FtrMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -30,6 +30,8 @@ module Karafka
 
             # We always need to pause prior to doing any jobs for LRJ
             def handle_before_enqueue
+              super
+
               # This ensures that when running LRJ with VP, things operate as expected run only
               # once for all the virtual partitions collectively
               coordinator.on_enqueued do

data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the LRJ starting strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Mom::Vp
+            include Strategies::Lrj::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Ftr enabled
+          # - Vp enabled
+          module FtrVp
+            include Strategies::Mom::Vp
+            include Strategies::Mom::Ftr
+
+            # Features of this strategy
+            FEATURES = %i[
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/{base_consumer.rb → processing/strategies/mom/vp.rb}

@@ -13,13 +13,23 @@
 
 module Karafka
   module Pro
-    # Karafka PRO consumer.
-    #
-    # If you use PRO, all your consumers should inherit (indirectly) from it.
-    #
-    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
-    #   after each batch is processed.
-    class BaseConsumer < Karafka::BaseConsumer
+    module Processing
+      module Strategies
+        module Mom
+          # - Mom enabled
+          # - Vp enabled
+          module Vp
+            include Strategies::Vp::Default
+            include Strategies::Mom::Default
+
+            # Features of this strategy
+            FEATURES = %i[
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
     end
   end
 end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -28,6 +28,44 @@ module Karafka
               virtual_partitions
             ].freeze
 
+            # @param message [Karafka::Messages::Message] marks message as consumed
+            # @note This virtual offset management uses a regular default marking API underneath.
+            #   We do not alter the "real" marking API, as VPs are just one of many cases we want
+            #   to support and we do not want to impact them with collective offsets management
+            def mark_as_consumed(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                # If this is last marking on a finished flow, we can use the original
+                # last message and in order to do so, we need to mark all previous messages as
+                # consumed as otherwise the computed offset could be different
+                # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                # want to mark all but until the expected location
+                manager.mark_until(message) if coordinator.finished?
+
+                return revoked? unless manager.markable?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
+            # @param message [Karafka::Messages::Message] blocking marks message as consumed
+            def mark_as_consumed!(message)
+              return super if collapsed?
+
+              manager = coordinator.virtual_offset_manager
+
+              coordinator.synchronize do
+                manager.mark(message)
+                manager.mark_until(message) if coordinator.finished?
+              end
+
+              manager.markable? ? super(manager.markable) : revoked?
+            end
+
             # @return [Boolean] is the virtual processing collapsed in the context of given
             #   consumer.
             def collapsed?
@@ -45,6 +83,19 @@ module Karafka
             def failing?
               coordinator.failure?
             end
+
+            private
+
+            # Prior to adding work to the queue, registers all the messages offsets into the
+            # virtual offset group.
+            #
+            # @note This can be done without the mutex, because it happens from the same thread
+            #   for all the work (listener thread)
+            def handle_before_enqueue
+              coordinator.virtual_offset_manager.register(
+                messages.map(&:offset)
+              )
+            end
           end
         end
       end

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manager that keeps track of our offsets with the virtualization layer that are local
+      # to given partition assignment. It allows for easier offset management for virtual
+      # virtual partition cases as it provides us ability to mark as consumed and move the
+      # real offset behind as expected.
+      #
+      # @note We still use the regular coordinator "real" offset management as we want to have
+      #   them as separated as possible because the real seek offset management is also used for
+      #   pausing, filtering and others and should not be impacted by the virtual one
+      #
+      # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+      #   locked locations.
+      class VirtualOffsetManager
+        attr_reader :groups
+
+        # @param topic [String]
+        # @param partition [Integer]
+        #
+        # @note We need topic and partition because we use a seek message (virtual) for real offset
+        #   management. We could keep real message reference but this can be memory consuming
+        #   and not worth it.
+        def initialize(topic, partition)
+          @topic = topic
+          @partition = partition
+          @groups = []
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Clears the manager for a next collective operation
+        def clear
+          @groups.clear
+          @marked = {}
+          @real_offset = -1
+        end
+
+        # Registers an offset group coming from one virtual consumer. In order to move the real
+        # underlying offset accordingly, we need to make sure to track the virtual consumers
+        # offsets groups independently and only materialize the end result.
+        #
+        # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+        def register(offsets_group)
+          @groups << offsets_group
+
+          offsets_group.each { |offset| @marked[offset] = false }
+        end
+
+        # Marks given message as marked (virtually consumed).
+        # We mark given message offset and other earlier offsets from the same group as done
+        # and we can refresh our real offset representation based on that as it might have changed
+        # to a newer real offset.
+        # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+        def mark(message)
+          offset = message.offset
+
+          group = @groups.find { |reg_group| reg_group.include?(offset) }
+
+          # This case can happen when someone uses MoM and wants to mark message from a previous
+          # batch as consumed. We can add it, since the real offset refresh will point to it
+          unless group
+            group = [offset]
+            @groups << group
+          end
+
+          position = group.index(offset)
+
+          # Mark all previous messages from the same group also as virtually consumed
+          group[0..position].each do |markable_offset|
+            @marked[markable_offset] = true
+          end
+
+          # Recompute the real offset representation
+          materialize_real_offset
+        end
+
+        # Mark all from all groups including the `message`.
+        # Useful when operating in a collapsed state for marking
+        # @param message [Karafka::Messages::Message]
+        def mark_until(message)
+          mark(message)
+
+          @groups.each do |group|
+            group.each do |offset|
+              next if offset > message.offset
+
+              @marked[offset] = true
+            end
+          end
+
+          materialize_real_offset
+        end
+
+        # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+        def marked
+          @marked.select { |_, status| status }.map(&:first).sort
+        end
+
+        # Is there a real offset we can mark as consumed
+        # @return [Boolean]
+        def markable?
+          !@real_offset.negative?
+        end
+
+        # @return [Messages::Seek] markable message for real offset marking
+        def markable
+          raise Errors::InvalidRealOffsetUsage unless markable?
+
+          Messages::Seek.new(
+            @topic,
+            @partition,
+            @real_offset
+          )
+        end
+
+        private
+
+        # Recomputes the biggest possible real offset we can have.
+        # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+        # consumed because this will be the collective offset.
+        def materialize_real_offset
+          @marked.to_a.sort_by(&:first).each do |offset, marked|
+            break unless marked
+
+            @real_offset = offset
+          end
+
+          @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -44,23 +44,6 @@ module Karafka
 
               [[%i[virtual_partitions partitioner], :respond_to_call]]
             end
-
-            # Make sure that manual offset management is not used together with Virtual Partitions
-            # This would not make any sense as there would be edge cases related to skipping
-            # messages even if there were errors.
-            virtual do |data, errors|
-              next unless errors.empty?
-
-              virtual_partitions = data[:virtual_partitions]
-              manual_offset_management = data[:manual_offset_management]
-              active_job = data[:active_job]
-
-              next unless virtual_partitions[:active]
-              next unless manual_offset_management[:active]
-              next if active_job[:active]
-
-              [[%i[manual_offset_management], :not_with_virtual_partitions]]
-            end
           end
         end
       end

data/lib/karafka/processing/strategies/default.rb

@@ -26,6 +26,7 @@ module Karafka
         def mark_as_consumed(message)
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
+          return false if revoked?
 
           unless client.mark_as_consumed(message)
             coordinator.revoke
@@ -46,6 +47,7 @@ module Karafka
         def mark_as_consumed!(message)
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
+          return false if revoked?
 
           unless client.mark_as_consumed!(message)
             coordinator.revoke