karafka 2.3.0 → 2.3.2

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Coordinators
+        # Manager that keeps track of our offsets with the virtualization layer that are local
+        # to given partition assignment. It allows for easier offset management for virtual
+        # virtual partition cases as it provides us ability to mark as consumed and move the
+        # real offset behind as expected.
+        #
+        # @note We still use the regular coordinator "real" offset management as we want to have
+        #   them as separated as possible because the real seek offset management is also used for
+        #   pausing, filtering and others and should not be impacted by the virtual one
+        #
+        # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+        #   locked locations.
+        class VirtualOffsetManager
+          attr_reader :groups
+
+          # @param topic [String]
+          # @param partition [Integer]
+          # @param offset_metadata_strategy [Symbol] what metadata should we select. That is,
+          #   should we use the most recent or one picked from the offset that is going to be
+          #   committed
+          #
+          # @note We need topic and partition because we use a seek message (virtual) for real
+          #   offset management. We could keep real message reference but this can be memory
+          #   consuming and not worth it.
+          def initialize(topic, partition, offset_metadata_strategy)
+            @topic = topic
+            @partition = partition
+            @groups = []
+            @marked = {}
+            @offsets_metadata = {}
+            @real_offset = -1
+            @offset_metadata_strategy = offset_metadata_strategy
+            @current_offset_metadata = nil
+          end
+
+          # Clears the manager for a next collective operation
+          def clear
+            @groups.clear
+            @offsets_metadata.clear
+            @current_offset_metadata = nil
+            @marked.clear
+            @real_offset = -1
+          end
+
+          # Registers an offset group coming from one virtual consumer. In order to move the real
+          # underlying offset accordingly, we need to make sure to track the virtual consumers
+          # offsets groups independently and only materialize the end result.
+          #
+          # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+          def register(offsets_group)
+            @groups << offsets_group
+
+            offsets_group.each { |offset| @marked[offset] = false }
+          end
+
+          # Marks given message as marked (virtually consumed).
+          # We mark given message offset and other earlier offsets from the same group as done
+          # and we can refresh our real offset representation based on that as it might have
+          # changed to a newer real offset.
+          # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+          # @param offset_metadata [String, nil] offset metadata. `nil` if none
+          def mark(message, offset_metadata)
+            offset = message.offset
+
+            # Store metadata when we materialize the most stable offset
+            @offsets_metadata[offset] = offset_metadata
+            @current_offset_metadata = offset_metadata
+
+            group = @groups.find { |reg_group| reg_group.include?(offset) }
+
+            # This case can happen when someone uses MoM and wants to mark message from a previous
+            # batch as consumed. We can add it, since the real offset refresh will point to it
+            unless group
+              group = [offset]
+              @groups << group
+            end
+
+            position = group.index(offset)
+
+            # Mark all previous messages from the same group also as virtually consumed
+            group[0..position].each do |markable_offset|
+              # Set previous messages metadata offset as the offset of higher one for overwrites
+              # unless a different metadata were set explicitely
+              @offsets_metadata[markable_offset] ||= offset_metadata
+              @marked[markable_offset] = true
+            end
+
+            # Recompute the real offset representation
+            materialize_real_offset
+          end
+
+          # Mark all from all groups including the `message`.
+          # Useful when operating in a collapsed state for marking
+          # @param message [Karafka::Messages::Message]
+          # @param offset_metadata [String, nil]
+          def mark_until(message, offset_metadata)
+            mark(message, offset_metadata)
+
+            @groups.each do |group|
+              group.each do |offset|
+                next if offset > message.offset
+
+                @offsets_metadata[offset] = offset_metadata
+                @marked[offset] = true
+              end
+            end
+
+            materialize_real_offset
+          end
+
+          # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+          def marked
+            @marked.select { |_, status| status }.map(&:first).sort
+          end
+
+          # Is there a real offset we can mark as consumed
+          # @return [Boolean]
+          def markable?
+            !@real_offset.negative?
+          end
+
+          # @return [Array<Messages::Seek, String>] markable message for real offset marking and
+          #   its associated metadata
+          def markable
+            raise Errors::InvalidRealOffsetUsageError unless markable?
+
+            offset_metadata = case @offset_metadata_strategy
+                              when :exact
+                                @offsets_metadata.fetch(@real_offset)
+                              when :current
+                                @current_offset_metadata
+                              else
+                                raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+                              end
+
+            [
+              Messages::Seek.new(
+                @topic,
+                @partition,
+                @real_offset
+              ),
+              offset_metadata
+            ]
+          end
+
+          private
+
+          # Recomputes the biggest possible real offset we can have.
+          # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+          # consumed because this will be the collective offset.
+          def materialize_real_offset
+            @marked.to_a.sort_by(&:first).each do |offset, marked|
+              break unless marked
+
+              @real_offset = offset
+            end
+
+            @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -51,14 +51,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -57,14 +57,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -44,8 +44,6 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 # If we've reached number of retries that we could, we need to skip the first
                 # message that was not marked as consumed, pause and continue, while also moving
                 # this message to the dead topic.
@@ -53,14 +51,14 @@ module Karafka
                 # For a Mom setup, this means, that user has to manage the checkpointing by
                 # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -48,23 +48,15 @@ module Karafka
                   mark_as_consumed(last_group_message)
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -47,14 +47,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -51,14 +51,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -42,17 +42,15 @@ module Karafka
                 if coordinator.success?
                   # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
                   coordinator.pause_tracker.reset
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -48,16 +48,14 @@ module Karafka
                   return if revoked?
 
                   mark_as_consumed(last_group_message)
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  # Here we are in a collapsed state, hence we can apply the same logic as
-                  # Aj::DlqMom
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    # Here we are in a collapsed state, hence we can apply the same logic as
+                    # Aj::DlqMom
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -76,16 +76,10 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message)
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-                  dispatch_if_needed_and_mark_as_consumed
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end
@@ -183,7 +177,10 @@ module Karafka
             #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
             #   message without taking any action.
             def dispatch_to_dlq?
-              topic.dead_letter_queue.topic
+              return false unless topic.dead_letter_queue.topic
+              return false unless @_dispatch_to_dlq
+
+              true
             end
 
             # @return [Boolean] should we use a transaction to move the data to the DLQ.
@@ -192,6 +189,35 @@ module Karafka
             def dispatch_in_a_transaction?
               producer.transactional? && topic.dead_letter_queue.transactional?
             end
+
+            # Runs the DLQ strategy and based on it it performs certain operations
+            #
+            # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
+            # In case of `:retry` always `#retry_after_pause` is applied
+            def apply_dlq_flow
+              flow = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+
+              case flow
+              when :retry
+                retry_after_pause
+
+                return
+              when :skip
+                @_dispatch_to_dlq = false
+              when :dispatch
+                @_dispatch_to_dlq = true
+              else
+                raise Karafka::UnsupportedCaseError, flow
+              end
+
+              # We reset the pause to indicate we will now consider it as "ok".
+              coordinator.pause_tracker.reset
+
+              yield
+
+              # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
+              pause(coordinator.seek_offset, nil, false)
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -42,14 +42,10 @@ module Karafka
                   mark_as_consumed(last_group_message)
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  dispatch_if_needed_and_mark_as_consumed
-
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -53,16 +53,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  dispatch_if_needed_and_mark_as_consumed
-
-                  pause(coordinator.seek_offset, nil, false)
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -48,18 +48,15 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -41,23 +41,13 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  apply_dlq_flow do
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -42,16 +42,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
-
-                  dispatch_if_needed_and_mark_as_consumed
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  pause(coordinator.seek_offset, nil, false)
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -42,18 +42,15 @@ module Karafka
                   end
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -35,28 +35,18 @@ module Karafka
 
                 if coordinator.success?
                   coordinator.pause_tracker.reset
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Save the next offset we want to go with after moving given message to DLQ
-                  # Without this, we would not be able to move forward and we would end up
-                  # in an infinite loop trying to un-pause from the message we've already processed
-                  # Of course, since it's a MoM a rebalance or kill, will move it back as no
-                  # offsets are being committed
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    # Save the next offset we want to go with after moving given message to DLQ
+                    # Without this, we would not be able to move forward and we would end up
+                    # in an infinite loop trying to un-pause from the message we've already
+                    # processed. Of course, since it's a MoM a rebalance or kill, will move it back
+                    # as no offsets are being committed
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -155,6 +155,13 @@ module Karafka
             def handle_before_schedule_consume
               super
 
+              # We should not register offsets in virtual manager when in collapse as virtual
+              # manager is not used then for offsets materialization.
+              #
+              # If we would do so, it would cause increased storage in cases of endless errors
+              # that are being retried in collapse without a DLQ.
+              return if collapsed?
+
               coordinator.virtual_offset_manager.register(
                 messages.map(&:offset)
               )