karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/processing/partitioner.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -16,39 +18,46 @@ module Karafka
       class Partitioner < ::Karafka::Processing::Partitioner
         # @param topic [String] topic name
         # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
+        #   will be used with those messages
         # @yieldparam [Integer] group id
         # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-        def call(topic, messages)
+        def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
-          @concurrency ||= ::Karafka::App.config.concurrency
-
-          # We only partition work if we have a virtual partitioner and more than one thread to
-          # process the data. With one thread it is not worth partitioning the work as the work
-          # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitioner? && @concurrency > 1
-            # We need to reduce it to number of threads, so the group_id is not a direct effect
+          # We only partition work if we have:
+          # - a virtual partitioner
+          # - more than one thread to process the data
+          # - collective is not collapsed via coordinator
+          #
+          # With one thread it is not worth partitioning the work as the work itself will be
+          # assigned to one thread (pointless work)
+          #
+          # We collapse the partitioning on errors because we "regain" full ordering on a batch
+          # that potentially contains the data that caused the error.
+          #
+          # This is great because it allows us to run things without the parallelization that adds
+          # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
+          if ktopic.virtual_partitions? &&
+             ktopic.virtual_partitions.max_partitions > 1 &&
+             !coordinator.collapsed?
+            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks
-            groupings = messages
-                        .group_by { |msg| ktopic.virtual_partitioner.call(msg) }
-                        .values
-
-            # Reduce the max concurrency to a size that matches the concurrency
-            # As mentioned above we cannot use the partitioning keys directly as it could cause
-            # memory leaks
             #
-            # The algorithm here is simple, we assume that the most costly in terms of processing,
-            # will be processing of the biggest group and we reduce the smallest once to have
-            # max of groups equal to concurrency
-            while groupings.size > @concurrency
-              groupings.sort_by! { |grouping| -grouping.size }
+            # This also needs to be consistent because the aggregation here needs to warrant, that
+            # the same partitioned message will always be assigned to the same virtual partition.
+            # Otherwise in case of a window aggregation with VP spanning across several polls, the
+            # data could not be complete.
+            groupings = messages.group_by do |msg|
+              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
 
-              # Offset order needs to be maintained for virtual partitions
-              groupings << (groupings.pop + groupings.pop).sort_by!(&:offset)
+              key % ktopic.virtual_partitions.max_partitions
             end
 
-            groupings.each_with_index { |messages_group, index| yield(index, messages_group) }
+            groupings.each do |key, messages_group|
+              yield(key, messages_group)
+            end
           else
             # When no virtual partitioner, works as regular one
             yield(0, messages)

data/lib/karafka/pro/processing/scheduler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -29,17 +31,15 @@ module Karafka
         # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
         #
         def schedule_consumption(queue, jobs_array)
-          pt = PerformanceTracker.instance
+          perf_tracker = PerformanceTracker.instance
 
           ordered = []
 
           jobs_array.each do |job|
-            messages = job.messages
-            message = messages.first
-
-            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-            ordered << [job, cost]
+            ordered << [
+              job,
+              processing_cost(perf_tracker, job)
+            ]
           end
 
           ordered.sort_by!(&:last)
@@ -50,6 +50,25 @@ module Karafka
             queue << job
           end
         end
+
+        private
+
+        # @param perf_tracker [PerformanceTracker]
+        # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+        # @return [Numeric] estimated cost of processing this job
+        def processing_cost(perf_tracker, job)
+          if job.is_a?(::Karafka::Processing::Jobs::Consume)
+            messages = job.messages
+            message = messages.first
+
+            perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+          else
+            # LJF will set first the most expensive, but we want to run the zero cost jobs
+            # related to the lifecycle always first. That is why we "emulate" that they
+            # the longest possible jobs that anyone can run
+            Float::INFINITY
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module DlqFtrLrjMom
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMom
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqFtrLrjMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          module DlqFtrMom
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # We write our own custom handler for after consume here, because we mark as consumed
+            # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
+            # dispatch, we can mark this message as consumed as well.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          # - VP
+          module DlqFtrMomVp
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::DlqFtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
+            # end of all VPs
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          #
+          # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+          # offset management works in this case.
+          module DlqLrjMom
+            include Strategies::Default
+            include Strategies::Dlq::Default
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # We cannot use a VP version of this, because non-VP can early stop on shutdown
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          #
+          # This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
+          # offset management works in this case.
+          module DlqLrjMomVp
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Manual offset management enabled
+          #
+          # AJ has manual offset management on by default and the offset management is delegated to
+          # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
+          # consumed when we skip given job upon errors. In all the other scenarios marking as
+          # consumed needs to happen in the AJ consumer on a per job basis.
+          module DlqMom
+            include Strategies::Dlq::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              manual_offset_management
+            ].freeze
+
+            # How should we post-finalize consumption.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+                  coordinator.pause_tracker.reset
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end