karafka 2.0.38 → 2.0.39

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Expirer for removing too old messages.
+        # It never moves offsets in any way and does not impact the processing flow. It always
+        # runs `:skip` action.
+        class Expirer < Base
+          # @param ttl [Integer] maximum age of a message (in ms)
+          def initialize(ttl)
+            super()
+
+            @ttl = ttl
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @ttl / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_old = message.timestamp < border
+
+              @applied = true if too_old
+
+              too_old
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace containing Pro out of the box filters used by various strategies
+      module Filters
+        # Throttler used to limit number of messages we can process in a given time interval
+        # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
+        # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
+        # not pause the partition and will fetch more data that we should not process.
+        #
+        # This is a special type of a filter that always throttles and makes us wait / seek if
+        # anything is applied out.
+        class Throttler < Base
+          # @param limit [Integer] how many messages we can process in a given time
+          # @param interval [Integer] interval in milliseconds for which we want to process
+          def initialize(limit, interval)
+            super()
+
+            @limit = limit
+            @interval = interval
+            @requests = Hash.new { |h, k| h[k] = 0 }
+          end
+
+          # Limits number of messages to a range that we can process (if needed) and keeps track
+          # of how many messages we've processed in a given time
+          # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
+          #   number we can accept in the context of throttling constraints
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+            @time = monotonic_now
+            @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
+            values = @requests.values.sum
+            accepted = 0
+
+            messages.delete_if do |message|
+              # +1 because of current
+              @applied = (values + accepted + 1) > @limit
+
+              @cursor = message if @applied && @cursor.nil?
+
+              next true if @applied
+
+              accepted += 1
+
+              false
+            end
+
+            @requests[@time] += accepted
+          end
+
+          # @return [Symbol] action to take upon throttler reaching certain state
+          def action
+            if applied?
+              timeout.zero? ? :seek : :pause
+            else
+              :skip
+            end
+          end
+
+          # @return [Integer] minimum number of milliseconds to wait before getting more messages
+          #   so we are no longer throttled and so we can process at least one message
+          def timeout
+            timeout = @interval - (monotonic_now - @time)
+            timeout <= 0 ? 0 : timeout
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Applier for all filters we want to have. Whether related to limiting messages based
+      # on the payload or any other things.
+      #
+      # From the outside world perspective, this encapsulates all the filters.
+      # This means that this is the API we expose as a single filter, allowing us to control
+      # the filtering via many filters easily.
+      class FiltersApplier
+        # @param coordinator [Pro::Coordinator] pro coordinator
+        def initialize(coordinator)
+          # Builds filters out of their factories
+          # We build it that way (providing topic and partition) because there may be a case where
+          # someone wants to have a specific logic that is per topic or partition. Like for example
+          # a case where there is a cache bypassing revocations for topic partition.
+          #
+          # We provide full Karafka routing topic here and not the name only, in case the filter
+          # would be customized based on other topic settings (like VPs, etc)
+          #
+          # This setup allows for biggest flexibility also because topic object holds the reference
+          # to the subscription group and consumer group
+          @filters = coordinator.topic.filtering.factories.map do |factory|
+            factory.call(coordinator.topic, coordinator.partition)
+          end
+        end
+
+        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+        #   partition
+        def apply!(messages)
+          return unless active?
+
+          @filters.each { |filter| filter.apply!(messages) }
+        end
+
+        # @return [Boolean] did we filter out any messages during filtering run
+        def applied?
+          return false unless active?
+
+          !applied.empty?
+        end
+
+        # @return [Symbol] consumer post-filtering action that should be taken
+        def action
+          return :skip unless applied?
+
+          # The highest priority is on a potential backoff from any of the filters because it is
+          # the less risky (delay and continue later)
+          return :pause if applied.any? { |filter| filter.action == :pause }
+
+          # If none of the filters wanted to pause, we can check for any that would want to seek
+          # and if there is any, we can go with this strategy
+          return :seek if applied.any? { |filter| filter.action == :seek }
+
+          :skip
+        end
+
+        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+        #   filters to satisfy all of them.
+        def timeout
+          applied.map(&:timeout).compact.min || 0
+        end
+
+        # The first message we do need to get next time we poll. We use the minimum not to jump
+        # accidentally by over any.
+        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+        def cursor
+          return nil unless active?
+
+          applied.map(&:cursor).compact.min_by(&:offset)
+        end
+
+        private
+
+        # @return [Boolean] is filtering active
+        def active?
+          !@filters.empty?
+        end
+
+        # @return [Array<Object>] filters that applied any sort of messages limiting
+        def applied
+          @filters.select(&:applied?)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -16,14 +16,18 @@ module Karafka
     module Processing
       # Pro jobs builder that supports lrj
       class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        def idle(executor)
+          Karafka::Processing::Jobs::Idle.new(executor)
+        end
+
         # @param executor [Karafka::Processing::Executor]
         # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-        # @param coordinator [Karafka::Processing::Coordinator]
         # @return [Karafka::Processing::Jobs::Consume] blocking job
         # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
-        def consume(executor, messages, coordinator)
+        def consume(executor, messages)
           if executor.topic.long_running_job?
-            Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
+            Jobs::ConsumeNonBlocking.new(executor, messages)
           else
             super
           end

data/lib/karafka/pro/processing/scheduler.rb

@@ -31,17 +31,15 @@ module Karafka
         # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
         #
         def schedule_consumption(queue, jobs_array)
-          pt = PerformanceTracker.instance
+          perf_tracker = PerformanceTracker.instance
 
           ordered = []
 
           jobs_array.each do |job|
-            messages = job.messages
-            message = messages.first
-
-            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-            ordered << [job, cost]
+            ordered << [
+              job,
+              processing_cost(perf_tracker, job)
+            ]
           end
 
           ordered.sort_by!(&:last)
@@ -52,6 +50,25 @@ module Karafka
             queue << job
           end
         end
+
+        private
+
+        # @param perf_tracker [PerformanceTracker]
+        # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+        # @return [Numeric] estimated cost of processing this job
+        def processing_cost(perf_tracker, job)
+          if job.is_a?(::Karafka::Processing::Jobs::Consume)
+            messages = job.messages
+            message = messages.first
+
+            perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+          else
+            # LJF will set first the most expensive, but we want to run the zero cost jobs
+            # related to the lifecycle always first. That is why we "emulate" that they
+            # the longest possible jobs that anyone can run
+            Float::INFINITY
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module DlqFtrLrjMom
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMom
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqFtrLrjMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          module DlqFtrMom
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # We write our own custom handler for after consume here, because we mark as consumed
+            # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
+            # dispatch, we can mark this message as consumed as well.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          # - VP
+          module DlqFtrMomVp
+            include Strategies::Vp::Default
+            include Strategies::Aj::DlqFtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
+            # end of all VPs
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          #
+          # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+          # offset management works in this case.
+          module DlqLrjMom
+            include Strategies::Aj::DlqLrjMomVp
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # We cannot use a VP version of this, because non-VP can early stop on shutdown
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end