karafka 2.0.37 → 2.0.39

data/lib/karafka/pro/active_job/consumer.rb

@@ -22,20 +22,25 @@ module Karafka
       #
       # It contains slightly better revocation warranties than the regular blocking consumer as
       # it can stop processing batch of jobs in the middle after the revocation.
-      class Consumer < Karafka::Pro::BaseConsumer
+      class Consumer < ::Karafka::ActiveJob::Consumer
         # Runs ActiveJob jobs processing and handles lrj if needed
         def consume
           messages.each do |message|
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
             break if revoked?
-            break if Karafka::App.stopping?
 
-            job = ::ActiveSupport::JSON.decode(message.raw_payload)
+            # We cannot early stop when running virtual partitions because the intermediate state
+            # would force us not to commit the offsets. This would cause extensive
+            # double-processing
+            break if Karafka::App.stopping? && !topic.virtual_partitions?
 
-            tags.add(:job_class, job['job_class'])
+            # Break if we already know, that one of virtual partitions has failed and we will
+            # be restarting processing all together after all VPs are done. This will minimize
+            # number of jobs that will be re-processed
+            break if topic.virtual_partitions? && failing?
 
-            ::ActiveJob::Base.execute(job)
+            consume_job(message)
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
             # this could create random markings.

data/lib/karafka/pro/processing/coordinator.rb

@@ -17,6 +17,8 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
+        attr_reader :filter
+
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
@@ -24,6 +26,7 @@ module Karafka
           @executed = []
           @flow_lock = Mutex.new
           @collapser = Collapser.new
+          @filter = FiltersApplier.new(self)
         end
 
         # Starts the coordination process
@@ -34,10 +37,10 @@ module Karafka
 
           @collapser.refresh!(messages.first.offset)
 
-          @mutex.synchronize do
-            @executed.clear
-            @last_message = messages.last
-          end
+          @filter.apply!(messages)
+
+          @executed.clear
+          @last_message = messages.last
         end
 
         # Sets the consumer failure status and additionally starts the collapse until
@@ -54,6 +57,12 @@ module Karafka
           @collapser.collapsed?
         end
 
+        # @return [Boolean] did any of the filters apply any logic that would cause use to run
+        #   the filtering flow
+        def filtered?
+          @filter.applied?
+        end
+
         # @return [Boolean] is the coordinated work finished or not
         def finished?
           @running_jobs.zero?

data/lib/karafka/pro/processing/filters/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Base for all the filters.
+        # All filters (including custom) need to use this API.
+        #
+        # Due to the fact, that filters can limit data in such a way, that we need to pause or
+        # seek (throttling for example), the api is not just "remove some things from batch" but
+        # also provides ways to control the post-filtering operations that may be needed.
+        class Base
+          # @return [Karafka::Messages::Message, nil] the message that we want to use as a cursor
+          #   one to pause or seek or nil if not applicable.
+          attr_reader :cursor
+
+          include Karafka::Core::Helpers::Time
+
+          def initialize
+            @applied = false
+            @cursor = nil
+          end
+
+          # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
+          #   in mind, this may already be partial due to execution of previous filters.
+          def apply!(messages)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or
+          #   `:seek`.
+          def action
+            :skip
+          end
+
+          # @return [Boolean] did this filter change messages in any way
+          def applied?
+            @applied
+          end
+
+          # @return [Integer] default timeout for pausing (if applicable)
+          def timeout
+            0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # A filter that allows us to delay processing by pausing until time is right.
+        class Delayer < Base
+          # @param delay [Integer] ms delay / minimum age of each message we want to process
+          def initialize(delay)
+            super()
+
+            @delay = delay
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @delay / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_young = message.timestamp > border
+
+              if too_young
+                @applied = true
+
+                @cursor ||= message
+              end
+
+              @applied
+            end
+          end
+
+          # @return [Integer] timeout delay in ms
+          def timeout
+            return 0 unless @cursor
+
+            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+
+            timeout <= 0 ? 0 : timeout * 1_000
+          end
+
+          # @return [Symbol] action to take on post-filtering
+          def action
+            return :skip unless applied?
+
+            timeout <= 0 ? :seek : :pause
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Expirer for removing too old messages.
+        # It never moves offsets in any way and does not impact the processing flow. It always
+        # runs `:skip` action.
+        class Expirer < Base
+          # @param ttl [Integer] maximum age of a message (in ms)
+          def initialize(ttl)
+            super()
+
+            @ttl = ttl
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @ttl / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_old = message.timestamp < border
+
+              @applied = true if too_old
+
+              too_old
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace containing Pro out of the box filters used by various strategies
+      module Filters
+        # Throttler used to limit number of messages we can process in a given time interval
+        # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
+        # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
+        # not pause the partition and will fetch more data that we should not process.
+        #
+        # This is a special type of a filter that always throttles and makes us wait / seek if
+        # anything is applied out.
+        class Throttler < Base
+          # @param limit [Integer] how many messages we can process in a given time
+          # @param interval [Integer] interval in milliseconds for which we want to process
+          def initialize(limit, interval)
+            super()
+
+            @limit = limit
+            @interval = interval
+            @requests = Hash.new { |h, k| h[k] = 0 }
+          end
+
+          # Limits number of messages to a range that we can process (if needed) and keeps track
+          # of how many messages we've processed in a given time
+          # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
+          #   number we can accept in the context of throttling constraints
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+            @time = monotonic_now
+            @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
+            values = @requests.values.sum
+            accepted = 0
+
+            messages.delete_if do |message|
+              # +1 because of current
+              @applied = (values + accepted + 1) > @limit
+
+              @cursor = message if @applied && @cursor.nil?
+
+              next true if @applied
+
+              accepted += 1
+
+              false
+            end
+
+            @requests[@time] += accepted
+          end
+
+          # @return [Symbol] action to take upon throttler reaching certain state
+          def action
+            if applied?
+              timeout.zero? ? :seek : :pause
+            else
+              :skip
+            end
+          end
+
+          # @return [Integer] minimum number of milliseconds to wait before getting more messages
+          #   so we are no longer throttled and so we can process at least one message
+          def timeout
+            timeout = @interval - (monotonic_now - @time)
+            timeout <= 0 ? 0 : timeout
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Applier for all filters we want to have. Whether related to limiting messages based
+      # on the payload or any other things.
+      #
+      # From the outside world perspective, this encapsulates all the filters.
+      # This means that this is the API we expose as a single filter, allowing us to control
+      # the filtering via many filters easily.
+      class FiltersApplier
+        # @param coordinator [Pro::Coordinator] pro coordinator
+        def initialize(coordinator)
+          # Builds filters out of their factories
+          # We build it that way (providing topic and partition) because there may be a case where
+          # someone wants to have a specific logic that is per topic or partition. Like for example
+          # a case where there is a cache bypassing revocations for topic partition.
+          #
+          # We provide full Karafka routing topic here and not the name only, in case the filter
+          # would be customized based on other topic settings (like VPs, etc)
+          #
+          # This setup allows for biggest flexibility also because topic object holds the reference
+          # to the subscription group and consumer group
+          @filters = coordinator.topic.filtering.factories.map do |factory|
+            factory.call(coordinator.topic, coordinator.partition)
+          end
+        end
+
+        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+        #   partition
+        def apply!(messages)
+          return unless active?
+
+          @filters.each { |filter| filter.apply!(messages) }
+        end
+
+        # @return [Boolean] did we filter out any messages during filtering run
+        def applied?
+          return false unless active?
+
+          !applied.empty?
+        end
+
+        # @return [Symbol] consumer post-filtering action that should be taken
+        def action
+          return :skip unless applied?
+
+          # The highest priority is on a potential backoff from any of the filters because it is
+          # the less risky (delay and continue later)
+          return :pause if applied.any? { |filter| filter.action == :pause }
+
+          # If none of the filters wanted to pause, we can check for any that would want to seek
+          # and if there is any, we can go with this strategy
+          return :seek if applied.any? { |filter| filter.action == :seek }
+
+          :skip
+        end
+
+        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+        #   filters to satisfy all of them.
+        def timeout
+          applied.map(&:timeout).compact.min || 0
+        end
+
+        # The first message we do need to get next time we poll. We use the minimum not to jump
+        # accidentally by over any.
+        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+        def cursor
+          return nil unless active?
+
+          applied.map(&:cursor).compact.min_by(&:offset)
+        end
+
+        private
+
+        # @return [Boolean] is filtering active
+        def active?
+          !@filters.empty?
+        end
+
+        # @return [Array<Object>] filters that applied any sort of messages limiting
+        def applied
+          @filters.select(&:applied?)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -16,14 +16,18 @@ module Karafka
     module Processing
       # Pro jobs builder that supports lrj
       class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        def idle(executor)
+          Karafka::Processing::Jobs::Idle.new(executor)
+        end
+
         # @param executor [Karafka::Processing::Executor]
         # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-        # @param coordinator [Karafka::Processing::Coordinator]
         # @return [Karafka::Processing::Jobs::Consume] blocking job
         # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
-        def consume(executor, messages, coordinator)
+        def consume(executor, messages)
           if executor.topic.long_running_job?
-            Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
+            Jobs::ConsumeNonBlocking.new(executor, messages)
           else
             super
           end

data/lib/karafka/pro/processing/scheduler.rb

@@ -31,17 +31,15 @@ module Karafka
         # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
         #
         def schedule_consumption(queue, jobs_array)
-          pt = PerformanceTracker.instance
+          perf_tracker = PerformanceTracker.instance
 
           ordered = []
 
           jobs_array.each do |job|
-            messages = job.messages
-            message = messages.first
-
-            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
-
-            ordered << [job, cost]
+            ordered << [
+              job,
+              processing_cost(perf_tracker, job)
+            ]
           end
 
           ordered.sort_by!(&:last)
@@ -52,6 +50,25 @@ module Karafka
             queue << job
           end
         end
+
+        private
+
+        # @param perf_tracker [PerformanceTracker]
+        # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+        # @return [Numeric] estimated cost of processing this job
+        def processing_cost(perf_tracker, job)
+          if job.is_a?(::Karafka::Processing::Jobs::Consume)
+            messages = job.messages
+            message = messages.first
+
+            perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+          else
+            # LJF will set first the most expensive, but we want to run the zero cost jobs
+            # related to the lifecycle always first. That is why we "emulate" that they
+            # the longest possible jobs that anyone can run
+            Float::INFINITY
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module DlqFtrLrjMom
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMom
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqFtrLrjMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end