karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -15,12 +17,30 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
+        attr_reader :filter, :virtual_offset_manager
+
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
-          @on_started_invoked = false
-          @on_finished_invoked = false
+
+          @executed = []
           @flow_lock = Mutex.new
+          @collapser = Collapser.new
+          @filter = FiltersApplier.new(self)
+
+          return unless topic.virtual_partitions?
+
+          @virtual_offset_manager = VirtualOffsetManager.new(
+            topic.name,
+            partition
+          )
+
+          # We register our own "internal" filter to support filtering of messages that were marked
+          # as consumed virtually
+          @filter.filters << Filters::VirtualLimiter.new(
+            @virtual_offset_manager,
+            @collapser
+          )
         end
 
         # Starts the coordination process
@@ -29,12 +49,37 @@ module Karafka
         def start(messages)
           super
 
-          @mutex.synchronize do
-            @on_started_invoked = false
-            @on_finished_invoked = false
-            @first_message = messages.first
-            @last_message = messages.last
-          end
+          @collapser.refresh!(messages.first.offset)
+
+          @filter.apply!(messages)
+
+          @executed.clear
+
+          # We keep the old processed offsets until the collapsing is done and regular processing
+          # with virtualization is restored
+          @virtual_offset_manager.clear if topic.virtual_partitions? && !@collapser.collapsed?
+
+          @last_message = messages.last
+        end
+
+        # Sets the consumer failure status and additionally starts the collapse until
+        #
+        # @param consumer [Karafka::BaseConsumer] consumer that failed
+        # @param error [StandardError] error from the failure
+        def failure!(consumer, error)
+          super
+          @collapser.collapse_until!(@last_message.offset + 1)
+        end
+
+        # @return [Boolean] are we in a collapsed state at the moment
+        def collapsed?
+          @collapser.collapsed?
+        end
+
+        # @return [Boolean] did any of the filters apply any logic that would cause use to run
+        #   the filtering flow
+        def filtered?
+          @filter.applied?
         end
 
         # @return [Boolean] is the coordinated work finished or not
@@ -42,14 +87,22 @@ module Karafka
           @running_jobs.zero?
         end
 
+        # Runs synchronized code once for a collective of virtual partitions prior to work being
+        # enqueued
+        def on_enqueued
+          @flow_lock.synchronize do
+            return unless executable?(:on_enqueued)
+
+            yield(@last_message)
+          end
+        end
+
         # Runs given code only once per all the coordinated jobs upon starting first of them
         def on_started
           @flow_lock.synchronize do
-            return if @on_started_invoked
+            return unless executable?(:on_started)
 
-            @on_started_invoked = true
-
-            yield(@first_message, @last_message)
+            yield(@last_message)
           end
         end
 
@@ -59,13 +112,36 @@ module Karafka
         def on_finished
           @flow_lock.synchronize do
             return unless finished?
-            return if @on_finished_invoked
+            return unless executable?(:on_finished)
+
+            yield(@last_message)
+          end
+        end
 
-            @on_finished_invoked = true
+        # Runs once after a partition is revoked
+        def on_revoked
+          @flow_lock.synchronize do
+            return unless executable?(:on_revoked)
 
-            yield(@first_message, @last_message)
+            yield(@last_message)
           end
         end
+
+        private
+
+        # Checks if given action is executable once. If it is and true is returned, this method
+        # will return false next time it is used.
+        #
+        # @param action [Symbol] what action we want to perform
+        # @return [Boolean] true if we can
+        # @note This method needs to run behind a mutex.
+        def executable?(action)
+          return false if @executed.include?(action)
+
+          @executed << action
+
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/processing/filters/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Base for all the filters.
+        # All filters (including custom) need to use this API.
+        #
+        # Due to the fact, that filters can limit data in such a way, that we need to pause or
+        # seek (throttling for example), the api is not just "remove some things from batch" but
+        # also provides ways to control the post-filtering operations that may be needed.
+        class Base
+          # @return [Karafka::Messages::Message, nil] the message that we want to use as a cursor
+          #   one to pause or seek or nil if not applicable.
+          attr_reader :cursor
+
+          include Karafka::Core::Helpers::Time
+
+          def initialize
+            @applied = false
+            @cursor = nil
+          end
+
+          # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
+          #   in mind, this may already be partial due to execution of previous filters.
+          def apply!(messages)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or
+          #   `:seek`.
+          def action
+            :skip
+          end
+
+          # @return [Boolean] did this filter change messages in any way
+          def applied?
+            @applied
+          end
+
+          # @return [Integer] default timeout for pausing (if applicable)
+          def timeout
+            0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # A filter that allows us to delay processing by pausing until time is right.
+        class Delayer < Base
+          # @param delay [Integer] ms delay / minimum age of each message we want to process
+          def initialize(delay)
+            super()
+
+            @delay = delay
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @delay / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_young = message.timestamp > border
+
+              if too_young
+                @applied = true
+
+                @cursor ||= message
+              end
+
+              @applied
+            end
+          end
+
+          # @return [Integer] timeout delay in ms
+          def timeout
+            return 0 unless @cursor
+
+            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+
+            timeout <= 0 ? 0 : timeout * 1_000
+          end
+
+          # @return [Symbol] action to take on post-filtering
+          def action
+            return :skip unless applied?
+
+            timeout <= 0 ? :seek : :pause
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Expirer for removing too old messages.
+        # It never moves offsets in any way and does not impact the processing flow. It always
+        # runs `:skip` action.
+        class Expirer < Base
+          # @param ttl [Integer] maximum age of a message (in ms)
+          def initialize(ttl)
+            super()
+
+            @ttl = ttl
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @ttl / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_old = message.timestamp < border
+
+              @applied = true if too_old
+
+              too_old
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace containing Pro out of the box filters used by various strategies
+      module Filters
+        # Throttler used to limit number of messages we can process in a given time interval
+        # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
+        # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
+        # not pause the partition and will fetch more data that we should not process.
+        #
+        # This is a special type of a filter that always throttles and makes us wait / seek if
+        # anything is applied out.
+        class Throttler < Base
+          # @param limit [Integer] how many messages we can process in a given time
+          # @param interval [Integer] interval in milliseconds for which we want to process
+          def initialize(limit, interval)
+            super()
+
+            @limit = limit
+            @interval = interval
+            @requests = Hash.new { |h, k| h[k] = 0 }
+          end
+
+          # Limits number of messages to a range that we can process (if needed) and keeps track
+          # of how many messages we've processed in a given time
+          # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
+          #   number we can accept in the context of throttling constraints
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+            @time = monotonic_now
+            @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
+            values = @requests.values.sum
+            accepted = 0
+
+            messages.delete_if do |message|
+              # +1 because of current
+              @applied = (values + accepted + 1) > @limit
+
+              @cursor = message if @applied && @cursor.nil?
+
+              next true if @applied
+
+              accepted += 1
+
+              false
+            end
+
+            @requests[@time] += accepted
+          end
+
+          # @return [Symbol] action to take upon throttler reaching certain state
+          def action
+            if applied?
+              timeout.zero? ? :seek : :pause
+            else
+              :skip
+            end
+          end
+
+          # @return [Integer] minimum number of milliseconds to wait before getting more messages
+          #   so we are no longer throttled and so we can process at least one message
+          def timeout
+            timeout = @interval - (monotonic_now - @time)
+            timeout <= 0 ? 0 : timeout
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Removes messages that are already marked as consumed in the virtual offset manager
+        # This should operate only when using virtual partitions.
+        #
+        # This cleaner prevents us from duplicated processing of messages that were virtually
+        # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
+        # to limit reprocessing when errors occur drastically when operating with virtual
+        # partitions
+        #
+        # @note It should be registered only when VPs are used
+        class VirtualLimiter < Base
+          # @param manager [Processing::VirtualOffsetManager]
+          # @param collapser [Processing::Collapser]
+          def initialize(manager, collapser)
+            @manager = manager
+            @collapser = collapser
+
+            super()
+          end
+
+          # Remove messages that we already marked as virtually consumed. Does nothing if not in
+          # the collapsed mode.
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            return unless @collapser.collapsed?
+
+            marked = @manager.marked
+
+            messages.delete_if { |message| marked.include?(message.offset) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Applier for all filters we want to have. Whether related to limiting messages based
+      # on the payload or any other things.
+      #
+      # From the outside world perspective, this encapsulates all the filters.
+      # This means that this is the API we expose as a single filter, allowing us to control
+      # the filtering via many filters easily.
+      class FiltersApplier
+        # @return [Array] registered filters array. Useful if we want to inject internal context
+        #   aware filters.
+        attr_reader :filters
+
+        # @param coordinator [Pro::Coordinator] pro coordinator
+        def initialize(coordinator)
+          # Builds filters out of their factories
+          # We build it that way (providing topic and partition) because there may be a case where
+          # someone wants to have a specific logic that is per topic or partition. Like for example
+          # a case where there is a cache bypassing revocations for topic partition.
+          #
+          # We provide full Karafka routing topic here and not the name only, in case the filter
+          # would be customized based on other topic settings (like VPs, etc)
+          #
+          # This setup allows for biggest flexibility also because topic object holds the reference
+          # to the subscription group and consumer group
+          @filters = coordinator.topic.filtering.factories.map do |factory|
+            factory.call(coordinator.topic, coordinator.partition)
+          end
+        end
+
+        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+        #   partition
+        def apply!(messages)
+          return unless active?
+
+          @filters.each { |filter| filter.apply!(messages) }
+        end
+
+        # @return [Boolean] did we filter out any messages during filtering run
+        def applied?
+          return false unless active?
+
+          !applied.empty?
+        end
+
+        # @return [Symbol] consumer post-filtering action that should be taken
+        def action
+          return :skip unless applied?
+
+          # The highest priority is on a potential backoff from any of the filters because it is
+          # the less risky (delay and continue later)
+          return :pause if applied.any? { |filter| filter.action == :pause }
+
+          # If none of the filters wanted to pause, we can check for any that would want to seek
+          # and if there is any, we can go with this strategy
+          return :seek if applied.any? { |filter| filter.action == :seek }
+
+          :skip
+        end
+
+        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+        #   filters to satisfy all of them.
+        def timeout
+          applied.map(&:timeout).compact.min || 0
+        end
+
+        # The first message we do need to get next time we poll. We use the minimum not to jump
+        # accidentally by over any.
+        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+        def cursor
+          return nil unless active?
+
+          applied.map(&:cursor).compact.min_by(&:offset)
+        end
+
+        private
+
+        # @return [Boolean] is filtering active
+        def active?
+          !@filters.empty?
+        end
+
+        # @return [Array<Object>] filters that applied any sort of messages limiting
+        def applied
+          @filters.select(&:applied?)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -16,8 +18,7 @@ module Karafka
       # Pro jobs
       module Jobs
         # The main job type in a non-blocking variant.
-        # This variant works "like" the regular consumption but pauses the partition for as long
-        # as it is needed until a job is done.
+        # This variant works "like" the regular consumption but does not block the queue.
         #
         # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
         # if would block.
@@ -25,8 +26,9 @@ module Karafka
         # @note It needs to be working with a proper consumer that will handle the partition
         #   management. This layer of the framework knows nothing about Kafka messages consumption.
         class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
-          # Releases the blocking lock after it is done with the preparation phase for this job
-          def before_call
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Consume`
+          def initialize(*args)
             super
             @non_blocking = true
           end

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The revoked job type in a non-blocking variant.
+        # This variant works "like" the regular revoked but does not block the queue.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # in scenarios where there are more jobs than threads, without this being async we
+        # would potentially stop polling
+        class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -14,14 +16,30 @@ module Karafka
     module Processing
       # Pro jobs builder that supports lrj
       class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        def idle(executor)
+          Karafka::Processing::Jobs::Idle.new(executor)
+        end
+
         # @param executor [Karafka::Processing::Executor]
         # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-        # @param coordinator [Karafka::Processing::Coordinator]
         # @return [Karafka::Processing::Jobs::Consume] blocking job
         # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
-        def consume(executor, messages, coordinator)
+        def consume(executor, messages)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages)
+          else
+            super
+          end
+        end
+
+        # @param executor [Karafka::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
+        # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def revoked(executor)
           if executor.topic.long_running_job?
-            Jobs::ConsumeNonBlocking.new(executor, messages, coordinator)
+            Jobs::RevokedNonBlocking.new(executor)
           else
             super
           end