karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/processing/collapser.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manages the collapse of virtual partitions
+      # Since any non-virtual partition is actually a virtual partition of size one, we can use
+      # it in a generic manner without having to distinguish between those cases.
+      #
+      # We need to have notion of the offset until we want to collapse because upon pause and retry
+      # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
+      # (different) dataset and without tracking the end of collapse, there would be a chance for
+      # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
+      # messages from the corrupted batch are processed.
+      class Collapser
+        # When initialized, nothing is collapsed
+        def initialize
+          @collapsed = false
+          @until_offset = -1
+          @mutex = Mutex.new
+        end
+
+        # @return [Boolean] Should we collapse into a single consumer
+        def collapsed?
+          @collapsed
+        end
+
+        # Collapse until given offset. Until given offset is encountered or offset bigger than that
+        # we keep collapsing.
+        # @param offset [Integer] offset until which we keep the collapse
+        def collapse_until!(offset)
+          @mutex.synchronize do
+            # We check it here in case after a pause and re-fetch we would get less messages and
+            # one of them would cause an error. We do not want to overwrite the offset here unless
+            # it is bigger.
+            @until_offset = offset if offset > @until_offset
+          end
+        end
+
+        # Sets the collapse state based on the first collective offset that we are going to process
+        # and makes the decision whether or not we need to still keep the collapse.
+        # @param first_offset [Integer] first offset from a collective batch
+        def refresh!(first_offset)
+          @mutex.synchronize do
+            @collapsed = first_offset < @until_offset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinator.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro coordinator that provides extra orchestration methods useful for parallel processing
+      # within the same partition
+      class Coordinator < ::Karafka::Processing::Coordinator
+        extend Forwardable
+
+        def_delegators :@collapser, :collapsed?, :collapse_until!
+
+        attr_reader :filter, :virtual_offset_manager
+
+        # @param args [Object] anything the base coordinator accepts
+        def initialize(*args)
+          super
+
+          @executed = []
+          @flow_lock = Mutex.new
+          @collapser = Collapser.new
+          @filter = FiltersApplier.new(self)
+
+          return unless topic.virtual_partitions?
+
+          @virtual_offset_manager = VirtualOffsetManager.new(
+            topic.name,
+            partition
+          )
+
+          # We register our own "internal" filter to support filtering of messages that were marked
+          # as consumed virtually
+          @filter.filters << Filters::VirtualLimiter.new(
+            @virtual_offset_manager,
+            @collapser
+          )
+        end
+
+        # Starts the coordination process
+        # @param messages [Array<Karafka::Messages::Message>] messages for which processing we are
+        #   going to coordinate.
+        def start(messages)
+          super
+
+          @collapser.refresh!(messages.first.offset)
+
+          @filter.apply!(messages)
+
+          @executed.clear
+
+          # We keep the old processed offsets until the collapsing is done and regular processing
+          # with virtualization is restored
+          @virtual_offset_manager.clear if topic.virtual_partitions? && !collapsed?
+
+          @last_message = messages.last
+        end
+
+        # Sets the consumer failure status and additionally starts the collapse until
+        #
+        # @param consumer [Karafka::BaseConsumer] consumer that failed
+        # @param error [StandardError] error from the failure
+        def failure!(consumer, error)
+          super
+          collapse_until!(@last_message.offset + 1)
+        end
+
+        # @return [Boolean] did any of the filters apply any logic that would cause use to run
+        #   the filtering flow
+        def filtered?
+          @filter.applied?
+        end
+
+        # @return [Boolean] is the coordinated work finished or not
+        def finished?
+          @running_jobs.zero?
+        end
+
+        # Runs synchronized code once for a collective of virtual partitions prior to work being
+        # enqueued
+        def on_enqueued
+          @flow_lock.synchronize do
+            return unless executable?(:on_enqueued)
+
+            yield(@last_message)
+          end
+        end
+
+        # Runs given code only once per all the coordinated jobs upon starting first of them
+        def on_started
+          @flow_lock.synchronize do
+            return unless executable?(:on_started)
+
+            yield(@last_message)
+          end
+        end
+
+        # Runs once when all the work that is suppose to be coordinated is finished
+        # It runs once per all the coordinated jobs and should be used to run any type of post
+        # jobs coordination processing execution
+        def on_finished
+          @flow_lock.synchronize do
+            return unless finished?
+            return unless executable?(:on_finished)
+
+            yield(@last_message)
+          end
+        end
+
+        # Runs once after a partition is revoked
+        def on_revoked
+          @flow_lock.synchronize do
+            return unless executable?(:on_revoked)
+
+            yield(@last_message)
+          end
+        end
+
+        private
+
+        # Checks if given action is executable once. If it is and true is returned, this method
+        # will return false next time it is used.
+        #
+        # @param action [Symbol] what action we want to perform
+        # @return [Boolean] true if we can
+        # @note This method needs to run behind a mutex.
+        def executable?(action)
+          return false if @executed.include?(action)
+
+          @executed << action
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Base for all the filters.
+        # All filters (including custom) need to use this API.
+        #
+        # Due to the fact, that filters can limit data in such a way, that we need to pause or
+        # seek (throttling for example), the api is not just "remove some things from batch" but
+        # also provides ways to control the post-filtering operations that may be needed.
+        class Base
+          # @return [Karafka::Messages::Message, nil] the message that we want to use as a cursor
+          #   one to pause or seek or nil if not applicable.
+          attr_reader :cursor
+
+          include Karafka::Core::Helpers::Time
+
+          def initialize
+            @applied = false
+            @cursor = nil
+          end
+
+          # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
+          #   in mind, this may already be partial due to execution of previous filters.
+          def apply!(messages)
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or
+          #   `:seek`.
+          def action
+            :skip
+          end
+
+          # @return [Boolean] did this filter change messages in any way
+          def applied?
+            @applied
+          end
+
+          # @return [Integer] default timeout for pausing (if applicable)
+          def timeout
+            0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # A filter that allows us to delay processing by pausing until time is right.
+        class Delayer < Base
+          # @param delay [Integer] ms delay / minimum age of each message we want to process
+          def initialize(delay)
+            super()
+
+            @delay = delay
+          end
+
+          # Removes too young messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @delay / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_young = message.timestamp > border
+
+              if too_young
+                @applied = true
+
+                @cursor ||= message
+              end
+
+              @applied
+            end
+          end
+
+          # @return [Integer] timeout delay in ms
+          def timeout
+            return 0 unless @cursor
+
+            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+
+            timeout <= 0 ? 0 : timeout * 1_000
+          end
+
+          # @return [Symbol] action to take on post-filtering
+          def action
+            return :skip unless applied?
+
+            timeout <= 0 ? :seek : :pause
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Expirer for removing too old messages.
+        # It never moves offsets in any way and does not impact the processing flow. It always
+        # runs `:skip` action.
+        class Expirer < Base
+          # @param ttl [Integer] maximum age of a message (in ms)
+          def initialize(ttl)
+            super()
+
+            @ttl = ttl
+          end
+
+          # Removes too old messages
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+
+            # Time on message is in seconds with ms precision, so we need to convert the ttl that
+            # is in ms to this format
+            border = ::Time.now.utc - @ttl / 1_000.to_f
+
+            messages.delete_if do |message|
+              too_old = message.timestamp < border
+
+              @applied = true if too_old
+
+              too_old
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Delayer that checks if we have appropriate insights available. If not, pauses for
+        # 5 seconds so the insights can be loaded from the broker.
+        #
+        # In case it would take more than five seconds to load insights, it will just pause again
+        #
+        # This filter ensures, that we always have inline insights that a consumer can use
+        #
+        # It is relevant in most cases only during the process start, when first poll may not
+        # yield statistics yet but will give some data.
+        class InlineInsightsDelayer < Base
+          # Minimum how long should we pause when there are no metrics
+          PAUSE_TIMEOUT = 5_000
+
+          private_constant :PAUSE_TIMEOUT
+
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer] partition
+          def initialize(topic, partition)
+            super()
+            @topic = topic
+            @partition = partition
+          end
+
+          # Pauses if inline insights would not be available. Does nothing otherwise
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = messages.first
+
+            # Nothing to do if there were no messages
+            # This can happen when we chain filters
+            return unless @cursor
+
+            insights = ::Karafka::Processing::InlineInsights::Tracker.find(
+              @topic,
+              @partition
+            )
+
+            # If insights are available, also nothing to do here and we can just process
+            return unless insights.empty?
+
+            messages.clear
+
+            @applied = true
+          end
+
+          # @return [Integer] ms timeout in case of pause
+          def timeout
+            @cursor && applied? ? PAUSE_TIMEOUT : 0
+          end
+
+          # Pause when we had to back-off or skip if delay is not needed
+          def action
+            applied? ? :pause : :skip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace containing Pro out of the box filters used by various strategies
+      module Filters
+        # Throttler used to limit number of messages we can process in a given time interval
+        # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
+        # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
+        # not pause the partition and will fetch more data that we should not process.
+        #
+        # This is a special type of a filter that always throttles and makes us wait / seek if
+        # anything is applied out.
+        class Throttler < Base
+          # @param limit [Integer] how many messages we can process in a given time
+          # @param interval [Integer] interval in milliseconds for which we want to process
+          def initialize(limit, interval)
+            super()
+
+            @limit = limit
+            @interval = interval
+            @requests = Hash.new { |h, k| h[k] = 0 }
+          end
+
+          # Limits number of messages to a range that we can process (if needed) and keeps track
+          # of how many messages we've processed in a given time
+          # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
+          #   number we can accept in the context of throttling constraints
+          def apply!(messages)
+            @applied = false
+            @cursor = nil
+            @time = monotonic_now
+            @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
+            values = @requests.values.sum
+            accepted = 0
+
+            messages.delete_if do |message|
+              # +1 because of current
+              @applied = (values + accepted + 1) > @limit
+
+              @cursor = message if @applied && @cursor.nil?
+
+              next true if @applied
+
+              accepted += 1
+
+              false
+            end
+
+            @requests[@time] += accepted
+          end
+
+          # @return [Symbol] action to take upon throttler reaching certain state
+          def action
+            if applied?
+              timeout.zero? ? :seek : :pause
+            else
+              :skip
+            end
+          end
+
+          # @return [Integer] minimum number of milliseconds to wait before getting more messages
+          #   so we are no longer throttled and so we can process at least one message
+          def timeout
+            timeout = @interval - (monotonic_now - @time)
+            timeout <= 0 ? 0 : timeout
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Removes messages that are already marked as consumed in the virtual offset manager
+        # This should operate only when using virtual partitions.
+        #
+        # This cleaner prevents us from duplicated processing of messages that were virtually
+        # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
+        # to limit reprocessing when errors occur drastically when operating with virtual
+        # partitions
+        #
+        # @note It should be registered only when VPs are used
+        class VirtualLimiter < Base
+          # @param manager [Processing::VirtualOffsetManager]
+          # @param collapser [Processing::Collapser]
+          def initialize(manager, collapser)
+            @manager = manager
+            @collapser = collapser
+
+            super()
+          end
+
+          # Remove messages that we already marked as virtually consumed. Does nothing if not in
+          # the collapsed mode.
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            return unless @collapser.collapsed?
+
+            marked = @manager.marked
+
+            messages.delete_if { |message| marked.include?(message.offset) }
+          end
+        end
+      end
+    end
+  end
+end