karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,197 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Pro coordinator that provides extra orchestration methods useful for parallel processing
-      # within the same partition
-      class Coordinator < Karafka::Processing::Coordinator
-        extend Forwardable
-        include Helpers::ConfigImporter.new(
-          errors_tracker_class: %i[internal processing errors_tracker_class]
-        )
-
-        def_delegators :@collapser, :collapsed?, :collapse_until!
-
-        attr_reader :filter, :virtual_offset_manager, :shared_mutex, :errors_tracker
-
-        # @param args [Object] anything the base coordinator accepts
-        def initialize(*args)
-          super
-
-          @executed = []
-          @errors_tracker = errors_tracker_class.new(topic, partition)
-          @flow_mutex = Mutex.new
-          # Lock for user code synchronization
-          # We do not want to mix coordinator lock with the user lock not to create cases where
-          # user imposed lock would lock the internal operations of Karafka
-          # This shared lock can be used by the end user as it is not used internally by the
-          # framework and can be used for user-facing locking
-          @shared_mutex = Mutex.new
-          @collapser = Collapser.new
-          @filter = Coordinators::FiltersApplier.new(self)
-
-          return unless topic.virtual_partitions?
-
-          @virtual_offset_manager = Coordinators::VirtualOffsetManager.new(
-            topic.name,
-            partition,
-            topic.virtual_partitions.offset_metadata_strategy
-          )
-
-          # We register our own "internal" filter to support filtering of messages that were marked
-          # as consumed virtually
-          @filter.filters << Filters::VirtualLimiter.new(
-            @virtual_offset_manager,
-            @collapser
-          )
-        end
-
-        # Starts the coordination process
-        # @param messages [Array<Karafka::Messages::Message>] messages for which processing we are
-        #   going to coordinate.
-        def start(messages)
-          super
-
-          @collapser.refresh!(messages.first.offset)
-
-          @filter.apply!(messages)
-
-          # Do not clear coordinator errors storage when we are retrying, so we can reference the
-          # errors that have happened during recovery. This can be useful for implementing custom
-          # flows. There can be more errors than one when running with virtual partitions so we
-          # need to make sure we collect them all. Under collapse when we reference a given
-          # consumer we should be able to get all the errors and not just first/last.
-          #
-          # @note We use zero as the attempt mark because we are not "yet" in the attempt 1
-          @errors_tracker.clear if attempt.zero?
-          @executed.clear
-
-          # We keep the old processed offsets until the collapsing is done and regular processing
-          # with virtualization is restored
-          @virtual_offset_manager.clear if topic.virtual_partitions? && !collapsed?
-
-          @last_message = messages.last
-        end
-
-        # Sets the consumer failure status and additionally starts the collapse until
-        #
-        # @param consumer [Karafka::BaseConsumer] consumer that failed
-        # @param error [StandardError] error from the failure
-        def failure!(consumer, error)
-          super
-          @errors_tracker << error
-          collapse_until!(@last_message.offset + 1)
-        end
-
-        # @return [Boolean] did any of the filters apply any logic that would cause use to run
-        #   the filtering flow
-        def filtered?
-          @filter.applied?
-        end
-
-        # @return [Boolean] is the coordinated work finished or not
-        # @note Used only in the consume operation context
-        def finished?
-          @running_jobs[:consume].zero?
-        end
-
-        # Runs synchronized code once for a collective of virtual partitions prior to work being
-        # enqueued
-        def on_enqueued
-          @flow_mutex.synchronize do
-            return unless executable?(:on_enqueued)
-
-            yield(@last_message)
-          end
-        end
-
-        # Runs given code only once per all the coordinated jobs upon starting first of them
-        def on_started
-          @flow_mutex.synchronize do
-            return unless executable?(:on_started)
-
-            yield(@last_message)
-          end
-        end
-
-        # Runs given code once when all the work that is suppose to be coordinated is finished
-        # It runs once per all the coordinated jobs and should be used to run any type of post
-        # jobs coordination processing execution
-        def on_finished
-          @flow_mutex.synchronize do
-            return unless finished?
-            return unless executable?(:on_finished)
-
-            yield(@last_message)
-          end
-        end
-
-        # Runs once after a partition is revoked
-        def on_revoked
-          @flow_mutex.synchronize do
-            return unless executable?(:on_revoked)
-
-            yield(@last_message)
-          end
-        end
-
-        # @param interval [Integer] milliseconds of activity
-        # @return [Boolean] was this partition in activity within last `interval` milliseconds
-        # @note Will return true also if currently active
-        def active_within?(interval)
-          # its always active if there's any job related to this coordinator that is still
-          # enqueued or running
-          return true if @running_jobs.values.any?(:positive?)
-
-          # Otherwise we check last time any job of this coordinator was active
-          @changed_at + interval > monotonic_now
-        end
-
-        private
-
-        # Checks if given action is executable once. If it is and true is returned, this method
-        # will return false next time it is used.
-        #
-        # @param action [Symbol] what action we want to perform
-        # @return [Boolean] true if we can
-        # @note This method needs to run behind a mutex.
-        def executable?(action)
-          return false if @executed.include?(action)
-
-          @executed << action
-
-          true
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -1,119 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Namespace for Pro coordinator related sub-components
-      module Coordinators
-        # Object used to track errors in between executions to be able to build error-type based
-        # recovery flows.
-        class ErrorsTracker
-          include Enumerable
-
-          # @return [Karafka::Routing::Topic] topic of this error tracker
-          attr_reader :topic
-
-          # @return [Integer] partition of this error tracker
-          attr_reader :partition
-
-          # @return [Hash]
-          attr_reader :counts
-
-          # @return [String]
-          attr_reader :trace_id
-
-          # Max errors we keep in memory.
-          # We do not want to keep more because for DLQ-less this would cause memory-leaks.
-          # We do however count per class for granular error counting
-          STORAGE_LIMIT = 100
-
-          private_constant :STORAGE_LIMIT
-
-          # @param topic [Karafka::Routing::Topic]
-          # @param partition [Integer]
-          # @param limit [Integer] max number of errors we want to keep for reference when
-          #   implementing custom error handling.
-          # @note `limit` does not apply to the counts. They will work beyond the number of errors
-          #   occurring
-          def initialize(topic, partition, limit: STORAGE_LIMIT)
-            @errors = []
-            @counts = Hash.new { |hash, key| hash[key] = 0 }
-            @topic = topic
-            @partition = partition
-            @limit = limit
-            @trace_id = SecureRandom.uuid
-          end
-
-          # Clears all the errors
-          def clear
-            @errors.clear
-            @counts.clear
-          end
-
-          # @param error [StandardError] adds the error to the tracker
-          def <<(error)
-            @errors.shift if @errors.size >= @limit
-            @errors << error
-            @counts[error.class] += 1
-            @trace_id = SecureRandom.uuid
-          end
-
-          # @return [Boolean] is the error tracker empty
-          def empty?
-            @errors.empty?
-          end
-
-          # @return [Integer] number of elements
-          def size
-            # We use counts reference of all errors and not the `@errors` array because it allows
-            # us to go beyond the whole errors storage limit
-            @counts.values.sum
-          end
-
-          # @return [StandardError, nil] last error that occurred or nil if no errors
-          def last
-            @errors.last
-          end
-
-          # Iterates over errors
-          def each(&)
-            @errors.each(&)
-          end
-
-          # @return [Array<StandardError>] array with all the errors that occurred
-          def all
-            @errors
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -1,152 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Coordinators
-        # Applier for all filters we want to have. Whether related to limiting messages based
-        # on the payload or any other things.
-        #
-        # From the outside world perspective, this encapsulates all the filters.
-        # This means that this is the API we expose as a single filter, allowing us to control
-        # the filtering via many filters easily.
-        class FiltersApplier
-          # @return [Array] registered filters array. Useful if we want to inject internal context
-          #   aware filters.
-          attr_reader :filters
-
-          # @param coordinator [Pro::Coordinator] pro coordinator
-          def initialize(coordinator)
-            # Builds filters out of their factories
-            # We build it that way (providing topic and partition) because there may be a case
-            # where someone wants to have a specific logic that is per topic or partition. Like for
-            # example a case where there is a cache bypassing revocations for topic partition.
-            #
-            # We provide full Karafka routing topic here and not the name only, in case the filter
-            # would be customized based on other topic settings (like VPs, etc)
-            #
-            # This setup allows for biggest flexibility also because topic object holds the
-            # reference to the subscription group and consumer group
-            @filters = coordinator.topic.filtering.factories.map do |factory|
-              factory.call(coordinator.topic, coordinator.partition)
-            end
-          end
-
-          # @param messages [Array<Karafka::Messages::Message>] array with messages from the
-          #   partition
-          def apply!(messages)
-            return unless active?
-
-            @filters.each { |filter| filter.apply!(messages) }
-          end
-
-          # @return [Boolean] did we filter out any messages during filtering run
-          def applied?
-            return false unless active?
-
-            !applied.empty?
-          end
-
-          # @return [Symbol] consumer post-filtering action that should be taken
-          def action
-            return :skip unless applied?
-
-            # The highest priority is on a potential backoff from any of the filters because it is
-            # the less risky (delay and continue later)
-            return :pause if applied.any? { |filter| filter.action == :pause }
-
-            # If none of the filters wanted to pause, we can check for any that would want to seek
-            # and if there is any, we can go with this strategy
-            return :seek if applied.any? { |filter| filter.action == :seek }
-
-            :skip
-          end
-
-          # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
-          #   filters to satisfy all of them.
-          def timeout
-            applied.filter_map(&:timeout).min || 0
-          end
-
-          # The first message we do need to get next time we poll. We use the minimum not to jump
-          # accidentally by over any.
-          # @return [Karafka::Messages::Message, nil] cursor message or nil if none
-          # @note Cursor message can also return the offset in the time format
-          def cursor
-            return nil unless active?
-
-            applied.filter_map(&:cursor).min_by(&:offset)
-          end
-
-          # @return [Boolean] did any of the filters requested offset storage during filter
-          #   application
-          def mark_as_consumed?
-            # We can manage filtering offset only when user wanted that and there is a cursor
-            # to use
-            applied.any?(&:mark_as_consumed?) && cursor
-          end
-
-          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
-          def marking_method
-            candidates = applied.map(&:marking_method)
-
-            return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
-
-            :mark_as_consumed
-          end
-
-          # The first (lowest) message we want to mark as consumed in marking. By default it uses
-          #   same position as cursor in case user wants to mark same message as consumed as the
-          #   one on which cursor action is applied.
-          # @return [Karafka::Messages::Message, nil] cursor marking message or nil if none
-          # @note It should not return position in time format, only numerical offset
-          def marking_cursor
-            return nil unless active?
-
-            applied.filter_map(&:marking_cursor).min_by(&:offset)
-          end
-
-          private
-
-          # @return [Boolean] is filtering active
-          def active?
-            !@filters.empty?
-          end
-
-          # @return [Array<Object>] filters that applied any sort of messages limiting
-          def applied
-            @filters.select(&:applied?)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -1,207 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Coordinators
-        # Manager that keeps track of our offsets with the virtualization layer that are local
-        # to given partition assignment. It allows for easier offset management for virtual
-        # virtual partition cases as it provides us ability to mark as consumed and move the
-        # real offset behind as expected.
-        #
-        # @note We still use the regular coordinator "real" offset management as we want to have
-        #   them as separated as possible because the real seek offset management is also used for
-        #   pausing, filtering and others and should not be impacted by the virtual one
-        #
-        # @note This manager is **not** thread-safe by itself. It should operate from coordinator
-        #   locked locations.
-        class VirtualOffsetManager
-          attr_reader :groups
-
-          # @param topic [String]
-          # @param partition [Integer]
-          # @param offset_metadata_strategy [Symbol] what metadata should we select. That is,
-          #   should we use the most recent or one picked from the offset that is going to be
-          #   committed
-          #
-          # @note We need topic and partition because we use a seek message (virtual) for real
-          #   offset management. We could keep real message reference but this can be memory
-          #   consuming and not worth it.
-          def initialize(topic, partition, offset_metadata_strategy)
-            @topic = topic
-            @partition = partition
-            @groups = []
-            @marked = {}
-            @offsets_metadata = {}
-            @real_offset = -1
-            @offset_metadata_strategy = offset_metadata_strategy
-            @current_offset_metadata = nil
-          end
-
-          # Clears the manager for a next collective operation
-          def clear
-            @groups.clear
-            @offsets_metadata.clear
-            @current_offset_metadata = nil
-            @marked.clear
-            @real_offset = -1
-          end
-
-          # Registers an offset group coming from one virtual consumer. In order to move the real
-          # underlying offset accordingly, we need to make sure to track the virtual consumers
-          # offsets groups independently and only materialize the end result.
-          #
-          # @param offsets_group [Array<Integer>] offsets from one virtual consumer
-          def register(offsets_group)
-            @groups << offsets_group
-
-            offsets_group.each { |offset| @marked[offset] = false }
-          end
-
-          # Marks given message as marked (virtually consumed).
-          # We mark given message offset and other earlier offsets from the same group as done
-          # and we can refresh our real offset representation based on that as it might have
-          # changed to a newer real offset.
-          # @param message [Karafka::Messages::Message] message coming from VP we want to mark
-          # @param offset_metadata [String, nil] offset metadata. `nil` if none
-          def mark(message, offset_metadata)
-            offset = message.offset
-
-            # Store metadata when we materialize the most stable offset
-            @offsets_metadata[offset] = offset_metadata
-            @current_offset_metadata = offset_metadata
-
-            group = nil
-            position = nil
-
-            @groups.each do |reg_group|
-              pos = reg_group.index(offset)
-
-              if pos
-                group = reg_group
-                position = pos
-                break
-              end
-            end
-
-            # This case can happen when someone uses MoM and wants to mark message from a previous
-            # batch as consumed. We can add it, since the real offset refresh will point to it
-            unless group
-              group = [offset]
-              position = 0
-              @groups << group
-            end
-
-            # Mark all previous messages from the same group also as virtually consumed
-            group[0..position].each do |markable_offset|
-              # Set previous messages metadata offset as the offset of higher one for overwrites
-              # unless a different metadata were set explicitely
-              @offsets_metadata[markable_offset] ||= offset_metadata
-              @marked[markable_offset] = true
-            end
-
-            # Recompute the real offset representation
-            materialize_real_offset
-          end
-
-          # Mark all from all groups including the `message`.
-          # Useful when operating in a collapsed state for marking
-          # @param message [Karafka::Messages::Message]
-          # @param offset_metadata [String, nil]
-          def mark_until(message, offset_metadata)
-            mark(message, offset_metadata)
-
-            @groups.each do |group|
-              group.each do |offset|
-                next if offset > message.offset
-
-                @offsets_metadata[offset] = offset_metadata
-                @marked[offset] = true
-              end
-            end
-
-            materialize_real_offset
-          end
-
-          # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
-          def marked
-            @marked.select { |_, status| status }.map { |offset, _| offset }.sort
-          end
-
-          # Is there a real offset we can mark as consumed
-          # @return [Boolean]
-          def markable?
-            !@real_offset.negative?
-          end
-
-          # @return [Array<Messages::Seek, String>] markable message for real offset marking and
-          #   its associated metadata
-          def markable
-            raise Errors::InvalidRealOffsetUsageError unless markable?
-
-            offset_metadata = case @offset_metadata_strategy
-            when :exact
-              @offsets_metadata.fetch(@real_offset)
-            when :current
-              @current_offset_metadata
-            else
-              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
-            end
-
-            [
-              Messages::Seek.new(
-                @topic,
-                @partition,
-                @real_offset
-              ),
-              offset_metadata
-            ]
-          end
-
-          private
-
-          # Recomputes the biggest possible real offset we can have.
-          # It picks the biggest offset that has uninterrupted stream of virtually marked as
-          # consumed because this will be the collective offset.
-          def materialize_real_offset
-            @marked.keys.sort.each do |offset|
-              break unless @marked[offset]
-
-              @real_offset = offset
-            end
-
-            @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
-          end
-        end
-      end
-    end
-  end
-end