karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Filtering related init strategies
-        module Lrj
-          # Filtering enabled
-          # LRJ enabled
-          # MoM enabled
-          # VPs enabled
-          module FtrMomVp
-            # Filtering + LRJ + Mom + VPs
-            FEATURES = %i[
-              filtering
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            include Strategies::Lrj::MomVp
-            include Strategies::Lrj::FtrMom
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Lrj
-          # Long-Running Job enabled
-          # Filtering enabled
-          # Virtual Partitions enabled
-          #
-          # Behaves same as non-VP because of the aggregated flow in the coordinator.
-          module FtrVp
-            include Strategies::Vp::Default
-            include Strategies::Lrj::Ftr
-
-            # Features for this strategy
-            FEATURES = %i[
-              filtering
-              long_running_job
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Namespace for all the LRJ starting strategies
-        module Lrj
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          module Mom
-            include Strategies::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              long_running_job
-              manual_offset_management
-            ].freeze
-
-            # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_schedule_consume
-              super
-
-              # This ensures that when running LRJ with VP, things operate as expected run only
-              # once for all the virtual partitions collectively
-              coordinator.on_enqueued do
-                pause(:consecutive, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
-              end
-            end
-
-            # No offset management, aside from that typical LRJ
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  retry_after_pause
-                end
-              end
-            end
-
-            # We do not un-pause on revokations for LRJ
-            def handle_revoked
-              coordinator.on_revoked do
-                coordinator.revoke
-              end
-
-              monitor.instrument("consumer.revoke", caller: self)
-              monitor.instrument("consumer.revoked", caller: self) do
-                revoked
-              end
-            ensure
-              coordinator.decrement(:revoked)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Mom
-          # - Mom enabled
-          # - Ftr enabled
-          # - Vp enabled
-          module FtrVp
-            include Strategies::Mom::Vp
-            include Strategies::Mom::Ftr
-
-            # Features of this strategy
-            FEATURES = %i[
-              filtering
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -1,192 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # VP starting strategies
-        module Vp
-          # Just Virtual Partitions enabled
-          module Default
-            # This flow is exactly the same as the default one because the default one is wrapper
-            # with `coordinator#on_finished`
-            include Strategies::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              virtual_partitions
-            ].freeze
-
-            # @param message [Karafka::Messages::Message] marks message as consumed
-            # @param offset_metadata [String, nil]
-            # @note This virtual offset management uses a regular default marking API underneath.
-            #   We do not alter the "real" marking API, as VPs are just one of many cases we want
-            #   to support and we do not want to impact them with collective offsets management
-            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
-              if @_in_transaction && !collapsed?
-                mark_in_transaction(message, offset_metadata, true)
-              elsif collapsed?
-                super
-              else
-                manager = coordinator.virtual_offset_manager
-
-                coordinator.synchronize do
-                  manager.mark(message, offset_metadata)
-                  # If this is last marking on a finished flow, we can use the original
-                  # last message and in order to do so, we need to mark all previous messages as
-                  # consumed as otherwise the computed offset could be different
-                  # We mark until our offset just in case of a DLQ flow or similar, where we do not
-                  # want to mark all but until the expected location
-                  manager.mark_until(message, offset_metadata) if coordinator.finished?
-
-                  return revoked? unless manager.markable?
-
-                  manager.markable? ? super(*manager.markable) : revoked?
-                end
-              end
-            ensure
-              @_current_offset_metadata = nil
-            end
-
-            # @param message [Karafka::Messages::Message] blocking marks message as consumed
-            # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
-              if @_in_transaction && !collapsed?
-                mark_in_transaction(message, offset_metadata, false)
-              elsif collapsed?
-                super
-              else
-                manager = coordinator.virtual_offset_manager
-
-                coordinator.synchronize do
-                  manager.mark(message, offset_metadata)
-                  manager.mark_until(message, offset_metadata) if coordinator.finished?
-                  manager.markable? ? super(*manager.markable) : revoked?
-                end
-              end
-            ensure
-              @_current_offset_metadata = nil
-            end
-
-            # Stores the next offset for processing inside of the transaction when collapsed and
-            # accumulates marking as consumed in the local buffer.
-            #
-            # Due to nature of VPs we cannot provide full EOS support but we can simulate it,
-            # making sure that no offset are stored unless transaction is finished. We do it by
-            # accumulating the post-transaction marking requests and after it is successfully done
-            # we mark each as consumed. This effectively on errors "rollbacks" the state and
-            # prevents offset storage.
-            #
-            # Since the EOS here is "weak", we do not have to worry about the race-conditions and
-            # we do not have to have any mutexes.
-            #
-            # @param message [Messages::Message] message we want to commit inside of a transaction
-            # @param offset_metadata [String, nil] offset metadata or nil if none
-            # @param async [Boolean] should we mark in async or sync way (applicable only to post
-            #   transaction state synchronization usage as within transaction it is always sync)
-            def mark_in_transaction(message, offset_metadata, async)
-              raise Errors::TransactionRequiredError unless @_in_transaction
-              # Prevent from attempts of offset storage when we no longer own the assignment
-              raise Errors::AssignmentLostError if revoked?
-
-              return super if collapsed?
-              # If this is user post-execution transaction (one initiated by the system) we should
-              # delegate to the original implementation that will store the offset via the producer
-              return super if @_transaction_internal
-
-              @_transaction_marked << [message, offset_metadata, async]
-            end
-
-            # @return [Boolean] is the virtual processing collapsed in the context of given
-            #   consumer.
-            def collapsed?
-              coordinator.collapsed?
-            end
-
-            # @param offset [Integer] first offset from which we should not operate in a collapsed
-            #   mode.
-            # @note Keep in mind, that if a batch contains this but also messages earlier messages
-            #   that should be collapsed, all will continue to operate in a collapsed mode until
-            #   first full batch with only messages that should not be collapsed.
-            def collapse_until!(offset)
-              coordinator.collapse_until!(offset)
-            end
-
-            # @return [Boolean] true if any of virtual partition we're operating in the entangled
-            #   mode has already failed and we know we are failing collectively.
-            #   Useful for early stop to minimize number of things processed twice.
-            #
-            # @note We've named it `#failing?` instead of `#failure?`  because it aims to be used
-            #   from within virtual partitions where we want to have notion of collective failing
-            #   not just "local" to our processing. We "are" failing with other virtual partitions
-            #   raising an error, but locally we are still processing.
-            def failing?
-              coordinator.failure?
-            end
-
-            # Allows for cross-virtual-partition consumers locks
-            #
-            # This is not needed in the non-VP flows except LRJ because there is always only one
-            # consumer per partition at the same time, so no coordination is needed directly for
-            # the end users. With LRJ it is needed and provided in the `LRJ::Default` strategy,
-            # because lifecycle events on revocation can run in parallel to the LRJ job as it is
-            # non-blocking.
-            def synchronize(&)
-              coordinator.shared_mutex.synchronize(&)
-            end
-
-            private
-
-            # Prior to adding work to the queue, registers all the messages offsets into the
-            # virtual offset group.
-            #
-            # @note This can be done without the mutex, because it happens from the same thread
-            #   for all the work (listener thread)
-            def handle_before_schedule_consume
-              super
-
-              # We should not register offsets in virtual manager when in collapse as virtual
-              # manager is not used then for offsets materialization.
-              #
-              # If we would do so, it would cause increased storage in cases of endless errors
-              # that are being retried in collapse without a DLQ.
-              return if collapsed?
-
-              coordinator.virtual_offset_manager.register(
-                messages.raw.map(&:offset)
-              )
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Pro processing strategies namespace
-      module Strategies
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategy_selector.rb

@@ -1,102 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Selector of appropriate processing strategy matching topic combinations
-      # When using Karafka Pro, there is a different set of strategies than for regular, as there
-      # are different features.
-      class StrategySelector
-        attr_reader :strategies
-
-        # Strategies that we support in the Pro offering
-        # They can be combined
-        SUPPORTED_FEATURES = %i[
-          active_job
-          long_running_job
-          manual_offset_management
-          virtual_partitions
-          dead_letter_queue
-          filtering
-        ].freeze
-
-        # Initializes the strategy selector and preloads all strategies
-        def initialize
-          # Preload the strategies
-          # We load them once for performance reasons not to do too many lookups
-          @strategies = find_all
-        end
-
-        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
-        #   the strategy
-        # @return [Module] module with proper strategy
-        def find(topic)
-          feature_set = SUPPORTED_FEATURES.map do |feature_name|
-            topic.public_send("#{feature_name}?") ? feature_name : nil
-          end
-
-          feature_set.compact!
-          feature_set.sort!
-
-          @strategies.find do |strategy|
-            strategy::FEATURES.sort == feature_set
-          end || raise(Errors::StrategyNotFoundError, topic.name)
-        end
-
-        private
-
-        # @return [Array<Module>] all available strategies
-        def find_all
-          scopes = [Strategies]
-          modules = Strategies.constants
-
-          modules.each do |const|
-            scopes << Strategies.const_get(const)
-            modules += scopes.last.constants
-          end
-
-          scopes.flat_map do |scope|
-            modules.map do |const|
-              next if const == :FEATURES
-              next unless scope.const_defined?(const)
-
-              candidate = scope.const_get(const)
-
-              next unless candidate.const_defined?(:FEATURES)
-
-              candidate
-            end
-          end.uniq.compact
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/subscription_groups_coordinator.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Uses the jobs queue API to lock (pause) and unlock (resume) operations of a given
-      # subscription group. It is abstracted away from jobs queue on this layer because we do
-      # not want to introduce jobs queue as a concept to the consumers layer
-      class SubscriptionGroupsCoordinator
-        include Singleton
-
-        # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group we
-        #   want to pause
-        # @param lock_id [Object] key we want to use if we want to set multiple locks on the same
-        #   subscription group
-        def pause(subscription_group, lock_id = nil, **)
-          jobs_queue.lock_async(
-            subscription_group.id,
-            lock_id,
-            **
-          )
-        end
-
-        # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group we
-        #   want to resume
-        # @param lock_id [Object] lock id (if it was used to pause)
-        def resume(subscription_group, lock_id = nil)
-          jobs_queue.unlock_async(subscription_group.id, lock_id)
-        end
-
-        private
-
-        # @return [Karafka::Pro::Processing::JobsQueue]
-        def jobs_queue
-          @jobs_queue ||= Karafka::Server.jobs_queue
-        end
-      end
-    end
-  end
-end