karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module VirtualPartitions
-        module Distributors
-          # Balanced distributor that groups messages by partition key
-          # and processes larger groups first while maintaining message order within groups
-          class Balanced < Base
-            # Distributes messages to virtual partitions ensuring balanced load across workers
-            # by grouping messages by partition key and assigning larger groups first
-            # @param messages [Array<Karafka::Messages::Message>]
-            # @return [Hash{Integer => Array<Karafka::Messages::Message>}] hash with group ids as
-            #   keys and message groups as values
-            def call(messages)
-              # Group messages by partition key
-              key_groupings = messages.group_by { |msg| config.partitioner.call(msg) }
-
-              worker_loads = Array.new(config.max_partitions, 0)
-              worker_assignments = Array.new(config.max_partitions) { [] }
-
-              # Sort keys by workload in descending order
-              sorted_keys = key_groupings.keys.sort_by { |key| -key_groupings[key].size }
-
-              # Assign each key to the worker with the least current load
-              sorted_keys.each do |key|
-                # Find worker with minimum current load
-                min_load_worker = worker_loads.each_with_index.min_by { |load, _| load }[1]
-                messages = key_groupings[key]
-
-                # Assign this key to that worker
-                worker_assignments[min_load_worker] += messages
-                worker_loads[min_load_worker] += messages.size
-              end
-
-              # Combine messages for each worker and sort by offset
-              worker_assignments
-                .each_with_index
-                .reject { |group_messages, _| group_messages.empty? }
-                .map! { |group_messages, index| [index, group_messages.sort_by!(&:offset)] }
-                .to_h
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/active_job.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Small extension to make ActiveJob work with pattern matching.
-        # Since our `#active_job_topic` is just a topic wrapper, we can introduce a similar
-        # `#active_job_pattern` to align with pattern building
-        class ActiveJob < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class AdaptiveIterator < Base
-          # Namespace for adaptive iterator contracts
-          module Contracts
-            # Contract to validate configuration of the adaptive iterator feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:adaptive_iterator) do
-                required(:active) { |val| [true, false].include?(val) }
-                required(:safety_margin) { |val| val.is_a?(Integer) && val.positive? && val < 100 }
-                required(:clean_after_yielding) { |val| [true, false].include?(val) }
-
-                required(:marking_method) do |val|
-                  %i[mark_as_consumed mark_as_consumed!].include?(val)
-                end
-              end
-
-              # Since adaptive iterator uses `#seek` and can break processing in the middle, we
-              # cannot use it with virtual partitions that can process data in a distributed
-              # manner
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                adaptive_iterator = data[:adaptive_iterator]
-                virtual_partitions = data[:virtual_partitions]
-
-                next unless adaptive_iterator[:active]
-                next unless virtual_partitions[:active]
-
-                [[%i[adaptive_iterator], :with_virtual_partitions]]
-              end
-
-              # There is no point of using the adaptive iterator with LRJ because of how LRJ works
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                adaptive_iterator = data[:adaptive_iterator]
-                long_running_jobs = data[:long_running_job]
-
-                next unless adaptive_iterator[:active]
-                next unless long_running_jobs[:active]
-
-                [[%i[adaptive_iterator], :with_long_running_job]]
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class AdaptiveIterator < Base
-          # Topic extension allowing us to enable and configure adaptive iterator
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @adaptive_iterator = nil
-            end
-
-            # @param active [Boolean] should we use the automatic adaptive iterator
-            # @param safety_margin [Integer]
-            #   How big of a margin we leave ourselves so we can safely communicate back with
-            #   Kafka, etc. We stop and seek back when we've burned 85% of the time by default.
-            #   We leave 15% of time for post-processing operations so we have space before we
-            #   hit max.poll.interval.ms.
-            # @param marking_method [Symbol] If we should, how should we mark
-            # @param clean_after_yielding [Boolean]  Should we clean post-yielding via the
-            #   cleaner API
-            def adaptive_iterator(
-              active: false,
-              safety_margin: 10,
-              marking_method: :mark_as_consumed,
-              clean_after_yielding: true
-            )
-              @adaptive_iterator ||= Config.new(
-                active: active,
-                safety_margin: safety_margin,
-                marking_method: marking_method,
-                clean_after_yielding: clean_after_yielding
-              )
-            end
-
-            # @return [Boolean] Is adaptive iterator active. It is always `true`, since we use it
-            #   via explicit messages batch wrapper
-            def adaptive_iterator?
-              adaptive_iterator.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus poll guarding
-            #   setup configuration.
-            def to_h
-              super.merge(
-                adaptive_iterator: adaptive_iterator.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/adaptive_iterator.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Feature that pro-actively monitors remaining time until max poll interval ms and
-        # cost of processing of each message in a batch. When there is no more time to process
-        # more messages from the batch, it will seek back so we do not reach max poll interval.
-        # It can be useful when we reach this once in a while. For a constant long-running jobs,
-        # please use the Long-Running Jobs feature instead.
-        #
-        # It also provides some wrapping over typical operations users do, like stopping if
-        # revoked, auto-marking, etc
-        class AdaptiveIterator < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class DeadLetterQueue < Base
-          # Namespace for DLQ contracts
-          module Contracts
-            # Extended rules for dead letter queue settings
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:dead_letter_queue) do
-                # We use strategy based DLQ for every case in Pro
-                # For default (when no strategy) a default `max_retries` based strategy is used
-                required(:strategy) { |val| val.respond_to?(:call) }
-              end
-
-              # Make sure that when we use virtual partitions with DLQ, at least one retry is set
-              # We cannot use VP with DLQ without retries as we in order to provide ordering
-              # warranties on errors with VP, we need to collapse the VPs concurrency and retry
-              # without any indeterministic work
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                dead_letter_queue = data[:dead_letter_queue]
-                virtual_partitions = data[:virtual_partitions]
-
-                next unless dead_letter_queue[:active]
-                next unless virtual_partitions[:active]
-                next if dead_letter_queue[:max_retries].positive?
-
-                [[%i[dead_letter_queue], :with_virtual_partitions]]
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class DeadLetterQueue < Base
-          # Expansions to the topic API in DLQ
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @dead_letter_queue = nil
-            end
-
-            # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
-            # (same as in OSS) should be applied
-            # @param args [Hash] Pro DLQ arguments
-            # @option args [String, nil] :topic name of the dead letter queue topic
-            # @option args [Integer] :max_retries maximum number of retries before dispatch to DLQ
-            # @option args [Boolean] :independent whether DLQ runs independently
-            def dead_letter_queue(strategy: nil, **args)
-              return @dead_letter_queue if @dead_letter_queue
-
-              super(**args).tap do |config|
-                # If explicit strategy is not provided, use the default approach from OSS
-                config.strategy = strategy || lambda do |_errors_tracker, attempt|
-                  (attempt > config.max_retries) ? :dispatch : :retry
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/dead_letter_queue.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Enhanced Dead Letter Queue
-        #
-        # This version behaves similar to the regular version but provides ordering warranties and
-        # allows for batch skipping
-        class DeadLetterQueue < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Delaying < Base
-          # Topic delaying API extensions
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @delaying = nil
-            end
-
-            # @param delay [Integer, nil] minimum age of a message we want to process
-            def delaying(delay = nil)
-              # Those settings are used for validation
-              @delaying ||= begin
-                config = Config.new(active: !delay.nil?, delay: delay)
-
-                if config.active?
-                  factory = ->(*) { Pro::Processing::Filters::Delayer.new(delay) }
-                  filter(factory)
-                end
-
-                config
-              end
-            end
-
-            # Just an alias for nice API
-            def delay_by(*)
-              delaying(*)
-            end
-
-            # @return [Boolean] is a given job delaying
-            def delaying?
-              delaying.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus delaying
-            def to_h
-              super.merge(
-                delaying: delaying.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/delaying.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Delaying allows us to delay processing of certain topics. This is useful when we want
-        # to for any reason to wait until processing data from a topic. It does not sleep and
-        # instead uses pausing to manage delays. This allows us to free up processing resources
-        # and not block the polling thread.
-        #
-        # Delaying is a virtual feature realized via the filters
-        class Delaying < Base
-        end
-      end
-    end
-  end
-end