karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/{inline_insights/config.rb → consumer_groups/active_job.rb}

@@ -32,15 +32,11 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class InlineInsights < Base
-          # Config of this feature
-          Config = Struct.new(
-            :active,
-            :required,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :required?, :required
+        module ConsumerGroups
+          # Small extension to make ActiveJob work with pattern matching.
+          # Since our `#active_job_topic` is just a topic wrapper, we can introduce a similar
+          # `#active_job_pattern` to align with pattern building
+          class ActiveJob < Base
           end
         end
       end

data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class AdaptiveIterator < Base
+            # Adaptive Iterator configuration
+            Config = Struct.new(
+              :active,
+              :safety_margin,
+              :marking_method,
+              :clean_after_yielding,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+              alias_method :clean_after_yielding?, :clean_after_yielding
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/contracts/topic.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class AdaptiveIterator < Base
+            # Namespace for adaptive iterator contracts
+            module Contracts
+              # Contract to validate configuration of the adaptive iterator feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:adaptive_iterator) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:safety_margin) { |val| val.is_a?(Integer) && val.positive? && val < 100 }
+                  required(:clean_after_yielding) { |val| [true, false].include?(val) }
+
+                  required(:marking_method) do |val|
+                    %i[mark_as_consumed mark_as_consumed!].include?(val)
+                  end
+                end
+
+                # Since adaptive iterator uses `#seek` and can break processing in the middle, we
+                # cannot use it with virtual partitions that can process data in a distributed
+                # manner
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  adaptive_iterator = data[:adaptive_iterator]
+                  virtual_partitions = data[:virtual_partitions]
+
+                  next unless adaptive_iterator[:active]
+                  next unless virtual_partitions[:active]
+
+                  [[%i[adaptive_iterator], :with_virtual_partitions]]
+                end
+
+                # There is no point of using the adaptive iterator with LRJ because of how LRJ works
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  adaptive_iterator = data[:adaptive_iterator]
+                  long_running_jobs = data[:long_running_job]
+
+                  next unless adaptive_iterator[:active]
+                  next unless long_running_jobs[:active]
+
+                  [[%i[adaptive_iterator], :with_long_running_job]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/topic.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class AdaptiveIterator < Base
+            # Topic extension allowing us to enable and configure adaptive iterator
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @adaptive_iterator = nil
+                super
+              end
+
+              # @param active [Boolean] should we use the automatic adaptive iterator
+              # @param safety_margin [Integer]
+              #   How big of a margin we leave ourselves so we can safely communicate back with
+              #   Kafka, etc. We stop and seek back when we've burned 85% of the time by default.
+              #   We leave 15% of time for post-processing operations so we have space before we
+              #   hit max.poll.interval.ms.
+              # @param marking_method [Symbol] If we should, how should we mark
+              # @param clean_after_yielding [Boolean]  Should we clean post-yielding via the
+              #   cleaner API
+              def adaptive_iterator(
+                active: false,
+                safety_margin: 10,
+                marking_method: :mark_as_consumed,
+                clean_after_yielding: true
+              )
+                @adaptive_iterator ||= Config.new(
+                  active: active,
+                  safety_margin: safety_margin,
+                  marking_method: marking_method,
+                  clean_after_yielding: clean_after_yielding
+                )
+              end
+
+              # @return [Boolean] Is adaptive iterator active. It is always `true`, since we use it
+              #   via explicit messages batch wrapper
+              def adaptive_iterator?
+                adaptive_iterator.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus poll guarding
+              #   setup configuration.
+              def to_h
+                super.merge(
+                  adaptive_iterator: adaptive_iterator.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Feature that pro-actively monitors remaining time until max poll interval ms and
+          # cost of processing of each message in a batch. When there is no more time to process
+          # more messages from the batch, it will seek back so we do not reach max poll interval.
+          # It can be useful when we reach this once in a while. For a constant long-running jobs,
+          # please use the Long-Running Jobs feature instead.
+          #
+          # It also provides some wrapping over typical operations users do, like stopping if
+          # revoked, auto-marking, etc
+          class AdaptiveIterator < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{patterns/contracts/consumer_group.rb → consumer_groups/dead_letter_queue/contracts/topic.rb}

@@ -32,44 +32,39 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Patterns < Base
-          module Contracts
-            # Contract to validate configuration of the filtering feature
-            class ConsumerGroup < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
-
-                required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
-
-                virtual do |data, errors|
-                  next unless errors.empty?
-
-                  validator = Pattern.new
-
-                  data[:patterns].each do |pattern|
-                    validator.validate!(pattern)
-                  end
+        module ConsumerGroups
+          class DeadLetterQueue < Base
+            # Namespace for DLQ contracts
+            module Contracts
+              # Extended rules for dead letter queue settings
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
 
-                  nil
+                nested(:dead_letter_queue) do
+                  # We use strategy based DLQ for every case in Pro
+                  # For default (when no strategy) a default `max_retries` based strategy is used
+                  required(:strategy) { |val| val.respond_to?(:call) }
                 end
 
-                # Make sure, that there are no same regular expressions with different names
-                # in a single consumer group
+                # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+                # We cannot use VP with DLQ without retries as we in order to provide ordering
+                # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+                # without any indeterministic work
                 virtual do |data, errors|
                   next unless errors.empty?
 
-                  regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
-                  regexp_names = data[:patterns].map { |pattern| pattern.fetch(:name) }
+                  dead_letter_queue = data[:dead_letter_queue]
+                  virtual_partitions = data[:virtual_partitions]
 
-                  # If all names are the same for the same regexp, it means its a multiplex and
-                  # we can allow it
-                  next if regexp_names.uniq.size == 1 && regexp_strings.uniq.size == 1
-                  next if regexp_strings.empty?
-                  next if regexp_strings.uniq.size == regexp_strings.size
+                  next unless dead_letter_queue[:active]
+                  next unless virtual_partitions[:active]
+                  next if dead_letter_queue[:max_retries].positive?
 
-                  [[%i[patterns], :regexps_not_unique]]
+                  [[%i[dead_letter_queue], :with_virtual_partitions]]
                 end
               end
             end

data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue/topic.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class DeadLetterQueue < Base
+            # Expansions to the topic API in DLQ
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @dead_letter_queue = nil
+                super
+              end
+
+              # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
+              # (same as in OSS) should be applied
+              # @param args [Hash] Pro DLQ arguments
+              # @option args [String, nil] :topic name of the dead letter queue topic
+              # @option args [Integer] :max_retries maximum number of retries before dispatch to DLQ
+              # @option args [Boolean] :independent whether DLQ runs independently
+              def dead_letter_queue(strategy: nil, **args)
+                return @dead_letter_queue if @dead_letter_queue
+
+                super(**args).tap do |config|
+                  # If explicit strategy is not provided, use the default approach from OSS
+                  config.strategy = strategy || lambda do |_errors_tracker, attempt|
+                    (attempt > config.max_retries) ? :dispatch : :retry
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Enhanced Dead Letter Queue
+          #
+          # This version behaves similar to the regular version but provides ordering warranties and
+          # allows for batch skipping
+          class DeadLetterQueue < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{direct_assignments → consumer_groups/delaying}/config.rb

@@ -32,10 +32,12 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class DirectAssignments < Base
-          # Direct assignments feature configuration
-          Config = Struct.new(:active, :partitions, keyword_init: true) do
-            alias_method :active?, :active
+        module ConsumerGroups
+          class Delaying < Base
+            # Delaying feature configuration
+            Config = Struct.new(:active, :delay, keyword_init: true) do
+              alias_method :active?, :active
+            end
           end
         end
       end

data/lib/karafka/pro/routing/features/{patterns/contracts/pattern.rb → consumer_groups/delaying/contracts/topic.rb}

@@ -32,24 +32,21 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Patterns < Base
-          # Namespace for patterns related contracts
-          module Contracts
-            # Contract used to validate pattern data
-            class Pattern < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("pattern")
-
-                required(:regexp) { |val| val.is_a?(Regexp) }
-
-                required(:regexp_string) do |val|
-                  val.is_a?(String) && val.start_with?("^") && val.size >= 2
+        module ConsumerGroups
+          class Delaying < Base
+            # Namespace for delaying feature contracts
+            module Contracts
+              # Contract to validate configuration of the expiring feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
                 end
 
-                required(:name) do |val|
-                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                nested(:delaying) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
                 end
               end
             end

data/lib/karafka/pro/routing/features/consumer_groups/delaying/topic.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Delaying < Base
+            # Topic delaying API extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @delaying = nil
+                super
+              end
+
+              # @param delay [Integer, nil] minimum age of a message we want to process
+              def delaying(delay = nil)
+                # Those settings are used for validation
+                @delaying ||= begin
+                  config = Config.new(active: !delay.nil?, delay: delay)
+
+                  if config.active?
+                    factory = ->(*) { Pro::Processing::ConsumerGroups::Filters::Delayer.new(delay) }
+                    filter(factory)
+                  end
+
+                  config
+                end
+              end
+
+              # Just an alias for nice API
+              def delay_by(*)
+                delaying(*)
+              end
+
+              # @return [Boolean] is a given job delaying
+              def delaying?
+                delaying.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus delaying
+              def to_h
+                super.merge(
+                  delaying: delaying.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{adaptive_iterator/config.rb → consumer_groups/delaying.rb}

@@ -32,17 +32,14 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class AdaptiveIterator < Base
-          # Adaptive Iterator configuration
-          Config = Struct.new(
-            :active,
-            :safety_margin,
-            :marking_method,
-            :clean_after_yielding,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :clean_after_yielding?, :clean_after_yielding
+        module ConsumerGroups
+          # Delaying allows us to delay processing of certain topics. This is useful when we want
+          # to for any reason to wait until processing data from a topic. It does not sleep and
+          # instead uses pausing to manage delays. This allows us to free up processing resources
+          # and not block the polling thread.
+          #
+          # Delaying is a virtual feature realized via the filters
+          class Delaying < Base
           end
         end
       end