karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/topic.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class DeadLetterQueue < Base
+          # DLQ topic extensions
+          module Topic
+            # After how many retries should data be moved to DLQ
+            DEFAULT_MAX_RETRIES = 3
+
+            private_constant :DEFAULT_MAX_RETRIES
+
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              @dead_letter_queue = nil
+              super
+            end
+
+            # @param max_retries [Integer] after how many retries should we move data to dlq
+            # @param topic [String, false] where the messages should be moved if failing or false
+            #   if we do not want to move it anywhere and just skip
+            # @param independent [Boolean] needs to be true in order for each marking as consumed
+            #   in a retry flow to reset the errors counter
+            # @param transactional [Boolean] if applicable, should transaction be used to move
+            #   given message to the dead-letter topic and mark it as consumed.
+            # @param dispatch_method [Symbol] `:produce_async` or `:produce_sync`. Describes
+            #   whether dispatch on dlq should be sync or async (async by default)
+            # @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
+            #   whether marking on DLQ should be async or sync (async by default)
+            # @param mark_after_dispatch [Boolean, nil] Should we mark after dispatch. `nil` means
+            #   that the default strategy approach to marking will be used. `true` or `false`
+            #   overwrites the default
+            # @return [Config] defined config
+            def dead_letter_queue(
+              max_retries: DEFAULT_MAX_RETRIES,
+              topic: nil,
+              independent: false,
+              transactional: true,
+              dispatch_method: :produce_async,
+              marking_method: :mark_as_consumed,
+              mark_after_dispatch: nil
+            )
+              @dead_letter_queue ||= Config.new(
+                active: !topic.nil?,
+                max_retries: max_retries,
+                topic: topic,
+                independent: independent,
+                transactional: transactional,
+                dispatch_method: dispatch_method,
+                marking_method: marking_method,
+                mark_after_dispatch: mark_after_dispatch
+              )
+            end
+
+            # @return [Boolean] is the dlq active or not
+            def dead_letter_queue?
+              dead_letter_queue.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus dlq settings
+            def to_h
+              super.merge(
+                dead_letter_queue: dead_letter_queue.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/dead_letter_queue.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        # This feature allows to continue processing when encountering errors.
+        # After certain number of retries, given messages will be moved to alternative topic,
+        # unclogging processing.
+        #
+        # @note This feature has an expanded version in the Pro mode. We do not use a new feature
+        #   injection in Pro (topic settings)
+        class DeadLetterQueue < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/eofed/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class Eofed < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/eofed/contracts/topic.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class Eofed < Base
+          # Eofed related contracts namespace
+          module Contracts
+            # Contract for eofed topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+              end
+
+              nested :eofed do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                eofed = data[:eofed]
+
+                next unless eofed[:active]
+
+                next if data[:kafka][:"enable.partition.eof"]
+
+                [[%i[eofed kafka], :enable]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/eofed/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class Eofed < Base
+          # Routing topic eofed API
+          module Topic
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              @eofed = nil
+              super
+            end
+
+            # @param active [Boolean] should the `#eofed` job run on eof
+            def eofed(active = false)
+              @eofed ||= Config.new(
+                active: active
+              )
+            end
+
+            # @return [Boolean] Are `#eofed` jobs active
+            def eofed?
+              eofed.active?
+            end
+
+            # @return [Hash] topic setup hash
+            def to_h
+              super.merge(
+                eofed: eofed.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/eofed.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        # Namespace for feature allowing to enable the `#eofed` jobs.
+        # We do not enable it always because users may only be interested in fast eofed yielding
+        # without running the `#eofed` operation at all. This saves on empty cycles of running
+        # pointless empty jobs.
+        class Eofed < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/inline_insights/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class InlineInsights < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/inline_insights/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class InlineInsights < Base
+          # Inline Insights related contracts namespace
+          module Contracts
+            # Contract for inline insights topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+              end
+
+              nested :inline_insights do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/inline_insights/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class InlineInsights < Base
+          # Routing topic inline insights API
+          module Topic
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              @inline_insights = nil
+              super
+            end
+
+            # @param active [Boolean] should inline insights be activated
+            def inline_insights(active = false)
+              @inline_insights ||= Config.new(
+                active: active
+              )
+            end
+
+            # @return [Boolean] Are inline insights active
+            def inline_insights?
+              inline_insights.active?
+            end
+
+            # @return [Hash] topic setup hash
+            def to_h
+              super.merge(
+                inline_insights: inline_insights.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/inline_insights.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        # Feature allowing us to get visibility during the consumption into metrics of particular
+        # partition we operate on. It can be useful when making context-aware consumers that change
+        # their behaviours based on the lag and other parameters.
+        class InlineInsights < Base
+          class << self
+            # If needed installs the needed listener and initializes tracker
+            #
+            # @param _config [Karafka::Core::Configurable::Node] app config
+            def post_setup(_config)
+              Karafka::App.monitor.subscribe("app.running") do
+                # Do not activate tracking of statistics if none of our active topics uses it
+                # This prevents us from tracking metrics when user just runs a subset of topics
+                # in a given process and none of those actually utilizes this feature
+                next unless Karafka::App
+                  .subscription_groups
+                  .values
+                  .flat_map(&:itself)
+                  .flat_map(&:topics)
+                  .flat_map(&:to_a)
+                  .any?(&:inline_insights?)
+
+                # Initialize the tracker prior to becoming multi-threaded
+                Karafka::Processing::ConsumerGroups::InlineInsights::Tracker.instance
+
+                # Subscribe to the statistics reports and collect them
+                Karafka.monitor.subscribe(
+                  Karafka::Processing::ConsumerGroups::InlineInsights::Listener.new
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/manual_offset_management/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ManualOffsetManagement < Base
+          # Config for manual offset management feature
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/manual_offset_management/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ManualOffsetManagement < Base
+          # This feature validation contracts
+          module Contracts
+            # Rules around manual offset management settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+              end
+
+              nested :manual_offset_management do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ManualOffsetManagement < Base
+          # Topic extensions to be able to manage manual offset management settings
+          module Topic
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              @manual_offset_management = nil
+              super
+            end
+
+            # @param active [Boolean] should we stop managing the offset in Karafka and make the user
+            #   responsible for marking messages as consumed.
+            # @return [Config] defined config
+            #
+            # @note Since this feature supports only one setting (active), we can use the old API
+            # where the boolean would be an argument
+            def manual_offset_management(active = false)
+              @manual_offset_management ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is manual offset management enabled for a given topic
+            def manual_offset_management?
+              manual_offset_management.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus manual offset
+            #   management namespace settings
+            def to_h
+              super.merge(
+                manual_offset_management: manual_offset_management.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/manual_offset_management.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        # All the things needed to be able to manage manual offset management from the routing
+        # perspective.
+        #
+        # Manual offset management allows users to completely disable automatic management of the
+        # offset. This can be used for implementing long-living window operations and other things
+        # where we do not want to commit the offset with each batch.
+        #
+        # Not all the Karafka and Karafka Pro features may be compatible with this feature being on.
+        class ManualOffsetManagement < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Consumer-group-specific routing features. A parallel `ShareGroups` namespace will hold
+      # share-group-specific feature implementations once KIP-932 lands.
+      module ConsumerGroups
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -4,26 +4,11 @@ module Karafka
   module Routing
     module Features
       class Declaratives < Base
-        # This feature validation contracts
+        # Contracts used by the routing Expander to validate topic declarations after draw
         module Contracts
-          # Basic validation of the Kafka expected config details
-          class Topic < Karafka::Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load_file(
-                File.join(Karafka.gem_root, "config", "locales", "errors.yml")
-              ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-            end
-
-            nested :declaratives do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
-              required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
-              required(:details) do |val|
-                val.is_a?(Hash) &&
-                  val.keys.all?(Symbol)
-              end
-            end
-          end
+          # Delegates to the canonical contract in the Declaratives namespace.
+          # Required here because the Expander looks up `scope::Contracts::Topic`.
+          Topic = Karafka::Declaratives::Contracts::Topic
         end
       end
     end

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -4,35 +4,51 @@ module Karafka
   module Routing
     module Features
       class Declaratives < Base
-        # Extension for managing Kafka topic configuration
+        # Bridge module prepended onto Karafka::Routing::Topic.
+        # The config(...) method forwards to the Declaratives subsystem, creating or retrieving
+        # a Karafka::Declaratives::Topic in the repository. This preserves backwards compatibility
+        # while the actual declarative state lives in Karafka::Declaratives.
         module Topic
-          # This method calls the parent class initializer and then sets up the
-          # extra instance variable to nil. The explicit initialization
+          # This method sets up the extra instance variable to nil before calling
+          # the parent class initializer. The explicit initialization
           # to nil is included as an optimization for Ruby's object shapes system,
           # which improves memory layout and access performance.
           def initialize(...)
-            super
             @declaratives = nil
+            super
           end
 
+          # Bridge: creates/retrieves a Declaratives::Topic in the repository and returns it.
+          # Preserves the ||= semantics (first call wins) for backwards compatibility.
+          #
           # @param active [Boolean] is the topic structure management feature active
           # @param partitions [Integer] number of partitions for the topic
           # @param replication_factor [Integer] replication factor for the topic
           # @param details [Hash] extra configuration for the topic
-          # @option details [String] :retention.ms retention time in milliseconds
-          # @option details [String] :compression.type compression type
+          # @option details [Symbol] :retention.ms retention time in milliseconds
+          # @option details [Symbol] :compression.type compression type
           #   (none, gzip, snappy, lz4, zstd)
-          # @return [Config] defined structure
+          # @return [Karafka::Declaratives::Topic] the declarative topic
           def config(active: true, partitions: 1, replication_factor: 1, **details)
-            @declaratives ||= Config.new(
-              active: active,
-              partitions: partitions,
-              replication_factor: replication_factor,
-              details: details
-            )
+            @declaratives ||= begin
+              repo = Karafka::App.declaratives.repository
+
+              repo.find(name) || begin
+                declaration = repo.find_or_create(name)
+                declaration.active(active)
+                declaration.partitions(partitions)
+                declaration.replication_factor(replication_factor)
+                declaration.config(details) unless details.empty?
+                declaration
+              end
+            end.tap do |declaration|
+              # Set bootstrap servers from the routing topic's kafka config so the CLI can
+              # filter by cluster. This runs on every call but is idempotent.
+              declaration.bootstrap_servers ||= kafka[:"bootstrap.servers"]
+            end
           end
 
-          # @return [Config] config details
+          # @return [Karafka::Declaratives::Topic] config details
           def declaratives
             config
           end

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -8,13 +8,13 @@ module Karafka
         # Routing topic deserializers API. It allows to configure deserializers for various
         # components of each message.
         module Topic
-          # This method calls the parent class initializer and then sets up the
-          # extra instance variable to nil. The explicit initialization
+          # This method sets up the extra instance variable to nil before calling
+          # the parent class initializer. The explicit initialization
           # to nil is included as an optimization for Ruby's object shapes system,
           # which improves memory layout and access performance.
           def initialize(...)
-            super
             @deserializers = nil
+            super
           end
 
           # Allows for setting all the deserializers with standard defaults

data/lib/karafka/routing/router.rb

@@ -11,8 +11,8 @@ module Karafka
       # @param lookup [Hash{Symbol => String}] hash with attribute - value key pairs
       # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
       def find_by(lookup)
-        App.consumer_groups.each do |consumer_group|
-          consumer_group.topics.each do |topic|
+        App.routes.each do |group|
+          group.topics.each do |topic|
             return topic if lookup.all? do |attribute, value|
               topic.public_send(attribute) == value
             end

data/lib/karafka/routing/subscription_group.rb

@@ -14,7 +14,12 @@ module Karafka
         node: %i[swarm node]
       )
 
-      attr_reader :id, :name, :topics, :kafka, :consumer_group, :position
+      attr_reader :id, :name, :topics, :kafka, :group, :position
+
+      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # unconditional alias and performs no type validation, so callers should prefer `#group`
+      # once additional group types (e.g. KIP-932 share groups) land.
+      alias_method :consumer_group, :group
 
       # Lock for generating new ids safely
       ID_MUTEX = Mutex.new
@@ -45,22 +50,26 @@ module Karafka
       def initialize(position, topics)
         @details = topics.first.subscription_group_details
         @name = @details.fetch(:name)
-        @consumer_group = topics.first.consumer_group
-        # We include the consumer group id here because we want to have unique ids of subscription
+        @group = topics.first.group
+        # We include the owning group id here because we want to have unique ids of subscription
         # groups across the system. Otherwise user could set the same name for multiple
-        # subscription groups in many consumer groups effectively having same id for different
-        # entities
-        @id = "#{@consumer_group.id}_#{@name}_#{position}"
+        # subscription groups in many groups effectively having same id for different entities
+        @id = "#{@group.id}_#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
       end
 
-      # @return [String] consumer group id
-      def consumer_group_id
+      # @return [String] group id (the Kafka `group.id` value assigned to this subscription
+      #   group's connection)
+      def group_id
         kafka[:"group.id"]
       end
 
+      # Backwards compatible alias. Will be retired in Karafka 3.0 once additional group types
+      # (e.g. KIP-932 share groups) land.
+      alias_method :consumer_group_id, :group_id
+
       # @return [Integer] max messages fetched in a single go
       def max_messages
         @topics.first.max_messages
@@ -124,7 +133,7 @@ module Karafka
         inject_group_instance_id(kafka)
         inject_client_id(kafka)
 
-        kafka[:"group.id"] ||= @consumer_group.id
+        kafka[:"group.id"] ||= @group.id
         kafka[:"auto.offset.reset"] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing