karafka 2.0.0 → 2.1.0

data/lib/karafka/routing/consumer_group.rb

@@ -10,19 +10,28 @@ module Karafka
     class ConsumerGroup
       attr_reader :id, :topics, :name
 
+      # This is a "virtual" attribute that is not building subscription groups.
+      # It allows us to store the "current" subscription group defined in the routing
+      # This subscription group id is then injected into topics, so we can compute the subscription
+      # groups
+      attr_accessor :current_subscription_group_id
+
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
       #   We add it to make a multi-system development easier for people that don't use
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
-        @name = name
+        @name = name.to_s
         @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = Topics.new([])
+        # Initialize the subscription group so there's always a value for it, since even if not
+        # defined directly, a subscription group will be created
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        Karafka::Server.consumer_groups.include?(name)
+        Karafka::App.config.internal.routing.activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -32,13 +41,38 @@ module Karafka
       def topic=(name, &block)
         topic = Topic.new(name, self)
         @topics << Proxy.new(topic, &block).target
-        @topics.last
+        built_topic = @topics.last
+        # We overwrite it conditionally in case it was not set by the user inline in the topic
+        # block definition
+        built_topic.subscription_group ||= current_subscription_group_id
+        built_topic
+      end
+
+      # Assigns the current subscription group id based on the defined one and allows for further
+      # topic definition
+      # @param name [String, Symbol] name of the current subscription group
+      # @param block [Proc] block that may include topics definitions
+      def subscription_group=(name = SubscriptionGroup.id, &block)
+        # We cast it here, so the routing supports symbol based but that's anyhow later on
+        # validated as a string
+        @current_subscription_group_id = name
+
+        Proxy.new(self, &block)
+
+        # We need to reset the current subscription group after it is used, so it won't leak
+        # outside to other topics that would be defined without a defined subscription group
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.routing.subscription_groups_builder.call(topics)
+        @subscription_groups ||= App
+                                 .config
+                                 .internal
+                                 .routing
+                                 .subscription_groups_builder
+                                 .call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/consumer_mapper.rb

@@ -12,16 +12,6 @@ module Karafka
     #       raw_consumer_group_name
     #     end
     #   end
-    #
-    # @example Mapper for replacing "_" with "." in topic names
-    #   class MyMapper
-    #     def call(raw_consumer_group_name)
-    #       [
-    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
-    #         raw_consumer_group_name
-    #       ].join('_').gsub('_', '.')
-    #     end
-    #   end
     class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name

data/lib/karafka/routing/features/active_job/builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing extensions for ActiveJob
+        module Builder
+          # This method simplifies routes definition for ActiveJob topics / queues by
+          # auto-injecting the consumer class
+          #
+          # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+          # @param block [Proc] block that we can use for some extra configuration
+          def active_job_topic(name, &block)
+            topic(name) do
+              consumer App.config.internal.active_job.consumer_class
+              active_job true
+
+              # This is handled by our custom ActiveJob consumer
+              # Without this, default behaviour would cause messages to skip upon shutdown as the
+              # offset would be committed for the last message
+              manual_offset_management true
+
+              next unless block
+
+              instance_eval(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Config for ActiveJob usage
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+        # in order to be able to use active job routing
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            # One should not define active job jobs without ActiveJob being available for usage
+            next if Object.const_defined?('ActiveJob::Base')
+
+            [[%i[consumer], :active_job_missing]]
+          end
+
+          # ActiveJob needs to always run with manual offset management
+          # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+          # consumed jobs that did not run because of shutdown.
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            next if data[:manual_offset_management][:active]
+
+            [[%i[manual_offset_management], :must_be_enabled]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Topic extensions to be able to check if given topic is ActiveJob topic
+        module Topic
+          # @param active [Boolean] should this topic be considered one working with ActiveJob
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def active_job(active = false)
+            @active_job ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is this an ActiveJob topic
+          def active_job?
+            active_job.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus active job
+          #   namespace settings
+          def to_h
+            super.merge(
+              active_job: active_job.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Active-Job related components
+      # @note We can load it always, despite someone not using ActiveJob as it just adds a method
+      #   to the routing, without actually breaking anything.
+      class ActiveJob < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Base
+        # Routing builder expander that injects feature related drawing operations into it
+        class Expander < Module
+          # @param scope [Module] feature scope in which contract and other things should be
+          # @return [Expander] builder expander instance
+          def initialize(scope)
+            super()
+            @scope = scope
+          end
+
+          # Builds anonymous module that alters how `#draw` behaves allowing the feature contracts
+          # to run.
+          # @param mod [::Karafka::Routing::Builder] builder we will prepend to
+          def prepended(mod)
+            super
+
+            mod.prepend(prepended_module)
+          end
+
+          private
+
+          # @return [Module] builds an anonymous module with `#draw` that will alter the builder
+          #   `#draw` allowing to run feature context aware code.
+          def prepended_module
+            scope = @scope
+
+            Module.new do
+              # Runs validations related to this feature on a topic
+              #
+              # @param block [Proc] routing defining block
+              define_method :draw do |&block|
+                result = super(&block)
+
+                each do |consumer_group|
+                  consumer_group.topics.each do |topic|
+                    scope::Contract.new.validate!(topic.to_h)
+                  end
+                end
+
+                result
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Namespace for all the topic related features we support
+    #
+    # @note Not all the Karafka features need to be defined here as only those that have routing
+    #   or other extensions need to be here. That is why we keep (for now) features under the
+    #   routing namespace.
+    module Features
+      # Base for all the features
+      class Base
+        class << self
+          # Extends topic and builder with given feature API
+          def activate
+            Topic.prepend(self::Topic) if const_defined?('Topic', false)
+            Proxy.prepend(self::Builder) if const_defined?('Builder', false)
+            Builder.prepend(self::Builder) if const_defined?('Builder', false)
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contract', false)
+          end
+
+          # Loads all the features and activates them
+          def load_all
+            ObjectSpace
+              .each_object(Class)
+              .select { |klass| klass < self }
+              .sort_by(&:to_s)
+              .each(&:activate)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Config for dead letter queue feature
+        Config = Struct.new(
+          :active,
+          # We add skip variants but in regular we support only `:one`
+          :max_retries,
+          # To what topic the skipped messages should be moved
+          :topic,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Rules around dead letter queue settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :dead_letter_queue do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+          end
+
+          # Validate topic name only if dlq is active
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            dead_letter_queue = data[:dead_letter_queue]
+
+            next unless dead_letter_queue[:active]
+
+            topic = dead_letter_queue[:topic]
+
+            # When topic is set to false, it means we just want to skip dispatch on DLQ
+            next if topic == false
+            next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
+
+            [[%i[dead_letter_queue topic], :format]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # DLQ topic extensions
+        module Topic
+          # After how many retries should be move data to DLQ
+          DEFAULT_MAX_RETRIES = 3
+
+          private_constant :DEFAULT_MAX_RETRIES
+
+          # @param max_retries [Integer] after how many retries should we move data to dlq
+          # @param topic [String, false] where the messages should be moved if failing or false
+          #   if we do not want to move it anywhere and just skip
+          # @return [Config] defined config
+          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+            @dead_letter_queue ||= Config.new(
+              active: !topic.nil?,
+              max_retries: max_retries,
+              topic: topic
+            )
+          end
+
+          # @return [Boolean] is the dlq active or not
+          def dead_letter_queue?
+            dead_letter_queue.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus dlq settings
+          def to_h
+            super.merge(
+              dead_letter_queue: dead_letter_queue.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to continue processing when encountering errors.
+      # After certain number of retries, given messages will be moved to alternative topic,
+      # unclogging processing.
+      #
+      # @note This feature has an expanded version in the Pro mode. We do not use a new feature
+      #   injection in Pro (topic settings)
+      class DeadLetterQueue < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/config.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Config for declarative topics feature
+        Config = Struct.new(
+          :active,
+          :partitions,
+          :replication_factor,
+          :details,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contract.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Basic validation of the Kafka expected config details
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :declaratives do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+            required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+            required(:details) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(Symbol) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Extension for managing Kafka topic configuration
+        module Topic
+          # @param active [Boolean] is the topic structure management feature active
+          # @param partitions [Integer]
+          # @param replication_factor [Integer]
+          # @param details [Hash] extra configuration for the topic
+          # @return [Config] defined structure
+          def config(active: true, partitions: 1, replication_factor: 1, **details)
+            @declaratives ||= Config.new(
+              active: active,
+              partitions: partitions,
+              replication_factor: replication_factor,
+              details: details
+            )
+          end
+
+          # @return [Config] config details
+          def declaratives
+            config
+          end
+
+          # @return [true] declaratives is always active
+          def declaratives?
+            declaratives.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus declaratives
+          #   settings
+          def to_h
+            super.merge(
+              declaratives: declaratives.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to store per topic structure that can be later on used to bootstrap
+      # topics structure for test/development, etc. This allows to share the same set of settings
+      # for topics despite the environment. Pretty much similar to how the `structure.sql` or
+      # `schema.rb` operate for SQL dbs.
+      class Declaratives < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Config for manual offset management feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/contract.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Rules around manual offset management settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :manual_offset_management do
+            required(:active) { |val| [true, false].include?(val) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/topic.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Topic extensions to be able to manage manual offset management settings
+        module Topic
+          # @param active [Boolean] should we stop managing the offset in Karafka and make the user
+          #   responsible for marking messages as consumed.
+          # @return [Config] defined config
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def manual_offset_management(active = false)
+            @manual_offset_management ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is manual offset management enabled for a given topic
+          def manual_offset_management?
+            manual_offset_management.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus manual offset
+          #   management namespace settings
+          def to_h
+            super.merge(
+              manual_offset_management: manual_offset_management.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # All the things needed to be able to manage manual offset management from the routing
+      # perspective.
+      #
+      # Manual offset management allows users to completely disable automatic management of the
+      # offset. This can be used for implementing long-living window operations and other things
+      # where we do not want to commit the offset with each batch.
+      #
+      # Not all the Karafka and Karafka Pro features may be compatible with this feature being on.
+      class ManualOffsetManagement < Base
+      end
+    end
+  end
+end