karafka 1.4.12 → 2.2.10

data/lib/karafka/routing/features/active_job/builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing extensions for ActiveJob
+        module Builder
+          # This method simplifies routes definition for ActiveJob topics / queues by
+          # auto-injecting the consumer class
+          #
+          # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+          # @param block [Proc] block that we can use for some extra configuration
+          def active_job_topic(name, &block)
+            topic(name) do
+              consumer App.config.internal.active_job.consumer_class
+              active_job true
+
+              # This is handled by our custom ActiveJob consumer
+              # Without this, default behaviour would cause messages to skip upon shutdown as the
+              # offset would be committed for the last message
+              manual_offset_management true
+
+              next unless block
+
+              instance_eval(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Config for ActiveJob usage
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+          # in order to be able to use active job routing
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              # One should not define active job jobs without ActiveJob being available for usage
+              next if Object.const_defined?('ActiveJob::Base')
+
+              [[%i[consumer], :active_job_missing]]
+            end
+
+            # ActiveJob needs to always run with manual offset management
+            # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+            # consumed jobs that did not run because of shutdown.
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              next if data[:manual_offset_management][:active]
+
+              [[%i[manual_offset_management], :must_be_enabled]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/proxy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing proxy extensions for ActiveJob
+        module Proxy
+          include Builder
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Topic extensions to be able to check if given topic is ActiveJob topic
+        module Topic
+          # @param active [Boolean] should this topic be considered one working with ActiveJob
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def active_job(active = false)
+            @active_job ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is this an ActiveJob topic
+          def active_job?
+            active_job.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus active job
+          #   namespace settings
+          def to_h
+            super.merge(
+              active_job: active_job.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Active-Job related components
+      # @note We can load it always, despite someone not using ActiveJob as it just adds a method
+      #   to the routing, without actually breaking anything.
+      class ActiveJob < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Base
+        # Routing builder expander that injects feature related drawing operations into it
+        class Expander < Module
+          # @param scope [Module] feature scope in which contract and other things should be
+          # @return [Expander] builder expander instance
+          def initialize(scope)
+            super()
+            @scope = scope
+          end
+
+          # Builds anonymous module that alters how `#draw` behaves allowing the feature contracts
+          # to run.
+          # @param mod [::Karafka::Routing::Builder] builder we will prepend to
+          def prepended(mod)
+            super
+
+            mod.prepend(prepended_module)
+          end
+
+          private
+
+          # @return [Module] builds an anonymous module with `#draw` that will alter the builder
+          #   `#draw` allowing to run feature context aware code.
+          def prepended_module
+            scope = @scope
+
+            Module.new do
+              # Runs validations related to this feature on a routing resources
+              #
+              # @param block [Proc] routing defining block
+              define_method :draw do |&block|
+                result = super(&block)
+
+                each do |consumer_group|
+                  if scope::Contracts.const_defined?('ConsumerGroup', false)
+                    scope::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+                  end
+
+                  next unless scope::Contracts.const_defined?('Topic', false)
+
+                  consumer_group.topics.each do |topic|
+                    scope::Contracts::Topic.new.validate!(topic.to_h)
+                  end
+                end
+
+                result
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Namespace for all the topic related features we support
+    #
+    # @note Not all the Karafka features need to be defined here as only those that have routing
+    #   or other extensions need to be here. That is why we keep (for now) features under the
+    #   routing namespace.
+    module Features
+      # Base for all the features
+      class Base
+        class << self
+          # Extends topic and builder with given feature API
+          def activate
+            Topic.prepend(self::Topic) if const_defined?('Topic', false)
+            Topics.prepend(self::Topics) if const_defined?('Topics', false)
+            ConsumerGroup.prepend(self::ConsumerGroup) if const_defined?('ConsumerGroup', false)
+            Proxy.prepend(self::Proxy) if const_defined?('Proxy', false)
+            Builder.prepend(self::Builder) if const_defined?('Builder', false)
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contracts', false)
+          end
+
+          # Loads all the features and activates them
+          def load_all
+            features.each(&:activate)
+          end
+
+          # @param config [Karafka::Core::Configurable::Node] app config that we can alter with
+          #   particular routing feature specific stuff if needed
+          def pre_setup_all(config)
+            features.each { |feature| feature.pre_setup(config) }
+          end
+
+          # Runs post setup routing features configuration operations
+          #
+          # @param config [Karafka::Core::Configurable::Node]
+          def post_setup_all(config)
+            features.each { |feature| feature.post_setup(config) }
+          end
+
+          private
+
+          # @return [Array<Class>] all available routing features
+          def features
+            ObjectSpace
+              .each_object(Class)
+              .select { |klass| klass < self }
+              .sort_by(&:to_s)
+          end
+
+          protected
+
+          # Runs pre-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def pre_setup(_config)
+            true
+          end
+
+          # Runs post-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def post_setup(_config)
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Config for dead letter queue feature
+        Config = Struct.new(
+          :active,
+          # We add skip variants but in regular we support only `:one`
+          :max_retries,
+          # To what topic the skipped messages should be moved
+          :topic,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around dead letter queue settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :dead_letter_queue do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+            end
+
+            # Validate topic name only if dlq is active
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              dead_letter_queue = data[:dead_letter_queue]
+
+              next unless dead_letter_queue[:active]
+
+              topic = dead_letter_queue[:topic]
+              topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
+
+              # When topic is set to false, it means we just want to skip dispatch on DLQ
+              next if topic == false
+              next if topic.is_a?(String) && topic_regexp.match?(topic)
+
+              [[%i[dead_letter_queue topic], :format]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # DLQ topic extensions
+        module Topic
+          # After how many retries should be move data to DLQ
+          DEFAULT_MAX_RETRIES = 3
+
+          private_constant :DEFAULT_MAX_RETRIES
+
+          # @param max_retries [Integer] after how many retries should we move data to dlq
+          # @param topic [String, false] where the messages should be moved if failing or false
+          #   if we do not want to move it anywhere and just skip
+          # @return [Config] defined config
+          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+            @dead_letter_queue ||= Config.new(
+              active: !topic.nil?,
+              max_retries: max_retries,
+              topic: topic
+            )
+          end
+
+          # @return [Boolean] is the dlq active or not
+          def dead_letter_queue?
+            dead_letter_queue.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus dlq settings
+          def to_h
+            super.merge(
+              dead_letter_queue: dead_letter_queue.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to continue processing when encountering errors.
+      # After certain number of retries, given messages will be moved to alternative topic,
+      # unclogging processing.
+      #
+      # @note This feature has an expanded version in the Pro mode. We do not use a new feature
+      #   injection in Pro (topic settings)
+      class DeadLetterQueue < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/config.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Config for declarative topics feature
+        Config = Struct.new(
+          :active,
+          :partitions,
+          :replication_factor,
+          :details,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # This feature validation contracts
+        module Contracts
+          # Basic validation of the Kafka expected config details
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :declaratives do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+              required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+              required(:details) do |val|
+                val.is_a?(Hash) &&
+                  val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # Extension for managing Kafka topic configuration
+        module Topic
+          # @param active [Boolean] is the topic structure management feature active
+          # @param partitions [Integer]
+          # @param replication_factor [Integer]
+          # @param details [Hash] extra configuration for the topic
+          # @return [Config] defined structure
+          def config(active: true, partitions: 1, replication_factor: 1, **details)
+            @declaratives ||= Config.new(
+              active: active,
+              partitions: partitions,
+              replication_factor: replication_factor,
+              details: details
+            )
+          end
+
+          # @return [Config] config details
+          def declaratives
+            config
+          end
+
+          # @return [true] declaratives is always active
+          def declaratives?
+            declaratives.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus declaratives
+          #   settings
+          def to_h
+            super.merge(
+              declaratives: declaratives.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to store per topic structure that can be later on used to bootstrap
+      # topics structure for test/development, etc. This allows to share the same set of settings
+      # for topics despite the environment. Pretty much similar to how the `structure.sql` or
+      # `schema.rb` operate for SQL dbs.
+      class Declaratives < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Inline Insights related contracts namespace
+        module Contracts
+          # Contract for inline insights topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :inline_insights do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Routing topic inline insights API
+        module Topic
+          # @param active [Boolean] should inline insights be activated
+          def inline_insights(active = false)
+            @inline_insights ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are inline insights active
+          def inline_insights?
+            inline_insights.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              inline_insights: inline_insights.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Feature allowing us to get visibility during the consumption into metrics of particular
+      # partition we operate on. It can be useful when making context-aware consumers that change
+      # their behaviours based on the lag and other parameters.
+      class InlineInsights < Base
+        class << self
+          # If needed installs the needed listener and initializes tracker
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config
+          def post_setup(_config)
+            ::Karafka::App.monitor.subscribe('app.running') do
+              # Do not activate tracking of statistics if none of our active topics uses it
+              # This prevents us from tracking metrics when user just runs a subset of topics
+              # in a given process and none of those actually utilizes this feature
+              next unless ::Karafka::App
+                          .subscription_groups
+                          .values
+                          .flat_map(&:itself)
+                          .flat_map(&:topics)
+                          .flat_map(&:to_a)
+                          .any?(&:inline_insights?)
+
+              # Initialize the tracker prior to becoming multi-threaded
+              ::Karafka::Processing::InlineInsights::Tracker.instance
+
+              # Subscribe to the statistics reports and collect them
+              ::Karafka.monitor.subscribe(
+                ::Karafka::Processing::InlineInsights::Listener.new
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Config for manual offset management feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end