karafka 1.4.12 → 2.2.10

data/lib/karafka/processing/strategies/mom.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using manual offset management, we do not mark as consumed after successful processing
+      module Mom
+        include Default
+
+        # Apply strategy when only manual offset management is turned on
+        FEATURES = %i[
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          else
+            retry_after_pause
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategy_selector.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate processing strategy matching topic combinations
+    class StrategySelector
+      attr_reader :strategies
+
+      # Features we support in the OSS offering.
+      SUPPORTED_FEATURES = %i[
+        active_job
+        manual_offset_management
+        dead_letter_queue
+      ].freeze
+
+      def initialize
+        # We load them once for performance reasons not to do too many lookups
+        @strategies = find_all
+      end
+
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
+      # @return [Module] module with proper strategy
+      def find(topic)
+        feature_set = SUPPORTED_FEATURES.map do |feature_name|
+          topic.public_send("#{feature_name}?") ? feature_name : nil
+        end
+
+        feature_set.compact!
+
+        @strategies.find do |strategy|
+          strategy::FEATURES.sort == feature_set.sort
+        end || raise(Errors::StrategyNotFoundError, topic.name)
+      end
+
+      private
+
+      # @return [Array<Module>] available strategies
+      def find_all
+        Strategies
+          .constants
+          .delete_if { |k| k == :Base }
+          .map { |k| Strategies.const_get(k) }
+          .uniq
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Workers are used to run jobs in separate threads.
+    # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
+    class Worker
+      include Helpers::Async
+
+      # @return [String] id of this worker
+      attr_reader :id
+
+      # @param jobs_queue [JobsQueue]
+      # @return [Worker]
+      def initialize(jobs_queue)
+        @id = SecureRandom.hex(6)
+        @jobs_queue = jobs_queue
+      end
+
+      private
+
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
+      # Fetches a single job, processes it and marks as completed.
+      #
+      # @note We do not have error handling here, as no errors should propagate this far. If they
+      #   do, it is a critical error and should bubble up.
+      #
+      # @note Upon closing the jobs queue, worker will close it's thread
+      def process
+        job = @jobs_queue.pop
+
+        instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+
+        if job
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
+            job.before_call
+
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
+
+            job.call
+
+            job.after_call
+
+            true
+          end
+        else
+          false
+        end
+      # We signal critical exceptions, notify and do not allow worker to fail
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+        # rubocop:enable Lint/RescueException
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          job: job,
+          jobs_queue: @jobs_queue,
+          error: e,
+          type: 'worker.process.error'
+        )
+      ensure
+        # job can be nil when the queue is being closed
+        @jobs_queue.complete(job) if job
+
+        # Always publish info, that we completed all the work despite its result
+        Karafka.monitor.instrument('worker.completed', instrument_details)
+      end
+    end
+  end
+end

data/lib/karafka/processing/workers_batch.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Abstraction layer around workers batch.
+    class WorkersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [WorkersBatch]
+      def initialize(jobs_queue)
+        @batch = Array.new(App.config.concurrency) { Processing::Worker.new(jobs_queue) }
+      end
+
+      # Iterates over available workers and yields each worker
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+# This file contains Railtie for auto-configuration
+
+unless Karafka.rails?
+  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
+  module Karafka
+    class Railtie
+    end
+  end
+end
+
+if Karafka.rails?
+  # Load ActiveJob adapter
+  require 'active_job/karafka'
+
+  # Setup env if configured (may be configured later by .net, etc)
+  ENV['KARAFKA_ENV'] ||= ENV['RAILS_ENV'] if ENV.key?('RAILS_ENV')
+
+  module Karafka
+    # Railtie for setting up Rails integration
+    class Railtie < Rails::Railtie
+      railtie_name :karafka
+
+      initializer 'karafka.active_job_integration' do
+        ActiveSupport.on_load(:active_job) do
+          # Extend ActiveJob with some Karafka specific ActiveJob magic
+          extend ::Karafka::ActiveJob::JobExtensions
+        end
+      end
+
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
+        # Make Karafka uses Rails logger
+        ::Karafka::App.config.logger = Rails.logger
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        # If we are already publishing to STDOUT, no need to add it again.
+        # If added again, would print stuff twice
+        next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
+
+        stdout_logger = ActiveSupport::Logger.new($stdout)
+        # Inherit the logger level from Rails, otherwise would always run with the debug level
+        stdout_logger.level = Rails.logger.level
+
+        rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
+
+        # Rails 7.1 replaced the broadcast module with a broadcast logger
+        if rails71plus
+          Rails.logger.broadcast_to(stdout_logger)
+        else
+          Rails.logger.extend(
+            ActiveSupport::Logger.broadcast(
+              stdout_logger
+            )
+          )
+        end
+      end
+
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
+
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # If consumer persistence is enabled, no reason to reload because we will still keep
+          # old consumer instances in memory.
+          next if Karafka::App.config.consumer_persistence
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
+        end
+      end
+
+      initializer 'karafka.release_active_record_connections' do
+        rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
+
+        ActiveSupport.on_load(:active_record) do
+          ::Karafka::App.monitor.subscribe('worker.completed') do
+            # Always release the connection after processing is done. Otherwise thread may hang
+            # blocking the reload and further processing
+            # @see https://github.com/rails/rails/issues/44183
+            #
+            # The change technically happens in 7.1 but 7.0 already supports this so we can make
+            # a proper change for 7.0+
+            if rails7plus
+              ActiveRecord::Base.connection_handler.clear_active_connections!
+            else
+              ActiveRecord::Base.clear_active_connections!
+            end
+          end
+        end
+      end
+
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/activity_manager.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Allows us to get track of which consumer groups, subscription groups and topics are enabled
+    # or disabled via CLI
+    class ActivityManager
+      # Supported types of inclusions and exclusions
+      SUPPORTED_TYPES = %i[
+        consumer_groups
+        subscription_groups
+        topics
+      ].freeze
+
+      def initialize
+        @included = Hash.new { |h, k| h[k] = [] }
+        @excluded = Hash.new { |h, k| h[k] = [] }
+      end
+
+      # Adds resource to included/active
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def include(type, name)
+        validate!(type)
+
+        @included[type] << name
+      end
+
+      # Adds resource to excluded
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def exclude(type, name)
+        validate!(type)
+
+        @excluded[type] << name
+      end
+
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      # @return [Boolean] is the given resource active or not
+      def active?(type, name)
+        validate!(type)
+
+        included = @included[type]
+        excluded = @excluded[type]
+
+        # If nothing defined, all active by default
+        return true if included.empty? && excluded.empty?
+        # Inclusion supersedes exclusion in case someone wrote both
+        return true if !included.empty? && included.include?(name)
+
+        # If there are exclusions but our is not excluded and no inclusions or included, it's ok
+        !excluded.empty? &&
+          !excluded.include?(name) &&
+          (included.empty? || included.include?(name))
+      end
+
+      # @return [Hash] accumulated data in a hash for validations
+      def to_h
+        (
+          SUPPORTED_TYPES.map { |type| ["include_#{type}".to_sym, @included[type]] } +
+          SUPPORTED_TYPES.map { |type| ["exclude_#{type}".to_sym, @excluded[type]] }
+        ).to_h
+      end
+
+      # Clears the manager
+      def clear
+        @included.clear
+        @excluded.clear
+      end
+
+      private
+
+      # Checks if the type we want to register is supported
+      #
+      # @param type [Symbol] type for inclusion
+      def validate!(type)
+        return if SUPPORTED_TYPES.include?(type)
+
+        raise(::Karafka::Errors::UnsupportedCaseError, type)
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -10,14 +10,15 @@ module Karafka
     #     end
     #   end
     class Builder < Concurrent::Array
-      # Consumer group consistency checking contract
-      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
+      # Empty default per-topic config
+      EMPTY_DEFAULTS = ->(_) {}.freeze
 
-      private_constant :CONTRACT
+      private_constant :EMPTY_DEFAULTS
 
       def initialize
-        super
         @draws = Concurrent::Array.new
+        @defaults = EMPTY_DEFAULTS
+        super
       end
 
       # Used to draw routes for Karafka
@@ -38,11 +39,16 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
-          hashed_group = consumer_group.to_h
-          validation_result = CONTRACT.call(hashed_group)
-          next if validation_result.success?
+          # Validate consumer group settings
+          Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+
+          # and then its topics settings
+          consumer_group.topics.each do |topic|
+            Contracts::Topic.new.validate!(topic.to_h)
+          end
 
-          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+          # Initialize subscription groups after all the routing is done
+          consumer_group.subscription_groups
         end
       end
 
@@ -55,17 +61,17 @@ module Karafka
 
       # Clears the builder and the draws memory
       def clear
+        @defaults = EMPTY_DEFAULTS
         @draws.clear
         super
       end
 
-      # Redraws all the routes for the in-process code reloading.
-      # @note This won't allow registration of new topics without process restart but will trigger
-      #   cache invalidation so all the classes, etc are re-fetched after code reload
-      def reload
-        draws = @draws.dup
-        clear
-        draws.each { |block| draw(&block) }
+      # @param block [Proc] block with per-topic evaluated defaults
+      # @return [Proc] defaults that should be evaluated per topic
+      def defaults(&block)
+        return @defaults unless block
+
+        @defaults = block
       end
 
       private
@@ -74,15 +80,35 @@ module Karafka
       # @param group_id [String, Symbol] name for consumer group
       # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
-        consumer_group = ConsumerGroup.new(group_id.to_s)
-        self << Proxy.new(consumer_group, &block).target
+        consumer_group = find { |cg| cg.name == group_id.to_s }
+
+        if consumer_group
+          Proxy.new(consumer_group, &block).target
+        else
+          consumer_group = ConsumerGroup.new(group_id.to_s)
+          self << Proxy.new(consumer_group, &block).target
+        end
+      end
+
+      # Handles the simple routing case where we create one consumer group and allow for further
+      # subscription group customization
+      # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
+      #   a random uuid will be used
+      # @param block [Proc] further topics definitions
+      def subscription_group(subscription_group_name = SubscriptionGroup.id, &block)
+        consumer_group('app') do
+          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
+        end
       end
 
+      # In case we use simple style of routing, all topics will be assigned to the same consumer
+      # group that will be based on the client_id
+      #
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
-        consumer_group(topic_name) do
-          topic(topic_name, &block).tap(&:build)
+        consumer_group('app') do
+          topic(topic_name, &block)
         end
       end
     end

data/lib/karafka/routing/consumer_group.rb

@@ -5,28 +5,33 @@ module Karafka
     # Object used to describe a single consumer group that is going to subscribe to
     # given topics
     # It is a part of Karafka's DSL
+    # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
+    #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
-      extend Helpers::ConfigRetriever
+      attr_reader :id, :topics, :name
 
-      attr_reader(
-        :topics,
-        :id,
-        :name
-      )
+      # This is a "virtual" attribute that is not building subscription groups.
+      # It allows us to store the "current" subscription group defined in the routing
+      # This subscription group id is then injected into topics, so we can compute the subscription
+      # groups
+      attr_accessor :current_subscription_group_name
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
       #   We add it to make a multi-system development easier for people that don't use
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
-        @name = name
-        @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @name = name.to_s
+        @id = config.consumer_mapper.call(name)
+        @topics = Topics.new([])
+        # Initialize the subscription group so there's always a value for it, since even if not
+        # defined directly, a subscription group will be created
+        @current_subscription_group_name = SubscriptionGroup.id
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        Karafka::Server.consumer_groups.include?(name)
+        config.internal.routing.activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -35,28 +40,59 @@ module Karafka
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &block)
         topic = Topic.new(name, self)
-        @topics << Proxy.new(topic, &block).target.tap(&:build)
-        @topics.last
+        @topics << Proxy.new(
+          topic,
+          config.internal.routing.builder.defaults,
+          &block
+        ).target
+        built_topic = @topics.last
+        # We overwrite it conditionally in case it was not set by the user inline in the topic
+        # block definition
+        built_topic.subscription_group_name ||= current_subscription_group_name
+        built_topic
       end
 
-      Karafka::AttributesMap.consumer_group.each do |attribute|
-        config_retriever_for(attribute)
+      # Assigns the current subscription group id based on the defined one and allows for further
+      # topic definition
+      # @param name [String, Symbol] name of the current subscription group
+      # @param block [Proc] block that may include topics definitions
+      def subscription_group=(name = SubscriptionGroup.id, &block)
+        # We cast it here, so the routing supports symbol based but that's anyhow later on
+        # validated as a string
+        @current_subscription_group_name = name.to_s
+
+        Proxy.new(self, &block)
+
+        # We need to reset the current subscription group after it is used, so it won't leak
+        # outside to other topics that would be defined without a defined subscription group
+        @current_subscription_group_name = SubscriptionGroup.id
+      end
+
+      # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
+      #   the consumer group topics
+      def subscription_groups
+        @subscription_groups ||= config
+                                 .internal
+                                 .routing
+                                 .subscription_groups_builder
+                                 .call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
       # @return [Hash] hash with consumer group attributes including serialized to hash
       # topics inside of it.
       def to_h
-        result = {
+        {
           topics: topics.map(&:to_h),
           id: id
-        }
+        }.freeze
+      end
 
-        Karafka::AttributesMap.consumer_group.each do |attribute|
-          result[attribute] = public_send(attribute)
-        end
+      private
 
-        result
+      # @return [Karafka::Core::Configurable::Node] root node config
+      def config
+        ::Karafka::App.config
       end
     end
   end

data/lib/karafka/routing/consumer_mapper.rb

@@ -12,22 +12,11 @@ module Karafka
     #       raw_consumer_group_name
     #     end
     #   end
-    #
-    # @example Mapper for replacing "_" with "." in topic names
-    #   class MyMapper
-    #     def call(raw_consumer_group_name)
-    #       [
-    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
-    #         raw_consumer_group_name
-    #       ].join('_').gsub('_', '.')
-    #     end
-    #   end
     class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
       def call(raw_consumer_group_name)
-        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
-        "#{client_name}_#{raw_consumer_group_name}"
+        "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
       end
     end
   end