karafka 2.0.0 → 2.1.0

data/lib/karafka/processing/strategies/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Our processing patterns differ depending on various features configurations
+    # In this namespace we collect strategies for particular feature combinations to simplify the
+    # design. Based on features combinations we can then select handling strategy for a given case.
+    #
+    # @note The lack of common code here is intentional. It would get complex if there would be
+    #   any type of composition, so each strategy is expected to be self-sufficient
+    module Strategies
+      # Base strategy that should be included in each strategy, just to ensure the API
+      module Base
+        # What should happen before jobs are enqueued
+        # @note This runs from the listener thread, not recommended to put anything slow here
+        def handle_before_enqueue
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # What should happen before we kick in the processing
+        def handle_before_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # What should happen in the processing
+        def handle_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Post-consumption handling
+        def handle_after_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Idle run handling
+        def handle_idle
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Revocation handling
+        def handle_revoked
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Shutdown handling
+        def handle_shutdown
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/default.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # No features enabled.
+      # No manual offset management
+      # No long running jobs
+      # Nothing. Just standard, automatic flow
+      module Default
+        include Base
+
+        # Apply strategy for a non-feature based flow
+        FEATURES = %i[].freeze
+
+        # Marks message as consumed in an async way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        #
+        # @note We keep track of this offset in case we would mark as consumed and got error when
+        #   processing another message. In case like this we do not pause on the message we've
+        #   already processed but rather at the next one. This applies to both sync and async
+        #   versions of this method.
+        def mark_as_consumed(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+          return false if revoked?
+
+          unless client.mark_as_consumed(message)
+            coordinator.revoke
+
+            return false
+          end
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
+        # Marks message as consumed in a sync way.
+        #
+        # @param message [Messages::Message] last successfully processed message.
+        # @return [Boolean] true if we were able to mark the offset, false otherwise.
+        #   False indicates that we were not able and that we have lost the partition.
+        def mark_as_consumed!(message)
+          # Ignore earlier offsets than the one we already committed
+          return true if coordinator.seek_offset > message.offset
+          return false if revoked?
+
+          unless client.mark_as_consumed!(message)
+            coordinator.revoke
+
+            return false
+          end
+
+          coordinator.seek_offset = message.offset + 1
+
+          true
+        end
+
+        # No actions needed for the standard flow here
+        def handle_before_enqueue
+          nil
+        end
+
+        # Increment number of attempts
+        def handle_before_consume
+          coordinator.pause_tracker.increment
+        end
+
+        # Run the user consumption code
+        def handle_consume
+          Karafka.monitor.instrument('consumer.consume', caller: self)
+          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+            consume
+          end
+
+          # Mark job as successful
+          coordinator.success!(self)
+        rescue StandardError => e
+          coordinator.failure!(self, e)
+
+          # Re-raise so reported in the consumer
+          raise e
+        ensure
+          # We need to decrease number of jobs that this coordinator coordinates as it has finished
+          coordinator.decrement
+        end
+
+        # Standard flow marks work as consumed and moves on if everything went ok.
+        # If there was a processing error, we will pause and continue from the next message
+        # (next that is +1 from the last one that was successfully marked as consumed)
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            # We should not move the offset automatically when the partition was paused
+            # If we would not do this upon a revocation during the pause time, a different process
+            # would pick not from the place where we paused but from the offset that would be
+            # automatically committed here
+            return if coordinator.manual_pause?
+
+            mark_as_consumed(messages.last)
+          else
+            retry_after_pause
+          end
+        end
+
+        # Code that should run on idle runs without messages available
+        def handle_idle
+          nil
+        end
+
+        # We need to always un-pause the processing in case we have lost a given partition.
+        # Otherwise the underlying librdkafka would not know we may want to continue processing and
+        # the pause could in theory last forever
+        def handle_revoked
+          resume
+
+          coordinator.revoke
+
+          Karafka.monitor.instrument('consumer.revoke', caller: self)
+          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            revoked
+          end
+        end
+
+        # Runs the shutdown code
+        def handle_shutdown
+          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
+          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+            shutdown
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using dead letter queue, processing won't stop after defined number of retries
+      # upon encountering non-critical errors but the messages that error will be moved to a
+      # separate topic with their payload and metadata, so they can be handled differently.
+      module Dlq
+        include Default
+
+        # Apply strategy when only dead letter queue is turned on
+        FEATURES = %i[
+          dead_letter_queue
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            return if coordinator.manual_pause?
+
+            mark_as_consumed(messages.last)
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message, = find_skippable_message
+
+            # Send skippable message to the dql topic
+            dispatch_to_dlq(skippable_message)
+
+            # We mark the broken message as consumed and move on
+            mark_as_consumed(skippable_message)
+
+            return if revoked?
+
+            # We pause to backoff once just in case.
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+
+        # Finds the message may want to skip (all, starting from first)
+        # @private
+        # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+        #   information if this message was from marked offset or figured out via mom flow
+        def find_skippable_message
+          skippable_message = messages.find do |msg|
+            coordinator.marked? && msg.offset == coordinator.seek_offset
+          end
+
+          # If we don't have the message matching the last comitted offset, it means that
+          # user operates with manual offsets and we're beyond the batch in which things
+          # broke for the first time. Then we skip the first (as no markings) and we
+          # move on one by one.
+          skippable_message ? [skippable_message, true] : [messages.first, false]
+        end
+
+        # Moves the broken message into a separate queue defined via the settings
+        # @private
+        # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
+        #   should go to the dlq topic
+        def dispatch_to_dlq(skippable_message)
+          producer.produce_async(
+            topic: topic.dead_letter_queue.topic,
+            payload: skippable_message.raw_payload
+          )
+
+          # Notify about dispatch on the events bus
+          Karafka.monitor.instrument(
+            'dead_letter_queue.dispatched',
+            caller: self,
+            message: skippable_message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # Same as pure dead letter queue but we do not marked failed message as consumed
+      module DlqMom
+        include Dlq
+
+        # Apply strategy when dlq is on with manual offset management
+        FEATURES = %i[
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            retry_after_pause
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message, = find_skippable_message
+
+            dispatch_to_dlq(skippable_message)
+
+            # Save the next offset we want to go with after moving given message to DLQ
+            # Without this, we would not be able to move forward and we would end up
+            # in an infinite loop trying to un-pause from the message we've already processed
+            # Of course, since it's a MoM a rebalance or kill, will move it back as no
+            # offsets are being committed
+            coordinator.seek_offset = skippable_message.offset + 1
+
+            pause(coordinator.seek_offset, nil, false)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/mom.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using manual offset management, we do not mark as consumed after successful processing
+      module Mom
+        include Default
+
+        # Apply strategy when only manual offset management is turned on
+        FEATURES = %i[
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          else
+            retry_after_pause
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategy_selector.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate processing strategy matching topic combinations
+    class StrategySelector
+      attr_reader :strategies
+
+      # Features we support in the OSS offering.
+      SUPPORTED_FEATURES = %i[
+        active_job
+        manual_offset_management
+        dead_letter_queue
+      ].freeze
+
+      def initialize
+        # We load them once for performance reasons not to do too many lookups
+        @strategies = find_all
+      end
+
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
+      # @return [Module] module with proper strategy
+      def find(topic)
+        feature_set = SUPPORTED_FEATURES.map do |feature_name|
+          topic.public_send("#{feature_name}?") ? feature_name : nil
+        end
+
+        feature_set.compact!
+
+        @strategies.find do |strategy|
+          strategy::FEATURES.sort == feature_set.sort
+        end || raise(Errors::StrategyNotFoundError, topic.name)
+      end
+
+      private
+
+      # @return [Array<Module>] available strategies
+      def find_all
+        Strategies
+          .constants
+          .delete_if { |k| k == :Base }
+          .map { |k| Strategies.const_get(k) }
+          .uniq
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -25,7 +25,7 @@ module Karafka
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
       end
 
@@ -46,9 +46,9 @@ module Karafka
       def process
         job = @jobs_queue.pop
 
-        if job
-          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+        instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
+        if job
           Karafka.monitor.instrument('worker.process', instrument_details)
 
           Karafka.monitor.instrument('worker.processed', instrument_details) do
@@ -76,12 +76,17 @@ module Karafka
         Karafka.monitor.instrument(
           'error.occurred',
           caller: self,
+          job: job,
+          jobs_queue: @jobs_queue,
           error: e,
           type: 'worker.process.error'
         )
       ensure
         # job can be nil when the queue is being closed
         @jobs_queue.complete(job) if job
+
+        # Always publish info, that we completed all the work despite its result
+        Karafka.monitor.instrument('worker.completed', instrument_details)
       end
     end
   end

data/lib/karafka/railtie.rb

@@ -2,13 +2,7 @@
 
 # This file contains Railtie for auto-configuration
 
-rails = false
-
-begin
-  require 'rails'
-
-  rails = true
-rescue LoadError
+unless Karafka.rails?
   # Without defining this in any way, Zeitwerk ain't happy so we do it that way
   module Karafka
     class Railtie
@@ -16,10 +10,7 @@ rescue LoadError
   end
 end
 
-if rails
-  # Load Karafka
-  require 'karafka'
-
+if Karafka.rails?
   # Load ActiveJob adapter
   require 'active_job/karafka'
 
@@ -47,10 +38,17 @@ if rails
 
         next unless Rails.env.development?
         next unless ENV.key?('KARAFKA_CLI')
+        # If we are already publishing to STDOUT, no need to add it again.
+        # If added again, would print stuff twice
+        next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
+
+        logger = ActiveSupport::Logger.new($stdout)
+        # Inherit the logger level from Rails, otherwise would always run with the debug level
+        logger.level = Rails.logger.level
 
         Rails.logger.extend(
           ActiveSupport::Logger.broadcast(
-            ActiveSupport::Logger.new($stdout)
+            logger
           )
         )
       end
@@ -74,11 +72,25 @@ if rails
         ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
           # Reload code each time there is a change in the code
           next unless Rails.application.reloaders.any?(&:updated?)
+          # If consumer persistence is enabled, no reason to reload because we will still keep
+          # old consumer instances in memory.
+          next if Karafka::App.config.consumer_persistence
 
           Rails.application.reloader.reload!
         end
       end
 
+      initializer 'karafka.release_active_record_connections' do
+        ActiveSupport.on_load(:active_record) do
+          ::Karafka::App.monitor.subscribe('worker.completed') do
+            # Always release the connection after processing is done. Otherwise thread may hang
+            # blocking the reload and further processing
+            # @see https://github.com/rails/rails/issues/44183
+            ActiveRecord::Base.clear_active_connections!
+          end
+        end
+      end
+
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 

data/lib/karafka/routing/activity_manager.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Allows us to get track of which consumer groups, subscription groups and topics are enabled
+    # or disabled via CLI
+    class ActivityManager
+      # Supported types of inclusions and exclusions
+      SUPPORTED_TYPES = %i[
+        consumer_groups
+        subscription_groups
+        topics
+      ].freeze
+
+      def initialize
+        @included = Hash.new { |h, k| h[k] = [] }
+        @excluded = Hash.new { |h, k| h[k] = [] }
+      end
+
+      # Adds resource to included/active
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def include(type, name)
+        validate!(type)
+
+        @included[type] << name
+      end
+
+      # Adds resource to excluded
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def exclude(type, name)
+        validate!(type)
+
+        @excluded[type] << name
+      end
+
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      # @return [Boolean] is the given resource active or not
+      def active?(type, name)
+        validate!(type)
+
+        included = @included[type]
+        excluded = @excluded[type]
+
+        # If nothing defined, all active by default
+        return true if included.empty? && excluded.empty?
+        # Inclusion supersedes exclusion in case someone wrote both
+        return true if !included.empty? && included.include?(name)
+
+        # If there are exclusions but our is not excluded and no inclusions or included, it's ok
+        !excluded.empty? &&
+          !excluded.include?(name) &&
+          (included.empty? || included.include?(name))
+      end
+
+      # @return [Hash] accumulated data in a hash for validations
+      def to_h
+        (
+          SUPPORTED_TYPES.map { |type| ["include_#{type}".to_sym, @included[type]] } +
+          SUPPORTED_TYPES.map { |type| ["exclude_#{type}".to_sym, @excluded[type]] }
+        ).to_h
+      end
+
+      # Clears the manager
+      def clear
+        @included.clear
+        @excluded.clear
+      end
+
+      private
+
+      # Checks if the type we want to register is supported
+      #
+      # @param type [Symbol] type for inclusion
+      def validate!(type)
+        return if SUPPORTED_TYPES.include?(type)
+
+        raise(::Karafka::Errors::UnsupportedCaseError, type)
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -33,7 +33,16 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
+          # Validate consumer group settings
           Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+
+          # and then its topics settings
+          consumer_group.topics.each do |topic|
+            Contracts::Topic.new.validate!(topic.to_h)
+          end
+
+          # Initialize subscription groups after all the routing is done
+          consumer_group.subscription_groups
         end
       end
 
@@ -66,6 +75,17 @@ module Karafka
         end
       end
 
+      # Handles the simple routing case where we create one consumer group and allow for further
+      # subscription group customization
+      # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
+      #   a random uuid will be used
+      # @param block [Proc] further topics definitions
+      def subscription_group(subscription_group_name = SubscriptionGroup.id, &block)
+        consumer_group('app') do
+          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
+        end
+      end
+
       # In case we use simple style of routing, all topics will be assigned to the same consumer
       # group that will be based on the client_id
       #