karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/strategies/dlq.rb

@@ -1,153 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # When using dead letter queue, processing won't stop after defined number of retries
-      # upon encountering non-critical errors but the messages that error will be moved to a
-      # separate topic with their payload and metadata, so they can be handled differently.
-      module Dlq
-        include Default
-
-        # Apply strategy when only dead letter queue is turned on
-        FEATURES = %i[
-          dead_letter_queue
-        ].freeze
-
-        # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
-        # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
-        # any offset marking causes the pause count tracker to reset. This is useful when
-        # the error is not due to the collective batch operations state but due to intermediate
-        # "crawling" errors that move with it
-        #
-        # @see `Strategies::Default#mark_as_consumed` for more details
-        # @param message [Messages::Message]
-        def mark_as_consumed(message)
-          # If we are not retrying pause count is already 0, no need to try to reset the state
-          return super unless retrying?
-          # If we do not use independent marking on DLQ, we just mark as consumed
-          return super unless topic.dead_letter_queue.independent?
-          # If we were not able to mark no need to reset
-          return false unless super
-
-          coordinator.pause_tracker.reset
-
-          true
-        end
-
-        # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
-        # DLQ was configured with the `independent` flag.
-        #
-        # @see `Strategies::Default#mark_as_consumed!` for more details
-        # @param message [Messages::Message]
-        def mark_as_consumed!(message)
-          return super unless retrying?
-          return super unless topic.dead_letter_queue.independent?
-          return false unless super
-
-          coordinator.pause_tracker.reset
-
-          true
-        end
-
-        # When manual offset management is on, we do not mark anything as consumed automatically
-        # and we rely on the user to figure things out
-        def handle_after_consume
-          return if revoked?
-
-          if coordinator.success?
-            coordinator.pause_tracker.reset
-
-            return if coordinator.manual_pause?
-
-            mark_as_consumed(messages.last)
-          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            retry_after_pause
-          # If we've reached number of retries that we could, we need to skip the first message
-          # that was not marked as consumed, pause and continue, while also moving this message
-          # to the dead topic
-          else
-            # We reset the pause to indicate we will now consider it as "ok".
-            coordinator.pause_tracker.reset
-
-            skippable_message, = find_skippable_message
-
-            # Send skippable message to the dql topic
-            dispatch_to_dlq(skippable_message)
-
-            # We mark the broken message as consumed and move on
-            if mark_after_dispatch?
-              mark_dispatched_to_dlq(skippable_message)
-
-              return if revoked?
-            else
-              self.seek_offset = skippable_message.offset + 1
-            end
-
-            # We pause to backoff once just in case.
-            pause(seek_offset, nil, false)
-          end
-        end
-
-        # Finds the message may want to skip (all, starting from first)
-        # @private
-        # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
-        #   information if this message was from marked offset or figured out via mom flow
-        def find_skippable_message
-          skippable_message = messages.raw.find do |msg|
-            coordinator.marked? && msg.offset == seek_offset
-          end
-
-          # If we don't have the message matching the last comitted offset, it means that
-          # user operates with manual offsets and we're beyond the batch in which things
-          # broke for the first time. Then we skip the first (as no markings) and we
-          # move on one by one.
-          skippable_message ? [skippable_message, true] : [messages.first, false]
-        end
-
-        # Moves the broken message into a separate queue defined via the settings
-        # @private
-        # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
-        #   should go to the dlq topic
-        def dispatch_to_dlq(skippable_message)
-          producer.public_send(
-            topic.dead_letter_queue.dispatch_method,
-            topic: topic.dead_letter_queue.topic,
-            payload: skippable_message.raw_payload
-          )
-
-          # Notify about dispatch on the events bus
-          monitor.instrument(
-            "dead_letter_queue.dispatched",
-            caller: self,
-            message: skippable_message
-          )
-        end
-
-        # @return [Boolean] should we mark given message as consumed after dispatch. For default
-        #  non MOM strategies if user did not explicitly tell us not to, we mark it. Default is
-        #  `nil`, which means `true` in this case. If user provided alternative value, we go
-        #  with it.
-        def mark_after_dispatch?
-          return true if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-          topic.dead_letter_queue.mark_after_dispatch
-        end
-
-        # Marks message that went to DLQ (if applicable) based on the requested method
-        # @param skippable_message [Karafka::Messages::Message]
-        def mark_dispatched_to_dlq(skippable_message)
-          case topic.dead_letter_queue.marking_method
-          when :mark_as_consumed
-            mark_as_consumed(skippable_message)
-          when :mark_as_consumed!
-            mark_as_consumed!(skippable_message)
-          else
-            # This should never happen. Bug if encountered. Please report
-            raise Karafka::Errors::UnsupportedCaseError
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # Same as pure dead letter queue but we do not marked failed message as consumed
-      module DlqMom
-        include Dlq
-
-        # Apply strategy when dlq is on with manual offset management
-        FEATURES = %i[
-          dead_letter_queue
-          manual_offset_management
-        ].freeze
-
-        # When manual offset management is on, we do not mark anything as consumed automatically
-        # and we rely on the user to figure things out
-        def handle_after_consume
-          return if revoked?
-
-          if coordinator.success?
-            coordinator.pause_tracker.reset
-          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            retry_after_pause
-          # If we've reached number of retries that we could, we need to skip the first message
-          # that was not marked as consumed, pause and continue, while also moving this message
-          # to the dead topic
-          else
-            # We reset the pause to indicate we will now consider it as "ok".
-            coordinator.pause_tracker.reset
-
-            skippable_message, = find_skippable_message
-
-            dispatch_to_dlq(skippable_message)
-
-            # We mark the broken message as consumed and move on
-            if mark_after_dispatch?
-              mark_dispatched_to_dlq(skippable_message)
-
-              return if revoked?
-            else
-              # Save the next offset we want to go with after moving given message to DLQ
-              # Without this, we would not be able to move forward and we would end up
-              # in an infinite loop trying to un-pause from the message we've already processed
-              # Of course, since it's a MoM a rebalance or kill, will move it back as no
-              # offsets are being committed
-              self.seek_offset = skippable_message.offset + 1
-            end
-
-            pause(seek_offset, nil, false)
-          end
-        end
-
-        # @return [Boolean] should we mark given message as consumed after dispatch. For
-        #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default is
-        #  `nil`, which means `false` in this case. If user provided alternative value, we go
-        #  with it.
-        #
-        # @note Please note, this is the opposite behavior than in case of AOM strategies.
-        def mark_after_dispatch?
-          return false if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-          topic.dead_letter_queue.mark_after_dispatch
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/strategies/mom.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module Strategies
-      # When using manual offset management, we do not mark as consumed after successful processing
-      module Mom
-        include Default
-
-        # Apply strategy when only manual offset management is turned on
-        FEATURES = %i[
-          manual_offset_management
-        ].freeze
-
-        # When manual offset management is on, we do not mark anything as consumed automatically
-        # and we rely on the user to figure things out
-        def handle_after_consume
-          return if revoked?
-
-          if coordinator.success?
-            coordinator.pause_tracker.reset
-          else
-            retry_after_pause
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/workers_batch.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Abstraction layer around workers batch.
-    class WorkersBatch
-      include Enumerable
-      include Helpers::ConfigImporter.new(
-        concurrency: %i[concurrency]
-      )
-
-      # @param jobs_queue [JobsQueue]
-      # @return [WorkersBatch]
-      def initialize(jobs_queue)
-        @batch = Array.new(concurrency) { Processing::Worker.new(jobs_queue) }
-      end
-
-      # Iterates over available workers and yields each worker
-      def each(&)
-        @batch.each(&)
-      end
-
-      # @return [Integer] number of workers in the batch
-      def size
-        @batch.size
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job/builder.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class ActiveJob < Base
-        # Routing extensions for ActiveJob
-        module Builder
-          # This method simplifies routes definition for ActiveJob topics / queues by
-          # auto-injecting the consumer class
-          #
-          # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-          # @param block [Proc] block that we can use for some extra configuration
-          def active_job_topic(name, &block)
-            topic(name) do
-              consumer App.config.internal.active_job.consumer_class
-              active_job true
-
-              # This is handled by our custom ActiveJob consumer
-              # Without this, default behaviour would cause messages to skip upon shutdown as the
-              # offset would be committed for the last message
-              manual_offset_management true
-
-              next unless block
-
-              instance_eval(&block)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job/config.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class ActiveJob < Base
-        # Config for ActiveJob usage
-        Config = Struct.new(
-          :active,
-          keyword_init: true
-        ) { alias_method :active?, :active }
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class ActiveJob < Base
-        # This feature validation contracts
-        module Contracts
-          # Rules around using ActiveJob routing - basically you need to have ActiveJob available
-          # in order to be able to use active job routing
-          class Topic < Karafka::Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load_file(
-                File.join(Karafka.gem_root, "config", "locales", "errors.yml")
-              ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-            end
-
-            virtual do |data, errors|
-              next unless errors.empty?
-              next unless data[:active_job][:active]
-              # One should not define active job jobs without ActiveJob being available for usage
-              next if Object.const_defined?("ActiveJob::Base")
-
-              [[%i[consumer], :active_job_missing]]
-            end
-
-            # ActiveJob needs to always run with manual offset management
-            # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
-            # consumed jobs that did not run because of shutdown.
-            virtual do |data, errors|
-              next unless errors.empty?
-              next unless data[:active_job][:active]
-              next if data[:manual_offset_management][:active]
-
-              [[%i[manual_offset_management], :must_be_enabled]]
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job/proxy.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class ActiveJob < Base
-        # Routing proxy extensions for ActiveJob
-        module Proxy
-          include Builder
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job/topic.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class ActiveJob < Base
-        # Topic extensions to be able to check if given topic is ActiveJob topic
-        #
-        # @note ActiveJob topics do not have per-topic deserializer configuration. The deserializer
-        #   is configured globally via `config.internal.active_job.deserializer` because Rails
-        #   serializes jobs before dispatching them, requiring a consistent serialization format
-        #   across all ActiveJob topics. If you need custom serialization (e.g., Avro, Protobuf),
-        #   configure it once at the application level rather than per-topic.
-        module Topic
-          # This method calls the parent class initializer and then sets up the
-          # extra instance variable to nil. The explicit initialization
-          # to nil is included as an optimization for Ruby's object shapes system,
-          # which improves memory layout and access performance.
-          def initialize(...)
-            super
-            @active_job = nil
-          end
-
-          # @param active [Boolean] should this topic be considered one working with ActiveJob
-          #
-          # @note Since this feature supports only one setting (active), we can use the old API
-          # where the boolean would be an argument
-          def active_job(active = false)
-            @active_job ||= Config.new(active: active)
-          end
-
-          # @return [Boolean] is this an ActiveJob topic
-          def active_job?
-            active_job.active?
-          end
-
-          # @return [Hash] topic with all its native configuration options plus active job
-          #   namespace settings
-          def to_h
-            super.merge(
-              active_job: active_job.to_h
-            ).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/active_job.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      # Active-Job related components
-      # @note We can load it always, despite someone not using ActiveJob as it just adds a method
-      #   to the routing, without actually breaking anything.
-      class ActiveJob < Base
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class DeadLetterQueue < Base
-        # Config for dead letter queue feature
-        Config = Struct.new(
-          :active,
-          # We add skip variants but in regular we support only `:one`
-          :max_retries,
-          # To what topic the skipped messages should be moved
-          :topic,
-          # Should retries be handled collectively on a batch or independently per message
-          :independent,
-          # Move to DLQ and mark as consumed in transactional mode (if applicable)
-          :transactional,
-          # Strategy to apply (if strategies supported)
-          :strategy,
-          # Should we use `#produce_sync` or `#produce_async`
-          :dispatch_method,
-          # Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
-          :marking_method,
-          # Should we mark as consumed after dispatch or not. True for most cases, except MOM where
-          # it is on user to decide (false by default)
-          :mark_after_dispatch,
-          # Initialize with kwargs
-          keyword_init: true
-        ) do
-          alias_method :active?, :active
-          alias_method :independent?, :independent
-          alias_method :transactional?, :transactional
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class DeadLetterQueue < Base
-        # This feature validation contracts
-        module Contracts
-          # Rules around dead letter queue settings
-          class Topic < Karafka::Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load_file(
-                File.join(Karafka.gem_root, "config", "locales", "errors.yml")
-              ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-            end
-
-            nested :dead_letter_queue do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:independent) { |val| [true, false].include?(val) }
-              required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:transactional) { |val| [true, false].include?(val) }
-              required(:mark_after_dispatch) { |val| [true, false, nil].include?(val) }
-
-              required(:dispatch_method) do |val|
-                %i[produce_async produce_sync].include?(val)
-              end
-
-              required(:marking_method) do |val|
-                %i[mark_as_consumed mark_as_consumed!].include?(val)
-              end
-            end
-
-            # Validate topic name only if dlq is active
-            virtual do |data, errors|
-              next unless errors.empty?
-
-              dead_letter_queue = data[:dead_letter_queue]
-
-              next unless dead_letter_queue[:active]
-
-              topic = dead_letter_queue[:topic]
-              topic_regexp = Karafka::Contracts::TOPIC_REGEXP
-
-              # When topic is set to false, it means we just want to skip dispatch on DLQ
-              next if topic == false
-              next if topic.is_a?(String) && topic_regexp.match?(topic)
-              next if topic == :strategy
-
-              [[%i[dead_letter_queue topic], :format]]
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class DeadLetterQueue < Base
-        # DLQ topic extensions
-        module Topic
-          # After how many retries should be move data to DLQ
-          DEFAULT_MAX_RETRIES = 3
-
-          private_constant :DEFAULT_MAX_RETRIES
-
-          # This method calls the parent class initializer and then sets up the
-          # extra instance variable to nil. The explicit initialization
-          # to nil is included as an optimization for Ruby's object shapes system,
-          # which improves memory layout and access performance.
-          def initialize(...)
-            super
-            @dead_letter_queue = nil
-          end
-
-          # @param max_retries [Integer] after how many retries should we move data to dlq
-          # @param topic [String, false] where the messages should be moved if failing or false
-          #   if we do not want to move it anywhere and just skip
-          # @param independent [Boolean] needs to be true in order for each marking as consumed
-          #   in a retry flow to reset the errors counter
-          # @param transactional [Boolean] if applicable, should transaction be used to move
-          #   given message to the dead-letter topic and mark it as consumed.
-          # @param dispatch_method [Symbol] `:produce_async` or `:produce_sync`. Describes
-          #   whether dispatch on dlq should be sync or async (async by default)
-          # @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
-          #   whether marking on DLQ should be async or sync (async by default)
-          # @param mark_after_dispatch [Boolean, nil] Should we mark after dispatch. `nil` means
-          #   that the default strategy approach to marking will be used. `true` or `false`
-          #   overwrites the default
-          # @return [Config] defined config
-          def dead_letter_queue(
-            max_retries: DEFAULT_MAX_RETRIES,
-            topic: nil,
-            independent: false,
-            transactional: true,
-            dispatch_method: :produce_async,
-            marking_method: :mark_as_consumed,
-            mark_after_dispatch: nil
-          )
-            @dead_letter_queue ||= Config.new(
-              active: !topic.nil?,
-              max_retries: max_retries,
-              topic: topic,
-              independent: independent,
-              transactional: transactional,
-              dispatch_method: dispatch_method,
-              marking_method: marking_method,
-              mark_after_dispatch: mark_after_dispatch
-            )
-          end
-
-          # @return [Boolean] is the dlq active or not
-          def dead_letter_queue?
-            dead_letter_queue.active?
-          end
-
-          # @return [Hash] topic with all its native configuration options plus dlq settings
-          def to_h
-            super.merge(
-              dead_letter_queue: dead_letter_queue.to_h
-            ).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/dead_letter_queue.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      # This feature allows to continue processing when encountering errors.
-      # After certain number of retries, given messages will be moved to alternative topic,
-      # unclogging processing.
-      #
-      # @note This feature has an expanded version in the Pro mode. We do not use a new feature
-      #   injection in Pro (topic settings)
-      class DeadLetterQueue < Base
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/declaratives/config.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class Declaratives < Base
-        # Config for declarative topics feature
-        Config = Struct.new(
-          :active,
-          :partitions,
-          :replication_factor,
-          :details,
-          keyword_init: true
-        ) { alias_method :active?, :active }
-      end
-    end
-  end
-end

data/lib/karafka/routing/features/eofed/config.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    module Features
-      class Eofed < Base
-        # Config of this feature
-        Config = Struct.new(
-          :active,
-          keyword_init: true
-        ) { alias_method :active?, :active }
-      end
-    end
-  end
-end