karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/strategies/default.rb

@@ -1,416 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # No features enabled.
-        # No manual offset management
-        # No long running jobs
-        # No virtual partitions
-        # Nothing. Just standard, automatic flow
-        module Default
-          include Base
-          include Karafka::Processing::Strategies::Default
-
-          # Apply strategy for a non-feature based flow
-          FEATURES = %i[].freeze
-
-          # Allows to set offset metadata that will be used with the upcoming marking as consumed
-          # as long as a different offset metadata was not used. After it was used either via
-          # `#mark_as_consumed` or `#mark_as_consumed!` it will be set back to `nil`. It is done
-          # that way to provide the end user with ability to influence metadata on the non-user
-          # initiated markings in complex flows.
-          #
-          # @param offset_metadata [String, nil] metadata we want to store with the upcoming
-          #   marking as consumed
-          #
-          # @note Please be aware, that offset metadata set this way will be passed to any marking
-          #   as consumed even if it was not user initiated. For example in the DLQ flow.
-          def store_offset_metadata(offset_metadata)
-            @_current_offset_metadata = offset_metadata
-          end
-
-          # Marks message as consumed in an async way.
-          #
-          # @param message [Messages::Message] last successfully processed message.
-          # @param offset_metadata [String, nil] offset metadata string or nil if nothing
-          # @return [Boolean] true if we were able to mark the offset, false otherwise.
-          #   False indicates that we were not able and that we have lost the partition.
-          #
-          # @note We keep track of this offset in case we would mark as consumed and got error when
-          #   processing another message. In case like this we do not pause on the message we've
-          #   already processed but rather at the next one. This applies to both sync and async
-          #   versions of this method.
-          def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
-            # If we are inside a transaction than we can just mark as consumed within it
-            if @_in_transaction
-              mark_in_transaction(message, offset_metadata, true)
-            elsif @_in_transaction_marked
-              mark_in_memory(message)
-            else
-              # seek offset can be nil only in case `#seek` was invoked with offset reset request
-              # In case like this we ignore marking
-              return true if seek_offset.nil?
-              # Ignore if it is the same offset as the one that is marked currently
-              # We ignore second marking because it changes nothing and in case of people using
-              # metadata storage but with automatic offset marking, this would cause metadata to be
-              # erased by automatic marking
-              return true if (seek_offset - 1) == message.offset
-              return false if revoked?
-
-              # If we are not inside a transaction but this is a transactional topic, we mark with
-              # artificially created transaction
-              stored = if producer.transactional?
-                mark_with_transaction(message, offset_metadata, true)
-              elsif @_transactional_marking
-                raise Errors::NonTransactionalMarkingAttemptError
-              else
-                client.mark_as_consumed(message, offset_metadata)
-              end
-
-              return revoked? unless stored
-
-              self.seek_offset = message.offset + 1
-            end
-
-            true
-          ensure
-            @_current_offset_metadata = nil
-          end
-
-          # Marks message as consumed in a sync way.
-          #
-          # @param message [Messages::Message] last successfully processed message.
-          # @param offset_metadata [String, nil] offset metadata string or nil if nothing
-          # @return [Boolean] true if we were able to mark the offset, false otherwise.
-          #   False indicates that we were not able and that we have lost the partition.
-          def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
-            if @_in_transaction
-              mark_in_transaction(message, offset_metadata, false)
-            elsif @_in_transaction_marked
-              mark_in_memory(message)
-            else
-              # seek offset can be nil only in case `#seek` was invoked with offset reset request
-              # In case like this we ignore marking
-              return true if seek_offset.nil?
-              # Ignore if it is the same offset as the one that is marked currently
-              # We ignore second marking because it changes nothing and in case of people using
-              # metadata storage but with automatic offset marking, this would cause metadata to be
-              # erased by automatic marking
-              return true if (seek_offset - 1) == message.offset
-              return false if revoked?
-
-              # If we are not inside a transaction but this is a transactional topic, we mark with
-              # artificially created transaction
-              stored = if producer.transactional?
-                mark_with_transaction(message, offset_metadata, false)
-              elsif @_transactional_marking
-                raise Errors::NonTransactionalMarkingAttemptError
-              else
-                client.mark_as_consumed!(message, offset_metadata)
-              end
-
-              return revoked? unless stored
-
-              self.seek_offset = message.offset + 1
-            end
-
-            true
-          ensure
-            @_current_offset_metadata = nil
-          end
-
-          # Starts producer transaction, saves the transaction context for transactional marking
-          # and runs user code in this context
-          #
-          # Transactions on a consumer level differ from those initiated by the producer as they
-          # allow to mark offsets inside of the transaction. If the transaction is initialized
-          # only from the consumer, the offset will be stored in a regular fashion.
-          #
-          # @param active_producer [WaterDrop::Producer] alternative producer instance we may want
-          #   to use. It is useful when we have connection pool or any other selective engine for
-          #   managing multiple producers. If not provided, default producer taken from `#producer`
-          #   will be used.
-          #
-          # @yield code that we want to run in a transaction
-          #
-          # @note Please note, that if you provide the producer, it will reassign the producer of
-          #   the consumer for the transaction time. This means, that in case you would even
-          #   accidentally refer to `Consumer#producer` from other threads, it will contain the
-          #   reassigned producer and not the initially used/assigned producer. It is done that
-          #   way, so the message producing aliases operate from within transactions and since the
-          #   producer in transaction is locked, it will prevent other threads from using it.
-          def transaction(active_producer = producer)
-            default_producer = nil
-            transaction_started = nil
-
-            monitor.instrument("consumer.consuming.transaction", caller: self) do
-              default_producer = producer
-              self.producer = active_producer
-
-              transaction_started = false
-              transaction_completed = false
-
-              # Prevent from nested transactions. It would not make any sense
-              raise Errors::TransactionAlreadyInitializedError if @_in_transaction
-
-              transaction_started = true
-              @_transaction_marked = []
-              @_in_transaction = true
-              @_in_transaction_marked = false
-
-              producer.transaction do
-                yield
-
-                # Ensure this transaction is rolled back if we have lost the ownership of this
-                # transaction. We do it only for transactions that contain offset management as for
-                # producer only, this is not relevant.
-                raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
-
-                # If we do not reach this, we should not move seek offsets because it means that
-                # either an error occured or transaction was aborted.
-                # In case of error, it will bubble up so no issue but in case of abort, while we
-                # do not reach this place, the code will continue
-                transaction_completed = true
-              end
-
-              @_in_transaction = false
-
-              # This offset is already stored in transaction but we set it here anyhow because we
-              # want to make sure our internal in-memory state is aligned with the transaction
-              #
-              # @note We never need to use the blocking `#mark_as_consumed!` here because the
-              #   offset anyhow was already stored during the transaction
-              #
-              # @note Since the offset could have been already stored in Kafka (could have because
-              #   you can have transactions without marking), we use the `@_in_transaction_marked`
-              #   state to decide if we need to dispatch the offset via client at all
-              #   (if post transaction, then we do not have to)
-              #
-              # @note In theory we could only keep reference to the most recent marking and reject
-              #   others. We however do not do it for two reasons:
-              #   - User may have non standard flow relying on some alternative order and we want
-              #     to mimic this
-              #   - Complex strategies like VPs can use this in VPs to mark in parallel without
-              #     having to redefine the transactional flow completely
-              #
-              # @note This should be applied only if transaction did not error and if it was not
-              #   aborted.
-              if transaction_completed
-                @_transaction_marked.each do |marking|
-                  marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
-                end
-              end
-
-              true
-            end
-          ensure
-            self.producer = default_producer
-
-            if transaction_started
-              @_transaction_marked.clear
-              @_in_transaction = false
-              @_in_transaction_marked = false
-            end
-          end
-
-          # Stores the next offset for processing inside of the transaction and stores it in a
-          # local accumulator for post-transaction status update
-          #
-          # @param message [Messages::Message] message we want to commit inside of a transaction
-          # @param offset_metadata [String, nil] offset metadata or nil if none
-          # @param async [Boolean] should we mark in async or sync way (applicable only to post
-          #   transaction state synchronization usage as within transaction it is always sync)
-          def mark_in_transaction(message, offset_metadata, async)
-            raise Errors::TransactionRequiredError unless @_in_transaction
-            raise Errors::AssignmentLostError if revoked?
-
-            producer.transaction_mark_as_consumed(
-              client,
-              message,
-              offset_metadata
-            )
-
-            # This one is long lived and used to make sure, that users do not mix transactional
-            # marking with non-transactional. When this happens we should raise error
-            @_transactional_marking = true
-            @_in_transaction_marked = true
-            @_transaction_marked ||= []
-            @_transaction_marked << [message, offset_metadata, async]
-          end
-
-          # @private
-          # @param message [Messages::Message] message we want to commit inside of a transaction
-          # @param offset_metadata [String, nil] offset metadata or nil if none
-          # @param async [Boolean] should we mark in async or sync way (applicable only to post
-          #   transaction state synchronization usage as within transaction it is always sync)
-          # @return [Boolean] false if marking failed otherwise true
-          def mark_with_transaction(message, offset_metadata, async)
-            # This flag is used by VPs to differentiate between user initiated transactions and
-            # post-execution system transactions.
-            @_transaction_internal = true
-
-            transaction do
-              mark_in_transaction(message, offset_metadata, async)
-            end
-
-            true
-          # We handle both cases here because this is a private API for internal usage and we want
-          # the post-user code execution marking with transactional producer to result in a
-          # boolean state of marking for further framework flow. This is a normalization to make it
-          # behave the same way as it would behave with a non-transactional one
-          rescue Rdkafka::RdkafkaError, Errors::AssignmentLostError
-            false
-          ensure
-            @_transaction_internal = false
-          end
-
-          # Marks the current state only in memory as the offset marking has already happened
-          # using the producer transaction
-          # @param message [Messages::Message] last successfully processed message.
-          # @return [Boolean] true if all good, false if we lost assignment and no point in marking
-          def mark_in_memory(message)
-            # seek offset can be nil only in case `#seek` was invoked with offset reset request
-            # In case like this we ignore marking
-            return true if seek_offset.nil?
-            # Ignore if it is the same offset as the one that is marked currently
-            # We ignore second marking because it changes nothing and in case of people using
-            # metadata storage but with automatic offset marking, this would cause metadata to be
-            # erased by automatic marking
-            return true if (seek_offset - 1) == message.offset
-            return false if revoked?
-
-            # If we have already marked this successfully in a transaction that was running
-            # we should not mark it again with the client offset delegation but instead we should
-            # just align the in-memory state
-            self.seek_offset = message.offset + 1
-
-            true
-          end
-
-          # No actions needed for the standard flow here
-          def handle_before_schedule_consume
-            monitor.instrument("consumer.before_schedule_consume", caller: self)
-
-            nil
-          end
-
-          # Increment number of attempts per one "full" job. For all VP on a single topic partition
-          # this also should run once.
-          def handle_before_consume
-            coordinator.on_started do
-              coordinator.pause_tracker.increment
-            end
-          end
-
-          # Run the user consumption code
-          def handle_consume
-            # We should not run the work at all on a partition that was revoked
-            # This can happen primarily when an LRJ job gets to the internal worker queue and
-            # this partition is revoked prior processing.
-            unless revoked?
-              monitor.instrument("consumer.consume", caller: self)
-              monitor.instrument("consumer.consumed", caller: self) do
-                consume
-              end
-            end
-
-            # Mark job as successful
-            coordinator.success!(self)
-          rescue => e
-            # If failed, mark as failed
-            coordinator.failure!(self, e)
-
-            # Re-raise so reported in the consumer
-            raise e
-          ensure
-            # We need to decrease number of jobs that this coordinator coordinates as it has
-            # finished
-            coordinator.decrement(:consume)
-          end
-
-          # Standard flow without any features
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              return if revoked?
-
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                # Do not mark last message if pause happened. This prevents a scenario where pause
-                # is overridden upon rebalance by marking
-                return if coordinator.manual_pause?
-
-                mark_as_consumed(last_group_message)
-              else
-                retry_after_pause
-              end
-            end
-          end
-
-          # Standard flow for revocation
-          def handle_revoked
-            coordinator.on_revoked do
-              resume
-
-              coordinator.revoke
-            end
-
-            monitor.instrument("consumer.revoke", caller: self)
-            monitor.instrument("consumer.revoked", caller: self) do
-              revoked
-            end
-          ensure
-            coordinator.decrement(:revoked)
-          end
-
-          # No action needed for the tick standard flow
-          def handle_before_schedule_tick
-            monitor.instrument("consumer.before_schedule_tick", caller: self)
-
-            nil
-          end
-
-          # Runs the consumer `#tick` method with reporting
-          def handle_tick
-            monitor.instrument("consumer.tick", caller: self)
-            monitor.instrument("consumer.ticked", caller: self) do
-              tick
-            end
-          ensure
-            coordinator.decrement(:periodic)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -1,280 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Namespace for all the strategies starting with DLQ
-        module Dlq
-          # Only dead letter queue enabled
-          module Default
-            include Strategies::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-            ].freeze
-
-            # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
-            # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
-            # any offset marking causes the pause count tracker to reset. This is useful when
-            # the error is not due to the collective batch operations state but due to intermediate
-            # "crawling" errors that move with it
-            #
-            # @see `Strategies::Default#mark_as_consumed` for more details
-            # @param message [Messages::Message]
-            # @param offset_metadata [String, nil]
-            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
-              return super unless retrying?
-              return super unless topic.dead_letter_queue.independent?
-              return false unless super
-
-              coordinator.pause_tracker.reset
-
-              true
-            ensure
-              @_current_offset_metadata = nil
-            end
-
-            # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
-            # DLQ was configured with the `independent` flag.
-            #
-            # @see `Strategies::Default#mark_as_consumed!` for more details
-            # @param message [Messages::Message]
-            # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
-              return super unless retrying?
-              return super unless topic.dead_letter_queue.independent?
-              return false unless super
-
-              coordinator.pause_tracker.reset
-
-              true
-            ensure
-              @_current_offset_metadata = nil
-            end
-
-            # When we encounter non-recoverable message, we skip it and go on with our lives
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  mark_as_consumed(last_group_message)
-                else
-                  apply_dlq_flow do
-                    dispatch_if_needed_and_mark_as_consumed
-                  end
-                end
-              end
-            end
-
-            # Finds the message may want to skip (all, starting from first)
-            # @private
-            # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
-            #   information if this message was from marked offset or figured out via mom flow
-            def find_skippable_message
-              skippable_message = messages.raw.find do |msg|
-                coordinator.marked? && msg.offset == seek_offset
-              end
-
-              # If we don't have the message matching the last comitted offset, it means that
-              # user operates with manual offsets and we're beyond the batch in which things
-              # broke for the first time. Then we skip the first (as no markings) and we
-              # move on one by one.
-              skippable_message ? [skippable_message, true] : [messages.first, false]
-            end
-
-            # Moves the broken message into a separate queue defined via the settings
-            #
-            # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
-            #   dispatch to DLQ
-            def dispatch_to_dlq(skippable_message)
-              # DLQ should never try to dispatch a message that was cleaned. It message was
-              # cleaned, we will not have all the needed data. If you see this error, it means
-              # that your processing flow is not as expected and you have cleaned message that
-              # should not be cleaned as it should go to the DLQ
-              raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
-
-              producer.public_send(
-                topic.dead_letter_queue.dispatch_method,
-                build_dlq_message(
-                  skippable_message
-                )
-              )
-
-              # Notify about dispatch on the events bus
-              monitor.instrument(
-                "dead_letter_queue.dispatched",
-                caller: self,
-                message: skippable_message
-              )
-            end
-
-            # Dispatches the message to the DLQ (when needed and when applicable based on settings)
-            #   and marks this message as consumed for non MOM flows.
-            #
-            # If producer is transactional and config allows, uses transaction to do that
-            def dispatch_if_needed_and_mark_as_consumed
-              skippable_message, = find_skippable_message
-
-              dispatch = lambda do
-                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-
-                if mark_after_dispatch?
-                  mark_dispatched_to_dlq(skippable_message)
-                else
-                  self.seek_offset = skippable_message.offset + 1
-                end
-              end
-
-              if dispatch_in_a_transaction?
-                transaction { dispatch.call }
-              else
-                dispatch.call
-              end
-            end
-
-            # @param skippable_message [Array<Karafka::Messages::Message>]
-            # @return [Hash] dispatch DLQ message
-            def build_dlq_message(skippable_message)
-              source_partition = skippable_message.partition.to_s
-
-              dlq_message = {
-                topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
-                key: skippable_message.raw_key,
-                partition_key: source_partition,
-                payload: skippable_message.raw_payload,
-                headers: skippable_message.raw_headers.merge(
-                  "source_topic" => topic.name,
-                  "source_partition" => source_partition,
-                  "source_offset" => skippable_message.offset.to_s,
-                  "source_consumer_group" => topic.consumer_group.id,
-                  "source_attempts" => attempt.to_s,
-                  "source_trace_id" => errors_tracker.trace_id
-                )
-              }
-
-              # Optional method user can define in consumer to enhance the dlq message hash with
-              # some extra details if needed or to replace payload, etc
-              if respond_to?(:enhance_dlq_message, true)
-                enhance_dlq_message(
-                  dlq_message,
-                  skippable_message
-                )
-              end
-
-              dlq_message
-            end
-
-            # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch
-            #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
-            #   message without taking any action.
-            def dispatch_to_dlq?
-              return false unless topic.dead_letter_queue.topic
-              return false unless @_dispatch_to_dlq
-
-              true
-            end
-
-            # @return [Boolean] should we use a transaction to move the data to the DLQ.
-            #   We can do it only when producer is transactional and configuration for DLQ
-            #   transactional dispatches is not set to false.
-            def dispatch_in_a_transaction?
-              producer.transactional? && topic.dead_letter_queue.transactional?
-            end
-
-            # @return [Boolean] should we mark given message as consumed after dispatch.
-            #  For default non MOM strategies if user did not explicitly tell us not to, we mark
-            #  it. Default is `nil`, which means `true` in this case. If user provided alternative
-            #  value, we go with it.
-            def mark_after_dispatch?
-              return true if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-              topic.dead_letter_queue.mark_after_dispatch
-            end
-
-            # Runs the DLQ strategy and based on it it performs certain operations
-            #
-            # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
-            # In case of `:retry` always `#retry_after_pause` is applied
-            def apply_dlq_flow
-              flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
-
-              case flow
-              when :retry
-                retry_after_pause
-
-                return
-              when :skip
-                @_dispatch_to_dlq = false
-              when :dispatch
-                @_dispatch_to_dlq = true
-                # Use custom topic if it was returned from the strategy
-                @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
-              else
-                raise Karafka::UnsupportedCaseError, flow
-              end
-
-              yield
-
-              # We reset the pause to indicate we will now consider it as "ok".
-              coordinator.pause_tracker.reset
-
-              # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
-              pause(seek_offset, nil, false)
-            ensure
-              @_dispatch_to_dlq_topic = nil
-            end
-
-            # Marks message that went to DLQ (if applicable) based on the requested method
-            # @param skippable_message [Karafka::Messages::Message]
-            def mark_dispatched_to_dlq(skippable_message)
-              case topic.dead_letter_queue.marking_method
-              when :mark_as_consumed
-                mark_as_consumed(skippable_message)
-              when :mark_as_consumed!
-                mark_as_consumed!(skippable_message)
-              else
-                # This should never happen. Bug if encountered. Please report
-                raise Karafka::Errors::UnsupportedCaseError
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end