lepus-saga 0.3.0

data/lib/lepus/saga/executor.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    # Runs one unit of saga work for a step message. The store is the source
+    # of truth: messages only point at the execution, so duplicate or stale
+    # deliveries are fenced out and lost publishes are healed by re-publishing
+    # whatever the execution record says is current.
+    #
+    # Returns :ack (message done) or :retry (transient failure — the consumer
+    # rejects the message into the TTL retry queue).
+    class Executor
+      class AbandonedAttempt < StandardError
+        def initialize = super("attempt abandoned (worker died or lost the message)")
+      end
+
+      class << self
+        # Switches an execution onto the compensation path (or straight to
+        # failed when there is nothing to undo). Caller must hold the row lock
+        # and is responsible for instrumentation and publishing afterwards.
+        def begin_compensation(execution, reason:)
+          target = compensation_target(execution)
+
+          if target
+            # Switching onto the compensation path is a cursor move, so the
+            # compensation gets its own stall window and resume budget.
+            execution.record_progress
+            store.update_execution(execution,
+              status: "compensating", direction: "compensate",
+              current_step: target.name, failure_reason: reason)
+          else
+            store.update_execution(execution,
+              status: "failed", failure_reason: reason, finished_at: Time.now.utc)
+          end
+          execution
+        end
+
+        # The nearest step before the current one that succeeded and declares
+        # a compensation. Skipped or never-run steps have nothing to undo.
+        def compensation_target(execution)
+          index = execution.definition_class.step_index(execution.current_step) ||
+            execution.steps.size
+          execution.steps.first(index).reverse.find do |step|
+            step.compensatable? && store.step_succeeded?(execution, step.name)
+          end
+        end
+
+        private
+
+        def store = Lepus::Saga.store
+      end
+
+      def initialize(payload)
+        @payload = payload.to_h.transform_keys(&:to_s)
+      end
+
+      def run
+        case prepare
+        when :proceed then run_step
+        when :exhausted then announce_failure
+        else :ack
+        end
+      end
+
+      private
+
+      attr_reader :payload, :execution, :attempt, :failure_outcome
+
+      def message_epoch = payload.fetch("epoch", 0).to_i
+
+      def message_step = payload["step"]
+
+      def message_direction = payload.fetch("direction", "execute")
+
+      def stale_running_attempt_after = Lepus::Saga.config.stale_running_attempt_after
+
+      def store = Lepus::Saga.store
+
+      # Fences the message against the execution record and opens a step
+      # attempt, all while holding the execution's lock.
+      def prepare
+        store.with_locked_execution(payload["execution_id"]) do |locked|
+          @execution = locked
+          next :ack unless execution
+          next :ack if execution.terminal?
+          next :ack if message_epoch < execution.epoch
+
+          unless message_step == execution.current_step && message_direction == execution.direction
+            StepProducer.publish_step(execution)
+            next :ack
+          end
+
+          next :ack if attempt_in_flight?
+
+          if attempts_so_far >= retry_policy.attempts
+            @failure_outcome = apply_failure(execution.failure_reason || "retries exhausted")
+            next :exhausted
+          end
+
+          open_attempt
+          :proceed
+        end
+      rescue DuplicateAttemptError
+        :ack
+      end
+
+      def attempts_so_far
+        @attempts_so_far ||= begin
+          close_abandoned_attempts
+          store.count_step_attempts(execution, **execution.current_step_scope)
+        end
+      end
+
+      def attempt_in_flight?
+        store.running_step_attempt?(execution, **execution.current_step_scope,
+          started_since: Time.now.utc - stale_running_attempt_after)
+      end
+
+      def close_abandoned_attempts
+        store.stale_step_attempts(execution, **execution.current_step_scope,
+          started_before: Time.now.utc - stale_running_attempt_after)
+          .each { |stale| store.finish_step_attempt(stale, "failed", error: AbandonedAttempt.new) }
+      end
+
+      def open_attempt
+        @attempt = store.create_step_attempt(execution,
+          step_name: execution.current_step,
+          direction: execution.direction,
+          epoch: execution.epoch,
+          attempt: attempts_so_far + 1,
+          started_at: Time.now.utc)
+        Instrumentation.saga_event("saga.step.started", execution,
+          step: attempt.step_name, attempt: attempt.attempt)
+      end
+
+      # Runs the step (or compensation) method lock-free — slow HTTP calls in
+      # user code must not hold the execution row.
+      def run_step
+        definition = execution.definition_class.new(execution, step_attempt: attempt)
+        step = execution.current_step_definition
+        definition.send(execution.compensate_direction? ? step.compensate : step.name)
+        finish_step("succeeded")
+      rescue Control::Skip => signal
+        finish_step("skipped", reason: signal.reason)
+      rescue Control::Complete => signal
+        finish_execution(signal.reason)
+      rescue Control::Fail => signal
+        store.finish_step_attempt(attempt, "failed", error: signal)
+        fail_permanently(signal.reason)
+      rescue => error
+        store.finish_step_attempt(attempt, "failed", error: error)
+        handle_step_error(error)
+      end
+
+      def finish_step(attempt_status, reason: nil)
+        new_context = execution.context
+        completed_event = nil
+        finished = with_fresh_lock do
+          store.finish_step_attempt(attempt, attempt_status, context_after: new_context)
+          execution.context = new_context
+          completed_event = advance_cursor
+          execution.record_progress
+          store.save_execution(execution)
+        end
+        return :ack unless finished
+
+        step_event = (attempt_status == "skipped") ? "saga.step.skipped" : "saga.step.finished"
+        Instrumentation.saga_event(step_event, execution,
+          step: attempt.step_name, attempt: attempt.attempt,
+          duration_ms: attempt.duration_ms, reason: reason)
+        if completed_event
+          Instrumentation.saga_event(completed_event, execution)
+        else
+          StepProducer.publish_step(execution)
+        end
+        :ack
+      end
+
+      def finish_execution(reason)
+        new_context = execution.context
+        new_context["completed_reason"] = reason if reason
+        finished = with_fresh_lock do
+          now = Time.now.utc
+          store.finish_step_attempt(attempt, "succeeded", context_after: new_context)
+          store.update_execution(execution, context: new_context, status: "completed", finished_at: now,
+            last_progress_at: now, resume_count: 0)
+        end
+        return :ack unless finished
+
+        Instrumentation.saga_event("saga.step.finished", execution,
+          step: attempt.step_name, attempt: attempt.attempt,
+          duration_ms: attempt.duration_ms)
+        Instrumentation.saga_event("saga.completed", execution, reason: reason)
+        :ack
+      end
+
+      # Re-acquires the lock on a fresh copy of the execution after user code
+      # ran lock-free. Yields with the lock held; returns false when a manual
+      # retry/compensate bumped the epoch in the meantime (this delivery's
+      # outcome must not move the cursor).
+      def with_fresh_lock
+        store.with_locked_execution(execution.id) do |fresh|
+          next false if fresh.nil? || fresh.epoch != message_epoch || fresh.terminal?
+
+          @execution = fresh
+          yield
+          true
+        end
+      end
+
+      def handle_step_error(error)
+        if attempt.attempt < retry_policy.attempts
+          Instrumentation.saga_event("saga.step.failed", execution,
+            step: attempt.step_name, attempt: attempt.attempt, error: error)
+          return :retry
+        end
+
+        Instrumentation.saga_event("saga.step.failed", execution,
+          step: attempt.step_name, attempt: attempt.attempt,
+          error: error, retries_exhausted: true)
+        fail_permanently(error.message)
+      end
+
+      def fail_permanently(reason)
+        with_fresh_lock do
+          @failure_outcome = apply_failure(reason)
+        end
+        announce_failure
+      end
+
+      def apply_failure(reason)
+        if execution.compensate_direction?
+          store.update_execution(execution, status: "compensation_failed", failure_reason: reason,
+            finished_at: Time.now.utc)
+          :compensation_failed
+        else
+          self.class.begin_compensation(execution, reason: reason)
+          execution.compensating? ? :compensating : :failed
+        end
+      end
+
+      def announce_failure
+        case failure_outcome
+        when :compensation_failed
+          Instrumentation.saga_event("saga.compensation_failed", execution)
+        when :compensating
+          Instrumentation.saga_event("saga.compensation_started", execution)
+          StepProducer.publish_step(execution)
+        when :failed
+          Instrumentation.saga_event("saga.failed", execution)
+        end
+        :ack
+      end
+
+      def advance_cursor
+        if execution.compensate_direction?
+          target = self.class.compensation_target(execution)
+          return complete_as("compensated") unless target
+
+          execution.current_step = target.name
+        else
+          index = execution.definition_class.step_index(execution.current_step)
+          next_step = execution.steps[index + 1]
+          return complete_as("completed") unless next_step
+
+          execution.current_step = next_step.name
+        end
+        nil
+      end
+
+      def complete_as(status)
+        execution.status = status
+        execution.finished_at = Time.now.utc
+        "saga.#{status}"
+      end
+
+      def retry_policy
+        execution.current_step_definition.retry_policy
+      end
+    end
+  end
+end

data/lib/lepus/saga/executor_consumer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    # Abstract base for the generic step-executor consumer. The host app
+    # declares one concrete subclass per partition:
+    #
+    #   class SagaExecutorConsumer < Lepus::Saga::ExecutorConsumer
+    #     saga_partition "default", worker: { name: "saga", threads: 1 }
+    #   end
+    #
+    # The exchange and queue names come from Lepus::Saga.config, so configure
+    # it before the consumer classes load.
+    class ExecutorConsumer < Lepus::Consumer
+      self.abstract_class = true
+
+      use(:json)
+
+      def self.saga_partition(name = "default", worker: {name: "default", threads: 1}, retry_delay: 10_000)
+        configure(
+          worker: worker,
+          queue: Lepus::Saga.queue_name(name),
+          exchange: {name: Lepus::Saga.exchange_name, type: :topic},
+          routing_key: Lepus::Saga.routing_key(name),
+          retry_queue: {delay: retry_delay},
+          error_queue: true
+        )
+      end
+
+      def perform(message)
+        case Executor.new(message.payload).run
+        when :retry then reject!
+        else ack!
+        end
+      end
+    end
+  end
+end

data/lib/lepus/saga/instrumentation.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    # All engine events go through Lepus.instrument, which appends the
+    # ".lepus" suffix — subscribe with e.g. "saga.completed.lepus" or
+    # /^saga\./ via ActiveSupport::Notifications.
+    module Instrumentation
+      EVENTS = %w[
+        saga.started
+        saga.step.started
+        saga.step.finished
+        saga.step.failed
+        saga.step.skipped
+        saga.completed
+        saga.compensation_started
+        saga.compensated
+        saga.failed
+        saga.compensation_failed
+        saga.resumed
+      ].freeze
+
+      def self.saga_event(name, execution, step: nil, attempt: nil, error: nil, **extra)
+        Lepus.instrument(
+          name,
+          execution: execution,
+          saga_name: execution.saga_name,
+          resource_type: execution.resource_type,
+          resource_id: execution.resource_id,
+          status: execution.status,
+          step: step,
+          attempt: attempt,
+          direction: execution.direction,
+          error_class: error&.class&.name,
+          error_message: error&.message,
+          **extra
+        )
+      end
+    end
+  end
+end

data/lib/lepus/saga/log_subscriber.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    # Writes one line per engine event to Lepus.logger. Lepus logs each
+    # delivery, but not which saga, step or resource it carried, nor a step
+    # failure the executor handles itself.
+    class LogSubscriber
+      LEVELS = {
+        "saga.step.started" => :debug,
+        "saga.step.finished" => :debug,
+        "saga.step.skipped" => :debug,
+        "saga.step.failed" => :warn,
+        "saga.failed" => :error,
+        "saga.compensation_failed" => :error
+      }.freeze
+
+      ATTRIBUTES = %i[saga_name step attempt duration_ms status reason resume_count retries_exhausted].freeze
+
+      def self.call(event, payload)
+        Lepus.logger.public_send(LEVELS.fetch(event, :info)) { format(event, payload) }
+      end
+
+      def self.format(event, payload)
+        attributes = {
+          execution: payload[:execution]&.id,
+          resource: "#{payload[:resource_type]}/#{payload[:resource_id]}",
+          **payload.slice(*ATTRIBUTES),
+          error: error(payload)
+        }.compact
+
+        "#{event} #{attributes.map { |key, value| "#{key}=#{value}" }.join(" ")}"
+      end
+
+      def self.error(payload)
+        message = [payload[:error_class], payload[:error_message]].compact.join(": ")
+        message unless message.empty?
+      end
+      private_class_method :error
+    end
+  end
+end

data/lib/lepus/saga/queue_depth.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    # Messages ready in a partition's execute queue. Under fake publishing the
+    # fake saga exchange stands in for the queue.
+    module QueueDepth
+      def self.call(partition)
+        return fake_depth(partition) if defined?(Lepus::Testing) && Lepus::Testing.fake_publisher_enabled?
+
+        Lepus.config.producer_config.with_connection do |connection|
+          channel = connection.create_channel
+          begin
+            channel.queue(Saga.queue_name(partition), passive: true).message_count
+          rescue Bunny::NotFound
+            0
+          ensure
+            channel.close if channel.open?
+          end
+        end
+      end
+
+      def self.fake_depth(partition)
+        Lepus::Testing.exchange(Saga.exchange_name).find_messages(routing_key: Saga.routing_key(partition)).size
+      end
+    end
+  end
+end

data/lib/lepus/saga/resume_stuck_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Lepus
+  module Saga
+    # Recurring job that heals stalled executions. Schedule it every few
+    # minutes, e.g. in Solid Queue's config/recurring.yml:
+    #
+    #   saga_resume_stuck:
+    #     class: Lepus::Saga::ResumeStuckJob
+    #     schedule: every 5 minutes
+    class ResumeStuckJob < ActiveJob::Base
+      def perform(older_than_minutes = 15)
+        Lepus::Saga.resume_stuck(older_than: older_than_minutes * 60)
+      end
+    end
+  end
+end

data/lib/lepus/saga/retry_policy.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Saga
+    RetryPolicy = Data.define(:attempts) do
+      self::DEFAULT_ATTEMPTS = 3
+
+      def self.build(options)
+        case options
+        when RetryPolicy then options
+        when Hash then new(attempts: options.fetch(:attempts, self::DEFAULT_ATTEMPTS))
+        when Integer then new(attempts: options)
+        when nil then new(attempts: self::DEFAULT_ATTEMPTS)
+        else raise ArgumentError, "Invalid retry policy: #{options.inspect}"
+        end
+      end
+    end
+  end
+end