active_orchestrator 0.1.0

data/lib/active_workflow/stores/active_record.rb

@@ -0,0 +1,695 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module ActiveWorkflow
+  module Stores
+    # ActiveRecord-backed persistence implementation.
+    class ActiveRecord < Base
+      TERMINAL_STATES = %w[completed failed cancelled timed_out].freeze
+
+      module Models
+        class Execution < ::ActiveRecord::Base
+          self.table_name = "aw_executions"
+
+          has_many :steps,
+            class_name: "ActiveWorkflow::Stores::ActiveRecord::Models::Step",
+            foreign_key: :execution_id,
+            inverse_of: :execution,
+            dependent: :destroy
+
+          has_many :events,
+            class_name: "ActiveWorkflow::Stores::ActiveRecord::Models::Event",
+            foreign_key: :execution_id,
+            inverse_of: :execution,
+            dependent: :destroy
+        end
+
+        class Step < ::ActiveRecord::Base
+          self.table_name = "aw_steps"
+
+          belongs_to :execution,
+            class_name: "ActiveWorkflow::Stores::ActiveRecord::Models::Execution",
+            foreign_key: :execution_id,
+            inverse_of: :steps
+
+          scope :ordered, -> { order(:position) }
+
+          enum :state, {
+            pending: "pending",
+            running: "running",
+            waiting: "waiting",
+            completed: "completed",
+            failed: "failed",
+            timed_out: "timed_out",
+            cancelled: "cancelled",
+            compensating: "compensating",
+            compensated: "compensated"
+          }
+        end
+
+        class Event < ::ActiveRecord::Base
+          self.table_name = "aw_events"
+
+          belongs_to :execution,
+            class_name: "ActiveWorkflow::Stores::ActiveRecord::Models::Execution",
+            foreign_key: :execution_id,
+            inverse_of: :events
+        end
+      end
+
+      def start_execution(workflow_class:, context:, steps:, idempotency_key:, timeout:, metadata: {})
+        now = clock.call
+        record = nil
+
+        ::ActiveRecord::Base.transaction do
+          if idempotency_key
+            existing = Models::Execution.lock.find_by(idempotency_key:)
+            return build_execution(existing) if existing
+          end
+
+          record = Models::Execution.create!(
+            workflow_class:,
+            state: "running",
+            ctx: serializer.dump(context),
+            cursor_step: steps.first && steps.first[:name],
+            idempotency_key:,
+            metadata: (metadata || {}).stringify_keys,
+            timeout_at: timeout ? now + timeout : nil,
+            last_enqueued_at: now
+          )
+
+          steps.each_with_index do |step, index|
+            Models::Step.create!(
+              execution: record,
+              name: step[:name].to_s,
+              style: step[:style].to_s,
+              options: serializable_step_options(step[:options]),
+              position: step[:position] || index,
+              state: index.zero? ? "pending" : "pending",
+              attempts: 0
+            )
+          end
+
+          if steps.empty?
+            record.update!(state: "completed", cursor_step: nil, completed_at: now)
+          end
+        end
+
+        enqueue_runner(record.id) unless steps.empty?
+        build_execution(record)
+      end
+
+      def serializable_step_options(options)
+        return {} unless options.is_a?(Hash)
+
+        options.each_with_object({}) do |(key, value), hash|
+          hash[key.to_s] = case value
+          when Proc, Method
+            nil
+          when Hash
+            serializable_step_options(value)
+          when Array
+            value.map { |item| item.is_a?(Hash) ? serializable_step_options(item) : item }
+          when Symbol, String, Numeric, TrueClass, FalseClass, NilClass
+            value
+          else
+            value.respond_to?(:name) ? value.name : value.to_s
+          end
+        end.compact
+      end
+
+      def load_execution(id)
+        record = Models::Execution.find_by(id:)
+        build_execution(record) if record
+      end
+
+      def enqueue_runner(execution_id, run_at: nil)
+        if run_at
+          ActiveWorkflow::Jobs::RunnerJob.set(wait_until: run_at).perform_later(execution_id)
+        else
+          ActiveWorkflow::Jobs::RunnerJob.perform_later(execution_id)
+        end
+      end
+
+      def with_execution_lock(execution_id)
+        ::ActiveRecord::Base.transaction do
+          record = Models::Execution.lock.find_by(id: execution_id)
+          return unless record
+
+          yield(record)
+        end
+      end
+
+      def process_execution(execution_id)
+        with_execution_lock(execution_id) do |execution|
+          Processor.new(self, execution).process!
+        end
+      end
+
+      def complete_step!(execution_id, step_name, payload:, idempotency_key: nil)
+        with_execution_lock(execution_id) do |execution|
+          step = execution.steps.lock.find_by(name: step_name.to_s)
+          if step&.completed? && idempotency_key && step.completion_idempotency_key == idempotency_key
+            return build_execution(execution)
+          end
+
+          raise ActiveWorkflow::Errors::StepNotWaiting, "Step #{step_name} is not waiting" unless step&.waiting?
+
+          if idempotency_key && step.completion_idempotency_key.present? && step.completion_idempotency_key != idempotency_key
+            raise ActiveWorkflow::Errors::AsyncCompletionConflict, "Completion idempotency mismatch"
+          end
+
+          return build_execution(execution) if step.completion_idempotency_key.present? && step.completion_idempotency_key == idempotency_key
+
+          definition = step_definition(execution, step)
+          ctx = context_from(execution)
+
+          unless definition.options[:fire_and_forget]
+            store_keys = [definition.name.to_sym]
+            store_result = definition.options[:store_result_as]&.to_sym
+            store_keys << store_result if store_result && store_result != definition.name.to_sym
+
+            store_keys.each do |key|
+              ctx[key] = payload unless payload.nil?
+            end
+          end
+
+          step.update!(
+            state: "completed",
+            completion_payload: payload,
+            completion_idempotency_key: idempotency_key,
+            waiting_since: nil,
+            timeout_at: nil,
+            completed_at: clock.call
+          )
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.completed_async",
+            execution_id: execution.id,
+            step: step.name,
+            workflow: execution.workflow_class)
+
+          persist_context(execution, ctx)
+          advance_cursor(execution, step)
+          enqueue_runner(execution.id)
+        end
+      end
+
+      def fail_step!(execution_id, step_name, error_class:, message:, details:, idempotency_key: nil)
+        with_execution_lock(execution_id) do |execution|
+          step = execution.steps.lock.find_by(name: step_name.to_s)
+          raise ActiveWorkflow::Errors::StepNotWaiting, "Step #{step_name} is not waiting" unless step&.waiting?
+
+          definition = step_definition(execution, step)
+          current_attempts = step.attempts.to_i
+          record_failure!(execution, step,
+            error_class:,
+            message:,
+            details: details || {},
+            attempts: current_attempts,
+            async: true)
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.failed_async",
+            execution_id: execution.id,
+            step: step.name,
+            workflow: execution.workflow_class,
+            error_class: error_class,
+            message: message)
+
+          schedule_retry_or_fail!(execution, step, definition)
+        end
+      end
+
+      def extend_timeout!(execution_id, step_name, by:)
+        with_execution_lock(execution_id) do |execution|
+          step = execution.steps.lock.find_by(name: step_name.to_s)
+          return unless step&.waiting?
+
+          step.update!(timeout_at: step.timeout_at && step.timeout_at + by)
+        end
+      end
+
+      def heartbeat!(execution_id, step_name, at: clock.call)
+        with_execution_lock(execution_id) do |execution|
+          step = execution.steps.lock.find_by(name: step_name.to_s)
+          return unless step&.waiting?
+
+          step.update!(last_heartbeat_at: at)
+        end
+      end
+
+      def signal!(execution_id, name, payload: nil)
+        with_execution_lock(execution_id) do |execution|
+          Models::Event.create!(execution:, name: name.to_s, payload: payload)
+
+          ActiveSupport::Notifications.instrument("active_workflow.signal.received",
+            execution_id: execution.id,
+            signal: name)
+
+          apply_signal_handler(execution, name, payload)
+
+          if execution.steps.where(name: name.to_s, state: "waiting").exists?
+            enqueue_runner(execution.id)
+          end
+        end
+      end
+
+      def context_from(execution)
+        ActiveWorkflow::Context.new(serializer.load(execution.ctx))
+      end
+
+      def persist_context(execution, ctx)
+        execution.update!(ctx: serializer.dump(ctx.to_h))
+      end
+
+      def build_execution(record)
+        ActiveWorkflow::Execution.new(
+          id: record.id,
+          workflow_class: record.workflow_class,
+          state: record.state,
+          ctx: serializer.load(record.ctx),
+          cursor_step: record.cursor_step&.to_sym,
+          created_at: record.created_at,
+          updated_at: record.updated_at,
+          store: self
+        )
+      end
+
+      def step_definition(execution, step)
+        execution.workflow_class.constantize.step_definition(step.name)
+      end
+
+      def apply_signal_handler(execution, name, payload)
+        workflow_class = execution.workflow_class.constantize
+        handler = workflow_class.signal_handler_for(name)
+        return unless handler
+
+        ctx = context_from(execution)
+        workflow = workflow_class.new(context: ctx, execution_id: execution.id)
+
+        workflow.send(handler, payload)
+        persist_context(execution, workflow.ctx)
+      end
+
+      def advance_cursor(execution, step)
+        next_step = execution.steps.ordered.where("position > ?", step.position).first
+        if next_step
+          execution.update!(
+            cursor_step: next_step.name,
+            state: next_step.waiting? ? "waiting" : "running"
+          )
+        else
+          execution.update!(
+            cursor_step: nil,
+            state: "completed",
+            completed_at: clock.call
+          )
+          ActiveSupport::Notifications.instrument("active_workflow.execution.completed",
+            execution_id: execution.id,
+            workflow: execution.workflow_class)
+        end
+      end
+
+      def record_failure!(execution, step, error_class:, message:, details:, attempts:, async: false)
+        step.update!(
+          state: async ? "failed" : "failed",
+          last_error_class: error_class,
+          last_error_message: message,
+          last_error_details: details,
+          attempts: attempts,
+          last_error_at: clock.call
+        )
+        execution.update!(state: "failed", last_error_class: error_class, last_error_message: message)
+      end
+
+      def schedule_retry_or_fail!(execution, step, definition)
+        retry_config = (definition.options[:retry] || {}).deep_symbolize_keys
+
+        return transition_to_failed!(execution, step) if retry_config.empty?
+
+        current_attempts = step.attempts.to_i
+        max_attempts = retry_config[:max] || Float::INFINITY
+
+        if current_attempts < max_attempts
+          retry_index = [current_attempts, 1].max
+          delay = Backoff.calculate(retry_config, attempts: retry_index)
+          scheduled_at = clock.call + delay
+
+          step.update!(state: "pending", attempts: current_attempts, scheduled_at: scheduled_at)
+          execution.update!(state: "running", cursor_step: definition.name)
+
+          ActiveSupport::Notifications.instrument("active_workflow.retry.scheduled",
+            execution_id: execution.id,
+            workflow: execution.workflow_class,
+            step: step.name,
+            attempts: current_attempts + 1,
+            wait: delay)
+
+          enqueue_runner(execution.id, run_at: scheduled_at)
+        else
+          transition_to_failed!(execution, step)
+        end
+      end
+
+      def transition_to_failed!(execution, step)
+        step.update!(state: "failed")
+        run_compensations!(execution, upto: step.position)
+        execution.update!(state: "failed", cursor_step: nil)
+        ActiveSupport::Notifications.instrument("active_workflow.execution.failed",
+          execution_id: execution.id,
+          workflow: execution.workflow_class,
+          step: step.name)
+      end
+
+      private
+
+      def run_compensations!(execution, upto:)
+        workflow_class = execution.workflow_class.constantize
+        ctx = context_from(execution)
+        workflow = workflow_class.new(context: ctx, execution_id: execution.id)
+
+        steps = execution.steps.where("position <= ?", upto).order(position: :desc)
+        steps.each do |step|
+          definition = workflow_class.step_definition(step.name)
+          compensate = definition.options[:compensate]
+          next unless compensate
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.compensating",
+            execution_id: execution.id,
+            step: definition.name) do
+            case compensate
+            when Symbol
+              workflow.send(compensate)
+            when Proc
+              workflow.instance_exec(workflow.ctx, &compensate)
+            else
+              workflow.send(compensate)
+            end
+          end
+          step.update!(state: "compensated")
+        end
+
+        persist_context(execution, workflow.ctx)
+      end
+
+      # Coordinates execution progression within a DB transaction.
+      class Processor
+        attr_reader :store, :execution
+
+        def initialize(store, execution)
+          @store = store
+          @execution = execution
+        end
+
+        def process!
+        if TERMINAL_STATES.include?(execution.state)
+          ActiveWorkflow.configuration.logger.debug("[ActiveWorkflow] process! early exit state=#{execution.state}")
+          ActiveSupport::Notifications.instrument("active_workflow.execution.#{execution.state}",
+            execution_id: execution.id,
+            workflow: execution.workflow_class)
+          return
+        end
+
+          check_timeouts!
+          step = next_step
+          if step.nil?
+            finalize_if_complete
+            return
+          end
+
+          workflow_class = execution.workflow_class.constantize
+          definition = workflow_class.step_definition(step.name)
+          ctx = store.context_from(execution)
+          workflow = workflow_class.new(context: ctx, execution_id: execution.id)
+
+          if skip_step?(workflow, definition)
+            mark_skipped(step, definition)
+            store.persist_context(execution, workflow.ctx)
+            ActiveSupport::Notifications.instrument("active_workflow.step.skipped",
+              execution_id: execution.id, step: definition.name, workflow: workflow_class.name)
+            process! # continue to next
+            return
+          end
+
+          attempt = step.attempts.to_i + 1
+          now = store.clock.call
+          step.update!(state: "running", attempts: attempt, started_at: now, scheduled_at: nil)
+          execution.update!(state: "running", cursor_step: definition.name)
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.started",
+            execution_id: execution.id,
+            workflow: workflow_class.name,
+            step: definition.name,
+            attempt:) do
+            result = execute_step(workflow, definition, step)
+
+            case result
+            when :waiting
+              # Wait steps or async initialization indicated waiting state.
+              store.persist_context(execution, workflow.ctx)
+            when :complete
+              store.persist_context(execution, workflow.ctx)
+              step.update!(state: "completed", completed_at: store.clock.call)
+              ActiveSupport::Notifications.instrument("active_workflow.step.completed",
+                execution_id: execution.id, workflow: workflow_class.name, step: definition.name)
+              execution.update!(state: next_state_after(step), cursor_step: next_step_name(step))
+              process!
+            else
+              # Synchronous result returned.
+              store.persist_context(execution, workflow.ctx)
+              step.update!(state: "completed", completed_at: store.clock.call)
+              ActiveSupport::Notifications.instrument("active_workflow.step.completed",
+                execution_id: execution.id, workflow: workflow_class.name, step: definition.name, result: result)
+              execution.update!(state: next_state_after(step), cursor_step: next_step_name(step))
+              process!
+            end
+          end
+        rescue => error
+          handle_error(step, definition, workflow, error)
+        end
+
+        private
+
+        def check_timeouts!
+          now = store.clock.call
+
+          execution.steps.waiting.each do |step|
+            next unless step.timeout_at && step.timeout_at <= now
+
+            definition = step_definition(step)
+            ActiveWorkflow.configuration.logger.debug("[ActiveWorkflow] timeout step=#{step.name} attempts=#{step.attempts}")
+          ActiveSupport::Notifications.instrument("active_workflow.step.timeout",
+            execution_id: execution.id,
+            workflow: execution.workflow_class,
+            step: step.name)
+
+            current_attempts = step.attempts.to_i
+            step.update!(state: "timed_out", attempts: current_attempts)
+            store.schedule_retry_or_fail!(execution, step, definition)
+          end
+        end
+
+        def next_step
+          now = store.clock.call
+          execution.steps.ordered.detect do |step|
+            case step.state
+            when "pending"
+              !step.scheduled_at || step.scheduled_at <= now
+            when "running"
+              true
+            when "waiting"
+              wait_step_ready?(step)
+            else
+              false
+            end
+          end
+        end
+
+        def wait_step_ready?(step)
+          return false unless step.style == "wait"
+
+          execution.events.where(name: step.name, consumed_at: nil).exists?
+        end
+
+        def finalize_if_complete
+          if execution.steps.all? { |step| step.completed? || step.compensated? }
+            execution.update!(state: "completed", cursor_step: nil, completed_at: store.clock.call)
+          end
+        end
+
+        def execute_step(workflow, definition, step)
+          case definition.style
+          when :wait
+            event = execution.events.where(name: step.name, consumed_at: nil).order(:created_at).first
+            if event
+              consume_event(workflow, definition, event)
+              :complete
+            else
+              mark_waiting(step, definition, nil)
+              :waiting
+            end
+          else
+            result = workflow.call_step_callable(definition)
+            if async_step?(definition)
+              assign_async_init_result(workflow, definition, result)
+              mark_waiting(step, definition, result)
+              :waiting
+            else
+              assign_result(workflow, definition, result)
+              :complete
+            end
+          end
+        end
+
+        def consume_event(workflow, definition, event)
+          payload = event.payload
+          key = definition.options[:as]&.to_sym || definition.name
+          workflow.ctx[key] = payload
+          event.update!(consumed_at: store.clock.call)
+        end
+
+        def async_step?(definition)
+          definition.options[:async]
+        end
+
+        def assign_result(workflow, definition, result)
+          return if definition.options[:fire_and_forget]
+
+          key = definition.options[:store_result_as]&.to_sym || definition.name
+          workflow.ctx[key] = result unless result.nil?
+        end
+
+        def assign_async_init_result(workflow, definition, result)
+          return if definition.options[:fire_and_forget]
+
+          keys = [definition.name.to_sym]
+          store_result = definition.options[:store_result_as]&.to_sym
+          keys << store_result if store_result && store_result != definition.name.to_sym
+          keys.each do |key|
+            workflow.ctx[key] = result unless result.nil?
+          end
+        end
+
+        def mark_waiting(step, definition, init_result)
+          now = store.clock.call
+          timeout = definition.options[:timeout]
+          timeout_at = timeout ? now + timeout : nil
+          step.update!(
+            state: "waiting",
+            waiting_since: now,
+            timeout_at: timeout_at,
+            init_result: init_result
+          )
+          execution.update!(state: "waiting", cursor_step: definition.name)
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.waiting",
+            execution_id: execution.id, workflow: execution.workflow_class, step: definition.name)
+          store.enqueue_runner(execution.id, run_at: timeout_at) if timeout_at
+        end
+
+        def mark_skipped(step, definition)
+          step.update!(state: "completed", completed_at: store.clock.call)
+          execution.update!(cursor_step: next_step_name(step), state: next_state_after(step))
+        end
+
+        def handle_error(step, definition, workflow, error)
+          attempts = step.attempts.to_i
+          step.update!(
+            state: "failed",
+            last_error_class: error.class.name,
+            last_error_message: error.message,
+            last_error_backtrace: error.backtrace&.take(20)&.join("\n"),
+            attempts: attempts
+          )
+
+          ActiveSupport::Notifications.instrument("active_workflow.step.failed",
+            execution_id: execution.id,
+            workflow: execution.workflow_class,
+            step: definition.name,
+            error_class: error.class.name,
+            message: error.message)
+
+          store.persist_context(execution, workflow.ctx)
+          store.schedule_retry_or_fail!(execution, step, definition)
+        end
+
+        def skip_step?(workflow, definition)
+          if_condition = definition.options[:if]
+          unless_condition = definition.options[:unless]
+
+          if if_condition
+            result = evaluate_condition(workflow, if_condition)
+            return true unless result
+          end
+
+          if unless_condition
+            result = evaluate_condition(workflow, unless_condition)
+            return true if result
+          end
+
+          false
+        end
+
+        def evaluate_condition(workflow, condition)
+          case condition
+          when Symbol
+            workflow.send(condition)
+          when Proc
+            workflow.instance_exec(workflow.ctx, &condition)
+          else
+            !!condition
+          end
+        end
+
+        def next_step_name(step)
+          next_step = execution.steps.ordered.where("position > ?", step.position).first
+          next_step&.name
+        end
+
+        def next_state_after(step)
+          next_step = execution.steps.ordered.where("position > ?", step.position).first
+          return "completed" unless next_step
+
+          next_step.waiting? ? "waiting" : "running"
+        end
+
+        def step_definition(step)
+          execution.workflow_class.constantize.step_definition(step.name)
+        end
+      end
+
+      class Backoff
+        class << self
+          def calculate(config, attempts:)
+            strategy = (config[:strategy] || config[:backoff] || :exponential).to_sym
+            base = base_delay(config, attempts, strategy)
+            config[:jitter] ? apply_jitter(base) : base
+          end
+
+          private
+
+          def base_delay(config, attempts, strategy)
+            first = config[:first_delay]&.to_f || config[:delay]&.to_f || 5.0
+            delay = config[:delay]&.to_f || first
+
+            case strategy
+            when :fixed
+              attempts <= 1 ? first : delay
+            else
+              return first if attempts <= 1
+
+              exponent = attempts - 1
+              delay * (2**exponent)
+            end
+          end
+
+          def apply_jitter(value)
+            low = value * 0.5
+            high = value * 1.5
+            rand(low..high)
+          end
+        end
+      end
+    end
+  end
+end