@abelxiaoxing/cadence 1.4.0 → 1.4.1

package/src/workflow-state-machine.ts

@@ -1,6 +1,7 @@
 import { readFileSync } from "node:fs";
 import path from "node:path";
 import { DatabaseSync } from "node:sqlite";
+import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
 import {
   assertPlanWithinChangeContract,
   normalizeChangeContract,
@@ -18,11 +19,11 @@ import {
   type ImplementPlan,
   type PlanTaskDraft,
 } from "./delivery-compiler.ts";
+import { compareDeliveryRevision } from "./delivery-revision.ts";
 import { snapshotFiles } from "./file-snapshot.ts";
 import type { RoutePolicy } from "./route-policy.ts";
 import {
   canonicalJson,
-  legalControlCommands,
   type RunProjection,
   type RunState,
 } from "./run-state.ts";
@@ -39,11 +40,9 @@ import {
   RECOVERY_SCHEMA,
   REJECTED_DELIVERY_SCHEMA,
 } from "./storage-schema.ts";
-import type { WorkflowActivityUpdate } from "./subagent-activity.ts";
 import { permitsContextRead } from "./submit-tool.ts";
 import {
   type ActiveOperation,
-  approvalRequirement,
   assertDelivery,
   type BootstrapRow,
   bootstrapAcceptanceHash,
@@ -51,7 +50,6 @@ import {
   cancellableRead,
   classifyPersistedContextRequest,
   decideRecoveryAction,
-  deliveryBoundPaths,
   type EngineOperationRow,
   type EngineRunRow,
   type EngineTaskRow,
@@ -66,16 +64,12 @@ import {
   normalizeBootstrapAcceptanceFacts,
   normalizeWorkflowContextRequest,
   normalizeWorkflowVerificationStatus,
-  parseDeliveryDiagnostics,
   parseJsonRecord,
   parsePlan,
-  permitsPlanAmendment,
   phases,
   recoveryKey,
   type SafeAttemptDiagnostic,
   SHA256,
-  taskApprovalBoundary,
-  taskConflicts,
   type WorkflowApplication,
   type WorkflowApplicationContext,
   type WorkflowAttemptOutcome,
@@ -90,6 +84,16 @@ import {
   type WorkflowVerificationStatus,
   type WorkflowWorker,
 } from "./workflow-policy.ts";
+import {
+  assessRecoveryGrant,
+  recoveryExhausted,
+} from "./workflow-recovery-policy.ts";
+import { selectRunnableTasks } from "./workflow-scheduling.ts";
+import {
+  firstPausedTask,
+  projectWorkflowStatus,
+  type WorkflowResourceBudget,
+} from "./workflow-status.ts";
 export class WorkflowEngine {
   readonly #consumerRoot: string;
   readonly #stateRoot: ResolvedStateRoot;
@@ -948,36 +952,13 @@ export class WorkflowEngine {
       engineRun.current_revision === null
         ? undefined
         : this.#planForRevision(runId, engineRun.current_revision);
-    const priorTasks = new Map(
-      (priorPlan?.tasks ?? []).map((task) => [task.taskId, task]),
-    );
-    const nextTasks = new Map(
-      delivery.plan.tasks.map((task) => [task.taskId, task]),
-    );
-    const priorBoundPaths = new Set(
-      priorPlan ? deliveryBoundPaths(priorPlan) : [],
-    );
-    const boundaryExpanded = deliveryBoundPaths(delivery.plan).some(
-      (relative) => !priorBoundPaths.has(relative),
-    );
-    const compatible = new Set<string>();
-    if (priorPlan) {
-      for (const [taskId, priorTask] of priorTasks) {
-        const nextTask = nextTasks.get(taskId);
-        if (
-          nextTask &&
-          taskApprovalBoundary(priorPlan, priorTask) ===
-            taskApprovalBoundary(delivery.plan, nextTask)
-        ) {
-          compatible.add(taskId);
-        }
-      }
-    }
-    let invalidated = new Set(
-      priorRows
-        .filter((row) => !compatible.has(row.task_id))
-        .map((row) => row.task_id),
+    const comparison = compareDeliveryRevision(
+      priorPlan,
+      delivery.plan,
+      priorRows,
     );
+    const { compatible, boundaryExpanded } = comparison;
+    let invalidated = comparison.invalidated;
     let revalidatedWorkspace:
       | {
           baselineRevisionId: string;
@@ -1416,62 +1397,6 @@ export class WorkflowEngine {
     });
   }
 
-  #hasConflict(
-    row: EngineTaskRow,
-    rows: EngineTaskRow[],
-    selectedTaskIds: ReadonlySet<string>,
-  ): boolean {
-    const candidate = this.#taskPlan(row);
-    return rows.some((other) => {
-      if (other.task_id === row.task_id || other.state === "verified") {
-        return false;
-      }
-      if (other.state === "queued") {
-        if (row.state !== "queued") return false;
-        const candidatePosition = row.queue_position ?? Number.MAX_SAFE_INTEGER;
-        const otherPosition = other.queue_position ?? Number.MAX_SAFE_INTEGER;
-        if (
-          otherPosition > candidatePosition ||
-          (otherPosition === candidatePosition &&
-            other.task_order > row.task_order)
-        ) {
-          return false;
-        }
-      }
-      if (other.task_order > row.task_order && other.state === "pending") {
-        return false;
-      }
-      if (
-        other.state === "pending" &&
-        !this.#dependenciesVerified(other, rows)
-      ) {
-        return false;
-      }
-      const conflict = taskConflicts(candidate, this.#taskPlan(other));
-      return (
-        conflict.taskLifetime ||
-        (conflict.verification &&
-          ([
-            "pending",
-            "phase-ready",
-            "phase-running",
-            "validating",
-            "queued",
-          ].includes(other.state) ||
-            selectedTaskIds.has(other.task_id)))
-      );
-    });
-  }
-
-  #dependenciesVerified(row: EngineTaskRow, rows: EngineTaskRow[]): boolean {
-    const byId = new Map(
-      rows.map((candidate) => [candidate.task_id, candidate]),
-    );
-    return this.#taskPlan(row).dependsOn.every(
-      (dependency) => byId.get(dependency)?.state === "verified",
-    );
-  }
-
   #recordWorkspaceFacts(
     runId: string,
     outcome: WorkflowAttemptOutcome,
@@ -1662,9 +1587,7 @@ export class WorkflowEngine {
         recoveryKey(task, phase, row, current),
       );
       const verificationOnly =
-        !granted &&
-        activeRecovery &&
-        activeRecovery.failures >= maxRecoveryAttempts &&
+        recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted) &&
         (await this.#worker.hasPendingVerification?.({
           runId,
           operationId,
@@ -1682,10 +1605,9 @@ export class WorkflowEngine {
             : {}),
         })) === true;
       if (
-        !granted &&
         !verificationOnly &&
         activeRecovery &&
-        activeRecovery.failures >= maxRecoveryAttempts
+        recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted)
       ) {
         this.#setTask(
           runId,
@@ -2141,10 +2063,7 @@ export class WorkflowEngine {
   }
 
   #firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
-    return (
-      rows.find((row) => row.state === "approval-needed") ??
-      rows.find((row) => ["paused", "retryable"].includes(row.state))
-    );
+    return firstPausedTask(rows);
   }
 
   #recoverLegacyContextApproval(runId: string): boolean {
@@ -2791,46 +2710,34 @@ export class WorkflowEngine {
     while (madeProgress && !signal.aborted) {
       madeProgress = false;
       const rows = this.#tasks(runId);
-      const runnable: Array<{
-        row: EngineTaskRow;
-        before: string;
-      }> = [];
-      const selectedTaskIds = new Set<string>();
-      let capacityBlocked = false;
-      for (const row of rows) {
-        if (row.state === "verified" || attempted.has(row.task_id)) continue;
-        if (!this.#dependenciesVerified(row, rows)) {
-          if (row.state !== "pending") {
-            this.#setTask(
-              runId,
-              row.task_id,
-              {
-                state: "pending",
-                pauseCode: null,
-                queuePosition: null,
-              },
-              lease,
-            );
-          }
-          continue;
-        }
-        if (this.#hasConflict(row, this.#tasks(runId), selectedTaskIds)) {
-          this.#queueTask(runId, row, lease);
-          continue;
-        }
-        if (
-          this.#activeTasks + runnable.length >=
-          LIMITS.maxActiveChildSessions
-        ) {
-          this.#queueTask(runId, row, lease, "capacity");
-          capacityBlocked = true;
-          continue;
-        }
-        const before = `${row.state}:${row.phase}`;
-        attempted.add(row.task_id);
-        selectedTaskIds.add(row.task_id);
-        runnable.push({ row, before });
+      const selected = selectRunnableTasks({
+        rows,
+        attempted,
+        activeTasks: this.#activeTasks,
+        capacity: LIMITS.maxActiveChildSessions,
+        nextQueuePosition: this.#engineRun(runId).next_queue_position,
+      });
+      const runnable = selected.runnable.map((taskId) => {
+        const row = rows.find((candidate) => candidate.task_id === taskId);
+        if (!row) throw new Error("scheduler-task-unavailable");
+        attempted.add(taskId);
+        return { row, before: `${row.state}:${row.phase}` };
+      });
+      for (const update of selected.updates) {
+        const row = rows.find(
+          (candidate) => candidate.task_id === update.taskId,
+        );
+        if (!row) throw new Error("scheduler-task-unavailable");
+        if (update.kind === "pending")
+          this.#setTask(
+            runId,
+            row.task_id,
+            { state: "pending", pauseCode: null, queuePosition: null },
+            lease,
+          );
+        else this.#queueTask(runId, row, lease, update.reason);
       }
+      const capacityBlocked = selected.capacityBlocked;
       if (runnable.length === 0 && capacityBlocked) {
         await this.#waitForCapacity(signal);
         madeProgress = true;
@@ -3019,20 +2926,22 @@ export class WorkflowEngine {
         .get(runId, request.incidentKey) as { conditions_json: string };
       const previous = JSON.parse(conditions.conditions_json);
       const observed = this.#recoveryConditions(runId, request.incidentKey);
-      if (
-        (request.reason === "route-changed" &&
-          (!previous.route || previous.route === observed.route)) ||
-        (request.reason === "context-extended" &&
-          (!previous.context || previous.context === observed.context))
-      )
-        throw new Error("recovery-evidence-unavailable");
       const budget = this.#database
         .prepare(
           "SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
         )
         .get(runId) as { used: number; max_work: number };
-      if (!budget || budget.used >= budget.max_work)
-        throw new Error("change-work-budget-exhausted");
+      const rejected = assessRecoveryGrant({
+        current,
+        incident,
+        failureSequence: this.#failureSequence(runId, request.incidentKey),
+        requestedSequence: request.failureSequence,
+        reason: request.reason,
+        previous,
+        observed,
+        budget,
+      });
+      if (rejected) throw new Error(rejected);
       this.#database
         .prepare(
           "INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
@@ -3147,100 +3056,22 @@ export class WorkflowEngine {
   #statusByRun(runId: string): Record<string, unknown> {
     const projection = this.#runStore.status(runId);
     const rows = this.#tasks(runId);
-    const queued = rows
-      .filter((row) => row.state === "queued")
-      .sort(
-        (left, right) =>
-          (left.queue_position ?? Number.MAX_SAFE_INTEGER) -
-            (right.queue_position ?? Number.MAX_SAFE_INTEGER) ||
-          left.task_order - right.task_order,
-      );
-    const firstPaused = this.#firstPaused(rows);
     let engineRun: EngineRunRow | undefined;
     try {
       engineRun = this.#engineRun(runId);
     } catch {
       engineRun = undefined;
     }
-    const contextRequest = firstPaused?.context_request_json
-      ? normalizeWorkflowContextRequest(
-          JSON.parse(firstPaused.context_request_json) as unknown,
-        )
-      : undefined;
-    const pauseCode =
-      projection.state === "approval-needed" &&
-      firstPaused?.state === "approval-needed"
-        ? (firstPaused.pause_code ?? projection.pauseCode ?? "task-paused")
-        : (projection.pauseCode ?? firstPaused?.pause_code ?? "task-paused");
-    const approval =
-      projection.state === "approval-needed"
-        ? approvalRequirement(pauseCode, contextRequest)
-        : undefined;
-    const attemptDiagnostic = this.#safeAttemptDiagnostic(runId, firstPaused);
-    const recovery =
-      rows
-        .map((row) => this.#safeAttemptDiagnostic(runId, row)?.recovery)
-        .find((fact) => fact && fact.failures >= fact.feedback.maxAttempts) ??
-      attemptDiagnostic?.recovery;
-    const recoveryExhausted =
-      recovery && recovery.failures >= recovery.feedback.maxAttempts;
-    const blockers = rows
-      .filter((row) =>
-        ["approval-needed", "paused", "retryable"].includes(row.state),
-      )
-      .map((row) => {
-        const context = row.context_request_json
-          ? normalizeWorkflowContextRequest(
-              JSON.parse(row.context_request_json),
-            )
-          : undefined;
-        const code = row.pause_code ?? "task-paused";
-        const requirement =
-          row.state === "approval-needed"
-            ? approvalRequirement(code, context)
-            : undefined;
-        return {
-          id: hash(runId, row.task_id, code, canonicalJson(context ?? {})),
-          taskId: row.task_id,
-          phase: row.phase,
-          code,
-          kind: requirement ? "decision" : "execution",
-          ...(requirement ?? {}),
-          ...(context ? { contextRequest: context } : {}),
-        };
-      });
-    if (
-      projection.state === "approval-needed" &&
-      !blockers.some((item) => item.kind === "decision") &&
-      approval
-    ) {
-      blockers.push({
-        id: hash(runId, pauseCode),
-        taskId: "change",
-        phase: "green",
-        code: pauseCode,
-        kind: "decision",
-        ...approval,
-      });
-    }
-    const decisions = blockers.filter((item) => item.kind === "decision");
-    const amendments = blockers.filter(
-      (item) => item.kind === "decision" || permitsPlanAmendment(item.code),
-    );
-    if (
-      projection.state === "paused" &&
-      permitsPlanAmendment(pauseCode) &&
-      !amendments.some((item) => item.code === pauseCode)
-    ) {
-      const item = {
-        id: hash(runId, pauseCode, engineRun?.delivery_diagnostics_json ?? ""),
-        taskId: "change",
-        phase: "red" as const,
-        code: pauseCode,
-        kind: "execution",
-      };
-      blockers.push(item);
-      amendments.push(item);
+    const attemptDiagnostics = new Map<string, SafeAttemptDiagnostic>();
+    const failureSequences = new Map<string, number>();
+    for (const row of rows) {
+      const diagnostic = this.#safeAttemptDiagnostic(runId, row);
+      if (!diagnostic) continue;
+      attemptDiagnostics.set(row.task_id, diagnostic);
+      if (diagnostic.recovery) {
+        const key = diagnostic.recovery.key;
+        failureSequences.set(key, this.#failureSequence(runId, key));
+      }
     }
     const amendmentUsed =
       (
@@ -3250,254 +3081,20 @@ export class WorkflowEngine {
           )
           .get(runId) as { used: number } | undefined
       )?.used ?? 0;
-    const amendmentBudget = {
-      used: amendmentUsed,
-      maximum: 64,
-      remaining: 64 - amendmentUsed,
-      exhausted: amendmentUsed >= 64,
-    };
-    const decisionBatch =
-      amendments.length &&
-      pauseCode !== "operation-cancelled" &&
-      ["approval-needed", "paused"].includes(projection.state)
-        ? {
-            id: hash(
-              runId,
-              String(projection.deliveryRevision ?? 0),
-              canonicalJson(amendments),
-            ),
-            continuation: {
-              action: "amend",
-              change: projection.change,
-              batchId: hash(
-                runId,
-                String(projection.deliveryRevision ?? 0),
-                canonicalJson(amendments),
-              ),
-            },
-            resolution: {
-              owner: "parent",
-              strategy: "recommended",
-              requiresUserInput: false,
-            },
-            items: amendments,
-            requiredGates: [
-              ...new Set(
-                amendments.flatMap((item) => item.requiredGates ?? ["gate-b"]),
-              ),
-            ].sort(),
-          }
-        : undefined;
     const resourceBudget = this.#database
       .prepare(
         "SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
       )
-      .get(runId) as { used: number; maximum: number } | undefined;
-    const budgetCode = rows.find(
-      (row) =>
-        row.pause_code === "change-work-budget-exhausted" ||
-        row.pause_code === "change-recovery-budget-exhausted",
-    )?.pause_code;
-    const budgetExhausted = budgetCode !== undefined;
-    const currentRevision = projection.deliveryRevision ?? 0;
-    return {
-      runId: projection.runId,
-      stage: projection.stage,
-      ...(projection.change ? { change: projection.change } : {}),
-      state: projection.state,
-      ...(projection.deliveryRevision === undefined
-        ? {}
-        : { deliveryRevision: projection.deliveryRevision }),
-      ...(projection.deliveryBindings.length > 0
-        ? { deliveryBindings: structuredClone(projection.deliveryBindings) }
-        : {}),
-      completed: projection.state === "completed",
-      blockers,
-      ...(resourceBudget
-        ? {
-            resourceBudget: {
-              ...resourceBudget,
-              remaining: resourceBudget.maximum - resourceBudget.used,
-            },
-          }
-        : {}),
-      ...(budgetExhausted
-        ? {
-            recovery: {
-              automatic: false,
-              exhausted: true,
-              code: budgetCode,
-            },
-          }
-        : {}),
-      ...(decisionBatch
-        ? {
-            decisionBatch,
-            amendmentBudget,
-            ...(!amendmentBudget.exhausted &&
-            !budgetExhausted &&
-            (!resourceBudget || resourceBudget.used < resourceBudget.maximum)
-              ? {
-                  continuation: {
-                    owner: "parent",
-                    automatic: true,
-                    ...decisionBatch.continuation,
-                  },
-                }
-              : {}),
-          }
-        : {}),
-      ...(recoveryExhausted
-        ? {
-            recovery: {
-              automatic: false,
-              exhausted: true,
-              code: recovery.feedback.code,
-              attempts: recovery.failures,
-              maxAttempts: recovery.feedback.maxAttempts,
-              automaticRetryExhausted: true,
-              ...(!budgetExhausted &&
-              resourceBudget &&
-              resourceBudget.used < resourceBudget.maximum
-                ? {
-                    additionalAttempt: {
-                      incidentKey: recovery.key,
-                      failureSequence: this.#failureSequence(
-                        runId,
-                        recovery.key,
-                      ),
-                      reason: "parent-directed-retry",
-                    },
-                  }
-                : {}),
-            },
-          }
-        : {}),
-      ...(projection.terminal ? { terminal: projection.terminal } : {}),
-      ...(projection.pauseCode || firstPaused?.pause_code
-        ? {
-            pause: {
-              code: pauseCode,
-              ...(attemptDiagnostic ? { diagnostic: attemptDiagnostic } : {}),
-            },
-          }
-        : {}),
-      legalCommands:
-        projection.state === "approval-needed"
-          ? ["status", "discard"]
-          : legalControlCommands(projection.state).filter(
-              (command) =>
-                (command !== "rebind" || firstPaused !== undefined) &&
-                (command !== "resume" ||
-                  (!recoveryExhausted && !budgetExhausted)),
-            ),
-      ...(approval && projection.change
-        ? {
-            approval: {
-              category: approval.category,
-              requiredGates:
-                decisionBatch?.requiredGates ?? approval.requiredGates,
-              categories: [
-                ...new Set(
-                  decisions.flatMap((item) =>
-                    item.category ? [item.category] : [],
-                  ),
-                ),
-              ],
-              refs: [...new Set(decisions.flatMap((item) => item.refs ?? []))],
-              continuation: decisionBatch?.continuation,
-              retainedRun: {
-                runId: projection.runId,
-                deliveryRevision: currentRevision,
-              },
-              receiptPrecondition: {
-                deliveryRevision: { greaterThan: currentRevision },
-                receiptHash: "matching-ready-receipt",
-              },
-            },
-            conditionalCommands: [
-              {
-                command: "resume",
-                stage: "abel-implement",
-                change: projection.change,
-                requires: {
-                  deliveryRevision: { greaterThan: currentRevision },
-                  receiptHash: "matching-ready-receipt",
-                },
-              },
-            ],
-          }
-        : {}),
-      tasks: rows.map((row) => ({
-        taskId: row.task_id,
-        state: row.state,
-        ...(row.state === "verified" ? {} : { phase: row.phase }),
-        ...(row.context_request_json
-          ? {
-              contextRequest: normalizeWorkflowContextRequest(
-                JSON.parse(row.context_request_json) as unknown,
-              ),
-            }
-          : {}),
-      })),
-      queue: queued.map((row, index) => ({
-        taskId: row.task_id,
-        position: index + 1,
-        reason: row.pause_code === "worker-capacity" ? "capacity" : "conflict",
-      })),
-      ...(engineRun?.verification_json
-        ? {
-            verification: normalizeWorkflowVerificationStatus(
-              JSON.parse(engineRun.verification_json) as unknown,
-            ),
-          }
-        : {}),
-      ...(engineRun?.delivery_diagnostics_json
-        ? {
-            delivery: {
-              code: "delivery-invalid",
-              diagnostics: parseDeliveryDiagnostics(
-                engineRun.delivery_diagnostics_json,
-              ),
-            },
-          }
-        : {}),
-      ...(engineRun?.route_id
-        ? {
-            routeBinding: {
-              routeId: engineRun.route_id,
-              ...(engineRun.route_fingerprint
-                ? { routeFingerprint: engineRun.route_fingerprint }
-                : {}),
-            },
-          }
-        : {}),
-      ...(engineRun?.cleanup_state === "retained"
-        ? {
-            privateData: {
-              retained: true,
-              cleanup: "retained",
-              ...(engineRun.baseline_workspace_revision
-                ? {
-                    baselineRevisionId: engineRun.baseline_workspace_revision,
-                  }
-                : {}),
-              ...(engineRun.current_workspace_revision
-                ? {
-                    currentRevisionId: engineRun.current_workspace_revision,
-                  }
-                : {}),
-            },
-          }
-        : engineRun?.cleanup_state === "complete"
-          ? {
-              privateData: {
-                retained: false,
-                cleanup: "complete",
-              },
-            }
-          : {}),
-    };
+      .get(runId) as WorkflowResourceBudget | undefined;
+    return projectWorkflowStatus({
+      projection,
+      rows,
+      engineRun,
+      attemptDiagnostics,
+      failureSequences,
+      amendmentUsed,
+      resourceBudget,
+    });
   }
 
   async #withAvailableDelivery(
@@ -3896,21 +3493,6 @@ export class WorkflowEngine {
           signal,
         );
         this.#admitDelivery(runId, delivery, lease);
-        this.#transition(
-          runId,
-          "ready",
-          `${command.operationId}:delivery-ready`,
-          undefined,
-          lease,
-        );
-      } else if (current.state !== "ready") {
-        this.#transition(
-          runId,
-          "ready",
-          `${command.operationId}:resume-ready`,
-          undefined,
-          lease,
-        );
       }
       if (command.recovery)
         this.#grantRecovery(
@@ -3919,6 +3501,13 @@ export class WorkflowEngine {
           command.recovery,
           lease,
         );
+      this.#transition(
+        runId,
+        "ready",
+        `${command.operationId}:resume-ready`,
+        undefined,
+        lease,
+      );
       return this.#launchAdvance(
         runId,
         command.operationId,
@@ -3943,6 +3532,25 @@ export class WorkflowEngine {
       if (failure) {
         return this.#commitOperation(runId, command.operationId, failure);
       }
+      // A revised delivery may invalidate a grant. Retain its admitted facts,
+      // but never leave a rejected resume looking ready or still validating.
+      if (
+        error instanceof Error &&
+        [
+          "recovery-request-stale",
+          "recovery-evidence-unavailable",
+          "change-work-budget-exhausted",
+        ].includes(error.message) &&
+        this.#runStore.status(runId).state === "validating-delivery"
+      ) {
+        this.#transition(
+          runId,
+          "paused",
+          `${command.operationId}:recovery-rejected`,
+          error.message,
+          lease,
+        );
+      }
       this.#interruptOperation(runId, command.operationId);
       throw error;
     }