@desplega.ai/agent-swarm 2.5.2 → 2.5.4

package/src/workflows/executors/sub-workflow.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 import { getChildWorkflowRunId, getWorkflowRunLineage } from "../../be/db/workflow-runs";
 import type { ExecutorMeta, WorkflowRun } from "../../types";
-import { startWorkflowExecution } from "../engine";
+import { ParentStepNotCurrentError, startWorkflowExecution } from "../engine";
 import type { ExecutorDependencies, ExecutorResult } from "./base";
 import { BaseExecutor } from "./base";
 import type { ExecutorRegistry } from "./registry";
@@ -35,15 +35,21 @@ const NON_NODE_CONTEXT_KEYS = new Set(["trigger", "input", "swarm", "workflow"])
 /**
  * The parent step's outcome for a child run: its result once the child
  * completed, an error once it failed, was cancelled or was skipped, and null
- * while it is still running or waiting.
+ * while it is still running or waiting. Only a failed child is retryable: a
+ * cancel or a skip is a decision, and a node retry must not override it. The
+ * synchronous (executor) and the parked (`resumeFromChildRun`) paths both
+ * apply this, so the same child end gets the same policy whenever it lands.
  */
 export function childRunOutcome(
   child: WorkflowRun,
-): { output: SubWorkflowOutput } | { error: string } | null {
+): { output: SubWorkflowOutput } | { error: string; retryable: boolean } | null {
   if (child.status === "running" || child.status === "waiting") return null;
   if (child.status !== "completed") {
     const reason = child.error ? `: ${child.error}` : "";
-    return { error: `Child workflow run ${child.id} ${child.status}${reason}` };
+    return {
+      error: `Child workflow run ${child.id} ${child.status}${reason}`,
+      retryable: child.status === "failed",
+    };
   }
   const outputs: Record<string, unknown> = {};
   for (const [key, value] of Object.entries(child.context ?? {})) {
@@ -56,8 +62,9 @@ export function childRunOutcome(
 
 /**
  * Runs another workflow as a child run and waits for it. The child run records
- * this step as its parent, so a re-executed step reconnects to it instead of
- * starting a second child. The resume path is `resumeFromChildRun`.
+ * this step and its attempt (`retryCount`) as its parent, so a re-executed or
+ * recovered step reconnects to it instead of starting a second child, while a
+ * retry starts a new one. The resume path is `resumeFromChildRun`.
  */
 export class SubWorkflowExecutor extends BaseExecutor<
   typeof SubWorkflowConfigSchema,
@@ -82,9 +89,12 @@ export class SubWorkflowExecutor extends BaseExecutor<
   ): Promise<ExecutorResult<SubWorkflowOutput>> {
     const { db } = this.deps;
 
-    const existingId = await getChildWorkflowRunId(meta.stepId);
+    // The retry poller re-executes a failed step under the same id with a
+    // higher retryCount; that attempt must not settle on the previous child.
+    const attempt = (await db.getWorkflowRunStep(meta.stepId))?.retryCount ?? 0;
+    const existingId = await getChildWorkflowRunId(meta.stepId, attempt);
     const existing = existingId ? await db.getWorkflowRun(existingId) : null;
-    if (existing) return this.settle(existing);
+    if (existing) return this.settle(existing, config.workflowId);
 
     // Capped at the max depth, so a long chain fails closed instead of
     // hiding a cycle past the end of the scan.
@@ -109,17 +119,34 @@ export class SubWorkflowExecutor extends BaseExecutor<
     }
 
     // Returns once the child run exists; the child walks on its own. A child
-    // that already ended (cooldown, readiness, input failure) settles here.
-    const childRunId = await startWorkflowExecution(workflow, config.inputs ?? {}, this.registry, {
-      requestedByUserId: meta.requestedByUserId,
-      parentStepId: meta.stepId,
-    });
+    // that already ended (cooldown, readiness, input failure) settles here,
+    // and a concurrent execution of this attempt gets the child it created.
+    let childRunId: string;
+    try {
+      childRunId = await startWorkflowExecution(workflow, config.inputs ?? {}, this.registry, {
+        requestedByUserId: meta.requestedByUserId,
+        parentStepId: meta.stepId,
+        parentStepAttempt: attempt,
+      });
+    } catch (err) {
+      if (err instanceof ParentStepNotCurrentError) return { status: "failed", error: err.message };
+      throw err;
+    }
     const child = await db.getWorkflowRun(childRunId);
     if (!child) return { status: "failed", error: `Child workflow run ${childRunId} not found` };
-    return this.settle(child);
+    return this.settle(child, config.workflowId);
   }
 
-  private settle(child: WorkflowRun): ExecutorResult<SubWorkflowOutput> {
+  private settle(child: WorkflowRun, workflowId: string): ExecutorResult<SubWorkflowOutput> {
+    // The attempt already started a child of another workflow (the config
+    // changed under a recovered step). That is a conflict, not a reason to
+    // start a replacement next to it.
+    if (child.workflowId !== workflowId) {
+      return {
+        status: "failed",
+        error: `Child workflow run ${child.id} runs workflow ${child.workflowId}, not ${workflowId}`,
+      };
+    }
     const outcome = childRunOutcome(child);
     if (!outcome) {
       return {
@@ -129,7 +156,9 @@ export class SubWorkflowExecutor extends BaseExecutor<
         correlationId: child.id,
       } as ExecutorResult<SubWorkflowOutput>;
     }
-    if ("error" in outcome) return { status: "failed", error: outcome.error };
+    if ("error" in outcome) {
+      return { status: "failed", error: outcome.error, retryable: outcome.retryable };
+    }
     return { status: "success", output: outcome.output, nextPort: "success" };
   }
 }

package/src/workflows/foreach-driver.ts

@@ -11,6 +11,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import { getRunLineageBlock, getWorkflowRunStatus } from "../be/db/workflow-runs";
 import type {
   Workflow,
   WorkflowDefinition,
@@ -41,6 +42,7 @@ import {
   cancelQueuedForeachChildren,
   FOREACH_TERMINAL_STEP_STATUSES,
   type ForeachChildOutcome,
+  isRetryPending,
   latestForeachChildren,
   settleForeachChild,
 } from "./foreach-join";
@@ -352,8 +354,8 @@ async function parkRunIfRunning(runId: string): Promise<void> {
 
 /**
  * Flip `pending` children to `running`, in item order, while fewer than the
- * fan-out's `concurrency` cap are in flight (`running` or `waiting`, latest
- * attempt rows only), and collect unowned `running` ones. Ownership is registered before commit, so no other
+ * fan-out's `concurrency` cap are in flight (`running`, `waiting` or queued
+ * for a retry, latest attempt rows only), and collect unowned `running` ones. Ownership is registered before commit, so no other
  * drive can see a claimed row without also seeing it owned.
  */
 async function claimChildren(
@@ -367,6 +369,10 @@ async function claimChildren(
     // so concurrent drives (a settlement's drive and the recovery sweep)
     // serialize here and never exceed the cap together.
     await getDbClient().transaction(async () => {
+      // A cancel of the run (or of an ancestor) that committed first wins.
+      const status = await getWorkflowRunStatus(runId);
+      if (status !== "running" && status !== "waiting") return;
+      if (await getRunLineageBlock(runId)) return;
       const children = latestForeachChildren(
         def,
         parentNode.id,
@@ -379,7 +385,8 @@ async function claimChildren(
         }
       }
       const inFlight = children.filter(
-        (child) => child.status === "running" || child.status === "waiting",
+        (child) =>
+          child.status === "running" || child.status === "waiting" || isRetryPending(child),
       ).length;
       let free = foreachConcurrencyCap(parentNode) - inFlight;
       for (const child of children) {

package/src/workflows/foreach-join.ts

@@ -7,6 +7,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import { getRunLineageBlock } from "../be/db/workflow-runs";
 import type {
   WorkflowDefinition,
   WorkflowNode,
@@ -27,6 +28,22 @@ export const FOREACH_TERMINAL_STEP_STATUSES = new Set([
   "skipped",
 ]);
 
+/**
+ * A failed step queued for the retry poller (`nextRetryAt` set). A foreach
+ * child in this state is still in flight: it is not settled and holds its
+ * concurrency slot until the poller re-queues it.
+ */
+export function isRetryPending(step: Pick<WorkflowRunStep, "status" | "nextRetryAt">): boolean {
+  return step.status === "failed" && step.nextRetryAt != null;
+}
+
+/** A foreach child whose outcome is final: terminal and not queued for a retry. */
+export function isForeachChildSettled(
+  step: Pick<WorkflowRunStep, "status" | "nextRetryAt">,
+): boolean {
+  return FOREACH_TERMINAL_STEP_STATUSES.has(step.status) && !isRetryPending(step);
+}
+
 export interface SyntheticNodeId {
   parentNodeId: string;
   itemKey: string;
@@ -74,8 +91,9 @@ export async function joinForeach(
 
   const children = latestForeachChildren(def, parent.id, await getWorkflowRunStepsByRunId(runId));
 
-  // `pending` (queued, not started) is non-terminal like running and waiting.
-  if (children.some((step) => !FOREACH_TERMINAL_STEP_STATUSES.has(step.status))) {
+  // `pending` (queued, not started) and a child queued for a retry are
+  // non-terminal like running and waiting.
+  if (children.some((step) => !isForeachChildSettled(step))) {
     return { joined: false, parentNodeId: parent.id, successors: [] };
   }
 
@@ -244,6 +262,8 @@ export async function settleForeachChild(
     });
 
     if (run.status !== "running" && run.status !== "waiting") return unclaimed;
+    // The outcome stays on the item as history; a cancelled tree routes nothing.
+    if (await getRunLineageBlock(runId)) return unclaimed;
 
     if (outcome.status !== "completed" && effectiveFailurePolicy(parent, def) === "fail-fast") {
       const reason = options.runError ?? outcome.error ?? `foreach item ${current.nodeId} failed`;
@@ -270,8 +290,9 @@ export async function settleForeachChild(
 
 /**
  * Fail-fast: cancel every queued (`pending`) child of the fan-out so none of
- * them starts. In-flight children are left to finish and record their outcome.
- * A run retry gives each cancelled item a new attempt.
+ * them starts, and every child queued for a retry so the retry poller does not
+ * revive the failed run. In-flight children are left to finish and record their
+ * outcome. A run retry gives each cancelled item a new attempt.
  */
 export async function cancelQueuedForeachChildren(
   def: WorkflowDefinition,
@@ -294,9 +315,11 @@ export async function cancelQueuedForeachChildren(
       await getWorkflowRunStepsByRunId(runId),
     );
     for (const child of children) {
-      if (child.status !== "pending") continue;
+      if (child.status !== "pending" && !isRetryPending(child)) continue;
       const row = await getDbClient().get<{ id: string }>(
-        "UPDATE workflow_run_steps SET status = 'cancelled', error = ?, finishedAt = ? WHERE id = ? AND status = 'pending' RETURNING id",
+        `UPDATE workflow_run_steps SET status = 'cancelled', error = ?, finishedAt = ?, nextRetryAt = NULL
+          WHERE id = ? AND (status = 'pending' OR (status = 'failed' AND nextRetryAt IS NOT NULL))
+          RETURNING id`,
         [error, finishedAt, child.id],
       );
       if (row) cancelled++;

package/src/workflows/recovery.ts

@@ -12,7 +12,11 @@ import {
   resolveApprovalRequest,
   updateWorkflowRun,
 } from "../be/db";
-import { getSettledChildRunParentSteps } from "../be/db/workflow-runs";
+import {
+  getRunLineageBlock,
+  getRunningSubWorkflowSteps,
+  getSettledChildRunParentSteps,
+} from "../be/db/workflow-runs";
 import type { WorkflowRunStep } from "../types";
 import { shapeApprovalResolution } from "./approval-resolution";
 import { loadCompletedStepRouting } from "./completed-step-routing";
@@ -20,6 +24,7 @@ import { FAILED_TASK_OUTPUT_PREFIX } from "./constants";
 import {
   failRunOnUnreadableReplay,
   findReadyNodes,
+  isStepExecuting,
   isWorkflowRunActive,
   walkGraph,
 } from "./engine";
@@ -76,6 +81,9 @@ export async function recoverIncompleteRuns(registry: ExecutorRegistry): Promise
   // --- Case 6: foreach fan-outs with queued or orphaned items ---
   recovered += await recoverForeachFanouts(registry);
 
+  // --- Case 7: sub-workflow steps a crash left running ---
+  recovered += await recoverOrphanedSubWorkflowSteps(registry);
+
   if (recovered > 0) {
     console.log(`[workflows] Recovered ${recovered} incomplete run(s)`);
   }
@@ -98,6 +106,13 @@ async function recoverRunningRuns(registry: ExecutorRegistry): Promise<number> {
       if (isWorkflowRunActive(runId)) continue;
       const run = await getWorkflowRun(runId);
       if (!run || run.status !== "running") continue;
+      // A child run commits with its initial context. One without it was
+      // created before that held and lost its trigger and inputs in a crash;
+      // walking it from an empty context would run its nodes on blanks.
+      if (run.parentStepId && run.context === undefined) {
+        await failRunningRun(runId, "Child workflow run lost its initial context in a restart");
+        continue;
+      }
 
       const workflow = await getWorkflow(run.workflowId);
       if (!workflow) continue;
@@ -146,6 +161,57 @@ async function recoverRunningRuns(registry: ExecutorRegistry): Promise<number> {
   return recovered;
 }
 
+/** Fail a run that is still `running`; a cancel or finish that committed first wins. */
+async function failRunningRun(runId: string, error: string): Promise<void> {
+  await getDbClient().transaction(async () => {
+    const run = await getWorkflowRun(runId);
+    if (run?.status !== "running") return;
+    await updateWorkflowRun(runId, {
+      status: "failed",
+      error,
+      finishedAt: new Date().toISOString(),
+    });
+  });
+}
+
+/**
+ * Re-enter `sub-workflow` steps a crash left `running`. The running-run case
+ * misses two shapes: a run another branch already parked in `waiting`, and a
+ * loop iteration of a node that completed before. Walking the node lets
+ * executeStep re-claim the same step row, so the executor reconnects to the
+ * child that attempt already started. Foreach items are left to the fan-out
+ * case: their synthetic node ids are not in the definition.
+ */
+async function recoverOrphanedSubWorkflowSteps(registry: ExecutorRegistry): Promise<number> {
+  let recovered = 0;
+  for (const row of await getRunningSubWorkflowSteps()) {
+    try {
+      if (isWorkflowRunActive(row.runId) || isStepExecuting(row.stepId)) continue;
+      const run = await getWorkflowRun(row.runId);
+      if (!run || (run.status !== "running" && run.status !== "waiting")) continue;
+      const workflow = await getWorkflow(run.workflowId);
+      const node = workflow?.definition.nodes.find((n) => n.id === row.nodeId);
+      if (!workflow || !node) continue;
+      // DB reads above yield; a resume may have acquired the run meanwhile.
+      if (isWorkflowRunActive(row.runId)) continue;
+      await walkGraph(
+        workflow.definition,
+        row.runId,
+        (run.context ?? {}) as Record<string, unknown>,
+        [node],
+        registry,
+        workflow.id,
+        getSecretInputKeys(workflow.input),
+      );
+      recovered++;
+    } catch (err) {
+      if (await failRunOnUnreadableReplay(row.runId, err)) continue;
+      console.error(`[workflows] Failed to recover sub-workflow step ${row.stepId}:`, err);
+    }
+  }
+  return recovered;
+}
+
 /** Nodes with a failed step the retry poller will pick up (`nextRetryAt` set). */
 function retryPendingNodeIds(steps: WorkflowRunStep[]): Set<string> {
   const nodeIds = new Set<string>();
@@ -165,6 +231,7 @@ async function completeIfSettled(runId: string, ctx: Record<string, unknown>): P
   return getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (run?.status !== "running") return false;
+    if (await getRunLineageBlock(runId)) return false;
     const steps = await getWorkflowRunStepsByRunId(runId);
     if (retryPendingNodeIds(steps).size > 0) return false;
     const latest = new Map<string, WorkflowRunStep>();