@desplega.ai/agent-swarm 2.4.4 → 2.5.0

package/src/workflows/foreach-join.ts

@@ -1,8 +1,24 @@
-import { getLatestStepForNode, getWorkflowRunStepsByRunId } from "../be/db";
-import type { WorkflowDefinition, WorkflowNode, WorkflowRunStep } from "../types";
+import {
+  getDbClient,
+  getLatestStepForNode,
+  getWorkflowRun,
+  getWorkflowRunStep,
+  getWorkflowRunStepsByRunId,
+  updateWorkflowRun,
+  updateWorkflowRunStep,
+} from "../be/db";
+import type {
+  WorkflowDefinition,
+  WorkflowNode,
+  WorkflowRunStep,
+  WorkflowRunStepStatus,
+} from "../types";
 import { checkpointStep } from "./checkpoint";
 import { getSuccessors } from "./definition";
 import type { ForeachOutput } from "./executors/foreach";
+import type { ExecutorRegistry } from "./executors/registry";
+import { effectiveFailurePolicy } from "./foreach-body";
+import { startForeachDrive } from "./foreach-driver";
 
 export const FOREACH_TERMINAL_STEP_STATUSES = new Set([
   "completed",
@@ -29,7 +45,10 @@ export function parseSyntheticNodeId(nodeId: string): SyntheticNodeId | null {
   };
 }
 
-export function resolveForeachParent(def: WorkflowDefinition, nodeId: string): WorkflowNode | null {
+export function resolveForeachParent(
+  def: Pick<WorkflowDefinition, "nodes">,
+  nodeId: string,
+): WorkflowNode | null {
   const parsed = parseSyntheticNodeId(nodeId);
   if (!parsed) return null;
   const parent = def.nodes.find((node) => node.id === parsed.parentNodeId);
@@ -53,10 +72,9 @@ export async function joinForeach(
     throw new Error(`Step "${childStep.nodeId}" is not a foreach child`);
   }
 
-  const children = (await getWorkflowRunStepsByRunId(runId))
-    .filter((step) => resolveForeachParent(def, step.nodeId)?.id === parent.id)
-    .sort((a, b) => childIndex(a) - childIndex(b));
+  const children = latestForeachChildren(def, parent.id, await getWorkflowRunStepsByRunId(runId));
 
+  // `pending` (queued, not started) is non-terminal like running and waiting.
   if (children.some((step) => !FOREACH_TERMINAL_STEP_STATUSES.has(step.status))) {
     return { joined: false, parentNodeId: parent.id, successors: [] };
   }
@@ -81,8 +99,28 @@ export async function joinForeach(
   };
 }
 
+/**
+ * The latest step row per child node id of foreach `parentNodeId`, in item
+ * order. `steps` must be in insert order (as `getWorkflowRunStepsByRunId`
+ * returns them), so a later attempt row supersedes an earlier one.
+ */
+export function latestForeachChildren(
+  def: WorkflowDefinition,
+  parentNodeId: string,
+  steps: WorkflowRunStep[],
+): WorkflowRunStep[] {
+  const latest = new Map<string, WorkflowRunStep>();
+  for (const step of steps) {
+    if (resolveForeachParent(def, step.nodeId)?.id === parentNodeId) latest.set(step.nodeId, step);
+  }
+  return [...latest.values()].sort((a, b) => childIndex(a) - childIndex(b));
+}
+
 export function buildForeachAggregate(children: WorkflowRunStep[]): ForeachOutput {
-  const orderedChildren = [...children].sort((a, b) => childIndex(a) - childIndex(b));
+  // Only the latest row per child node id counts (insert order).
+  const latest = new Map<string, WorkflowRunStep>();
+  for (const step of children) latest.set(step.nodeId, step);
+  const orderedChildren = [...latest.values()].sort((a, b) => childIndex(a) - childIndex(b));
   const results: ForeachOutput["results"] = orderedChildren.map((step) => {
     const parsed = parseSyntheticNodeId(step.nodeId)!;
     const status = resultStatus(step);
@@ -92,11 +130,15 @@ export function buildForeachAggregate(children: WorkflowRunStep[]): ForeachOutpu
       output: step.output,
     };
   });
+  // Only `completed` without an error is ok: failed, cancelled, skipped and the
+  // legacy completed+error shape all count as failed.
   const failedCount = results.filter((result) => result.status !== "completed").length;
+  const cancelledCount = results.filter((result) => result.status === "cancelled").length;
   const aggregate: ForeachOutput = {
     results,
     okCount: results.length - failedCount,
     failedCount,
+    ...(cancelledCount > 0 ? { cancelledCount } : {}),
   };
   return aggregate;
 }
@@ -115,3 +157,160 @@ function resultStatus(step: WorkflowRunStep): ForeachOutput["results"][number]["
   // (e.g. an agent quoting a log line) is user-controlled text, not a failure.
   return step.error ? "failed" : "completed";
 }
+
+/** The terminal outcome of one foreach item. */
+export interface ForeachChildOutcome {
+  status: "completed" | "failed" | "cancelled";
+  output?: unknown;
+  /**
+   * The failure reason. On a `completed` outcome it marks the legacy
+   * `onNodeFailure: "continue"` shape (completed with error), which the
+   * aggregate counts as failed.
+   */
+  error?: string;
+  /** Stored on the child row for display only: a body's ports do not route. */
+  nextPort?: string;
+}
+
+export interface ForeachSettleResult {
+  /** False when the child was not in an expected status, or the run is no longer live. */
+  claimed: boolean;
+  foreachChild: true;
+  /** True when this settlement closed the join and checkpointed the parent. */
+  joined: boolean;
+  /** The parent's successors to walk, when `joined`. */
+  successors: WorkflowNode[];
+  /** True when this settlement failed the run (a failed item under fail-fast). */
+  runFailed: boolean;
+}
+
+/**
+ * The one place a foreach child reaches a terminal status. Every resume path
+ * (task, approval, wait, child run, and their failure paths) and the foreach
+ * driver's inline items settle through here.
+ *
+ * In one transaction:
+ * 1. Re-read the child; claim it only while it is in one of `from` (default
+ *    `waiting`, the parked status every resume path expects).
+ * 2. Write the terminal status, output, error and port.
+ * 3. A run that is no longer live (failed, cancelled) only records the
+ *    outcome: a late sibling must not revive a failed run or route its join.
+ * 4. A failed or cancelled item under fail-fast (see effectiveFailurePolicy)
+ *    cancels the queued siblings and fails the run; in-flight siblings still
+ *    record their outcomes. Under collect-outcomes it counts as failed in the
+ *    aggregate and the join proceeds.
+ * 5. Otherwise run the join and set the run `running`.
+ *
+ * When the join stays open with a queued (`pending`) item and a `registry` is
+ * passed, a drive of the parent starts after commit, so the next item starts.
+ * Orphaned `running` items are the recovery sweep's.
+ */
+export async function settleForeachChild(
+  def: WorkflowDefinition,
+  runId: string,
+  childStepId: string,
+  outcome: ForeachChildOutcome,
+  ctx: Record<string, unknown>,
+  options: {
+    from?: readonly WorkflowRunStepStatus[];
+    registry?: ExecutorRegistry;
+    /** Run error when this outcome fails the run. Defaults to the outcome error. */
+    runError?: string;
+  } = {},
+): Promise<ForeachSettleResult> {
+  const from = options.from ?? ["waiting"];
+  return await getDbClient().transaction(async (): Promise<ForeachSettleResult> => {
+    const unclaimed: ForeachSettleResult = {
+      claimed: false,
+      foreachChild: true,
+      joined: false,
+      successors: [],
+      runFailed: false,
+    };
+    const current = await getWorkflowRunStep(childStepId);
+    if (!current || !from.includes(current.status)) return unclaimed;
+    const parent = resolveForeachParent(def, current.nodeId);
+    if (!parent) throw new Error(`Step "${current.nodeId}" is not a foreach child`);
+    const run = await getWorkflowRun(runId);
+    if (!run) return unclaimed;
+
+    const finishedAt = new Date().toISOString();
+    await updateWorkflowRunStep(childStepId, {
+      status: outcome.status,
+      ...(outcome.output !== undefined ? { output: outcome.output } : {}),
+      ...(outcome.error !== undefined ? { error: outcome.error } : {}),
+      ...(outcome.nextPort ? { nextPort: outcome.nextPort } : {}),
+      finishedAt,
+    });
+
+    if (run.status !== "running" && run.status !== "waiting") return unclaimed;
+
+    if (outcome.status !== "completed" && effectiveFailurePolicy(parent, def) === "fail-fast") {
+      const reason = options.runError ?? outcome.error ?? `foreach item ${current.nodeId} failed`;
+      await cancelQueuedForeachChildren(def, runId, parent.id, current.nodeId);
+      await updateWorkflowRun(runId, { status: "failed", error: reason, finishedAt });
+      return { ...unclaimed, claimed: true, runFailed: true };
+    }
+
+    const join = await joinForeach(def, runId, current, ctx);
+    await updateWorkflowRun(runId, { status: "running" });
+    const registry = options.registry;
+    if (!join.joined && registry && (await hasQueuedChild(def, runId, parent.id))) {
+      getDbClient().afterCommit(() => startForeachDrive(runId, parent.id, registry));
+    }
+    return {
+      claimed: true,
+      foreachChild: true,
+      joined: join.joined,
+      successors: join.successors,
+      runFailed: false,
+    };
+  });
+}
+
+/**
+ * Fail-fast: cancel every queued (`pending`) child of the fan-out so none of
+ * them starts. In-flight children are left to finish and record their outcome.
+ * A run retry gives each cancelled item a new attempt.
+ */
+export async function cancelQueuedForeachChildren(
+  def: WorkflowDefinition,
+  runId: string,
+  parentNodeId: string,
+  failedChildNodeId?: string,
+): Promise<number> {
+  const failedKey = failedChildNodeId
+    ? (parseSyntheticNodeId(failedChildNodeId)?.itemKey ?? failedChildNodeId)
+    : undefined;
+  const error = failedKey
+    ? `not started: fail-fast after item ${failedKey} failed`
+    : "not started: fail-fast after the run failed";
+  const finishedAt = new Date().toISOString();
+  let cancelled = 0;
+  await getDbClient().transaction(async () => {
+    const children = latestForeachChildren(
+      def,
+      parentNodeId,
+      await getWorkflowRunStepsByRunId(runId),
+    );
+    for (const child of children) {
+      if (child.status !== "pending") continue;
+      const row = await getDbClient().get<{ id: string }>(
+        "UPDATE workflow_run_steps SET status = 'cancelled', error = ?, finishedAt = ? WHERE id = ? AND status = 'pending' RETURNING id",
+        [error, finishedAt, child.id],
+      );
+      if (row) cancelled++;
+    }
+  });
+  return cancelled;
+}
+
+async function hasQueuedChild(
+  def: WorkflowDefinition,
+  runId: string,
+  parentNodeId: string,
+): Promise<boolean> {
+  return latestForeachChildren(def, parentNodeId, await getWorkflowRunStepsByRunId(runId)).some(
+    (child) => child.status === "pending",
+  );
+}

package/src/workflows/node-timeout.ts

@@ -0,0 +1,21 @@
+import type { WorkflowNode } from "../types";
+
+/** Watchdog budget for a node that configures none. */
+export const DEFAULT_NODE_TIMEOUT_MS = 30_000;
+
+/**
+ * The workflow watchdog budget for one executor call of `node`.
+ *
+ * Inline script nodes historically expose their wall-clock budget as
+ * `config.timeout`, while the workflow watchdog reads `config.timeoutMs` for
+ * every other executor. Keep the outer watchdog aligned with the script
+ * executor so a script configured for (for example) 90s is not killed by the
+ * 30s default first. The foreach driver applies this to each child with the
+ * synthesized body node, so every item gets its own budget.
+ */
+export function resolveNodeTimeoutMs(node: Pick<WorkflowNode, "type" | "config">): number {
+  const config = node.config ?? {};
+  if (typeof config.timeoutMs === "number") return config.timeoutMs;
+  if (node.type === "script" && typeof config.timeout === "number") return config.timeout;
+  return DEFAULT_NODE_TIMEOUT_MS;
+}

package/src/workflows/recovery.ts

@@ -24,6 +24,8 @@ import {
   walkGraph,
 } from "./engine";
 import type { ExecutorRegistry } from "./executors/registry";
+import { stepFailureMode } from "./foreach-body";
+import { isForeachChildOwned, runForeachDrive } from "./foreach-driver";
 import { getSecretInputKeys } from "./input";
 import {
   finalizeOrWait,
@@ -71,6 +73,9 @@ export async function recoverIncompleteRuns(registry: ExecutorRegistry): Promise
     }
   }
 
+  // --- Case 6: foreach fan-outs with queued or orphaned items ---
+  recovered += await recoverForeachFanouts(registry);
+
   if (recovered > 0) {
     console.log(`[workflows] Recovered ${recovered} incomplete run(s)`);
   }
@@ -211,20 +216,22 @@ async function recoverWaitingRuns(registry: ExecutorRegistry): Promise<number> {
         if (retry === "scheduled") recovered++;
         if (retry !== "not-eligible") continue;
       }
-      if (!taskCompleted && (workflow.definition.onNodeFailure ?? "fail") === "fail") {
+      if (!taskCompleted && stepFailureMode(workflow.definition, stuck.nodeId) === "fail") {
         // Preserve the fail-fast recovery policy for failed/cancelled tasks.
         // Claimed: this sweep runs on every heartbeat, so the live task.failed
         // bus event may already have routed this step — a blind write here
         // would kill a run that is already advancing.
         const reason =
           stuck.taskStatus === "failed" ? "Task failed (recovered)" : "Task cancelled (recovered)";
-        const claimed = await failStepAndRunIfWaiting(
+        const failed = await failStepAndRunIfWaiting(
+          workflow.definition,
           stuck.stepId,
           stuck.runId,
           reason,
           stuck.taskId,
+          { registry },
         );
-        if (!claimed) continue;
+        if (!failed.claimed) continue;
         recovered++;
         continue;
       }
@@ -248,6 +255,7 @@ async function recoverWaitingRuns(registry: ExecutorRegistry): Promise<number> {
         ctx,
         taskCompleted ? undefined : reason,
         stuck.taskId,
+        registry,
       );
       if (!routing.claimed) continue;
       if (routing.foreachChild && !routing.joined) {
@@ -351,6 +359,7 @@ async function recoverApprovalWaitingRuns(registry: ExecutorRegistry): Promise<n
           stepOutput,
           nextPort,
           ctx,
+          registry,
         );
       });
       if (!routing?.claimed) continue;
@@ -413,6 +422,62 @@ async function recoverWaitStates(registry: ExecutorRegistry): Promise<number> {
   return recovered;
 }
 
+/**
+ * Re-drive foreach fan-outs of live runs whose parent is still open and that
+ * have a queued (`pending`) item, or a `running` item no drive in this process
+ * owns. Catches a crash after a settlement committed and before the next
+ * drive, and items a crash left between their claim and their settlement. An
+ * orphaned instant item runs again (at-least-once, like a top-level instant
+ * node); an async item reconnects, because its executor dedupes on the child
+ * step id.
+ */
+async function recoverForeachFanouts(registry: ExecutorRegistry): Promise<number> {
+  const rows = await getDbClient().query<{
+    runId: string;
+    workflowId: string;
+    stepId: string;
+    nodeId: string;
+    status: string;
+  }>(
+    `SELECT s.runId AS runId, r.workflowId AS workflowId, s.id AS stepId, s.nodeId AS nodeId, s.status AS status
+       FROM workflow_run_steps s
+       JOIN workflow_runs r ON r.id = s.runId
+      WHERE r.status IN ('running', 'waiting')
+        AND s.status IN ('pending', 'running')
+        AND instr(s.nodeId, '#') > 1`,
+  );
+  const fanouts = new Map<string, { runId: string; workflowId: string; parentNodeId: string }>();
+  for (const row of rows) {
+    if (row.status === "running" && isForeachChildOwned(row.stepId)) continue;
+    const parentNodeId = row.nodeId.slice(0, row.nodeId.indexOf("#"));
+    fanouts.set(`${row.runId}\u0000${parentNodeId}`, {
+      runId: row.runId,
+      workflowId: row.workflowId,
+      parentNodeId,
+    });
+  }
+
+  let recovered = 0;
+  for (const fanout of fanouts.values()) {
+    try {
+      // The drive re-checks the run, the parent step and each child's latest
+      // row, so a stale snapshot row claims nothing.
+      const workflow = await getWorkflow(fanout.workflowId);
+      const parent = workflow?.definition.nodes.find((node) => node.id === fanout.parentNodeId);
+      if (parent?.type !== "foreach") continue;
+      const drive = await runForeachDrive(fanout.runId, fanout.parentNodeId, registry);
+      if (drive.claimed > 0) recovered++;
+    } catch (err) {
+      if (await failRunOnUnreadableReplay(fanout.runId, err)) continue;
+      console.error(
+        `[workflows] Failed to re-drive foreach ${fanout.parentNodeId} of run ${fanout.runId}:`,
+        err,
+      );
+    }
+  }
+  return recovered;
+}
+
 /**
  * Get run IDs by status. Simple query since there's no dedicated function for this.
  */