@desplega.ai/agent-swarm 2.5.3 → 2.6.0

package/src/workflows/foreach-join.ts

@@ -7,6 +7,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import { getRunLineageBlock } from "../be/db/workflow-runs";
 import type {
   WorkflowDefinition,
   WorkflowNode,
@@ -27,6 +28,22 @@ export const FOREACH_TERMINAL_STEP_STATUSES = new Set([
   "skipped",
 ]);
 
+/**
+ * A failed step queued for the retry poller (`nextRetryAt` set). A foreach
+ * child in this state is still in flight: it is not settled and holds its
+ * concurrency slot until the poller re-queues it.
+ */
+export function isRetryPending(step: Pick<WorkflowRunStep, "status" | "nextRetryAt">): boolean {
+  return step.status === "failed" && step.nextRetryAt != null;
+}
+
+/** A foreach child whose outcome is final: terminal and not queued for a retry. */
+export function isForeachChildSettled(
+  step: Pick<WorkflowRunStep, "status" | "nextRetryAt">,
+): boolean {
+  return FOREACH_TERMINAL_STEP_STATUSES.has(step.status) && !isRetryPending(step);
+}
+
 export interface SyntheticNodeId {
   parentNodeId: string;
   itemKey: string;
@@ -74,8 +91,9 @@ export async function joinForeach(
 
   const children = latestForeachChildren(def, parent.id, await getWorkflowRunStepsByRunId(runId));
 
-  // `pending` (queued, not started) is non-terminal like running and waiting.
-  if (children.some((step) => !FOREACH_TERMINAL_STEP_STATUSES.has(step.status))) {
+  // `pending` (queued, not started) and a child queued for a retry are
+  // non-terminal like running and waiting.
+  if (children.some((step) => !isForeachChildSettled(step))) {
     return { joined: false, parentNodeId: parent.id, successors: [] };
   }
 
@@ -244,6 +262,8 @@ export async function settleForeachChild(
     });
 
     if (run.status !== "running" && run.status !== "waiting") return unclaimed;
+    // The outcome stays on the item as history; a cancelled tree routes nothing.
+    if (await getRunLineageBlock(runId)) return unclaimed;
 
     if (outcome.status !== "completed" && effectiveFailurePolicy(parent, def) === "fail-fast") {
       const reason = options.runError ?? outcome.error ?? `foreach item ${current.nodeId} failed`;
@@ -270,8 +290,9 @@ export async function settleForeachChild(
 
 /**
  * Fail-fast: cancel every queued (`pending`) child of the fan-out so none of
- * them starts. In-flight children are left to finish and record their outcome.
- * A run retry gives each cancelled item a new attempt.
+ * them starts, and every child queued for a retry so the retry poller does not
+ * revive the failed run. In-flight children are left to finish and record their
+ * outcome. A run retry gives each cancelled item a new attempt.
  */
 export async function cancelQueuedForeachChildren(
   def: WorkflowDefinition,
@@ -294,9 +315,11 @@ export async function cancelQueuedForeachChildren(
       await getWorkflowRunStepsByRunId(runId),
     );
     for (const child of children) {
-      if (child.status !== "pending") continue;
+      if (child.status !== "pending" && !isRetryPending(child)) continue;
       const row = await getDbClient().get<{ id: string }>(
-        "UPDATE workflow_run_steps SET status = 'cancelled', error = ?, finishedAt = ? WHERE id = ? AND status = 'pending' RETURNING id",
+        `UPDATE workflow_run_steps SET status = 'cancelled', error = ?, finishedAt = ?, nextRetryAt = NULL
+          WHERE id = ? AND (status = 'pending' OR (status = 'failed' AND nextRetryAt IS NOT NULL))
+          RETURNING id`,
         [error, finishedAt, child.id],
       );
       if (row) cancelled++;

package/src/workflows/recovery.ts

@@ -12,7 +12,11 @@ import {
   resolveApprovalRequest,
   updateWorkflowRun,
 } from "../be/db";
-import { getRunningSubWorkflowSteps, getSettledChildRunParentSteps } from "../be/db/workflow-runs";
+import {
+  getRunLineageBlock,
+  getRunningSubWorkflowSteps,
+  getSettledChildRunParentSteps,
+} from "../be/db/workflow-runs";
 import type { WorkflowRunStep } from "../types";
 import { shapeApprovalResolution } from "./approval-resolution";
 import { loadCompletedStepRouting } from "./completed-step-routing";
@@ -227,6 +231,7 @@ async function completeIfSettled(runId: string, ctx: Record<string, unknown>): P
   return getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (run?.status !== "running") return false;
+    if (await getRunLineageBlock(runId)) return false;
     const steps = await getWorkflowRunStepsByRunId(runId);
     if (retryPendingNodeIds(steps).size > 0) return false;
     const latest = new Map<string, WorkflowRunStep>();

package/src/workflows/resume.ts

@@ -1,6 +1,8 @@
 import {
+  type ApprovalRequest,
   cancelPendingApprovalRequestsForRun,
   cancelTask,
+  getApprovalRequestById,
   getCompletedStepNodeIds,
   getDbClient,
   getPendingEventWaitNames,
@@ -11,13 +13,25 @@ import {
   getWorkflowRun,
   getWorkflowRunStep,
   getWorkflowRunStepsByRunId,
-  listCancelledApprovalRequestsForRun,
   listCancelledApprovalRequestsForStep,
   resolveWaitState,
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
-import { createForeachAttemptSteps, getChildWorkflowRunId } from "../be/db/workflow-runs";
+import { listUndeliveredCancelledApprovalRequestIds } from "../be/db/approvals";
+import {
+  cancelLiveWorkflowTreeRuns,
+  cancelOpenWorkflowTreeSteps,
+  createForeachAttemptSteps,
+  disarmWorkflowTreeRetries,
+  getChildWorkflowRunId,
+  getOpenWorkflowTreeTaskIds,
+  getRunLineageBlock,
+  getTaskIdsWithDependents,
+  getWorkflowRunStatus,
+  getWorkflowRunTreeIds,
+  workflowRunTreeHasOpenWork,
+} from "../be/db/workflow-runs";
 import {
   type RetryPolicy,
   RetryPolicySchema,
@@ -48,6 +62,7 @@ import { startForeachDrive } from "./foreach-driver";
 import {
   FOREACH_TERMINAL_STEP_STATUSES,
   type ForeachChildOutcome,
+  isRetryPending,
   parseSyntheticNodeId,
   resolveForeachParent,
   settleForeachChild,
@@ -278,12 +293,16 @@ export async function finalizeOrWait(runId: string): Promise<void> {
   // move a step into `waiting` between the read and the write, and a run
   // finalized from a stale snapshot strands that branch.
   await getDbClient().transaction(async () => {
+    // A cancel (or another finalizer) that committed after the caller's claim
+    // wins: a terminal run is never re-parked or completed from here.
+    const status = await getWorkflowRunStatus(runId);
+    if (status !== "running" && status !== "waiting") return;
+    if (await getRunLineageBlock(runId)) return;
     const steps = await getWorkflowRunStepsByRunId(runId);
     // A branch still executing belongs to a live walk, which finalizes the
     // run itself; its finalizer only acts on a `running` run.
     if (hasRunningStep(steps)) {
-      const run = await getWorkflowRun(runId);
-      if (run?.status === "waiting") await updateWorkflowRun(runId, { status: "running" });
+      if (status === "waiting") await updateWorkflowRun(runId, { status: "running" });
       return;
     }
     // A step queued for the retry poller is still live, like a waiting one.
@@ -521,6 +540,10 @@ export async function retryFailedRun(runId: string, registry: ExecutorRegistry):
   const attempts = await getDbClient().transaction(async () => {
     const current = await getWorkflowRun(runId);
     if (!current || current.status !== "failed") return null;
+    // A failed run inside a cancelled tree keeps its failure; the cancel
+    // forbids starting it again.
+    const blocked = await getRunLineageBlock(runId);
+    if (blocked) throw new Error(`Retry not started, run left failed: ${blocked}`);
     // Fan-outs with failed or cancelled items get one new attempt row per such
     // item; completed items are never rerun. Selected from the rows as of this
     // transaction: an in-flight sibling of a fail-fast fan-out can record its
@@ -531,6 +554,13 @@ export async function retryFailedRun(runId: string, registry: ExecutorRegistry):
     ]);
     const capError = foreachRetryCapError(currentSteps.length, attempts);
     if (capError) throw new Error(`Retry not started, run left failed: ${capError}`);
+    // An item retry still queued when the run failed gets a new attempt row
+    // below. Retire the queued one, or the poller starts the old row beside it.
+    for (const step of currentSteps) {
+      if (isRetryPending(step) && resolveForeachParent(workflow.definition, step.nodeId)) {
+        await updateWorkflowRunStep(step.id, { nextRetryAt: null });
+      }
+    }
     // A foreach item retries as a new attempt row instead of a reset: a fresh
     // step id is a fresh invocation identity for every body type, and the
     // failed row stays as history.
@@ -639,82 +669,162 @@ function foreachRetryCapError(
   return `foreach retry would reach ${maxSteps} total steps (WORKFLOW_MAX_STEPS_PER_RUN): ${existingSteps} existing + ${added} new attempts leaves no room for the retry walk`;
 }
 
+const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "cancelled", "skipped"]);
+
+/** What one `cancelWorkflowRunRows` call covered. */
+export interface WorkflowRunCancelResult {
+  /** True when this call moved the root run to `cancelled`. */
+  cancelled: boolean;
+  /**
+   * The root and every run below it, root first, for the caller's Slack
+   * posts. Also set for a repeat cancel that found nothing open, so posts a
+   * failed earlier cancel left undelivered are retried. Empty when the root
+   * is missing or completed, failed or skipped.
+   */
+  runIds: string[];
+}
+
 /**
- * The DB writes of a run cancel, with no Slack post. Returns false when the
- * run is missing or already terminal. Called inside a caller's transaction,
- * the writes join it (as a SAVEPOINT) and commit or roll back with it.
+ * The DB writes of a run cancel, with no Slack post. Cancels the run and its
+ * whole child tree (every attempt's child, through terminal runs) in one
+ * transaction:
+ * - every non-terminal step, its task, and the run's other open tasks;
+ * - pending approvals and queued step retries;
+ * - every non-terminal run. A terminal descendant keeps its status, result
+ *   and reason; only its leftover open work is cancelled.
+ *
+ * A missing root, or one that is completed, failed or skipped, cancels
+ * nothing. An already cancelled root is walked again, so a repeat cancel
+ * finishes descendants an earlier cancel missed. A tree with a cycle or past
+ * its bounds throws and nothing commits.
+ *
+ * Called inside a caller's transaction, the writes join it (as a SAVEPOINT)
+ * and commit or roll back with it. Task lifecycle events and worker nudges
+ * queue behind COMMIT (afterCommit) and drop on rollback.
  */
-export async function cancelWorkflowRunRows(runId: string, cancelReason: string): Promise<boolean> {
-  const terminalStatuses = ["completed", "failed", "cancelled", "skipped"];
+export async function cancelWorkflowRunRows(
+  runId: string,
+  cancelReason: string,
+): Promise<WorkflowRunCancelResult> {
   const now = new Date().toISOString();
 
-  // Step snapshot, task cancels, and both status writes commit together so a
-  // step created after the snapshot cannot survive the cancel and a
-  // concurrent cancel cannot interleave. Task lifecycle events queue behind
-  // this transaction's COMMIT (afterCommit) and drop on rollback.
-  return await getDbClient().transaction(async () => {
-    const current = await getWorkflowRun(runId);
-    if (!current || terminalStatuses.includes(current.status)) return false;
-
-    // Cancel non-terminal steps and their associated tasks
-    const steps = await getWorkflowRunStepsByRunId(runId);
-    for (const step of steps) {
-      if (terminalStatuses.includes(step.status)) continue;
-
-      // Cancel any task linked to this step
-      const task = await getTaskByWorkflowRunStepId(step.id);
-      if (task) {
-        await cancelTask(task.id, cancelReason);
-      }
+  // A repeat cancel of a tree with nothing left open is answered by reads
+  // alone, without the write lock. The guards keep a cancelled tree from
+  // gaining new work, so the answer still holds when this returns.
+  // Its run ids still go back, so the caller retries Slack posts an earlier
+  // cancel could not deliver.
+  const settledTree = await getDbClient().transaction(
+    async () => {
+      if ((await getWorkflowRunStatus(runId)) !== "cancelled") return null;
+      const ids = await getWorkflowRunTreeIds(runId);
+      return (await workflowRunTreeHasOpenWork(ids)) ? null : ids;
+    },
+    { readOnly: true },
+  );
+  if (settledTree) return { cancelled: false, runIds: settledTree };
 
-      await updateWorkflowRunStep(step.id, {
-        status: "cancelled",
-        error: cancelReason,
-        finishedAt: now,
-      });
+  // The tree snapshot and every write commit together. A child run, task or
+  // step that commits first is in the snapshot; one that commits after sees
+  // the cancelled rows in its own guard (getRunLineageBlock and the step
+  // claims) and is refused.
+  return await getDbClient().transaction(async () => {
+    const rootStatus = await getWorkflowRunStatus(runId);
+    if (!rootStatus || (TERMINAL_RUN_STATUSES.has(rootStatus) && rootStatus !== "cancelled")) {
+      return { cancelled: false, runIds: [] };
     }
 
-    await cancelPendingApprovalRequestsForRun(runId, cancelReason);
-
-    // Mark the run itself as cancelled
-    await updateWorkflowRun(runId, {
-      status: "cancelled",
-      error: cancelReason,
-      finishedAt: now,
-    });
-    return true;
+    const runIds = await getWorkflowRunTreeIds(runId);
+    // Every step task carries the run link (indexed), and a retry detaches
+    // only the step link, so this covers current and earlier attempts' tasks.
+    // Dependents are looked up once for the whole tree, and only when needed.
+    const openTaskIds = await getOpenWorkflowTreeTaskIds(runIds);
+    const withDependents =
+      openTaskIds.length > 0 ? await getTaskIdsWithDependents() : new Set<string>();
+    for (const taskId of openTaskIds) {
+      await cancelTask(taskId, cancelReason, { cascadeDependents: withDependents.has(taskId) });
+    }
+    await cancelOpenWorkflowTreeSteps(runIds, cancelReason, now);
+    await disarmWorkflowTreeRetries(runIds, now);
+    for (const id of runIds) await cancelPendingApprovalRequestsForRun(id, cancelReason);
+    // Descendants' parent steps were just cancelled, so only the root's own
+    // parent (when the root is a child) needs the child-finished wake-up.
+    await cancelLiveWorkflowTreeRuns(runIds.slice(1), cancelReason, now);
+    if (rootStatus !== "cancelled") {
+      await updateWorkflowRun(runId, { status: "cancelled", error: cancelReason, finishedAt: now });
+    }
+    return { cancelled: rootStatus !== "cancelled", runIds };
   });
 }
 
+// Cancels in flight in this process, by run id. The API process owns the
+// database, so this bounds concurrent walks of one tree.
+const inFlightCancels = new Map<string, Promise<void>>();
+
 /**
- * Cancel a workflow run and all its non-terminal steps.
- * Also cancels any in-progress tasks spawned by waiting/running steps.
+ * Cancel a workflow run and its child tree (see `cancelWorkflowRunRows`), then
+ * post the Slack updates for every approval the cancel closed. The posts run
+ * after COMMIT, and a failed post is logged without undoing the cancel.
+ * Concurrent calls for the same run share one cancel and its result.
  */
-export async function cancelWorkflowRun(runId: string, reason?: string): Promise<void> {
-  const run = await getWorkflowRun(runId);
+export function cancelWorkflowRun(runId: string, reason?: string): Promise<void> {
+  const inFlight = inFlightCancels.get(runId);
+  if (inFlight) return inFlight;
+  const cancel = cancelWorkflowRunOnce(runId, reason).finally(() => {
+    inFlightCancels.delete(runId);
+  });
+  inFlightCancels.set(runId, cancel);
+  return cancel;
+}
+
+async function cancelWorkflowRunOnce(runId: string, reason?: string): Promise<void> {
+  const run = await getWorkflowRun(runId, "display");
   if (!run) throw new Error("Workflow run not found");
 
-  const terminalStatuses = ["completed", "failed", "cancelled", "skipped"];
-  if (terminalStatuses.includes(run.status) && run.status !== "cancelled") {
+  if (TERMINAL_RUN_STATUSES.has(run.status) && run.status !== "cancelled") {
     throw new Error(`Cannot cancel run in '${run.status}' state`);
   }
 
   const cancelReason = reason ?? "Cancelled by user";
-  const applied = await cancelWorkflowRunRows(runId, cancelReason);
+  const { cancelled, runIds } = await cancelWorkflowRunRows(runId, cancelReason);
 
-  // The migration trigger also catches direct step cancellation. Re-read after
-  // commit so its rows are included alongside the run-wide sweep.
-  const cancelledApprovals = await listCancelledApprovalRequestsForRun(runId);
-  const latestRun = await getWorkflowRun(runId);
-  if (!applied && latestRun?.status !== "cancelled") {
+  const latestRun = await getWorkflowRun(runId, "display");
+  if (!cancelled && latestRun?.status !== "cancelled") {
     throw new Error(`Cannot cancel run in '${latestRun?.status ?? "missing"}' state`);
   }
-  await postApprovalCancellationUpdates(
-    cancelledApprovals,
-    applied ? cancelReason : (latestRun?.error ?? cancelReason),
+  await postTreeApprovalCancellations(
+    runIds,
+    cancelled ? cancelReason : (latestRun?.error ?? cancelReason),
   );
 }
 
+/**
+ * Post the cancellation update of every cancelled approval in the given runs
+ * whose Slack post is not delivered yet. Re-read after commit: the migration
+ * trigger also cancels the approvals of a directly cancelled step. Each post
+ * is claimed, and a failed post releases its claim, so the next cancel of the
+ * tree retries it and a delivered post is never repeated.
+ */
+export async function postTreeApprovalCancellations(
+  runIds: string[],
+  reason: string,
+  options: { exceptRequestId?: string } = {},
+): Promise<void> {
+  try {
+    const approvals: ApprovalRequest[] = [];
+    for (const id of await listUndeliveredCancelledApprovalRequestIds(runIds)) {
+      if (id === options.exceptRequestId) continue;
+      const approval = await getApprovalRequestById(id);
+      if (approval) approvals.push(approval);
+    }
+    if (approvals.length > 0) await postApprovalCancellationUpdates(approvals, reason);
+  } catch (err) {
+    console.error(
+      "[workflows] approval cancellation updates not posted:",
+      scrubSecrets(err instanceof Error ? err.message : String(err)),
+    );
+  }
+}
+
 /** Cancel one HITL step and close any approval request it gates. */
 export async function cancelWorkflowRunStep(
   stepId: string,
@@ -993,6 +1103,8 @@ export async function resumeFromChildRun(
     const routing = await getDbClient().transaction(async () => {
       const current = await getWorkflowRunStep(step.id);
       if (!current || current.status !== "waiting") return null;
+      // Inside a cancelled tree a child's end settles nothing and queues no retry.
+      if (await getRunLineageBlock(run.id)) return null;
       const childId = await getChildWorkflowRunId(step.id, current.retryCount);
       const child = childId ? await getWorkflowRun(childId) : null;
       // The child row exists before the step parks, so a missing one was deleted.

package/src/workflows/retry-poller.ts

@@ -6,6 +6,7 @@ import {
   getWorkflowRunStepsByRunId,
   updateWorkflowRun,
 } from "../be/db";
+import { disarmWorkflowTreeRetries, getRunLineageBlock } from "../be/db/workflow-runs";
 import type { RetryPolicy } from "../types";
 import { checkpointStep, checkpointStepFailure, checkpointStepWaiting } from "./checkpoint";
 import { getSuccessors, resolveValidationPort } from "./definition";
@@ -17,7 +18,8 @@ import {
 } from "./engine";
 import type { AsyncExecutorResult } from "./executors/base";
 import type { ExecutorRegistry } from "./executors/registry";
-import { FOREACH_ITEMS_WAIT_FOR, parkAndDriveForeach } from "./foreach-driver";
+import { FOREACH_ITEMS_WAIT_FOR, parkAndDriveForeach, runForeachDrive } from "./foreach-driver";
+import { resolveForeachParent } from "./foreach-join";
 import { rebuildNodeExecutionInput } from "./step-input";
 import { runStepValidation } from "./validation";
 
@@ -45,6 +47,19 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
           const workflow = await getWorkflow(run.workflowId);
           if (!workflow) continue;
 
+          // A foreach child goes back to its fan-out's queue: the driver
+          // rebuilds the per-item config and starts it under the concurrency
+          // cap, and the item settles into the join like any other.
+          const foreachParent = resolveForeachParent(workflow.definition, step.nodeId);
+          if (foreachParent) {
+            if (!(await claimRetry(run.id, step.id, { requeueForeachChild: true }))) continue;
+            console.log(
+              `[workflows] Retrying foreach item ${step.nodeId} (attempt ${step.retryCount}) for run ${step.runId}`,
+            );
+            await runForeachDrive(run.id, foreachParent.id, registry);
+            continue;
+          }
+
           // Find the node definition for this step
           const node = workflow.definition.nodes.find((n) => n.id === step.nodeId);
           if (!node) continue;
@@ -264,14 +279,44 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
 /**
  * Flip a retry-pending step to `running` only while it is still retry-pending
  * and its run is live, and revive a run failed by this step.
+ *
+ * `requeueForeachChild` flips a foreach child to `pending` instead, for the
+ * driver to claim, and clears the cut's error: the join reads `error` on a
+ * completed child as a failed item, so a stale one would count the retry's
+ * success as a failure.
+ *
+ * A queued child never revives a failed run: its retry does not settle, so
+ * something else failed the run. The queued retry is retired instead, leaving
+ * the item `failed` with the cut's error for a run retry to give a new attempt.
+ *
+ * A run inside a cancelled tree is never revived: its queued retry is dropped.
  */
-async function claimRetry(runId: string, stepId: string): Promise<boolean> {
+async function claimRetry(
+  runId: string,
+  stepId: string,
+  options: { requeueForeachChild?: boolean } = {},
+): Promise<boolean> {
   return getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (!run || !RETRYABLE_RUN_STATUSES.has(run.status)) return false;
+    if (await getRunLineageBlock(runId)) {
+      await disarmWorkflowTreeRetries([runId], new Date().toISOString());
+      return false;
+    }
+    if (options.requeueForeachChild && run.status === "failed") {
+      await getDbClient().run(
+        `UPDATE workflow_run_steps SET nextRetryAt = NULL, finishedAt = ?
+          WHERE id = ? AND runId = ? AND status = 'failed' AND nextRetryAt IS NOT NULL`,
+        [new Date().toISOString(), stepId, runId],
+      );
+      return false;
+    }
+    const set = options.requeueForeachChild
+      ? "status = 'pending', nextRetryAt = NULL, error = NULL"
+      : "status = 'running', nextRetryAt = NULL";
     const claimed = await getDbClient().get<{ id: string }>(
       `UPDATE workflow_run_steps
-          SET status = 'running', nextRetryAt = NULL
+          SET ${set}
         WHERE id = ? AND runId = ? AND status = 'failed' AND nextRetryAt IS NOT NULL
         RETURNING id`,
       [stepId, runId],
@@ -293,6 +338,7 @@ async function completeRunIfSettled(runId: string, ctx: Record<string, unknown>)
   await getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (!run || (run.status !== "running" && run.status !== "waiting")) return;
+    if (await getRunLineageBlock(runId)) return;
     const steps = await getWorkflowRunStepsByRunId(runId);
     const live = steps.some(
       (s) =>

package/src/workflows/task-step-routing.ts

@@ -7,6 +7,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import { getRunLineageBlock } from "../be/db/workflow-runs";
 import {
   type RetryPolicy,
   RetryPolicySchema,
@@ -18,6 +19,7 @@ import {
 import { checkpointStep, checkpointStepFailure } from "./checkpoint";
 import { getSuccessors } from "./definition";
 import type { ExecutorRegistry } from "./executors/registry";
+import { resolveStepNode } from "./foreach-body";
 import { type ForeachSettleResult, resolveForeachParent, settleForeachChild } from "./foreach-join";
 
 export interface TaskStepRoutingResult {
@@ -146,6 +148,7 @@ export async function checkpointPortStepAndResolveSuccessors(
     if (!run || (run.status !== "waiting" && run.status !== "running")) {
       return UNCLAIMED;
     }
+    if (await getRunLineageBlock(runId)) return UNCLAIMED;
     // A foreach child's port (approval rejected/timeout, wait event/timeout)
     // is stored for display and still counts as a completed item: the body's
     // ports do not route, the item's consumer reads the output.
@@ -182,8 +185,8 @@ export async function checkpointPortStepAndResolveSuccessors(
  *   or the failed task is no longer the one bound to the step (it was
  *   already retried and a fresh task now owns it); the caller stops.
  * - `not-eligible`: no retry applies (no `node.retry` and not a supersede
- *   cut, not an agent-task node, or retries exhausted); the caller applies
- *   `onNodeFailure` as before.
+ *   cut, not an agent-task node, retries exhausted, or a foreach child whose
+ *   run is no longer live); the caller applies `onNodeFailure` as before.
  */
 export type TaskStepRetryOutcome = "scheduled" | "not-claimed" | "not-eligible";
 
@@ -217,8 +220,13 @@ export const SUPERSEDE_RETRY_POLICY: RetryPolicy = {
  * only after ownership is confirmed: a stale caller's step snapshot may carry
  * the counter of a newer attempt.
  *
- * Foreach children (`<foreachId>#<key>`) are not matched here: the retry
- * poller resolves nodes by exact id and cannot re-dispatch a child yet.
+ * A foreach child (`<foreachId>#<key>`) with an `agent-task` body takes the
+ * parent foreach node's `retry`, else the same supersede default. While queued
+ * it does not settle, so the join stays open; the retry poller re-queues it and
+ * the foreach driver starts it under the fan-out's concurrency cap. A child is
+ * queued only while its run is live. Exhausted retries, a run that is no longer
+ * live and every other failure settle the item through `onNodeFailure` and the
+ * fan-out's failure policy as before.
  */
 export async function scheduleTaskStepRetry(
   def: WorkflowDefinition,
@@ -227,11 +235,14 @@ export async function scheduleTaskStepRetry(
   taskId: string,
   reason: string,
 ): Promise<TaskStepRetryOutcome> {
-  const node = def.nodes.find((n) => n.id === step.nodeId);
+  const foreachParent = resolveForeachParent(def, step.nodeId);
+  const node = resolveStepNode(def, step.nodeId);
   if (!node || node.type !== "agent-task") return "not-eligible";
+  // A synthesized child node carries only the body; its retry lives on the parent.
+  const declaredRetry = foreachParent ? foreachParent.retry : node.retry;
   let policy: RetryPolicy;
-  if (node.retry) {
-    const parsed = RetryPolicySchema.safeParse(node.retry);
+  if (declaredRetry) {
+    const parsed = RetryPolicySchema.safeParse(declaredRetry);
     if (!parsed.success) return "not-eligible";
     policy = parsed.data;
   } else {
@@ -243,6 +254,15 @@ export async function scheduleTaskStepRetry(
     const current = await getWorkflowRunStep(step.id);
     if (!current || current.status !== "waiting") return "not-claimed";
     if (!(await isTaskBoundToStep(taskId, step.id))) return "not-claimed";
+    // A sibling's fail-fast or another branch can fail the run while the
+    // caller awaited. A child queued then would revive the failed run, so the
+    // caller records the cut as the item's late outcome instead.
+    if (foreachParent) {
+      const run = await getWorkflowRun(runId);
+      if (run?.status !== "running" && run?.status !== "waiting") return "not-eligible";
+    }
+    // No retry is queued inside a cancelled tree.
+    if (await getRunLineageBlock(runId)) return "not-claimed";
     if (current.retryCount >= policy.maxRetries) return "not-eligible";
     await detachTaskFromWorkflowRunStep(taskId);
     await checkpointStepFailure(runId, step.id, reason, current.retryCount, policy);
@@ -280,6 +300,8 @@ export async function completeTaskStepAndResolveSuccessors(
     if (!current || current.status !== "waiting") return UNCLAIMED;
     // A stale snapshot of a step since redriven to a fresh task must not route it.
     if (ownerTaskId && !(await isTaskBoundToStep(ownerTaskId, step.id))) return UNCLAIMED;
+    // A late result inside a cancelled tree routes nothing and revives nothing.
+    if (await getRunLineageBlock(runId)) return UNCLAIMED;
 
     if (resolveForeachParent(def, step.nodeId)) {
       // onNodeFailure:"continue" completions persist the failure reason as

package/templates/ai-toolbox.manifest.json

@@ -1,5 +1,5 @@
 {
-  "commit": "77abe63716623f28177ebe29679c37b5859ef40e",
+  "commit": "c81f2bb62f49b45603e9e702558e1ca22e98fcdd",
   "excludedSkills": [
     "feedback"
   ],
@@ -16,8 +16,8 @@
     "templates/skills/comms/files/references/ste-rules.md": "6ee2ed5d6b49d2e421b6c9d22310a5b589fa1abe81de06213e4b89fbc1271e74",
     "templates/skills/comms/files/references/visual-shapes.md": "5e07e9250ad4cdc877af2281525185f0de89c01f90a077463a2c77c8a7b998e8",
     "templates/skills/delegate-work/config.json": "5b1542eb0be359bb8a17117fcf87917e0607d1b939ae3266ddffacdc14efb281",
-    "templates/skills/delegate-work/content.md": "c9562bd17846d3595d40172ecfdc6f85bb99eaf23597eb9d5aa9635d5c14d079",
-    "templates/skills/delegate-work/files/scripts/codex-exec.sh": "be9b33ba111afc8b38c20a4190eae19099025474d4ed3278d944ee0e6b6621af",
+    "templates/skills/delegate-work/content.md": "1046da2fbc5603c4deed92beb6d98c443df21e87cf83a7d7b3ce58dbc9d01997",
+    "templates/skills/delegate-work/files/scripts/codex-exec.sh": "4524a224fe0657d2e2872d97a70357c4a3ce84d2241bd935ed83febf1fac6e37",
     "templates/skills/design-docs/config.json": "b78141763541486d67a105e8fa23873c9ffee6a682832c4b7266b8320d55ead7",
     "templates/skills/design-docs/content.md": "fc860031defbd192c1b8e1c024b46509d78dcf10dbf1b8c93cc4d62396153a52",
     "templates/skills/design-docs/files/template.md": "72d2cdcf8713c08d525d69892c5fed536107594522e3e799ae44f51df7015a36",
@@ -38,13 +38,13 @@
     "templates/skills/planning/content.md": "543476e4dbb5c352acd686c3929c45cc992d7d033d06f7e9a30f78009b50785c",
     "templates/skills/planning/files/template.md": "c0d5ca63c07934546172322bbd1180e2901dc45e27b5d12ce3dc5c30f5e7a98b",
     "templates/skills/qa/config.json": "c53b01558a691c0343b6270a548e6172af86600a429b789eedb518849f7ea741",
-    "templates/skills/qa/content.md": "c885095ea814b4c410dbc7aac1e2fde09b94b735d818b6f06153901311a232a0",
+    "templates/skills/qa/content.md": "46b1147d8af1a292d9bcd7091385d2c98264f10bb5afe4fb61073113eb0a6574",
     "templates/skills/qa/files/template.md": "1902e4bb65b19ab58c54ab5b63693ad09adecee69079c2c70f58d7d6c6597896",
     "templates/skills/questioning/config.json": "da2d5fe6dfdac1915a0476a7063e62ebeb996b84caff07cc7527a55477656a80",
     "templates/skills/questioning/content.md": "66f826a191e96d0041adca00a7c197a458de8e6fe164c1599b8708098d4109ab",
     "templates/skills/questioning/files/template.md": "a68d72801a0b10380bf773b07e9d2f8d4c5b4b1c609ce954289e38ffc78cc223",
     "templates/skills/researching/config.json": "921c062cab599097cc7677d2dad4797ec4d5844d2ee73607fbd417ec3fa487d4",
-    "templates/skills/researching/content.md": "0e8f98a7a5c75059fba6cd2c908b8268efce13192c1220ba594f2a9e3bf1e60a",
+    "templates/skills/researching/content.md": "87ced42d15f1fd63d1b4e8d5ed8161537202f5b05a85df38dea273e9e3d6ad08",
     "templates/skills/researching/files/template.md": "411d34b98c610ef715edf0f24d09f828ba2d0f65f688e7879ef33a9453007949",
     "templates/skills/reviewing/config.json": "09c40bb098e9738c60a848995fa9fe7568d158631a089c76ba02c0dd955445c7",
     "templates/skills/reviewing/content.md": "1bbc285c9ad8cc3312f1ee2c558b04ffee914ffb6f5c63fb3741fcbb3e300fa5",
@@ -130,13 +130,13 @@
       },
       {
         "from": "`${CLAUDE_PLUGIN_ROOT}/skills/delegate-work/scripts/codex-exec.sh`",
-        "line": 66,
+        "line": 70,
         "skill": "delegate-work",
         "to": "`bash ~/.claude/skills/delegate-work/scripts/codex-exec.sh`"
       },
       {
         "from": "| codex-exec.sh -m",
-        "line": 69,
+        "line": 73,
         "skill": "delegate-work",
         "to": "| bash ~/.claude/skills/delegate-work/scripts/codex-exec.sh -m"
       },
@@ -178,7 +178,7 @@
       },
       {
         "from": "cc-plugin/base/skills/researching/template.md",
-        "line": 113,
+        "line": 115,
         "skill": "researching",
         "to": "template.md"
       },