@desplega.ai/agent-swarm 2.5.2 → 2.5.4

package/src/workflows/resume.ts

@@ -1,6 +1,8 @@
 import {
+  type ApprovalRequest,
   cancelPendingApprovalRequestsForRun,
   cancelTask,
+  getApprovalRequestById,
   getCompletedStepNodeIds,
   getDbClient,
   getPendingEventWaitNames,
@@ -11,19 +13,33 @@ import {
   getWorkflowRun,
   getWorkflowRunStep,
   getWorkflowRunStepsByRunId,
-  listCancelledApprovalRequestsForRun,
   listCancelledApprovalRequestsForStep,
   resolveWaitState,
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
-import { createForeachAttemptSteps, getChildWorkflowRunId } from "../be/db/workflow-runs";
-import type {
-  WaitStateRow,
-  WorkflowDefinition,
-  WorkflowNode,
-  WorkflowRun,
-  WorkflowRunStep,
+import { listUndeliveredCancelledApprovalRequestIds } from "../be/db/approvals";
+import {
+  cancelLiveWorkflowTreeRuns,
+  cancelOpenWorkflowTreeSteps,
+  createForeachAttemptSteps,
+  disarmWorkflowTreeRetries,
+  getChildWorkflowRunId,
+  getOpenWorkflowTreeTaskIds,
+  getRunLineageBlock,
+  getTaskIdsWithDependents,
+  getWorkflowRunStatus,
+  getWorkflowRunTreeIds,
+  workflowRunTreeHasOpenWork,
+} from "../be/db/workflow-runs";
+import {
+  type RetryPolicy,
+  RetryPolicySchema,
+  type WaitStateRow,
+  type WorkflowDefinition,
+  type WorkflowNode,
+  type WorkflowRun,
+  type WorkflowRunStep,
 } from "../types";
 import { scrubSecrets } from "../utils/secret-scrubber";
 import {
@@ -31,6 +47,7 @@ import {
   postApprovalCancellationUpdates,
 } from "./approval-notifications";
 import { shapeApprovalResolution } from "./approval-resolution";
+import { checkpointStepFailure } from "./checkpoint";
 import { loadCompletedStepRouting } from "./completed-step-routing";
 import { FAILED_TASK_OUTPUT_PREFIX } from "./constants";
 import { getNextTargets } from "./definition";
@@ -45,6 +62,7 @@ import { startForeachDrive } from "./foreach-driver";
 import {
   FOREACH_TERMINAL_STEP_STATUSES,
   type ForeachChildOutcome,
+  isRetryPending,
   parseSyntheticNodeId,
   resolveForeachParent,
   settleForeachChild,
@@ -275,12 +293,16 @@ export async function finalizeOrWait(runId: string): Promise<void> {
   // move a step into `waiting` between the read and the write, and a run
   // finalized from a stale snapshot strands that branch.
   await getDbClient().transaction(async () => {
+    // A cancel (or another finalizer) that committed after the caller's claim
+    // wins: a terminal run is never re-parked or completed from here.
+    const status = await getWorkflowRunStatus(runId);
+    if (status !== "running" && status !== "waiting") return;
+    if (await getRunLineageBlock(runId)) return;
     const steps = await getWorkflowRunStepsByRunId(runId);
     // A branch still executing belongs to a live walk, which finalizes the
     // run itself; its finalizer only acts on a `running` run.
     if (hasRunningStep(steps)) {
-      const run = await getWorkflowRun(runId);
-      if (run?.status === "waiting") await updateWorkflowRun(runId, { status: "running" });
+      if (status === "waiting") await updateWorkflowRun(runId, { status: "running" });
       return;
     }
     // A step queued for the retry poller is still live, like a waiting one.
@@ -518,6 +540,10 @@ export async function retryFailedRun(runId: string, registry: ExecutorRegistry):
   const attempts = await getDbClient().transaction(async () => {
     const current = await getWorkflowRun(runId);
     if (!current || current.status !== "failed") return null;
+    // A failed run inside a cancelled tree keeps its failure; the cancel
+    // forbids starting it again.
+    const blocked = await getRunLineageBlock(runId);
+    if (blocked) throw new Error(`Retry not started, run left failed: ${blocked}`);
     // Fan-outs with failed or cancelled items get one new attempt row per such
     // item; completed items are never rerun. Selected from the rows as of this
     // transaction: an in-flight sibling of a fail-fast fan-out can record its
@@ -528,6 +554,13 @@ export async function retryFailedRun(runId: string, registry: ExecutorRegistry):
     ]);
     const capError = foreachRetryCapError(currentSteps.length, attempts);
     if (capError) throw new Error(`Retry not started, run left failed: ${capError}`);
+    // An item retry still queued when the run failed gets a new attempt row
+    // below. Retire the queued one, or the poller starts the old row beside it.
+    for (const step of currentSteps) {
+      if (isRetryPending(step) && resolveForeachParent(workflow.definition, step.nodeId)) {
+        await updateWorkflowRunStep(step.id, { nextRetryAt: null });
+      }
+    }
     // A foreach item retries as a new attempt row instead of a reset: a fresh
     // step id is a fresh invocation identity for every body type, and the
     // failed row stays as history.
@@ -636,82 +669,162 @@ function foreachRetryCapError(
   return `foreach retry would reach ${maxSteps} total steps (WORKFLOW_MAX_STEPS_PER_RUN): ${existingSteps} existing + ${added} new attempts leaves no room for the retry walk`;
 }
 
+const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "cancelled", "skipped"]);
+
+/** What one `cancelWorkflowRunRows` call covered. */
+export interface WorkflowRunCancelResult {
+  /** True when this call moved the root run to `cancelled`. */
+  cancelled: boolean;
+  /**
+   * The root and every run below it, root first, for the caller's Slack
+   * posts. Also set for a repeat cancel that found nothing open, so posts a
+   * failed earlier cancel left undelivered are retried. Empty when the root
+   * is missing or completed, failed or skipped.
+   */
+  runIds: string[];
+}
+
 /**
- * The DB writes of a run cancel, with no Slack post. Returns false when the
- * run is missing or already terminal. Called inside a caller's transaction,
- * the writes join it (as a SAVEPOINT) and commit or roll back with it.
+ * The DB writes of a run cancel, with no Slack post. Cancels the run and its
+ * whole child tree (every attempt's child, through terminal runs) in one
+ * transaction:
+ * - every non-terminal step, its task, and the run's other open tasks;
+ * - pending approvals and queued step retries;
+ * - every non-terminal run. A terminal descendant keeps its status, result
+ *   and reason; only its leftover open work is cancelled.
+ *
+ * A missing root, or one that is completed, failed or skipped, cancels
+ * nothing. An already cancelled root is walked again, so a repeat cancel
+ * finishes descendants an earlier cancel missed. A tree with a cycle or past
+ * its bounds throws and nothing commits.
+ *
+ * Called inside a caller's transaction, the writes join it (as a SAVEPOINT)
+ * and commit or roll back with it. Task lifecycle events and worker nudges
+ * queue behind COMMIT (afterCommit) and drop on rollback.
  */
-export async function cancelWorkflowRunRows(runId: string, cancelReason: string): Promise<boolean> {
-  const terminalStatuses = ["completed", "failed", "cancelled", "skipped"];
+export async function cancelWorkflowRunRows(
+  runId: string,
+  cancelReason: string,
+): Promise<WorkflowRunCancelResult> {
   const now = new Date().toISOString();
 
-  // Step snapshot, task cancels, and both status writes commit together so a
-  // step created after the snapshot cannot survive the cancel and a
-  // concurrent cancel cannot interleave. Task lifecycle events queue behind
-  // this transaction's COMMIT (afterCommit) and drop on rollback.
-  return await getDbClient().transaction(async () => {
-    const current = await getWorkflowRun(runId);
-    if (!current || terminalStatuses.includes(current.status)) return false;
-
-    // Cancel non-terminal steps and their associated tasks
-    const steps = await getWorkflowRunStepsByRunId(runId);
-    for (const step of steps) {
-      if (terminalStatuses.includes(step.status)) continue;
-
-      // Cancel any task linked to this step
-      const task = await getTaskByWorkflowRunStepId(step.id);
-      if (task) {
-        await cancelTask(task.id, cancelReason);
-      }
+  // A repeat cancel of a tree with nothing left open is answered by reads
+  // alone, without the write lock. The guards keep a cancelled tree from
+  // gaining new work, so the answer still holds when this returns.
+  // Its run ids still go back, so the caller retries Slack posts an earlier
+  // cancel could not deliver.
+  const settledTree = await getDbClient().transaction(
+    async () => {
+      if ((await getWorkflowRunStatus(runId)) !== "cancelled") return null;
+      const ids = await getWorkflowRunTreeIds(runId);
+      return (await workflowRunTreeHasOpenWork(ids)) ? null : ids;
+    },
+    { readOnly: true },
+  );
+  if (settledTree) return { cancelled: false, runIds: settledTree };
 
-      await updateWorkflowRunStep(step.id, {
-        status: "cancelled",
-        error: cancelReason,
-        finishedAt: now,
-      });
+  // The tree snapshot and every write commit together. A child run, task or
+  // step that commits first is in the snapshot; one that commits after sees
+  // the cancelled rows in its own guard (getRunLineageBlock and the step
+  // claims) and is refused.
+  return await getDbClient().transaction(async () => {
+    const rootStatus = await getWorkflowRunStatus(runId);
+    if (!rootStatus || (TERMINAL_RUN_STATUSES.has(rootStatus) && rootStatus !== "cancelled")) {
+      return { cancelled: false, runIds: [] };
     }
 
-    await cancelPendingApprovalRequestsForRun(runId, cancelReason);
-
-    // Mark the run itself as cancelled
-    await updateWorkflowRun(runId, {
-      status: "cancelled",
-      error: cancelReason,
-      finishedAt: now,
-    });
-    return true;
+    const runIds = await getWorkflowRunTreeIds(runId);
+    // Every step task carries the run link (indexed), and a retry detaches
+    // only the step link, so this covers current and earlier attempts' tasks.
+    // Dependents are looked up once for the whole tree, and only when needed.
+    const openTaskIds = await getOpenWorkflowTreeTaskIds(runIds);
+    const withDependents =
+      openTaskIds.length > 0 ? await getTaskIdsWithDependents() : new Set<string>();
+    for (const taskId of openTaskIds) {
+      await cancelTask(taskId, cancelReason, { cascadeDependents: withDependents.has(taskId) });
+    }
+    await cancelOpenWorkflowTreeSteps(runIds, cancelReason, now);
+    await disarmWorkflowTreeRetries(runIds, now);
+    for (const id of runIds) await cancelPendingApprovalRequestsForRun(id, cancelReason);
+    // Descendants' parent steps were just cancelled, so only the root's own
+    // parent (when the root is a child) needs the child-finished wake-up.
+    await cancelLiveWorkflowTreeRuns(runIds.slice(1), cancelReason, now);
+    if (rootStatus !== "cancelled") {
+      await updateWorkflowRun(runId, { status: "cancelled", error: cancelReason, finishedAt: now });
+    }
+    return { cancelled: rootStatus !== "cancelled", runIds };
   });
 }
 
+// Cancels in flight in this process, by run id. The API process owns the
+// database, so this bounds concurrent walks of one tree.
+const inFlightCancels = new Map<string, Promise<void>>();
+
 /**
- * Cancel a workflow run and all its non-terminal steps.
- * Also cancels any in-progress tasks spawned by waiting/running steps.
+ * Cancel a workflow run and its child tree (see `cancelWorkflowRunRows`), then
+ * post the Slack updates for every approval the cancel closed. The posts run
+ * after COMMIT, and a failed post is logged without undoing the cancel.
+ * Concurrent calls for the same run share one cancel and its result.
  */
-export async function cancelWorkflowRun(runId: string, reason?: string): Promise<void> {
-  const run = await getWorkflowRun(runId);
+export function cancelWorkflowRun(runId: string, reason?: string): Promise<void> {
+  const inFlight = inFlightCancels.get(runId);
+  if (inFlight) return inFlight;
+  const cancel = cancelWorkflowRunOnce(runId, reason).finally(() => {
+    inFlightCancels.delete(runId);
+  });
+  inFlightCancels.set(runId, cancel);
+  return cancel;
+}
+
+async function cancelWorkflowRunOnce(runId: string, reason?: string): Promise<void> {
+  const run = await getWorkflowRun(runId, "display");
   if (!run) throw new Error("Workflow run not found");
 
-  const terminalStatuses = ["completed", "failed", "cancelled", "skipped"];
-  if (terminalStatuses.includes(run.status) && run.status !== "cancelled") {
+  if (TERMINAL_RUN_STATUSES.has(run.status) && run.status !== "cancelled") {
     throw new Error(`Cannot cancel run in '${run.status}' state`);
   }
 
   const cancelReason = reason ?? "Cancelled by user";
-  const applied = await cancelWorkflowRunRows(runId, cancelReason);
+  const { cancelled, runIds } = await cancelWorkflowRunRows(runId, cancelReason);
 
-  // The migration trigger also catches direct step cancellation. Re-read after
-  // commit so its rows are included alongside the run-wide sweep.
-  const cancelledApprovals = await listCancelledApprovalRequestsForRun(runId);
-  const latestRun = await getWorkflowRun(runId);
-  if (!applied && latestRun?.status !== "cancelled") {
+  const latestRun = await getWorkflowRun(runId, "display");
+  if (!cancelled && latestRun?.status !== "cancelled") {
     throw new Error(`Cannot cancel run in '${latestRun?.status ?? "missing"}' state`);
   }
-  await postApprovalCancellationUpdates(
-    cancelledApprovals,
-    applied ? cancelReason : (latestRun?.error ?? cancelReason),
+  await postTreeApprovalCancellations(
+    runIds,
+    cancelled ? cancelReason : (latestRun?.error ?? cancelReason),
   );
 }
 
+/**
+ * Post the cancellation update of every cancelled approval in the given runs
+ * whose Slack post is not delivered yet. Re-read after commit: the migration
+ * trigger also cancels the approvals of a directly cancelled step. Each post
+ * is claimed, and a failed post releases its claim, so the next cancel of the
+ * tree retries it and a delivered post is never repeated.
+ */
+export async function postTreeApprovalCancellations(
+  runIds: string[],
+  reason: string,
+  options: { exceptRequestId?: string } = {},
+): Promise<void> {
+  try {
+    const approvals: ApprovalRequest[] = [];
+    for (const id of await listUndeliveredCancelledApprovalRequestIds(runIds)) {
+      if (id === options.exceptRequestId) continue;
+      const approval = await getApprovalRequestById(id);
+      if (approval) approvals.push(approval);
+    }
+    if (approvals.length > 0) await postApprovalCancellationUpdates(approvals, reason);
+  } catch (err) {
+    console.error(
+      "[workflows] approval cancellation updates not posted:",
+      scrubSecrets(err instanceof Error ? err.message : String(err)),
+    );
+  }
+}
+
 /** Cancel one HITL step and close any approval request it gates. */
 export async function cancelWorkflowRunStep(
   stepId: string,
@@ -963,10 +1076,11 @@ async function resumeClaimedWait(
 }
 
 /**
- * Resume a waiting `sub-workflow` step once its child run is terminal: a
- * completed child completes the step, any other end (or a deleted child) fails
- * the step and its run. Returns true when this call claimed the step. Safe to call repeatedly:
- * the live event and the recovery sweep both route here.
+ * Resume a waiting `sub-workflow` step once its current attempt's child run is
+ * terminal: a completed child completes the step, any other end (or a deleted
+ * child) fails the step and its run. Returns true when this call claimed the
+ * step. Safe to call repeatedly: the live event and the recovery sweep both
+ * route here, and only the call that claims the step walks its successors.
  */
 export async function resumeFromChildRun(
   parentStepId: string,
@@ -974,13 +1088,6 @@ export async function resumeFromChildRun(
 ): Promise<boolean> {
   const step = await getWorkflowRunStep(parentStepId);
   if (!step || step.status !== "waiting" || step.nodeType !== "sub-workflow") return false;
-  const childId = await getChildWorkflowRunId(parentStepId);
-  const child = childId ? await getWorkflowRun(childId) : null;
-  // The child row exists before the step parks, so a missing one was deleted.
-  const outcome = child
-    ? childRunOutcome(child)
-    : { error: `Child workflow run of step ${parentStepId} was deleted` };
-  if (!outcome) return false;
 
   let claimed = false;
   await failClosedOnUnreadableReplay(step.runId, step.id, async () => {
@@ -990,10 +1097,34 @@ export async function resumeFromChildRun(
     if (!workflow) return;
 
     const ctx = (run.context ?? {}) as Record<string, unknown>;
-    // A foreach item's failure settles the item: under `onNodeFailure:
-    // "continue"` it can close the join, so it routes like a success.
-    const routing =
-      "error" in outcome
+    // The step's attempt, that attempt's child and the claim share one
+    // transaction: a retry that re-parks the step under a newer attempt in
+    // between must not be settled by the previous attempt's child.
+    const routing = await getDbClient().transaction(async () => {
+      const current = await getWorkflowRunStep(step.id);
+      if (!current || current.status !== "waiting") return null;
+      // Inside a cancelled tree a child's end settles nothing and queues no retry.
+      if (await getRunLineageBlock(run.id)) return null;
+      const childId = await getChildWorkflowRunId(step.id, current.retryCount);
+      const child = childId ? await getWorkflowRun(childId) : null;
+      // The child row exists before the step parks, so a missing one was deleted.
+      const outcome = child
+        ? childRunOutcome(child)
+        : { error: `Child workflow run of step ${parentStepId} was deleted`, retryable: false };
+      if (!outcome) return null;
+      // A retryable end (a failed child) is this attempt failing: the node's
+      // retry policy applies exactly as on the executor path. The poller
+      // re-executes the step under the next attempt, which starts a new child.
+      if ("error" in outcome && outcome.retryable) {
+        const policy = childStepRetryPolicy(workflow.definition, step.nodeId);
+        if (policy && current.retryCount < policy.maxRetries) {
+          await checkpointStepFailure(run.id, step.id, outcome.error, current.retryCount, policy);
+          return CHILD_RETRY_SCHEDULED;
+        }
+      }
+      // A foreach item's failure settles the item: under `onNodeFailure:
+      // "continue"` it can close the join, so it routes like a success.
+      return "error" in outcome
         ? await failStepAndRunIfWaiting(
             workflow.definition,
             step.id,
@@ -1012,6 +1143,12 @@ export async function resumeFromChildRun(
             ctx,
             registry,
           );
+    });
+    if (!routing) return;
+    if (routing === CHILD_RETRY_SCHEDULED) {
+      claimed = true;
+      return;
+    }
     if (!routing.claimed || routing.runFailed) {
       claimed = routing.claimed;
       return;
@@ -1036,6 +1173,20 @@ export async function resumeFromChildRun(
   return claimed;
 }
 
+const CHILD_RETRY_SCHEDULED = {
+  claimed: true,
+  runFailed: false,
+  successors: [] as WorkflowNode[],
+} as const;
+
+/** The `node.retry` of a top-level `sub-workflow` node; foreach items retry through their parent. */
+function childStepRetryPolicy(def: WorkflowDefinition, nodeId: string): RetryPolicy | null {
+  const node = def.nodes.find((candidate) => candidate.id === nodeId);
+  if (!node?.retry) return null;
+  const parsed = RetryPolicySchema.safeParse(node.retry);
+  return parsed.success ? parsed.data : null;
+}
+
 // ─── 64KB firedPayload cap ──────────────────────────────────────────────────
 
 const FIRED_PAYLOAD_BYTE_CAP = 64 * 1024; // 64KB

package/src/workflows/retry-poller.ts

@@ -6,6 +6,7 @@ import {
   getWorkflowRunStepsByRunId,
   updateWorkflowRun,
 } from "../be/db";
+import { disarmWorkflowTreeRetries, getRunLineageBlock } from "../be/db/workflow-runs";
 import type { RetryPolicy } from "../types";
 import { checkpointStep, checkpointStepFailure, checkpointStepWaiting } from "./checkpoint";
 import { getSuccessors, resolveValidationPort } from "./definition";
@@ -17,7 +18,8 @@ import {
 } from "./engine";
 import type { AsyncExecutorResult } from "./executors/base";
 import type { ExecutorRegistry } from "./executors/registry";
-import { FOREACH_ITEMS_WAIT_FOR, parkAndDriveForeach } from "./foreach-driver";
+import { FOREACH_ITEMS_WAIT_FOR, parkAndDriveForeach, runForeachDrive } from "./foreach-driver";
+import { resolveForeachParent } from "./foreach-join";
 import { rebuildNodeExecutionInput } from "./step-input";
 import { runStepValidation } from "./validation";
 
@@ -45,6 +47,19 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
           const workflow = await getWorkflow(run.workflowId);
           if (!workflow) continue;
 
+          // A foreach child goes back to its fan-out's queue: the driver
+          // rebuilds the per-item config and starts it under the concurrency
+          // cap, and the item settles into the join like any other.
+          const foreachParent = resolveForeachParent(workflow.definition, step.nodeId);
+          if (foreachParent) {
+            if (!(await claimRetry(run.id, step.id, { requeueForeachChild: true }))) continue;
+            console.log(
+              `[workflows] Retrying foreach item ${step.nodeId} (attempt ${step.retryCount}) for run ${step.runId}`,
+            );
+            await runForeachDrive(run.id, foreachParent.id, registry);
+            continue;
+          }
+
           // Find the node definition for this step
           const node = workflow.definition.nodes.find((n) => n.id === step.nodeId);
           if (!node) continue;
@@ -61,6 +76,9 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
           // the run were read outside it: a cancel committing in between must
           // win, or the step re-executes inside a cancelled run.
           if (!(await claimRetry(run.id, step.id))) continue;
+          // Every result write below is fenced on this claim, so a cancel or
+          // another settle that commits while the executor runs wins.
+          const fence = { retryCount: step.retryCount };
 
           // Rebuild `run.id`, upstream outputs and the node's declared-inputs
           // context the way the original execution saw them. Without the alias
@@ -78,6 +96,8 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
               step.id,
               scriptBodyInterpolationError(node.id, scriptBodyUnresolved),
               step.retryCount,
+              undefined,
+              { fence },
             );
             continue;
           }
@@ -104,14 +124,17 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
             if (result.status === "failed") {
               // Re-failed — use the EXISTING retryCount from the step.
               // checkpointStepFailure handles marking run as failed if no retries left,
-              // or setting nextRetryAt for the next poll cycle.
-              const retryPolicy = node.retry || executor.retryPolicy;
+              // or setting nextRetryAt for the next poll cycle. A non-retryable
+              // failure skips the policy, as in executeStep.
+              const retryPolicy =
+                result.retryable === false ? undefined : node.retry || executor.retryPolicy;
               await checkpointStepFailure(
                 run.id,
                 step.id,
                 result.error || "Retry failed",
                 step.retryCount,
                 retryPolicy,
+                { fence, output: result.output },
               );
             } else if (
               "async" in result &&
@@ -150,6 +173,12 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
               // find the parent already advanced.
               const waiting = await checkpointStepWaiting(run.id, step.id, ctx);
               if (!waiting) continue;
+              // As in executeStep: a child run that finished before the step
+              // parked had its event dropped.
+              if ((result as AsyncExecutorResult).waitFor === "workflow.child.finished") {
+                const { resumeFromChildRun } = await import("./resume");
+                await resumeFromChildRun(step.id, registry);
+              }
             } else {
               // Success! Re-run validation if configured before checkpointing.
               let validationPassed: boolean | undefined;
@@ -168,6 +197,8 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
                     step.id,
                     "Validation failed (mustPass)",
                     step.retryCount,
+                    undefined,
+                    { fence },
                   );
                   continue;
                 }
@@ -186,6 +217,7 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
                     "Validation failed, retrying",
                     step.retryCount,
                     retryPolicy,
+                    { fence },
                   );
                   continue;
                 }
@@ -197,7 +229,9 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
               result.nextPort = resolveValidationPort(node.next, result.nextPort, validationPassed);
 
               // Validation passed (or no validation) — checkpoint and continue
-              await checkpointStep(run.id, step.id, step.nodeId, result, ctx);
+              if (!(await checkpointStep(run.id, step.id, step.nodeId, result, ctx, { fence }))) {
+                continue;
+              }
 
               const port = result.nextPort || "default";
               const successors = getSuccessors(workflow.definition, step.nodeId, port);
@@ -219,7 +253,9 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
             if (await failRunOnUnreadableReplay(run.id, err, step.id)) continue;
             const errorMsg = err instanceof Error ? err.message : String(err);
             const retryPolicy = node.retry || executor.retryPolicy;
-            await checkpointStepFailure(run.id, step.id, errorMsg, step.retryCount, retryPolicy);
+            await checkpointStepFailure(run.id, step.id, errorMsg, step.retryCount, retryPolicy, {
+              fence,
+            });
           }
         } catch (err) {
           if (await failRunOnUnreadableReplay(step.runId, err, step.id)) continue;
@@ -243,14 +279,44 @@ export function startRetryPoller(registry: ExecutorRegistry, intervalMs = 5000):
 /**
  * Flip a retry-pending step to `running` only while it is still retry-pending
  * and its run is live, and revive a run failed by this step.
+ *
+ * `requeueForeachChild` flips a foreach child to `pending` instead, for the
+ * driver to claim, and clears the cut's error: the join reads `error` on a
+ * completed child as a failed item, so a stale one would count the retry's
+ * success as a failure.
+ *
+ * A queued child never revives a failed run: its retry does not settle, so
+ * something else failed the run. The queued retry is retired instead, leaving
+ * the item `failed` with the cut's error for a run retry to give a new attempt.
+ *
+ * A run inside a cancelled tree is never revived: its queued retry is dropped.
  */
-async function claimRetry(runId: string, stepId: string): Promise<boolean> {
+async function claimRetry(
+  runId: string,
+  stepId: string,
+  options: { requeueForeachChild?: boolean } = {},
+): Promise<boolean> {
   return getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (!run || !RETRYABLE_RUN_STATUSES.has(run.status)) return false;
+    if (await getRunLineageBlock(runId)) {
+      await disarmWorkflowTreeRetries([runId], new Date().toISOString());
+      return false;
+    }
+    if (options.requeueForeachChild && run.status === "failed") {
+      await getDbClient().run(
+        `UPDATE workflow_run_steps SET nextRetryAt = NULL, finishedAt = ?
+          WHERE id = ? AND runId = ? AND status = 'failed' AND nextRetryAt IS NOT NULL`,
+        [new Date().toISOString(), stepId, runId],
+      );
+      return false;
+    }
+    const set = options.requeueForeachChild
+      ? "status = 'pending', nextRetryAt = NULL, error = NULL"
+      : "status = 'running', nextRetryAt = NULL";
     const claimed = await getDbClient().get<{ id: string }>(
       `UPDATE workflow_run_steps
-          SET status = 'running', nextRetryAt = NULL
+          SET ${set}
         WHERE id = ? AND runId = ? AND status = 'failed' AND nextRetryAt IS NOT NULL
         RETURNING id`,
       [stepId, runId],
@@ -272,6 +338,7 @@ async function completeRunIfSettled(runId: string, ctx: Record<string, unknown>)
   await getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (!run || (run.status !== "running" && run.status !== "waiting")) return;
+    if (await getRunLineageBlock(runId)) return;
     const steps = await getWorkflowRunStepsByRunId(runId);
     const live = steps.some(
       (s) =>