@desplega.ai/agent-swarm 2.5.3 → 2.6.0

package/src/utils/provider-key-probes.ts

@@ -0,0 +1,328 @@
+/** Shared credential probes. This module does not import worker harnesses. */
+import {
+  type OpenAICompatibleProvider,
+  providerBaseUrl,
+  providerForKeyVar,
+} from "@desplega/model-catalog";
+import {
+  createScopedFetch,
+  deriveDefaultRoute,
+  getRouteProvider,
+  type ModelRoute,
+} from "@desplega/model-routing";
+import { assertUrlSafe, publicEndpointSsrfOptions } from "../oauth/mcp-wrapper";
+import { getOpenRouterAttributionHeaders, getOpenRouterBaseUrl } from "./openrouter-base-url";
+import { scrubSecrets } from "./secret-scrubber";
+
+interface ProbeOptions {
+  env?: Record<string, string | undefined>;
+  publicEndpoint?: boolean;
+}
+
+const LIVE_TEST_TIMEOUT_MS = 5_000;
+
+function scrubProbeBody(body: string, init: RequestInit): string {
+  const headers = new Headers(init.headers);
+  const apiKey =
+    headers.get("x-api-key") ?? headers.get("authorization")?.replace(/^Bearer\s+/i, "");
+  return scrubSecrets(apiKey ? body.replaceAll(apiKey, "[REDACTED]") : body);
+}
+
+export interface LiveKeyResult {
+  keyType: string;
+  index: number;
+  suffix: string;
+  ok: boolean;
+  error?: string;
+  httpStatus?: number;
+  latency_ms: number;
+}
+
+export interface LiveValidationResult {
+  ok: boolean;
+  error?: string;
+  httpStatus?: number;
+  warning?: string;
+  keys?: LiveKeyResult[];
+  latency_ms: number;
+  /** No free live check exists for this credential (cloud routes): report presence only. */
+  skipped?: boolean;
+}
+
+export async function timedFetch(
+  url: string,
+  init: RequestInit,
+  options: ProbeOptions = {},
+): Promise<{ ok: boolean; status: number; bodyText: string; latency_ms: number }> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), LIVE_TEST_TIMEOUT_MS);
+  const start = Date.now();
+  try {
+    if (options.publicEndpoint) assertUrlSafe(url, publicEndpointSsrfOptions());
+    const res = await fetch(url, {
+      ...init,
+      ...(options.publicEndpoint ? { redirect: "manual" } : {}),
+      signal: controller.signal,
+    });
+    let bodyText = "";
+    try {
+      bodyText = scrubProbeBody(await res.text(), init);
+    } catch {
+      // The HTTP status is sufficient when the response body cannot be read.
+    }
+    return {
+      ok: res.ok,
+      status: res.status,
+      bodyText,
+      latency_ms: Date.now() - start,
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function checkAnthropicApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.anthropic.com/v1/models",
+    {
+      method: "GET",
+      headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01" },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+export async function checkOpenAiApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.openai.com/v1/models",
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/** `GET /v1/models` is the call the Grok CLI makes first; it runs no inference. */
+export async function checkXaiApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.x.ai/v1/models",
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/** Fetch a key-authenticated GET and normalize it into a `LiveValidationResult`. */
+async function checkBearerGet(
+  url: string,
+  init: RequestInit,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(url, { method: "GET", ...init }, options);
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/**
+ * OpenRouter's `/models` is public and answers 200 for any key, so the check
+ * reads `GET /key` (the key's own info, 401 for a bad key). A custom
+ * `OPENROUTER_BASE_URL` without `/key` (404/405) falls back to `/models`.
+ */
+export async function checkOpenRouter(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const base = getOpenRouterBaseUrl(options.env);
+  const get = (url: string) =>
+    checkBearerGet(
+      url,
+      {
+        headers: {
+          ...getOpenRouterAttributionHeaders(url, options.env),
+          Authorization: `Bearer ${apiKey}`,
+        },
+      },
+      options,
+    );
+  const keyResult = await get(`${base}/key`);
+  if (keyResult.httpStatus === 404 || keyResult.httpStatus === 405) return get(`${base}/models`);
+  return keyResult;
+}
+
+/** `GET /models` on the official DeepSeek API: lists models, runs no inference. */
+export function checkDeepSeekApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  return checkBearerGet(
+    "https://api.deepseek.com/models",
+    {
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+}
+
+/**
+ * `GET <baseUrl>/models` with a Bearer key against an OpenAI-compatible
+ * registry provider (NaN, Helmcode). Lists models, runs no inference.
+ * `baseUrl` defaults to the registry entry; the probe passes the
+ * `<ID>_BASE_URL` override, and tests inject a fake server.
+ */
+export async function checkOpenAICompatibleProvider(
+  provider: OpenAICompatibleProvider,
+  apiKey: string,
+  baseUrl: string = provider.baseUrl,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    `${baseUrl.replace(/\/+$/, "")}/models`,
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/**
+ * Run the route provider's own check. Requests stay on the route's origin and
+ * path prefix, go to public hosts only, and never follow a redirect.
+ */
+async function checkRoute(
+  route: ModelRoute,
+  env: Record<string, string | undefined>,
+): Promise<LiveValidationResult> {
+  const startedAt = Date.now();
+  const provider = getRouteProvider(route.provider);
+  const baseUrl = route.baseUrl ?? provider?.defaultBaseUrl;
+  if (!provider?.validate || !baseUrl) return { ok: true, skipped: true, latency_ms: 0 };
+  const publicFetch = async (url: string | URL | Request, init?: RequestInit) => {
+    assertUrlSafe(String(url), publicEndpointSsrfOptions());
+    const res = await fetch(url, init);
+    if (res.ok || !res.body) return res;
+    return new Response(scrubProbeBody(await res.text(), init ?? {}), {
+      status: res.status,
+      statusText: res.statusText,
+      headers: res.headers,
+    });
+  };
+  const result = await provider.validate({
+    route,
+    env,
+    signal: AbortSignal.timeout(LIVE_TEST_TIMEOUT_MS),
+    fetch: createScopedFetch(baseUrl, publicFetch as typeof fetch),
+  });
+  const latency_ms = Date.now() - startedAt;
+  if (result.status === "verified") return { ok: true, latency_ms };
+  return {
+    ok: result.status === "configured",
+    ...(result.status === "configured" ? { skipped: true } : { httpStatus: result.httpStatus }),
+    error: scrubSecrets(result.reason),
+    latency_ms,
+  };
+}
+
+/** Check one newly submitted pool member without loading a worker harness. */
+export async function checkProviderPoolKey(
+  keyType: string,
+  apiKey: string,
+  env: Record<string, string | undefined> = process.env,
+): Promise<LiveValidationResult> {
+  const startedAt = Date.now();
+  const options: ProbeOptions = { env, publicEndpoint: true };
+  try {
+    switch (keyType) {
+      case "ANTHROPIC_API_KEY": {
+        // Behind a gateway (`ANTHROPIC_BASE_URL`), Claude Code sends this key
+        // to the gateway, so check it there and not at api.anthropic.com.
+        const keyEnv = { ...env, ANTHROPIC_API_KEY: apiKey };
+        const route = deriveDefaultRoute("claude", keyEnv);
+        if (
+          route?.provider === "anthropic-gateway" &&
+          "secretKey" in route.auth &&
+          route.auth.secretKey === "ANTHROPIC_API_KEY"
+        ) {
+          return await checkRoute(route, keyEnv);
+        }
+        return await checkAnthropicApiKey(apiKey, options);
+      }
+      case "OPENAI_API_KEY":
+        return await checkOpenAiApiKey(apiKey, options);
+      case "OPENROUTER_API_KEY":
+        return await checkOpenRouter(apiKey, options);
+      case "DEEPSEEK_API_KEY":
+        return await checkDeepSeekApiKey(apiKey, options);
+      case "XAI_API_KEY":
+        return await checkXaiApiKey(apiKey, options);
+      case "CLAUDE_CODE_OAUTH_TOKEN": {
+        const tokenEnv = { CLAUDE_CODE_OAUTH_TOKEN: apiKey };
+        return await checkRoute(deriveDefaultRoute("claude", tokenEnv)!, tokenEnv);
+      }
+      default: {
+        const provider = providerForKeyVar(keyType);
+        if (provider) {
+          return await checkOpenAICompatibleProvider(
+            provider,
+            apiKey,
+            providerBaseUrl(provider, env),
+            options,
+          );
+        }
+        return { ok: true, skipped: true, latency_ms: 0 };
+      }
+    }
+  } catch (err) {
+    const raw = err instanceof Error ? err.message : String(err);
+    return {
+      ok: false,
+      error: scrubSecrets(raw.split(apiKey).join("[REDACTED]")) || "Unknown error",
+      latency_ms: Date.now() - startedAt,
+    };
+  }
+}

package/src/utils/transcription.ts

@@ -20,6 +20,7 @@
  * auto-detected key always goes to its provider's own endpoint. Deepgram and AssemblyAI use other request shapes and are not
  * supported. Read at call time, so a config reload applies without a restart.
  */
+import { selectCredential } from "./credentials";
 import { createValueScrubber, scrubSecrets } from "./secret-scrubber";
 
 const TRANSCRIPTION_TIMEOUT_MS = 60_000;
@@ -119,7 +120,9 @@ export function resolveTranscriptionConfig(): TranscriptionConfig | null {
     : AUTO_DETECT_ORDER;
   for (const name of candidates) {
     const { keyEnv } = PROVIDERS[name];
-    const apiKey = envValue(keyEnv);
+    const value = envValue(keyEnv);
+    // OPENAI_API_KEY may be a comma pool: send one member.
+    const apiKey = value ? selectCredential(value, undefined, keyEnv).selected : undefined;
     if (apiKey) return { provider: name, apiKey, keyEnv, source: "auto" };
   }
   return null;

package/src/workflows/approval-cancel.ts

@@ -4,16 +4,15 @@ import {
   getApprovalRequestById,
   getDbClient,
   getTaskById,
-  getWorkflowRun,
-  listCancelledApprovalRequestsForRun,
   resolveApprovalRequest,
 } from "../be/db";
+import { getWorkflowRunStatus } from "../be/db/workflow-runs";
 import { can, type RbacPrincipal } from "../rbac";
 import {
   createApprovalFollowUpTask,
   postApprovalCancellationUpdates,
 } from "./approval-notifications";
-import { cancelWorkflowRunRows } from "./resume";
+import { cancelWorkflowRunRows, postTreeApprovalCancellations } from "./resume";
 
 export type CancelApprovalRequestResult =
   | { ok: true; request: ApprovalRequest; alreadyCancelled: boolean; runCancelled: boolean }
@@ -126,25 +125,28 @@ export async function cancelApprovalRequestAndRun(
     if (!request) return null;
 
     // The request is cancelled first so it keeps this reason; the run cancel
-    // then skips it because it is no longer pending.
+    // then skips it because it is no longer pending. The run's child tree is
+    // cancelled with it.
     let runCancelled = false;
+    let treeRunIds: string[] = [];
     const runId = request.workflowRunId;
     if (runId) {
-      const run = await getWorkflowRun(runId);
-      if (run && LIVE_RUN_STATUSES.has(run.status)) {
-        runCancelled = await cancelWorkflowRunRows(runId, data.reason);
+      const status = await getWorkflowRunStatus(runId);
+      if (status && LIVE_RUN_STATUSES.has(status)) {
+        const result = await cancelWorkflowRunRows(runId, data.reason);
+        runCancelled = result.cancelled;
+        treeRunIds = result.runIds;
       }
     }
 
     const slackReason = data.slackReason ?? data.reason;
     client.afterCommit(async () => {
       await postApprovalCancellationUpdates([request], slackReason);
-      if (runCancelled && runId) {
-        // Other requests the run cancel closed get the run's reason.
-        const others = (await listCancelledApprovalRequestsForRun(runId)).filter(
-          (row) => row.id !== request.id,
-        );
-        await postApprovalCancellationUpdates(others, data.reason);
+      if (runCancelled) {
+        // Other requests the tree cancel closed get the run's reason.
+        await postTreeApprovalCancellations(treeRunIds, data.reason, {
+          exceptRequestId: request.id,
+        });
       }
     });
     return { request, runCancelled };

package/src/workflows/approval-notifications.ts

@@ -9,6 +9,7 @@ import {
   releaseApprovalCancellationNotificationClaim,
   updateApprovalRequestNotifications,
 } from "../be/db";
+import { approvalCancellationNotificationKey } from "../be/db/approvals";
 import { resolveTemplate } from "../prompts/resolver";
 import { getSlackApp } from "../slack/app";
 import { resolveSlackConversationId } from "../slack/dm";
@@ -263,7 +264,10 @@ export async function postApprovalCancellationUpdates(
 
     for (const notification of slackThreads) {
       if (notification.channel !== "slack" || !notification.messageTs) continue;
-      const notificationKey = `${notification.target}:${notification.messageTs}`;
+      const notificationKey = approvalCancellationNotificationKey(
+        notification.target,
+        notification.messageTs,
+      );
       const claim = await claimApprovalCancellationNotification(approval.id, notificationKey);
       if (!claim) continue;
       try {

package/src/workflows/checkpoint.ts

@@ -1,12 +1,14 @@
 import { getDbClient, getWorkflowRun, updateWorkflowRun, updateWorkflowRunStep } from "../be/db";
+import { getRunLineageBlock } from "../be/db/workflow-runs";
 import type { RetryPolicy } from "../types";
 
 /**
  * The execution a checkpoint belongs to: the step attempt (`retryCount`) that
  * claimed the row. A fenced checkpoint writes only while that attempt still
  * owns the row: the step is `running` with that `retryCount` in a `running` or
- * `waiting` run. A cancel, a newer attempt or another settle that committed
- * meanwhile wins, and the stale execution writes nothing.
+ * `waiting` run with no cancelled ancestor. A cancel, a newer attempt or
+ * another settle that committed meanwhile wins, and the stale execution
+ * writes nothing.
  */
 export type StepFence = { retryCount: number };
 
@@ -19,7 +21,7 @@ async function holdsFence(runId: string, stepId: string, fence: StepFence): Prom
         AND r.status IN ('running', 'waiting')`,
     [stepId, runId, fence.retryCount],
   );
-  return row != null;
+  return row != null && (await getRunLineageBlock(runId)) == null;
 }
 
 /**
@@ -129,6 +131,7 @@ export async function checkpointStepWaiting(
   options: { keepRunStatus?: boolean } = {},
 ): Promise<boolean> {
   return getDbClient().transaction(async () => {
+    if (await getRunLineageBlock(runId)) return false;
     const step = await getDbClient().get<{ id: string }>(
       `UPDATE workflow_run_steps
          SET status = 'waiting'

package/src/workflows/engine.ts

@@ -21,7 +21,11 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
-import { getChildWorkflowRunId, isParentStepAttemptCurrent } from "../be/db/workflow-runs";
+import {
+  getChildWorkflowRunId,
+  getRunLineageBlock,
+  isParentStepAttemptCurrent,
+} from "../be/db/workflow-runs";
 import { WorkflowReplayStateError } from "../be/workflow-replay";
 import { telemetry } from "../telemetry";
 import type { Workflow, WorkflowDefinition, WorkflowNode, WorkflowRunStep } from "../types";
@@ -67,6 +71,13 @@ export interface WorkflowExecutionOptions {
   parentStepId?: string;
   /** The parent step's `retryCount`; each attempt gets its own child run. */
   parentStepAttempt?: number;
+  /**
+   * Runs inside the transaction that commits a top-level run (or reuses the
+   * day's setup-failure run), after the run row is written. A throw rolls the
+   * run back and nothing else happens: no walk, no alert, no telemetry. The
+   * HTTP trigger records its `Idempotency-Key` here.
+   */
+  accept?: (run: { runId: string; skipped: boolean }) => Promise<void>;
 }
 
 /**
@@ -111,12 +122,16 @@ export async function startWorkflowExecution(
     await getAutomationSetupStates(),
   );
   if (preflight.state === "needs_setup" && !options.parentStepId) {
-    const { runId, recorded } = await recordWorkflowPreflightFailure({
-      workflowId: workflow.id,
-      triggerType: options.triggerType ?? "manual",
-      triggerData,
-      failureReason: preflight.failureReason!,
-      createdBy: options.requestedByUserId,
+    const { runId, recorded } = await getDbClient().transaction(async () => {
+      const failure = await recordWorkflowPreflightFailure({
+        workflowId: workflow.id,
+        triggerType: options.triggerType ?? "manual",
+        triggerData,
+        failureReason: preflight.failureReason!,
+        createdBy: options.requestedByUserId,
+      });
+      await options.accept?.({ runId: failure.runId, skipped: false });
+      return failure;
     });
     if (recorded) await notifyAutomationPreflightFailure(preflight);
     return runId;
@@ -276,7 +291,8 @@ async function buildInitialRunState(
  * executing and returns the child that attempt already started instead of a
  * second one; the unique (parentStepId, parentStepAttempt) index backs this
  * across processes. A setup failure claims the day's alert in the same
- * transaction, so concurrent failures alert exactly once.
+ * transaction, so concurrent failures alert exactly once. `options.accept`
+ * runs last in it.
  */
 async function createRunWithInitialState(
   workflow: Workflow,
@@ -314,6 +330,7 @@ async function createRunWithInitialState(
       initial.status === "failed" && initial.preflight
         ? await isFirstWorkflowPreflightFailureToday(workflow.id, runId)
         : undefined;
+    await options.accept?.({ runId, skipped: initial.status === "skipped" });
     return { runId, created: true, preflightAlert };
   });
 }
@@ -606,6 +623,8 @@ async function walkGraphOwned(
   await getDbClient().transaction(async () => {
     const run = await getWorkflowRun(runId);
     if (!run || run.status !== "running") return;
+    // Inside a cancelled tree the run's outcome is the cancel, not its steps.
+    if (await getRunLineageBlock(runId)) return;
 
     const finalSteps = await getWorkflowRunStepsByRunId(runId);
     // Another walker is still executing a branch; it finalizes the run.
@@ -793,6 +812,9 @@ async function executeStep(
     if (!run || (run.status !== "running" && run.status !== "waiting")) {
       return { halted: true as const };
     }
+    // A live run under a cancelled ancestor (or with a broken parent link)
+    // starts no step: the cancel covers its whole tree.
+    if (await getRunLineageBlock(runId)) return { halted: true as const };
 
     // Another walker already started or finished this node for the same
     // predecessor completions (two branch completions both walking the join,

package/src/workflows/executors/foreach.ts

@@ -6,7 +6,7 @@ import { getSuccessors } from "../definition";
 import { foreachBodyTypeError } from "../foreach-body";
 import {
   buildForeachAggregate,
-  FOREACH_TERMINAL_STEP_STATUSES,
+  isForeachChildSettled,
   parseSyntheticNodeId,
 } from "../foreach-join";
 import { getMaxWorkflowStepsPerRun } from "../limits";
@@ -144,7 +144,7 @@ export class ForeachExecutor extends BaseExecutor<
     const childSteps = (await this.deps.db.getWorkflowRunStepsByRunId(meta.runId)).filter(
       (step) => parseSyntheticNodeId(step.nodeId)?.parentNodeId === meta.nodeId,
     );
-    if (childSteps.every((step) => FOREACH_TERMINAL_STEP_STATUSES.has(step.status))) {
+    if (childSteps.every(isForeachChildSettled)) {
       return { status: "success", output: buildForeachAggregate(childSteps) };
     }
 

package/src/workflows/executors/sub-workflow.ts

@@ -133,7 +133,14 @@ export class SubWorkflowExecutor extends BaseExecutor<
       throw err;
     }
     const child = await db.getWorkflowRun(childRunId);
-    if (!child) return { status: "failed", error: `Child workflow run ${childRunId} not found` };
+    // The start committed the row, so a missing one was deleted with its workflow.
+    if (!child) {
+      return {
+        status: "failed",
+        error: `Child workflow run ${childRunId} not found`,
+        retryable: false,
+      };
+    }
     return this.settle(child, config.workflowId);
   }
 

package/src/workflows/foreach-driver.ts

@@ -11,6 +11,7 @@ import {
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
+import { getRunLineageBlock, getWorkflowRunStatus } from "../be/db/workflow-runs";
 import type {
   Workflow,
   WorkflowDefinition,
@@ -41,6 +42,7 @@ import {
   cancelQueuedForeachChildren,
   FOREACH_TERMINAL_STEP_STATUSES,
   type ForeachChildOutcome,
+  isRetryPending,
   latestForeachChildren,
   settleForeachChild,
 } from "./foreach-join";
@@ -352,8 +354,8 @@ async function parkRunIfRunning(runId: string): Promise<void> {
 
 /**
  * Flip `pending` children to `running`, in item order, while fewer than the
- * fan-out's `concurrency` cap are in flight (`running` or `waiting`, latest
- * attempt rows only), and collect unowned `running` ones. Ownership is registered before commit, so no other
+ * fan-out's `concurrency` cap are in flight (`running`, `waiting` or queued
+ * for a retry, latest attempt rows only), and collect unowned `running` ones. Ownership is registered before commit, so no other
  * drive can see a claimed row without also seeing it owned.
  */
 async function claimChildren(
@@ -367,6 +369,10 @@ async function claimChildren(
     // so concurrent drives (a settlement's drive and the recovery sweep)
     // serialize here and never exceed the cap together.
     await getDbClient().transaction(async () => {
+      // A cancel of the run (or of an ancestor) that committed first wins.
+      const status = await getWorkflowRunStatus(runId);
+      if (status !== "running" && status !== "waiting") return;
+      if (await getRunLineageBlock(runId)) return;
       const children = latestForeachChildren(
         def,
         parentNode.id,
@@ -379,7 +385,8 @@ async function claimChildren(
         }
       }
       const inFlight = children.filter(
-        (child) => child.status === "running" || child.status === "waiting",
+        (child) =>
+          child.status === "running" || child.status === "waiting" || isRetryPending(child),
       ).length;
       let free = foreachConcurrencyCap(parentNode) - inFlight;
       for (const child of children) {