@desplega.ai/agent-swarm 2.5.4 → 2.6.0

package/src/be/db/workflow-runs.ts

@@ -1,12 +1,161 @@
-import { TERMINAL_TASK_STATUSES, type WorkflowRunStatus } from "../../types";
+import {
+  TERMINAL_TASK_STATUSES,
+  type WorkflowRun,
+  type WorkflowRunStatus,
+  type WorkflowRunStep,
+  type WorkflowRunStepSlim,
+  type WorkflowRunStepStatus,
+  type WorkflowRunSummary,
+} from "../../types";
 import { scrubSecrets } from "../../utils/secret-scrubber";
+import { normalizeDate, normalizeDateRequired } from "../date-utils";
 import { scrubJsonValue } from "../scrub-json";
+import { defineWorkflowPayload, type WorkflowPayloadView } from "../workflow-replay";
 import { getDbClient } from "./runtime";
 
 // Parent/child links between workflow runs (migration 203): a `sub-workflow`
 // step starts a child run that records the step in `parentStepId`, and the
 // step's `retryCount` at that time in `parentStepAttempt` (migration 240).
 
+export type WorkflowRunRow = {
+  id: string;
+  workflowId: string;
+  status: string;
+  triggerType: string;
+  triggerData: string | null;
+  /** Absent from list rows: see `WORKFLOW_RUN_SUMMARY_COLUMNS`. */
+  context?: string | null;
+  /** Sealed exact `context` (migration 201). Absent from list rows. */
+  context_replay?: string | null;
+  error: string | null;
+  created_by: string | null;
+  startedAt: string;
+  lastUpdatedAt: string;
+  finishedAt: string | null;
+  /** The `sub-workflow` step that started this run (migration 203). */
+  parentStepId?: string | null;
+  /** That step's attempt (`retryCount`) when it started this run (migration 240). */
+  parentStepAttempt?: number | null;
+};
+
+/**
+ * `view` picks the `context` copy: `replay` (default) is the exact sealed value
+ * the engine resumes from; `display` is the scrubbed column for API, MCP and UI
+ * responses. See `src/be/workflow-replay.ts`.
+ */
+export function rowToWorkflowRun(
+  row: WorkflowRunRow,
+  view: WorkflowPayloadView = "replay",
+): WorkflowRun {
+  const run: WorkflowRun = {
+    id: row.id,
+    workflowId: row.workflowId,
+    status: row.status as WorkflowRunStatus,
+    triggerData: row.triggerData ? JSON.parse(row.triggerData) : undefined,
+    context: undefined,
+    error: row.error ?? undefined,
+    createdBy: row.created_by ?? undefined,
+    startedAt: normalizeDateRequired(row.startedAt),
+    lastUpdatedAt: normalizeDateRequired(row.lastUpdatedAt),
+    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
+    parentStepId: row.parentStepId ?? undefined,
+    parentStepAttempt: row.parentStepId ? (row.parentStepAttempt ?? 0) : undefined,
+  };
+  return defineWorkflowPayload(run, "context", row.context, row.context_replay, view, {
+    runId: row.id,
+    target: "its context",
+  });
+}
+
+// Everything but `context`. `triggerData` stays: it is small, and stored
+// scripts read it off list rows (dedupe by trigger, failure reports). The
+// parent link and attempt stay so a list row identifies a child's invocation.
+export const WORKFLOW_RUN_SUMMARY_COLUMNS =
+  "id, workflowId, status, triggerType, triggerData, error, created_by, startedAt, lastUpdatedAt, finishedAt, parentStepId, parentStepAttempt";
+
+export type WorkflowRunStepRow = {
+  id: string;
+  runId: string;
+  nodeId: string;
+  nodeType: string;
+  status: string;
+  /** Absent from slim rows: see `WORKFLOW_RUN_STEP_SLIM_COLUMNS`. */
+  input?: string | null;
+  output: string | null;
+  /** Sealed exact `output` (migration 201). Absent from slim rows. */
+  output_replay?: string | null;
+  error: string | null;
+  startedAt: string;
+  finishedAt: string | null;
+  retryCount: number;
+  maxRetries: number;
+  nextRetryAt: string | null;
+  idempotencyKey: string | null;
+  diagnostics: string | null;
+  nextPort: string | null;
+};
+
+/** `view` picks the `output` copy, as in `rowToWorkflowRun`. */
+export function rowToWorkflowRunStep(
+  row: WorkflowRunStepRow,
+  view: WorkflowPayloadView = "replay",
+): WorkflowRunStep {
+  const step: WorkflowRunStep = {
+    id: row.id,
+    runId: row.runId,
+    nodeId: row.nodeId,
+    nodeType: row.nodeType,
+    status: row.status as WorkflowRunStepStatus,
+    input: row.input ? JSON.parse(row.input) : undefined,
+    output: undefined,
+    error: row.error ?? undefined,
+    startedAt: normalizeDateRequired(row.startedAt),
+    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
+    retryCount: row.retryCount,
+    maxRetries: row.maxRetries,
+    nextRetryAt: normalizeDate(row.nextRetryAt) ?? undefined,
+    idempotencyKey: row.idempotencyKey ?? undefined,
+    diagnostics: row.diagnostics ?? undefined,
+    nextPort: row.nextPort ?? undefined,
+  };
+  return defineWorkflowPayload(step, "output", row.output, row.output_replay, view, {
+    runId: row.runId,
+    target: `the output of step ${row.id} (node ${row.nodeId})`,
+  });
+}
+
+// Everything but `input` (a context snapshot per step, so loops grow it with
+// every iteration) and `output_replay` (the display view never opens it).
+const WORKFLOW_RUN_STEP_SLIM_COLUMNS =
+  "id, runId, nodeId, nodeType, status, output, error, startedAt, finishedAt, retryCount, maxRetries, nextRetryAt, idempotencyKey, diagnostics, nextPort";
+
+/**
+ * `getWorkflowRun` without `context`, in the display view. Backs
+ * `GET /api/workflow-runs/{id}?fields=slim`: the large column is neither read
+ * nor parsed.
+ */
+export async function getWorkflowRunSummary(id: string): Promise<WorkflowRunSummary | null> {
+  const row = await getDbClient().get<WorkflowRunRow>(
+    `SELECT ${WORKFLOW_RUN_SUMMARY_COLUMNS} FROM workflow_runs WHERE id = ?`,
+    [id],
+  );
+  return row ? rowToWorkflowRun(row, "display") : null;
+}
+
+/**
+ * `getWorkflowRunStepsByRunId` without `input`, in the display view. Same
+ * order, including the rowid tie-break for steps that share `startedAt`.
+ */
+export async function getWorkflowRunStepsSlimByRunId(
+  runId: string,
+): Promise<WorkflowRunStepSlim[]> {
+  const rows = await getDbClient().query<WorkflowRunStepRow>(
+    `SELECT ${WORKFLOW_RUN_STEP_SLIM_COLUMNS} FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC, rowid ASC`,
+    [runId],
+  );
+  return rows.map((row) => rowToWorkflowRunStep(row, "display"));
+}
+
 const TERMINAL_STATUSES = new Set<WorkflowRunStatus>([
   "completed",
   "failed",
@@ -242,6 +391,39 @@ export async function workflowRunTreeHasOpenWork(runIds: string[]): Promise<bool
   return open;
 }
 
+export type WorkflowRunTree = {
+  rootRunId: string;
+  /** Root first, in `getWorkflowRunTreeIds` order. List rows: no `context`. */
+  runs: WorkflowRun[];
+  hasOpenWork: boolean;
+};
+
+/**
+ * The run and every run below it, with whether any of them still has open
+ * work (see `workflowRunTreeHasOpenWork`). One read transaction, so `runs` and
+ * `hasOpenWork` come from the same snapshot. Null when the root is missing.
+ * Throws WorkflowRunTreeError on a cycle or a chain past the depth bound.
+ */
+export async function getWorkflowRunTree(rootRunId: string): Promise<WorkflowRunTree | null> {
+  return getDbClient().transaction(
+    async () => {
+      if ((await getWorkflowRunStatus(rootRunId)) === null) return null;
+      const ids = await getWorkflowRunTreeIds(rootRunId);
+      const byId = new Map<string, WorkflowRun>();
+      await forEachChunk(ids, async (chunk, placeholders) => {
+        const rows = await getDbClient().query<WorkflowRunRow>(
+          `SELECT ${WORKFLOW_RUN_SUMMARY_COLUMNS} FROM workflow_runs WHERE id IN (${placeholders})`,
+          chunk,
+        );
+        for (const row of rows) byId.set(row.id, rowToWorkflowRun(row, "display"));
+      });
+      const runs = ids.flatMap((id) => byId.get(id) ?? []);
+      return { rootRunId, runs, hasOpenWork: await workflowRunTreeHasOpenWork(ids) };
+    },
+    { readOnly: true },
+  );
+}
+
 /**
  * Cancel the pending, running and waiting steps of the given runs. The
  * migration-140 trigger closes each cancelled step's pending approval.

package/src/be/db-retention.ts

@@ -1,6 +1,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import type { Attributes, DbRetentionSweepMetric, SpanStatus, SwarmSpan } from "../otel";
 import { recordDbRetentionStatement, recordDbRetentionSweep, startSpan, withSpan } from "../otel";
+import { TERMINAL_SCRIPT_RUN_STATUSES } from "../types";
 import { isEnvFlagEnabled } from "../utils/env-flag";
 import { scrubSecrets } from "../utils/secret-scrubber";
 import { getDbClient } from "./db";
@@ -23,7 +24,11 @@ const YIELD_MS = 25;
  *
  * Two policies exist:
  *  - `age`: the env value is a number of days; rows whose `timeColumn` is older
- *    than that horizon are deleted.
+ *    than that horizon are deleted. Optional narrowing: `terminalStatuses`
+ *    deletes only rows whose `statusColumn` is in that list, `keepWhenSet`
+ *    keeps every row where that column is non-NULL, and
+ *    `cutoffFormat: "sqliteDatetime"` compares against a `YYYY-MM-DD HH:MM:SS`
+ *    cutoff for a column that mixes that format with ISO strings.
  *  - `keepLatest`: the env value is a count; within each `partitionBy` group,
  *    only the newest `rankBy` rows are kept. The newest row of a group is
  *    never deleted, because the smallest accepted count is 1.
@@ -65,6 +70,29 @@ export const DB_RETENTION_TABLES = [
     initialBatchSize: 50,
     maxBatchSize: 100,
   },
+  {
+    // Every inline script run stores its full source, args and output. Only
+    // terminal runs are eligible: a running or paused durable run must survive
+    // to be resumed. Runs with an idempotencyKey are kept, so a retried launch
+    // still gets its 409 instead of executing twice. The status filter, the
+    // time range and the key exclusion all resolve on indexes
+    // (idx_script_runs_status_startedAt, idx_script_runs_idempotencyKey).
+    // Workflow runs write `datetime('now')` (space separator) while inline runs
+    // write ISO strings, hence the sqliteDatetime cutoff. script_run_journal
+    // rows go with their run through ON DELETE CASCADE on idx_srj_runId.
+    // Rows average ~30 KB and reach ~1.3 MB, so batches stay small.
+    table: "script_runs",
+    policy: "age",
+    timeColumn: "startedAt",
+    cutoffFormat: "sqliteDatetime",
+    statusColumn: "status",
+    terminalStatuses: TERMINAL_SCRIPT_RUN_STATUSES,
+    keepWhenSet: "idempotencyKey",
+    envKey: "SCRIPT_RUNS_RETENTION_DAYS",
+    metricsKey: "scriptRuns",
+    initialBatchSize: 100,
+    maxBatchSize: 500,
+  },
 ] as const satisfies ReadonlyArray<
   {
     table: string;
@@ -73,7 +101,14 @@ export const DB_RETENTION_TABLES = [
     initialBatchSize?: number;
     maxBatchSize?: number;
   } & (
-    | { policy: "age"; timeColumn: string }
+    | {
+        policy: "age";
+        timeColumn: string;
+        cutoffFormat?: "sqliteDatetime";
+        statusColumn?: string;
+        terminalStatuses?: readonly string[];
+        keepWhenSet?: string;
+      }
     | { policy: "keepLatest"; partitionBy: string; rankBy: string }
   )
 >;
@@ -326,13 +361,33 @@ export function sweepStatements(table: RetentionTable, value: number, now: Date)
       params: [value],
     };
   }
-  const cutoff = new Date(now.getTime() - value * DAY_MS).toISOString();
+  const iso = new Date(now.getTime() - value * DAY_MS).toISOString();
+  // Text comparison of the two formats: ' ' sorts before 'T', so against a
+  // space-separated cutoff an ISO row on the cutoff's own date is kept, never
+  // deleted early. It is swept on the next day's ticks instead.
+  const cutoff = "cutoffFormat" in table ? `${iso.slice(0, 10)} ${iso.slice(11, 19)}` : iso;
+  const conditions: string[] = [];
+  const params: Array<string | number> = [];
+  if ("terminalStatuses" in table) {
+    conditions.push(
+      `${table.statusColumn} IN (${table.terminalStatuses.map(() => "?").join(", ")})`,
+    );
+    params.push(...table.terminalStatuses);
+  }
+  conditions.push(`${table.timeColumn} < ?`);
+  params.push(cutoff);
+  if ("keepWhenSet" in table) {
+    conditions.push(
+      `rowid NOT IN (SELECT rowid FROM ${table.table} WHERE ${table.keepWhenSet} IS NOT NULL)`,
+    );
+  }
+  const where = conditions.join(" AND ");
   return {
-    backlogSql: `SELECT COUNT(*) AS n FROM ${table.table} WHERE ${table.timeColumn} < ?`,
+    backlogSql: `SELECT COUNT(*) AS n FROM ${table.table} WHERE ${where}`,
     deleteSql: `DELETE FROM ${table.table} WHERE rowid IN (
-         SELECT rowid FROM ${table.table} WHERE ${table.timeColumn} < ? ORDER BY ${table.timeColumn} LIMIT ?
+         SELECT rowid FROM ${table.table} WHERE ${where} ORDER BY ${table.timeColumn} LIMIT ?
        )`,
-    params: [cutoff],
+    params,
   };
 }
 

package/src/be/db.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite";
 import pkg from "../../package.json";
-import { defaultAssetKey, normalizeAssetKey, sessionGroupKey } from "../assets/key";
+import { defaultAssetKey, normalizeAssetKey } from "../assets/key";
+import { sessionSourceKey, sessionThreadGroupKey } from "../assets/source-key";
 import {
   generateDefaultClaudeMd,
   generateDefaultIdentityMd,
@@ -207,6 +208,7 @@ import {
   sessionCountQuery,
   sessionPageFields,
   sessionSummaryQuery,
+  threadRootsSql,
 } from "./db/session-tree";
 import { searchSkillRowsRanked } from "./db/skill-search";
 import { classifyTaskHumanFree, reclassifyTaskHumanFree } from "./db/tasks/human-free";
@@ -227,6 +229,11 @@ import {
   emitChildRunFinished,
   getRunLineageBlock,
   getStepExecutionBlock,
+  rowToWorkflowRun,
+  rowToWorkflowRunStep,
+  WORKFLOW_RUN_SUMMARY_COLUMNS,
+  type WorkflowRunRow,
+  type WorkflowRunStepRow,
   WorkflowStepNotExecutableError,
   wakeParentsOfDeletedRuns,
 } from "./db/workflow-runs";
@@ -242,7 +249,7 @@ import {
 } from "./swarm-config-guard";
 import { emitTaskStarted } from "./task-lifecycle-events";
 import { publishWorkerConfigNudge, publishWorkerNudge } from "./worker-nudges";
-import { defineWorkflowPayload, type WorkflowPayloadView } from "./workflow-replay";
+import type { WorkflowPayloadView } from "./workflow-replay";
 
 export {
   refreshActiveSessionOnActivity,
@@ -285,7 +292,7 @@ export {
   updateAgentStatus,
   updateAgentStatusFromCapacity,
 } from "./db/agents";
-export { recordKeySeatMismatch } from "./db/api-keys";
+export { recordKeySeatMismatch, recordKeyValidation } from "./db/api-keys";
 export {
   type ApprovalRequestSummary,
   type ApprovalVote,
@@ -2497,6 +2504,8 @@ export async function createTaskExtended(
     );
   }
   let requestedByUserIdInherited = false;
+  // The parent key is read again inside the INSERT transaction (see below).
+  let keyInherited = false;
   // True only when `options.routingAffinity` ends up populated purely via
   // the plain parent-fallback inherit below (child declared no affinity of
   // its own, the parent wasn't lead-only, AND the parent's affinity actually
@@ -2661,6 +2670,7 @@ export async function createTaskExtended(
       }
       if (parent.key && !options.key) {
         options.key = parent.key;
+        keyInherited = true;
       }
       if (parent.contextKey && !options.contextKey) {
         options.contextKey = parent.contextKey;
@@ -2850,7 +2860,18 @@ export async function createTaskExtended(
   }
 
   const auditUserId = getCurrentRequestUserId() ?? null;
-  const assetKey = normalizeAssetKey(options?.key ?? defaultAssetKey("task", id));
+  const assetKey = normalizeAssetKey(
+    options?.key ??
+      (options?.parentTaskId
+        ? null
+        : sessionSourceKey({
+            id,
+            slackChannelId: options?.slackChannelId,
+            slackThreadTs: options?.slackThreadTs,
+            contextKey: options?.contextKey,
+          })) ??
+      defaultAssetKey("task", id),
+  );
   // The Linear tracker dedup guard and the INSERT commit as one unit. Two
   // distinct webhook deliveries for the same issue are dispatched
   // fire-and-forget (src/linear/webhook.ts), nothing serializes them, and
@@ -2875,6 +2896,19 @@ export async function createTaskExtended(
       : await findExistingLinearTrackerContextWork(options?.contextKey);
     if (existingTrackerWork) return { existing: existingTrackerWork };
 
+    // A chain move (a user move, the source key backfill) can commit between
+    // the parent read above and this write lock. Its chain query did not see
+    // this child, so take the parent's key as it is now.
+    const parentKey =
+      keyInherited && options?.parentTaskId
+        ? (
+            await getDbClient().get<{ key: string }>('SELECT "key" FROM agent_tasks WHERE id = ?', [
+              options.parentTaskId,
+            ])
+          )?.key
+        : undefined;
+    const insertKey = parentKey ? normalizeAssetKey(parentKey) : assetKey;
+
     let routingDecisionSnapshot: RoutingDecisionSnapshot | null = null;
     try {
       routingDecisionSnapshot = await captureRoutingDecisionSnapshot(options);
@@ -2905,7 +2939,7 @@ export async function createTaskExtended(
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
       [
         id,
-        assetKey,
+        insertKey,
         options?.agentId ?? null,
         options?.creatorAgentId ?? null,
         // Free text from every task source; scrub at the single INSERT.
@@ -7859,53 +7893,6 @@ export async function getWorkflowsByScheduleId(scheduleId: string): Promise<Work
 // Workflow Run CRUD
 // ============================================================================
 
-type WorkflowRunRow = {
-  id: string;
-  workflowId: string;
-  status: string;
-  triggerType: string;
-  triggerData: string | null;
-  /** Absent from list rows: see `WORKFLOW_RUN_SUMMARY_COLUMNS`. */
-  context?: string | null;
-  /** Sealed exact `context` (migration 201). Absent from list rows. */
-  context_replay?: string | null;
-  error: string | null;
-  created_by: string | null;
-  startedAt: string;
-  lastUpdatedAt: string;
-  finishedAt: string | null;
-  /** The `sub-workflow` step that started this run (migration 203). */
-  parentStepId?: string | null;
-  /** That step's attempt (`retryCount`) when it started this run (migration 240). */
-  parentStepAttempt?: number | null;
-};
-
-/**
- * `view` picks the `context` copy: `replay` (default) is the exact sealed value
- * the engine resumes from; `display` is the scrubbed column for API, MCP and UI
- * responses. See `src/be/workflow-replay.ts`.
- */
-function rowToWorkflowRun(row: WorkflowRunRow, view: WorkflowPayloadView = "replay"): WorkflowRun {
-  const run: WorkflowRun = {
-    id: row.id,
-    workflowId: row.workflowId,
-    status: row.status as WorkflowRunStatus,
-    triggerData: row.triggerData ? JSON.parse(row.triggerData) : undefined,
-    context: undefined,
-    error: row.error ?? undefined,
-    createdBy: row.created_by ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    lastUpdatedAt: normalizeDateRequired(row.lastUpdatedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    parentStepId: row.parentStepId ?? undefined,
-    parentStepAttempt: row.parentStepId ? (row.parentStepAttempt ?? 0) : undefined,
-  };
-  return defineWorkflowPayload(run, "context", row.context, row.context_replay, view, {
-    runId: row.id,
-    target: "its context",
-  });
-}
-
 export async function createWorkflowRun(data: {
   id: string;
   workflowId: string;
@@ -8031,12 +8018,6 @@ export type WorkflowRunListOptions = {
   includeContext?: boolean;
 };
 
-// Everything but `context`. `triggerData` stays: it is small, and stored
-// scripts read it off list rows (dedupe by trigger, failure reports). The
-// parent link and attempt stay so a list row identifies a child's invocation.
-const WORKFLOW_RUN_SUMMARY_COLUMNS =
-  "id, workflowId, status, triggerType, triggerData, error, created_by, startedAt, lastUpdatedAt, finishedAt, parentStepId, parentStepAttempt";
-
 export type WorkflowRunPage = {
   runs: WorkflowRun[];
   page: {
@@ -8121,56 +8102,6 @@ export async function listWorkflowRunsPage(
 // Workflow Run Step CRUD
 // ============================================================================
 
-type WorkflowRunStepRow = {
-  id: string;
-  runId: string;
-  nodeId: string;
-  nodeType: string;
-  status: string;
-  input: string | null;
-  output: string | null;
-  /** Sealed exact `output` (migration 201). */
-  output_replay: string | null;
-  error: string | null;
-  startedAt: string;
-  finishedAt: string | null;
-  retryCount: number;
-  maxRetries: number;
-  nextRetryAt: string | null;
-  idempotencyKey: string | null;
-  diagnostics: string | null;
-  nextPort: string | null;
-};
-
-/** `view` picks the `output` copy, as in `rowToWorkflowRun`. */
-function rowToWorkflowRunStep(
-  row: WorkflowRunStepRow,
-  view: WorkflowPayloadView = "replay",
-): WorkflowRunStep {
-  const step: WorkflowRunStep = {
-    id: row.id,
-    runId: row.runId,
-    nodeId: row.nodeId,
-    nodeType: row.nodeType,
-    status: row.status as WorkflowRunStepStatus,
-    input: row.input ? JSON.parse(row.input) : undefined,
-    output: undefined,
-    error: row.error ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    retryCount: row.retryCount,
-    maxRetries: row.maxRetries,
-    nextRetryAt: normalizeDate(row.nextRetryAt) ?? undefined,
-    idempotencyKey: row.idempotencyKey ?? undefined,
-    diagnostics: row.diagnostics ?? undefined,
-    nextPort: row.nextPort ?? undefined,
-  };
-  return defineWorkflowPayload(step, "output", row.output, row.output_replay, view, {
-    runId: row.runId,
-    target: `the output of step ${row.id} (node ${row.nodeId})`,
-  });
-}
-
 export async function createWorkflowRunStep(data: {
   id: string;
   runId: string;
@@ -8284,7 +8215,7 @@ export async function getWorkflowRunStepsByRunId(
   view: WorkflowPayloadView = "replay",
 ): Promise<WorkflowRunStep[]> {
   const rows = await getDbClient().query<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC, rowid ASC",
     [runId],
   );
   return rows.map((row) => rowToWorkflowRunStep(row, view));
@@ -8400,7 +8331,7 @@ export async function getLatestStepForNode(
   nodeId: string,
 ): Promise<WorkflowRunStep | null> {
   const row = await getDbClient().get<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC LIMIT 1",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC, rowid DESC LIMIT 1",
     [runId, nodeId],
   );
   return row ? rowToWorkflowRunStep(row) : null;
@@ -11742,6 +11673,8 @@ export interface ApiKeyStatus {
   /** Auth failures in a row since the last success or clear. */
   consecutiveAuthFailures: number;
   lastAuthFailureAt: string | null;
+  lastValidationError: string | null;
+  lastValidationFailedAt: string | null;
   createdAt: string;
   updatedAt: string;
 }
@@ -11767,6 +11700,7 @@ function rowToApiKeyStatus(row: ApiKeyStatusRow): ApiKeyStatus {
 
 export interface AvailableKeyIndicesResult {
   availableIndices: number[];
+  rejectedIndices: number[];
   /** Indices excluded only by an active model-scoped window block (not key-wide). */
   modelBlockedIndices: number[];
   /** ISO of the earliest resetsAt among modelBlockedIndices, or null when none. */
@@ -11815,8 +11749,9 @@ export async function getAvailableKeyIndices(
     rateLimitWindows: string | null;
     plan: string | null;
     authFailureSeq: number;
+    lastValidationFailedAt: string | null;
   }>(
-    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq FROM api_key_status
+    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq, lastValidationFailedAt FROM api_key_status
        WHERE keyType = ? AND scope = ? AND scopeId = ?`,
     [keyType, scope, effectiveScopeId],
   );
@@ -11824,6 +11759,12 @@ export async function getAvailableKeyIndices(
   const blockedIndices = new Set(
     rows.filter((r) => r.status === "rate_limited").map((r) => r.keyIndex),
   );
+  const rejectedIndices = [
+    ...new Set(
+      rows.filter((row) => row.lastValidationFailedAt !== null).map((row) => row.keyIndex),
+    ),
+  ];
+  for (const index of rejectedIndices) blockedIndices.add(index);
 
   const modelBlockedIndices: number[] = [];
   let earliestModelResetsAtSec: number | undefined;
@@ -11866,6 +11807,7 @@ export async function getAvailableKeyIndices(
 
   return {
     availableIndices,
+    rejectedIndices,
     modelBlockedIndices,
     seatBlockedIndices,
     earliestModelResetAt:
@@ -12997,6 +12939,7 @@ export async function listRecentSessions(
   opts: ListRecentSessionsOpts = {},
 ): Promise<SessionListItem[] | SessionListItemSummary[]> {
   const query = sessionSummaryQuery(opts);
+  const threadRoots = threadRootsSql();
   const rootRows = await getDbClient().query<
     AgentTaskRow & {
       __chainCount: number;
@@ -13005,15 +12948,17 @@ export async function listRecentSessions(
       __readState: SessionReadState | null;
       __snoozeUntil: string | null;
       __needsYou: number;
+      __threadRoots: number;
     }
   >(
     `SELECT r.*, a.chainCount AS __chainCount, a.lastActivityAt AS __lastActivityAt,
       a.latestStatus AS __latestStatus, a.readState AS __readState,
-      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou
+      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou,
+      ${threadRoots.sql} AS __threadRoots
     FROM (${query.sql}) a CROSS JOIN agent_tasks r ON r.id = a.root_id
     ORDER BY ${opts.triage === "later" ? "a.snoozeUntil ASC," : ""}
       a.lastActivityAt DESC, r.createdAt DESC, r.id DESC LIMIT ? OFFSET ?`,
-    [...query.params, opts.limit ?? 25, opts.offset ?? 0],
+    [...threadRoots.params, ...query.params, opts.limit ?? 25, opts.offset ?? 0],
   );
   const fields = await sessionPageFields(
     rootRows.map((row) => ({
@@ -13029,7 +12974,7 @@ export async function listRecentSessions(
     ...(row.__readState ? { readState: row.__readState } : {}),
     snoozeUntil: row.__snoozeUntil,
     needsYou: !!row.__needsYou,
-    groupKey: sessionGroupKey(row.key),
+    groupKey: sessionThreadGroupKey(row.key, row.__threadRoots),
     ...fields.get(row.id),
   }));
 }

package/src/be/memory/providers/openai-embedding.ts

@@ -1,4 +1,5 @@
 import OpenAI from "openai";
+import { selectCredential } from "../../../utils/credentials";
 import { getOpenRouterAttributionHeaders } from "../../../utils/openrouter-base-url";
 import { DEFAULT_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS } from "../constants";
 import type { EmbeddingProvider } from "../types";
@@ -46,7 +47,10 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
   private readonly apiKey: string | undefined;
 
   constructor(config?: OpenAIEmbeddingConfig) {
-    this.apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    const apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    this.apiKey = apiKey
+      ? selectCredential(apiKey, undefined, "EMBEDDING_API_KEY").selected
+      : undefined;
 
     this.model = config?.model ?? process.env.EMBEDDING_MODEL ?? "text-embedding-3-small";
 

package/src/be/migrations/244_api_key_validation.sql

@@ -0,0 +1,4 @@
+-- Last provider rejection of a pooled key by a worker live test (HTTP 401).
+-- A key with lastValidationFailedAt set is left out of /api/keys/available.
+ALTER TABLE api_key_status ADD COLUMN lastValidationError TEXT;
+ALTER TABLE api_key_status ADD COLUMN lastValidationFailedAt TEXT;

package/src/be/onboarding.ts

@@ -130,6 +130,10 @@ export const OnboardingSignalsSchema = z.object({
       state: z.enum(["unverified", "configured", "verified"]),
       workers: z.number().int().nonnegative(),
       verifiedWorkers: z.number().int().nonnegative(),
+      /** The latest live test failed: its error. */
+      error: z.string().nullable(),
+      /** The latest live test passed, but some pool keys failed: names them. */
+      warning: z.string().nullable(),
     }),
   ),
   embeddings: z.object({

package/src/be/scripts/typecheck.ts

@@ -250,7 +250,7 @@ export interface SwarmSdk {
   workflow_list(args?: { enabled?: boolean; includeFull?: boolean; consecutiveErrorsMin?: number; lastRunStatus?: "running" | "waiting" | "completed" | "failed" | "skipped" | "cancelled" }): Promise<unknown>;
   workflow_get(args: { id: string }): Promise<unknown>;
   workflow_listRuns(args: { workflowId: string; status?: "running" | "waiting" | "completed" | "failed" | "skipped" | "cancelled"; limit?: number; offset?: number }): Promise<unknown>;
-  workflow_getRun(args: { id: string }): Promise<unknown>;
+  workflow_getRun(args: { id: string; fields?: "full" | "slim" }): Promise<unknown>;
   // --- prompt templates ---
   prompt_list(args?: { eventType?: string; scope?: "global" | "agent" | "repo"; scopeId?: string; isDefault?: boolean }): Promise<unknown>;
   prompt_get(args: { id: string }): Promise<unknown>;