@desplega.ai/agent-swarm 2.5.3 → 2.6.0

package/src/be/db.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite";
 import pkg from "../../package.json";
-import { defaultAssetKey, normalizeAssetKey, sessionGroupKey } from "../assets/key";
+import { defaultAssetKey, normalizeAssetKey } from "../assets/key";
+import { sessionSourceKey, sessionThreadGroupKey } from "../assets/source-key";
 import {
   generateDefaultClaudeMd,
   generateDefaultIdentityMd,
@@ -207,6 +208,7 @@ import {
   sessionCountQuery,
   sessionPageFields,
   sessionSummaryQuery,
+  threadRootsSql,
 } from "./db/session-tree";
 import { searchSkillRowsRanked } from "./db/skill-search";
 import { classifyTaskHumanFree, reclassifyTaskHumanFree } from "./db/tasks/human-free";
@@ -222,7 +224,19 @@ import {
   rowToAgentTaskSummary,
 } from "./db/tasks/read";
 import { configureTaskWriteDependencies, failTask } from "./db/tasks/write";
-import { emitChildRunFinished, wakeParentsOfDeletedRuns } from "./db/workflow-runs";
+import { tombstoneWorkflowIdempotencyKeys } from "./db/workflow-idempotency";
+import {
+  emitChildRunFinished,
+  getRunLineageBlock,
+  getStepExecutionBlock,
+  rowToWorkflowRun,
+  rowToWorkflowRunStep,
+  WORKFLOW_RUN_SUMMARY_COLUMNS,
+  type WorkflowRunRow,
+  type WorkflowRunStepRow,
+  WorkflowStepNotExecutableError,
+  wakeParentsOfDeletedRuns,
+} from "./db/workflow-runs";
 import { scrubJsonValue } from "./scrub-json";
 import { openSealedJson, sealedJsonForDisplay, sealJson } from "./sealed-json";
 import { configSecretName, registerStoredSecret } from "./secret-registry";
@@ -235,7 +249,7 @@ import {
 } from "./swarm-config-guard";
 import { emitTaskStarted } from "./task-lifecycle-events";
 import { publishWorkerConfigNudge, publishWorkerNudge } from "./worker-nudges";
-import { defineWorkflowPayload, type WorkflowPayloadView } from "./workflow-replay";
+import type { WorkflowPayloadView } from "./workflow-replay";
 
 export {
   refreshActiveSessionOnActivity,
@@ -278,7 +292,7 @@ export {
   updateAgentStatus,
   updateAgentStatusFromCapacity,
 } from "./db/agents";
-export { recordKeySeatMismatch } from "./db/api-keys";
+export { recordKeySeatMismatch, recordKeyValidation } from "./db/api-keys";
 export {
   type ApprovalRequestSummary,
   type ApprovalVote,
@@ -2490,6 +2504,8 @@ export async function createTaskExtended(
     );
   }
   let requestedByUserIdInherited = false;
+  // The parent key is read again inside the INSERT transaction (see below).
+  let keyInherited = false;
   // True only when `options.routingAffinity` ends up populated purely via
   // the plain parent-fallback inherit below (child declared no affinity of
   // its own, the parent wasn't lead-only, AND the parent's affinity actually
@@ -2654,6 +2670,7 @@ export async function createTaskExtended(
       }
       if (parent.key && !options.key) {
         options.key = parent.key;
+        keyInherited = true;
       }
       if (parent.contextKey && !options.contextKey) {
         options.contextKey = parent.contextKey;
@@ -2843,7 +2860,18 @@ export async function createTaskExtended(
   }
 
   const auditUserId = getCurrentRequestUserId() ?? null;
-  const assetKey = normalizeAssetKey(options?.key ?? defaultAssetKey("task", id));
+  const assetKey = normalizeAssetKey(
+    options?.key ??
+      (options?.parentTaskId
+        ? null
+        : sessionSourceKey({
+            id,
+            slackChannelId: options?.slackChannelId,
+            slackThreadTs: options?.slackThreadTs,
+            contextKey: options?.contextKey,
+          })) ??
+      defaultAssetKey("task", id),
+  );
   // The Linear tracker dedup guard and the INSERT commit as one unit. Two
   // distinct webhook deliveries for the same issue are dispatched
   // fire-and-forget (src/linear/webhook.ts), nothing serializes them, and
@@ -2855,11 +2883,32 @@ export async function createTaskExtended(
   const claim = await getDbClient().transaction<
     { existing: ExistingTrackerContextWork } | { row: AgentTaskRow }
   >(async () => {
+    // A workflow step's task commits only while the step may still start
+    // work. Checked in the INSERT's transaction: a cascade cancel of the
+    // step's run tree either commits first and is seen here, or sees and
+    // cancels this task.
+    if (options?.workflowRunStepId) {
+      const blocked = await getStepExecutionBlock(options.workflowRunStepId);
+      if (blocked) throw new WorkflowStepNotExecutableError(options.workflowRunStepId, blocked);
+    }
     const existingTrackerWork = options?.bypassTrackerContextDedup
       ? null
       : await findExistingLinearTrackerContextWork(options?.contextKey);
     if (existingTrackerWork) return { existing: existingTrackerWork };
 
+    // A chain move (a user move, the source key backfill) can commit between
+    // the parent read above and this write lock. Its chain query did not see
+    // this child, so take the parent's key as it is now.
+    const parentKey =
+      keyInherited && options?.parentTaskId
+        ? (
+            await getDbClient().get<{ key: string }>('SELECT "key" FROM agent_tasks WHERE id = ?', [
+              options.parentTaskId,
+            ])
+          )?.key
+        : undefined;
+    const insertKey = parentKey ? normalizeAssetKey(parentKey) : assetKey;
+
     let routingDecisionSnapshot: RoutingDecisionSnapshot | null = null;
     try {
       routingDecisionSnapshot = await captureRoutingDecisionSnapshot(options);
@@ -2890,7 +2939,7 @@ export async function createTaskExtended(
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
       [
         id,
-        assetKey,
+        insertKey,
         options?.agentId ?? null,
         options?.creatorAgentId ?? null,
         // Free text from every task source; scrub at the single INSERT.
@@ -4739,7 +4788,7 @@ export async function createSessionLogs(logs: {
   for (let start = 0; start < logs.lines.length; start += BATCH_SIZE) {
     // Yield to the event loop between batches.
     if (start > 0) await new Promise<void>((r) => setTimeout(r, 5));
-    const batch: { content: string; lineNumber: number }[] = [];
+    const batch: string[] = [];
     const end = Math.min(start + BATCH_SIZE, logs.lines.length);
     for (let i = start; i < end; i++) {
       const line = logs.lines[i];
@@ -4748,10 +4797,22 @@ export async function createSessionLogs(logs: {
       // content that is already scrubbed at the adapter emit site. We scrub
       // again here so any future write path that bypasses the adapter still
       // lands clean text in the persistent session_logs table.
-      batch.push({ content: scrubSecrets(line), lineNumber: i });
+      batch.push(scrubSecrets(line));
     }
     await getDbClient().transaction(async (tx) => {
-      for (const { content, lineNumber } of batch) {
+      // The worker flushes one (sessionId, iteration) in many POSTs, so the
+      // lineNumber continues from the newest stored row instead of restarting
+      // at the POST's index. Reading the newest row by rowid walks the
+      // sessionId index backwards and stops at the first hit, where MAX()
+      // would scan the whole session. Rows written here are monotonic in
+      // rowid, so this equals MAX + 1 for them.
+      const last = await tx.get<{ lineNumber: number }>(
+        `SELECT lineNumber FROM session_logs WHERE sessionId = ? AND iteration = ?
+         ORDER BY rowid DESC LIMIT 1`,
+        [logs.sessionId, logs.iteration],
+      );
+      let lineNumber = last ? last.lineNumber + 1 : 0;
+      for (const content of batch) {
         await tx.run(
           `INSERT INTO session_logs (id, taskId, sessionId, iteration, cli, content, lineNumber, createdAt)
            VALUES (?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`,
@@ -4762,7 +4823,7 @@ export async function createSessionLogs(logs: {
             logs.iteration,
             logs.cli,
             content,
-            lineNumber,
+            lineNumber++,
           ],
         );
       }
@@ -4770,6 +4831,10 @@ export async function createSessionLogs(logs: {
   }
 }
 
+// Session log reads order by rowid, which is insert order. Rows written before
+// lineNumber was assigned server-side restart at 0 on every worker flush, so
+// only rowid orders them. The API never runs VACUUM (runbooks/db-retention.md);
+// a manual VACUUM may renumber rowids but copies rows in rowid order.
 export async function getSessionLogsByTaskId(
   taskId: string,
   limit?: number,
@@ -4777,16 +4842,16 @@ export async function getSessionLogsByTaskId(
   if (typeof limit === "number" && limit > 0) {
     const rows = await getDbClient().query<SessionLogRow>(
       `SELECT * FROM (
-         SELECT * FROM session_logs WHERE taskId = ?
-         ORDER BY iteration DESC, lineNumber DESC
+         SELECT rowid AS seq, * FROM session_logs WHERE taskId = ?
+         ORDER BY rowid DESC
          LIMIT ?
-       ) ORDER BY iteration ASC, lineNumber ASC`,
+       ) ORDER BY seq ASC`,
       [taskId, limit],
     );
     return rows.map(rowToSessionLog);
   }
   const rows = await getDbClient().query<SessionLogRow>(
-    "SELECT * FROM session_logs WHERE taskId = ? ORDER BY iteration ASC, lineNumber ASC",
+    "SELECT * FROM session_logs WHERE taskId = ? ORDER BY rowid ASC",
     [taskId],
   );
   return rows.map(rowToSessionLog);
@@ -4797,7 +4862,7 @@ export async function getSessionLogsBySession(
   iteration: number,
 ): Promise<SessionLog[]> {
   const rows = await getDbClient().query<SessionLogRow>(
-    "SELECT * FROM session_logs WHERE sessionId = ? AND iteration = ? ORDER BY lineNumber ASC",
+    "SELECT * FROM session_logs WHERE sessionId = ? AND iteration = ? ORDER BY rowid ASC",
     [sessionId, iteration],
   );
   return rows.map(rowToSessionLog);
@@ -7790,7 +7855,9 @@ async function deleteWorkflowRows(id: string, source?: "api" | "mcp"): Promise<b
   await reclassifyTaskHumanFree(linkedTaskIds);
   // 4. Delete every user's favorite of it (no FK to the workflow)
   await client.run("DELETE FROM user_favorites WHERE itemType = 'workflow' AND itemId = ?", [id]);
-  // 5. Delete workflow
+  // 5. Keep Idempotency-Keys that created or triggered it taken, without the stored response
+  await tombstoneWorkflowIdempotencyKeys(id);
+  // 6. Delete workflow
   const result = await client.run("DELETE FROM workflows WHERE id = ?", [id]);
   const deleted = result.changes > 0;
   if (deleted) {
@@ -7826,53 +7893,6 @@ export async function getWorkflowsByScheduleId(scheduleId: string): Promise<Work
 // Workflow Run CRUD
 // ============================================================================
 
-type WorkflowRunRow = {
-  id: string;
-  workflowId: string;
-  status: string;
-  triggerType: string;
-  triggerData: string | null;
-  /** Absent from list rows: see `WORKFLOW_RUN_SUMMARY_COLUMNS`. */
-  context?: string | null;
-  /** Sealed exact `context` (migration 201). Absent from list rows. */
-  context_replay?: string | null;
-  error: string | null;
-  created_by: string | null;
-  startedAt: string;
-  lastUpdatedAt: string;
-  finishedAt: string | null;
-  /** The `sub-workflow` step that started this run (migration 203). */
-  parentStepId?: string | null;
-  /** That step's attempt (`retryCount`) when it started this run (migration 240). */
-  parentStepAttempt?: number | null;
-};
-
-/**
- * `view` picks the `context` copy: `replay` (default) is the exact sealed value
- * the engine resumes from; `display` is the scrubbed column for API, MCP and UI
- * responses. See `src/be/workflow-replay.ts`.
- */
-function rowToWorkflowRun(row: WorkflowRunRow, view: WorkflowPayloadView = "replay"): WorkflowRun {
-  const run: WorkflowRun = {
-    id: row.id,
-    workflowId: row.workflowId,
-    status: row.status as WorkflowRunStatus,
-    triggerData: row.triggerData ? JSON.parse(row.triggerData) : undefined,
-    context: undefined,
-    error: row.error ?? undefined,
-    createdBy: row.created_by ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    lastUpdatedAt: normalizeDateRequired(row.lastUpdatedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    parentStepId: row.parentStepId ?? undefined,
-    parentStepAttempt: row.parentStepId ? (row.parentStepAttempt ?? 0) : undefined,
-  };
-  return defineWorkflowPayload(run, "context", row.context, row.context_replay, view, {
-    runId: row.id,
-    target: "its context",
-  });
-}
-
 export async function createWorkflowRun(data: {
   id: string;
   workflowId: string;
@@ -7998,12 +8018,6 @@ export type WorkflowRunListOptions = {
   includeContext?: boolean;
 };
 
-// Everything but `context`. `triggerData` stays: it is small, and stored
-// scripts read it off list rows (dedupe by trigger, failure reports). The
-// parent link and attempt stay so a list row identifies a child's invocation.
-const WORKFLOW_RUN_SUMMARY_COLUMNS =
-  "id, workflowId, status, triggerType, triggerData, error, created_by, startedAt, lastUpdatedAt, finishedAt, parentStepId, parentStepAttempt";
-
 export type WorkflowRunPage = {
   runs: WorkflowRun[];
   page: {
@@ -8088,56 +8102,6 @@ export async function listWorkflowRunsPage(
 // Workflow Run Step CRUD
 // ============================================================================
 
-type WorkflowRunStepRow = {
-  id: string;
-  runId: string;
-  nodeId: string;
-  nodeType: string;
-  status: string;
-  input: string | null;
-  output: string | null;
-  /** Sealed exact `output` (migration 201). */
-  output_replay: string | null;
-  error: string | null;
-  startedAt: string;
-  finishedAt: string | null;
-  retryCount: number;
-  maxRetries: number;
-  nextRetryAt: string | null;
-  idempotencyKey: string | null;
-  diagnostics: string | null;
-  nextPort: string | null;
-};
-
-/** `view` picks the `output` copy, as in `rowToWorkflowRun`. */
-function rowToWorkflowRunStep(
-  row: WorkflowRunStepRow,
-  view: WorkflowPayloadView = "replay",
-): WorkflowRunStep {
-  const step: WorkflowRunStep = {
-    id: row.id,
-    runId: row.runId,
-    nodeId: row.nodeId,
-    nodeType: row.nodeType,
-    status: row.status as WorkflowRunStepStatus,
-    input: row.input ? JSON.parse(row.input) : undefined,
-    output: undefined,
-    error: row.error ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    retryCount: row.retryCount,
-    maxRetries: row.maxRetries,
-    nextRetryAt: normalizeDate(row.nextRetryAt) ?? undefined,
-    idempotencyKey: row.idempotencyKey ?? undefined,
-    diagnostics: row.diagnostics ?? undefined,
-    nextPort: row.nextPort ?? undefined,
-  };
-  return defineWorkflowPayload(step, "output", row.output, row.output_replay, view, {
-    runId: row.runId,
-    target: `the output of step ${row.id} (node ${row.nodeId})`,
-  });
-}
-
 export async function createWorkflowRunStep(data: {
   id: string;
   runId: string;
@@ -8251,7 +8215,7 @@ export async function getWorkflowRunStepsByRunId(
   view: WorkflowPayloadView = "replay",
 ): Promise<WorkflowRunStep[]> {
   const rows = await getDbClient().query<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC, rowid ASC",
     [runId],
   );
   return rows.map((row) => rowToWorkflowRunStep(row, view));
@@ -8367,7 +8331,7 @@ export async function getLatestStepForNode(
   nodeId: string,
 ): Promise<WorkflowRunStep | null> {
   const row = await getDbClient().get<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC LIMIT 1",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC, rowid DESC LIMIT 1",
     [runId, nodeId],
   );
   return row ? rowToWorkflowRunStep(row) : null;
@@ -9787,15 +9751,29 @@ export async function createApprovalRequest(data: {
   const row = await getDbClient().transaction(async () => {
     let status: ApprovalRequest["status"] = "pending";
     let resolutionReason: string | null = null;
-    if (data.requireActionableWorkflow && data.workflowRunId && data.workflowRunStepId) {
-      const run = await getWorkflowRun(data.workflowRunId);
-      const step = await getWorkflowRunStep(data.workflowRunStepId);
+    // A request linked to an existing run, with or without a step, is born
+    // cancelled unless the run is live and its tree is not cancelled: a
+    // pending request in a cancelled tree would be open work no cancel may
+    // leave behind. A missing run is refused only on the workflow (HITL) path.
+    const actionable = data.requireActionableWorkflow && data.workflowRunStepId;
+    const run = data.workflowRunId ? await getWorkflowRun(data.workflowRunId, "display") : null;
+    if (data.workflowRunId && (run || actionable)) {
+      const step = actionable ? await getWorkflowRunStep(data.workflowRunStepId!) : undefined;
       if (!run || (run.status !== "running" && run.status !== "waiting")) {
         status = "cancelled";
         resolutionReason = run?.error ?? `Workflow run is ${run?.status ?? "missing"}`;
-      } else if (!step || step.runId !== run.id || step.status !== "running") {
+      } else if (
+        step !== undefined &&
+        (!step || step.runId !== run.id || step.status !== "running")
+      ) {
         status = "cancelled";
         resolutionReason = `Human-in-the-loop step is ${step?.status ?? "missing"}`;
+      } else {
+        const blocked = await getRunLineageBlock(run.id);
+        if (blocked) {
+          status = "cancelled";
+          resolutionReason = blocked;
+        }
       }
     }
 
@@ -11695,6 +11673,8 @@ export interface ApiKeyStatus {
   /** Auth failures in a row since the last success or clear. */
   consecutiveAuthFailures: number;
   lastAuthFailureAt: string | null;
+  lastValidationError: string | null;
+  lastValidationFailedAt: string | null;
   createdAt: string;
   updatedAt: string;
 }
@@ -11720,6 +11700,7 @@ function rowToApiKeyStatus(row: ApiKeyStatusRow): ApiKeyStatus {
 
 export interface AvailableKeyIndicesResult {
   availableIndices: number[];
+  rejectedIndices: number[];
   /** Indices excluded only by an active model-scoped window block (not key-wide). */
   modelBlockedIndices: number[];
   /** ISO of the earliest resetsAt among modelBlockedIndices, or null when none. */
@@ -11768,8 +11749,9 @@ export async function getAvailableKeyIndices(
     rateLimitWindows: string | null;
     plan: string | null;
     authFailureSeq: number;
+    lastValidationFailedAt: string | null;
   }>(
-    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq FROM api_key_status
+    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq, lastValidationFailedAt FROM api_key_status
        WHERE keyType = ? AND scope = ? AND scopeId = ?`,
     [keyType, scope, effectiveScopeId],
   );
@@ -11777,6 +11759,12 @@ export async function getAvailableKeyIndices(
   const blockedIndices = new Set(
     rows.filter((r) => r.status === "rate_limited").map((r) => r.keyIndex),
   );
+  const rejectedIndices = [
+    ...new Set(
+      rows.filter((row) => row.lastValidationFailedAt !== null).map((row) => row.keyIndex),
+    ),
+  ];
+  for (const index of rejectedIndices) blockedIndices.add(index);
 
   const modelBlockedIndices: number[] = [];
   let earliestModelResetsAtSec: number | undefined;
@@ -11819,6 +11807,7 @@ export async function getAvailableKeyIndices(
 
   return {
     availableIndices,
+    rejectedIndices,
     modelBlockedIndices,
     seatBlockedIndices,
     earliestModelResetAt:
@@ -12892,7 +12881,8 @@ export async function listTaskTemplates(opts?: {
 /**
  * Walk the chain rooted at `rootTaskId`, ordered by `createdAt` (root first; `rowid` breaks
  * ties as idx_agent_tasks_created did). `CROSS JOIN` keeps the chain as the outer loop: with
- * a plain JOIN, SQLite scanned all of agent_tasks to skip the sort.
+ * a plain JOIN, SQLite scanned all of agent_tasks to skip the sort. Each task carries its
+ * `totalCostUsd`, so the session header can total the whole chain, not only the root.
  */
 export async function getRootTaskChain(rootTaskId: string): Promise<AgentTask[]> {
   const rows = await getDbClient().query<AgentTaskRow>(
@@ -12902,7 +12892,9 @@ export async function getRootTaskChain(rootTaskId: string): Promise<AgentTask[]>
          SELECT t.id FROM agent_tasks t
          JOIN chain c ON t.parentTaskId = c.id
        )
-       SELECT t.* FROM chain
+       SELECT t.*,
+         (SELECT SUM(totalCostUsd) FROM session_costs WHERE session_costs.taskId = t.id) AS totalCostUsd
+       FROM chain
        CROSS JOIN agent_tasks t ON t.id = chain.id
        ORDER BY t.createdAt, t.rowid`,
     [rootTaskId],
@@ -12947,6 +12939,7 @@ export async function listRecentSessions(
   opts: ListRecentSessionsOpts = {},
 ): Promise<SessionListItem[] | SessionListItemSummary[]> {
   const query = sessionSummaryQuery(opts);
+  const threadRoots = threadRootsSql();
   const rootRows = await getDbClient().query<
     AgentTaskRow & {
       __chainCount: number;
@@ -12955,15 +12948,17 @@ export async function listRecentSessions(
       __readState: SessionReadState | null;
       __snoozeUntil: string | null;
       __needsYou: number;
+      __threadRoots: number;
     }
   >(
     `SELECT r.*, a.chainCount AS __chainCount, a.lastActivityAt AS __lastActivityAt,
       a.latestStatus AS __latestStatus, a.readState AS __readState,
-      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou
+      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou,
+      ${threadRoots.sql} AS __threadRoots
     FROM (${query.sql}) a CROSS JOIN agent_tasks r ON r.id = a.root_id
     ORDER BY ${opts.triage === "later" ? "a.snoozeUntil ASC," : ""}
       a.lastActivityAt DESC, r.createdAt DESC, r.id DESC LIMIT ? OFFSET ?`,
-    [...query.params, opts.limit ?? 25, opts.offset ?? 0],
+    [...threadRoots.params, ...query.params, opts.limit ?? 25, opts.offset ?? 0],
   );
   const fields = await sessionPageFields(
     rootRows.map((row) => ({
@@ -12979,7 +12974,7 @@ export async function listRecentSessions(
     ...(row.__readState ? { readState: row.__readState } : {}),
     snoozeUntil: row.__snoozeUntil,
     needsYou: !!row.__needsYou,
-    groupKey: sessionGroupKey(row.key),
+    groupKey: sessionThreadGroupKey(row.key, row.__threadRoots),
     ...fields.get(row.id),
   }));
 }

package/src/be/memory/providers/openai-embedding.ts

@@ -1,4 +1,5 @@
 import OpenAI from "openai";
+import { selectCredential } from "../../../utils/credentials";
 import { getOpenRouterAttributionHeaders } from "../../../utils/openrouter-base-url";
 import { DEFAULT_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS } from "../constants";
 import type { EmbeddingProvider } from "../types";
@@ -46,7 +47,10 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
   private readonly apiKey: string | undefined;
 
   constructor(config?: OpenAIEmbeddingConfig) {
-    this.apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    const apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    this.apiKey = apiKey
+      ? selectCredential(apiKey, undefined, "EMBEDDING_API_KEY").selected
+      : undefined;
 
     this.model = config?.model ?? process.env.EMBEDDING_MODEL ?? "text-embedding-3-small";
 

package/src/be/migrations/242_workflow_idempotency_keys.sql

@@ -0,0 +1,40 @@
+-- Idempotency-Key records for POST /api/workflows and POST /api/workflows/{id}/trigger.
+--
+-- One row is one operation the server accepted under a client key. It commits
+-- in the same transaction as the workflow (create) or the run and its initial
+-- state (trigger), so a retry after a lost response, a restart or a concurrent
+-- duplicate finds it and gets the original result instead of a second
+-- workflow or run.
+--
+-- The scope is (principal, operation, targetId, keyHash): the caller's stable
+-- identity (never the bearer value), the operation, the triggered workflow id
+-- ('' for create) and a SHA-256 of the client key. requestFingerprint is a
+-- SHA-256 of the canonical request; a different request under the same key is
+-- a conflict. responseBody is the original response, sealed (sealJson). When
+-- the workflow is deleted, responseBody is cleared and resourceDeletedAt set:
+-- the key stays taken and is never accepted again.
+--
+-- No FK to workflows: a row outlives the workflow as that tombstone.
+
+CREATE TABLE IF NOT EXISTS workflow_idempotency_keys (
+  id TEXT PRIMARY KEY,
+  principal TEXT NOT NULL,
+  operation TEXT NOT NULL CHECK (operation IN ('workflow.create', 'workflow.trigger')),
+  targetId TEXT NOT NULL DEFAULT '',
+  keyHash TEXT NOT NULL,
+  requestFingerprint TEXT NOT NULL,
+  workflowId TEXT NOT NULL,
+  runId TEXT,
+  responseStatus INTEGER NOT NULL,
+  responseBody TEXT,
+  resourceDeletedAt TEXT,
+  createdAt TEXT NOT NULL,
+  created_by TEXT,
+  updated_by TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_workflow_idempotency_keys_scope
+  ON workflow_idempotency_keys(principal, operation, targetId, keyHash);
+
+CREATE INDEX IF NOT EXISTS idx_workflow_idempotency_keys_workflowId
+  ON workflow_idempotency_keys(workflowId);

package/src/be/migrations/244_api_key_validation.sql

@@ -0,0 +1,4 @@
+-- Last provider rejection of a pooled key by a worker live test (HTTP 401).
+-- A key with lastValidationFailedAt set is left out of /api/keys/available.
+ALTER TABLE api_key_status ADD COLUMN lastValidationError TEXT;
+ALTER TABLE api_key_status ADD COLUMN lastValidationFailedAt TEXT;

package/src/be/onboarding.ts

@@ -130,6 +130,10 @@ export const OnboardingSignalsSchema = z.object({
       state: z.enum(["unverified", "configured", "verified"]),
       workers: z.number().int().nonnegative(),
       verifiedWorkers: z.number().int().nonnegative(),
+      /** The latest live test failed: its error. */
+      error: z.string().nullable(),
+      /** The latest live test passed, but some pool keys failed: names them. */
+      warning: z.string().nullable(),
     }),
   ),
   embeddings: z.object({

package/src/be/scripts/typecheck.ts

@@ -250,7 +250,7 @@ export interface SwarmSdk {
   workflow_list(args?: { enabled?: boolean; includeFull?: boolean; consecutiveErrorsMin?: number; lastRunStatus?: "running" | "waiting" | "completed" | "failed" | "skipped" | "cancelled" }): Promise<unknown>;
   workflow_get(args: { id: string }): Promise<unknown>;
   workflow_listRuns(args: { workflowId: string; status?: "running" | "waiting" | "completed" | "failed" | "skipped" | "cancelled"; limit?: number; offset?: number }): Promise<unknown>;
-  workflow_getRun(args: { id: string }): Promise<unknown>;
+  workflow_getRun(args: { id: string; fields?: "full" | "slim" }): Promise<unknown>;
   // --- prompt templates ---
   prompt_list(args?: { eventType?: string; scope?: "global" | "agent" | "repo"; scopeId?: string; isDefault?: boolean }): Promise<unknown>;
   prompt_get(args: { id: string }): Promise<unknown>;

package/src/be/sealed-json.ts

@@ -30,6 +30,7 @@ export const SEALED_REPLAY_COLUMNS: readonly { table: string; column: string }[]
   { table: "workflow_run_steps", column: "output_replay" },
   { table: "workflow_runs", column: "context_replay" },
   { table: "approval_requests", column: "responses" },
+  { table: "workflow_idempotency_keys", column: "responseBody" },
 ];
 
 /** SQL `EXISTS` arms matching a sealed value in each `SEALED_REPLAY_COLUMNS` entry. */

package/src/be/seed-scripts/catalog/compound-insights.inline.ts

@@ -574,14 +574,14 @@ export default async function compoundInsights(args: any, ctx: any) {
     const rows = rowsToObjects(
       await ctx.swarm.db_query({
         sql: `WITH raw AS (
-                 SELECT sessionId, iteration, lineNumber, content,
+                 SELECT rowid AS seq, sessionId, iteration, lineNumber, content,
                         json_extract(content, '$.tool_name') as jsonToolName
                  FROM session_logs
                  WHERE createdAt > ${w}
                    AND (content LIKE '%"type":"tool_use"%' OR json_extract(content, '$.tool_name') IS NOT NULL)
                )
                SELECT sessionId, iteration, lineNumber, jsonToolName, content
-               FROM raw ORDER BY sessionId, iteration, lineNumber LIMIT 100`,
+               FROM raw ORDER BY sessionId, iteration, seq LIMIT 100`,
       }),
     );
     const bySession = new Map<string, string[]>();

package/src/be/seed-scripts/catalog/compound-insights.ts

@@ -574,14 +574,14 @@ export default async function compoundInsights(args: any, ctx: any) {
     const rows = rowsToObjects(
       await ctx.swarm.db_query({
         sql: `WITH raw AS (
-                 SELECT sessionId, iteration, lineNumber, content,
+                 SELECT rowid AS seq, sessionId, iteration, lineNumber, content,
                         json_extract(content, '$.tool_name') as jsonToolName
                  FROM session_logs
                  WHERE createdAt > ${w}
                    AND (content LIKE '%"type":"tool_use"%' OR json_extract(content, '$.tool_name') IS NOT NULL)
                )
                SELECT sessionId, iteration, lineNumber, jsonToolName, content
-               FROM raw ORDER BY sessionId, iteration, lineNumber LIMIT 100`,
+               FROM raw ORDER BY sessionId, iteration, seq LIMIT 100`,
       }),
     );
     const bySession = new Map<string, string[]>();

package/src/be/seed-skills/bundled-files.generated.json

@@ -18,7 +18,7 @@
   "delegate-work": [
     {
       "path": "scripts/codex-exec.sh",
-      "content": "#!/usr/bin/env bash\n#\n# codex-exec.sh — invoke `codex exec` (non-interactive) for delegated\n# implementation work. This is the SINGLE SOURCE OF TRUTH for the invocation\n# flags. The delegate-work skill calls only this script; its routing matrix\n# decides WHICH gpt-5.6 variant (-m) and effort (-e) each task gets:\n#   luna  = mechanical/bounded     terra = everyday frozen-spec slice\n#   sol   = hard/long-horizon      effort: medium|high|xhigh|max (never ultra)\n#\n# Usage:\n#   codex-exec.sh [-m <model>] [-e <effort>] -C <workdir> -o <last-message-file> [-l <log-file>] \"PROMPT\"\n#   printf '%s' \"$PROMPT\" | codex-exec.sh -C <workdir> -o <file>     # prompt via stdin (preferred)\n#\n# Notes:\n#   * -C sets codex's working root (the git worktree, for parallel isolation).\n#   * -o writes codex's final message (its report) to a file you can read back.\n#   * Reads the prompt from stdin when no positional PROMPT is given.\n#   * In `exec` mode approvals are already \"never\", so nothing hangs waiting for\n#     a human. The sandbox mode below bounds what codex may WRITE.\n#\n# Sandbox default: `workspace-write` — codex may freely edit files inside its\n# working root (the worktree) and read elsewhere on disk (so it can read the\n# plan by absolute path), but cannot write outside the workspace. This keeps\n# parallel worktree runs from clobbering each other or the main tree.\n#\n# Env overrides (flags -m/-e take precedence):\n#   CODEX_MODEL   default: gpt-5.6-sol          gpt-5.6-sol | gpt-5.6-terra | gpt-5.6-luna\n#   CODEX_EFFORT  default: high                 reasoning effort (never \"fast\"/low)\n#   CODEX_SANDBOX default: workspace-write (passed to codex's -s option)\n#   CODEX_BYPASS  set to 1 to use --dangerously-bypass-approvals-and-sandbox\n#                 (fully unsandboxed — opt in only when a phase must write\n#                 outside its worktree; overrides CODEX_SANDBOX)\nset -euo pipefail\n\nWORKDIR=\".\"\nOUTFILE=\"\"\nLOGFILE=\"\"\nPROMPT=\"\"\nMODEL_FLAG=\"\"\nEFFORT_FLAG=\"\"\n\nwhile [[ $# -gt 0 ]]; do\n  case \"$1\" in\n    -m|--model)  MODEL_FLAG=\"$2\"; shift 2 ;;\n    -e|--effort) EFFORT_FLAG=\"$2\"; shift 2 ;;\n    -C|--cd)  WORKDIR=\"$2\"; shift 2 ;;\n    -o|--out) OUTFILE=\"$2\"; shift 2 ;;\n    -l|--log) LOGFILE=\"$2\"; shift 2 ;;\n    --)       shift; PROMPT=\"${1:-}\"; break ;;\n    -*)       echo \"codex-exec.sh: unknown flag: $1\" >&2; exit 2 ;;\n    *)        PROMPT=\"$1\"; shift ;;\n  esac\ndone\n\nMODEL=\"${MODEL_FLAG:-${CODEX_MODEL:-gpt-5.6-sol}}\"\nEFFORT=\"${EFFORT_FLAG:-${CODEX_EFFORT:-high}}\"\nSANDBOX=\"${CODEX_SANDBOX:-workspace-write}\"\n\nargs=( exec\n  -m \"$MODEL\"\n  -c \"model_reasoning_effort=\\\"$EFFORT\\\"\"\n  -C \"$WORKDIR\"\n)\nif [[ \"${CODEX_BYPASS:-0}\" == \"1\" ]]; then\n  args+=( --dangerously-bypass-approvals-and-sandbox )\nelse\n  args+=( -s \"$SANDBOX\" )\nfi\n[[ -n \"$OUTFILE\" ]] && args+=( -o \"$OUTFILE\" )\n\n# Prompt source: positional arg if present, otherwise stdin (`-`).\nif [[ -n \"$PROMPT\" ]]; then\n  set -- \"$PROMPT\"\nelse\n  set -- \"-\"\nfi\n\nif [[ -n \"$LOGFILE\" ]]; then\n  codex \"${args[@]}\" \"$@\" 2>&1 | tee \"$LOGFILE\"\nelse\n  codex \"${args[@]}\" \"$@\"\nfi\n"
+      "content": "#!/usr/bin/env bash\n#\n# codex-exec.sh — invoke `codex exec` (non-interactive) for delegated\n# implementation work. This is the SINGLE SOURCE OF TRUTH for the invocation\n# flags. The delegate-work skill calls only this script; its routing matrix\n# decides WHICH gpt-6 variant (-m) and effort (-e) each task gets:\n#   gpt-6-luna  = trivial/mechanical   gpt-6.1-sol = default (medium) to hard (high/xhigh)\n#   gpt-6-astra = escalation only      effort: medium|high|xhigh|max (never ultra)\n#\n# Usage:\n#   codex-exec.sh [-m <model>] [-e <effort>] -C <workdir> -o <last-message-file> [-l <log-file>] \"PROMPT\"\n#   printf '%s' \"$PROMPT\" | codex-exec.sh -C <workdir> -o <file>     # prompt via stdin (preferred)\n#\n# Notes:\n#   * -C sets codex's working root (the git worktree, for parallel isolation).\n#   * -o writes codex's final message (its report) to a file you can read back.\n#   * Reads the prompt from stdin when no positional PROMPT is given.\n#   * In `exec` mode approvals are already \"never\", so nothing hangs waiting for\n#     a human. The sandbox mode below bounds what codex may WRITE.\n#\n# Sandbox default: `workspace-write` — codex may freely edit files inside its\n# working root (the worktree) and read elsewhere on disk (so it can read the\n# plan by absolute path), but cannot write outside the workspace. This keeps\n# parallel worktree runs from clobbering each other or the main tree.\n#\n# Env overrides (flags -m/-e take precedence):\n#   CODEX_MODEL   default: gpt-6.1-sol          gpt-6.1-sol | gpt-6-astra | gpt-6-luna\n#   CODEX_EFFORT  default: high                 reasoning effort (never \"fast\"/low)\n#   CODEX_SANDBOX default: workspace-write (passed to codex's -s option)\n#   CODEX_BYPASS  set to 1 to use --dangerously-bypass-approvals-and-sandbox\n#                 (fully unsandboxed — opt in only when a phase must write\n#                 outside its worktree; overrides CODEX_SANDBOX)\nset -euo pipefail\n\nWORKDIR=\".\"\nOUTFILE=\"\"\nLOGFILE=\"\"\nPROMPT=\"\"\nMODEL_FLAG=\"\"\nEFFORT_FLAG=\"\"\n\nwhile [[ $# -gt 0 ]]; do\n  case \"$1\" in\n    -m|--model)  MODEL_FLAG=\"$2\"; shift 2 ;;\n    -e|--effort) EFFORT_FLAG=\"$2\"; shift 2 ;;\n    -C|--cd)  WORKDIR=\"$2\"; shift 2 ;;\n    -o|--out) OUTFILE=\"$2\"; shift 2 ;;\n    -l|--log) LOGFILE=\"$2\"; shift 2 ;;\n    --)       shift; PROMPT=\"${1:-}\"; break ;;\n    -*)       echo \"codex-exec.sh: unknown flag: $1\" >&2; exit 2 ;;\n    *)        PROMPT=\"$1\"; shift ;;\n  esac\ndone\n\nMODEL=\"${MODEL_FLAG:-${CODEX_MODEL:-gpt-6.1-sol}}\"\nEFFORT=\"${EFFORT_FLAG:-${CODEX_EFFORT:-high}}\"\nSANDBOX=\"${CODEX_SANDBOX:-workspace-write}\"\n\nargs=( exec\n  -m \"$MODEL\"\n  -c \"model_reasoning_effort=\\\"$EFFORT\\\"\"\n  -C \"$WORKDIR\"\n)\nif [[ \"${CODEX_BYPASS:-0}\" == \"1\" ]]; then\n  args+=( --dangerously-bypass-approvals-and-sandbox )\nelse\n  args+=( -s \"$SANDBOX\" )\nfi\n[[ -n \"$OUTFILE\" ]] && args+=( -o \"$OUTFILE\" )\n\n# Prompt source: positional arg if present, otherwise stdin (`-`).\nif [[ -n \"$PROMPT\" ]]; then\n  set -- \"$PROMPT\"\nelse\n  set -- \"-\"\nfi\n\nif [[ -n \"$LOGFILE\" ]]; then\n  codex \"${args[@]}\" \"$@\" 2>&1 | tee \"$LOGFILE\"\nelse\n  codex \"${args[@]}\" \"$@\"\nfi\n"
     }
   ],
   "design-docs": [