@desplega.ai/agent-swarm 2.5.4 → 2.7.0

package/src/be/db.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite";
 import pkg from "../../package.json";
-import { defaultAssetKey, normalizeAssetKey, sessionGroupKey } from "../assets/key";
+import { defaultAssetKey, normalizeAssetKey } from "../assets/key";
+import { sessionSourceKey, sessionThreadGroupKey } from "../assets/source-key";
 import {
   generateDefaultClaudeMd,
   generateDefaultIdentityMd,
@@ -207,6 +208,7 @@ import {
   sessionCountQuery,
   sessionPageFields,
   sessionSummaryQuery,
+  threadRootsSql,
 } from "./db/session-tree";
 import { searchSkillRowsRanked } from "./db/skill-search";
 import { classifyTaskHumanFree, reclassifyTaskHumanFree } from "./db/tasks/human-free";
@@ -225,8 +227,15 @@ import { configureTaskWriteDependencies, failTask } from "./db/tasks/write";
 import { tombstoneWorkflowIdempotencyKeys } from "./db/workflow-idempotency";
 import {
   emitChildRunFinished,
+  emitRunFinished,
   getRunLineageBlock,
   getStepExecutionBlock,
+  isTerminalWorkflowRunStatus,
+  rowToWorkflowRun,
+  rowToWorkflowRunStep,
+  WORKFLOW_RUN_SUMMARY_COLUMNS,
+  type WorkflowRunRow,
+  type WorkflowRunStepRow,
   WorkflowStepNotExecutableError,
   wakeParentsOfDeletedRuns,
 } from "./db/workflow-runs";
@@ -242,7 +251,7 @@ import {
 } from "./swarm-config-guard";
 import { emitTaskStarted } from "./task-lifecycle-events";
 import { publishWorkerConfigNudge, publishWorkerNudge } from "./worker-nudges";
-import { defineWorkflowPayload, type WorkflowPayloadView } from "./workflow-replay";
+import type { WorkflowPayloadView } from "./workflow-replay";
 
 export {
   refreshActiveSessionOnActivity,
@@ -285,7 +294,7 @@ export {
   updateAgentStatus,
   updateAgentStatusFromCapacity,
 } from "./db/agents";
-export { recordKeySeatMismatch } from "./db/api-keys";
+export { recordKeySeatMismatch, recordKeyValidation } from "./db/api-keys";
 export {
   type ApprovalRequestSummary,
   type ApprovalVote,
@@ -2497,6 +2506,8 @@ export async function createTaskExtended(
     );
   }
   let requestedByUserIdInherited = false;
+  // The parent key is read again inside the INSERT transaction (see below).
+  let keyInherited = false;
   // True only when `options.routingAffinity` ends up populated purely via
   // the plain parent-fallback inherit below (child declared no affinity of
   // its own, the parent wasn't lead-only, AND the parent's affinity actually
@@ -2661,6 +2672,7 @@ export async function createTaskExtended(
       }
       if (parent.key && !options.key) {
         options.key = parent.key;
+        keyInherited = true;
       }
       if (parent.contextKey && !options.contextKey) {
         options.contextKey = parent.contextKey;
@@ -2850,7 +2862,18 @@ export async function createTaskExtended(
   }
 
   const auditUserId = getCurrentRequestUserId() ?? null;
-  const assetKey = normalizeAssetKey(options?.key ?? defaultAssetKey("task", id));
+  const assetKey = normalizeAssetKey(
+    options?.key ??
+      (options?.parentTaskId
+        ? null
+        : sessionSourceKey({
+            id,
+            slackChannelId: options?.slackChannelId,
+            slackThreadTs: options?.slackThreadTs,
+            contextKey: options?.contextKey,
+          })) ??
+      defaultAssetKey("task", id),
+  );
   // The Linear tracker dedup guard and the INSERT commit as one unit. Two
   // distinct webhook deliveries for the same issue are dispatched
   // fire-and-forget (src/linear/webhook.ts), nothing serializes them, and
@@ -2875,6 +2898,19 @@ export async function createTaskExtended(
       : await findExistingLinearTrackerContextWork(options?.contextKey);
     if (existingTrackerWork) return { existing: existingTrackerWork };
 
+    // A chain move (a user move, the source key backfill) can commit between
+    // the parent read above and this write lock. Its chain query did not see
+    // this child, so take the parent's key as it is now.
+    const parentKey =
+      keyInherited && options?.parentTaskId
+        ? (
+            await getDbClient().get<{ key: string }>('SELECT "key" FROM agent_tasks WHERE id = ?', [
+              options.parentTaskId,
+            ])
+          )?.key
+        : undefined;
+    const insertKey = parentKey ? normalizeAssetKey(parentKey) : assetKey;
+
     let routingDecisionSnapshot: RoutingDecisionSnapshot | null = null;
     try {
       routingDecisionSnapshot = await captureRoutingDecisionSnapshot(options);
@@ -2905,7 +2941,7 @@ export async function createTaskExtended(
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
       [
         id,
-        assetKey,
+        insertKey,
         options?.agentId ?? null,
         options?.creatorAgentId ?? null,
         // Free text from every task source; scrub at the single INSERT.
@@ -7859,53 +7895,6 @@ export async function getWorkflowsByScheduleId(scheduleId: string): Promise<Work
 // Workflow Run CRUD
 // ============================================================================
 
-type WorkflowRunRow = {
-  id: string;
-  workflowId: string;
-  status: string;
-  triggerType: string;
-  triggerData: string | null;
-  /** Absent from list rows: see `WORKFLOW_RUN_SUMMARY_COLUMNS`. */
-  context?: string | null;
-  /** Sealed exact `context` (migration 201). Absent from list rows. */
-  context_replay?: string | null;
-  error: string | null;
-  created_by: string | null;
-  startedAt: string;
-  lastUpdatedAt: string;
-  finishedAt: string | null;
-  /** The `sub-workflow` step that started this run (migration 203). */
-  parentStepId?: string | null;
-  /** That step's attempt (`retryCount`) when it started this run (migration 240). */
-  parentStepAttempt?: number | null;
-};
-
-/**
- * `view` picks the `context` copy: `replay` (default) is the exact sealed value
- * the engine resumes from; `display` is the scrubbed column for API, MCP and UI
- * responses. See `src/be/workflow-replay.ts`.
- */
-function rowToWorkflowRun(row: WorkflowRunRow, view: WorkflowPayloadView = "replay"): WorkflowRun {
-  const run: WorkflowRun = {
-    id: row.id,
-    workflowId: row.workflowId,
-    status: row.status as WorkflowRunStatus,
-    triggerData: row.triggerData ? JSON.parse(row.triggerData) : undefined,
-    context: undefined,
-    error: row.error ?? undefined,
-    createdBy: row.created_by ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    lastUpdatedAt: normalizeDateRequired(row.lastUpdatedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    parentStepId: row.parentStepId ?? undefined,
-    parentStepAttempt: row.parentStepId ? (row.parentStepAttempt ?? 0) : undefined,
-  };
-  return defineWorkflowPayload(run, "context", row.context, row.context_replay, view, {
-    runId: row.id,
-    target: "its context",
-  });
-}
-
 export async function createWorkflowRun(data: {
   id: string;
   workflowId: string;
@@ -8007,16 +7996,33 @@ export async function updateWorkflowRun(
   updates.push("lastUpdatedAt = ?");
   params.push(new Date().toISOString());
   params.push(id);
-  const row = await getDbClient().get<WorkflowRunRow>(
-    `UPDATE workflow_runs SET ${updates.join(", ")} WHERE id = ? RETURNING *`,
-    params,
-  );
+  const update = () =>
+    getDbClient().get<WorkflowRunRow>(
+      `UPDATE workflow_runs SET ${updates.join(", ")} WHERE id = ? RETURNING *`,
+      params,
+    );
+  const endsRun = data.status !== undefined && isTerminalWorkflowRunStatus(data.status);
+  // A run that ends reads the status it replaces in the same transaction, so
+  // only a change out of a live status announces the end: a later terminal
+  // write, same status or not, does not announce it again.
+  const [previousStatus, row] = endsRun
+    ? await getDbClient().transaction(async () => {
+        const previous = await getDbClient().get<{ status: WorkflowRunStatus }>(
+          "SELECT status FROM workflow_runs WHERE id = ?",
+          [id],
+        );
+        return [previous?.status, await update()] as const;
+      })
+    : [undefined, await update()];
   if (!row) return null;
   const run = rowToWorkflowRun(row);
   if (data.status === "completed" || data.status === "failed") {
     emitWorkflowTerminalTelemetry(run);
   }
   if (row.parentStepId) emitChildRunFinished(id, row.parentStepId, run.status);
+  if (endsRun && previousStatus && !isTerminalWorkflowRunStatus(previousStatus)) {
+    emitRunFinished(id, run.workflowId, run.status);
+  }
   return run;
 }
 
@@ -8031,12 +8037,6 @@ export type WorkflowRunListOptions = {
   includeContext?: boolean;
 };
 
-// Everything but `context`. `triggerData` stays: it is small, and stored
-// scripts read it off list rows (dedupe by trigger, failure reports). The
-// parent link and attempt stay so a list row identifies a child's invocation.
-const WORKFLOW_RUN_SUMMARY_COLUMNS =
-  "id, workflowId, status, triggerType, triggerData, error, created_by, startedAt, lastUpdatedAt, finishedAt, parentStepId, parentStepAttempt";
-
 export type WorkflowRunPage = {
   runs: WorkflowRun[];
   page: {
@@ -8121,56 +8121,6 @@ export async function listWorkflowRunsPage(
 // Workflow Run Step CRUD
 // ============================================================================
 
-type WorkflowRunStepRow = {
-  id: string;
-  runId: string;
-  nodeId: string;
-  nodeType: string;
-  status: string;
-  input: string | null;
-  output: string | null;
-  /** Sealed exact `output` (migration 201). */
-  output_replay: string | null;
-  error: string | null;
-  startedAt: string;
-  finishedAt: string | null;
-  retryCount: number;
-  maxRetries: number;
-  nextRetryAt: string | null;
-  idempotencyKey: string | null;
-  diagnostics: string | null;
-  nextPort: string | null;
-};
-
-/** `view` picks the `output` copy, as in `rowToWorkflowRun`. */
-function rowToWorkflowRunStep(
-  row: WorkflowRunStepRow,
-  view: WorkflowPayloadView = "replay",
-): WorkflowRunStep {
-  const step: WorkflowRunStep = {
-    id: row.id,
-    runId: row.runId,
-    nodeId: row.nodeId,
-    nodeType: row.nodeType,
-    status: row.status as WorkflowRunStepStatus,
-    input: row.input ? JSON.parse(row.input) : undefined,
-    output: undefined,
-    error: row.error ?? undefined,
-    startedAt: normalizeDateRequired(row.startedAt),
-    finishedAt: normalizeDate(row.finishedAt) ?? undefined,
-    retryCount: row.retryCount,
-    maxRetries: row.maxRetries,
-    nextRetryAt: normalizeDate(row.nextRetryAt) ?? undefined,
-    idempotencyKey: row.idempotencyKey ?? undefined,
-    diagnostics: row.diagnostics ?? undefined,
-    nextPort: row.nextPort ?? undefined,
-  };
-  return defineWorkflowPayload(step, "output", row.output, row.output_replay, view, {
-    runId: row.runId,
-    target: `the output of step ${row.id} (node ${row.nodeId})`,
-  });
-}
-
 export async function createWorkflowRunStep(data: {
   id: string;
   runId: string;
@@ -8284,7 +8234,7 @@ export async function getWorkflowRunStepsByRunId(
   view: WorkflowPayloadView = "replay",
 ): Promise<WorkflowRunStep[]> {
   const rows = await getDbClient().query<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? ORDER BY startedAt ASC, rowid ASC",
     [runId],
   );
   return rows.map((row) => rowToWorkflowRunStep(row, view));
@@ -8400,7 +8350,7 @@ export async function getLatestStepForNode(
   nodeId: string,
 ): Promise<WorkflowRunStep | null> {
   const row = await getDbClient().get<WorkflowRunStepRow>(
-    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC LIMIT 1",
+    "SELECT * FROM workflow_run_steps WHERE runId = ? AND nodeId = ? ORDER BY startedAt DESC, rowid DESC LIMIT 1",
     [runId, nodeId],
   );
   return row ? rowToWorkflowRunStep(row) : null;
@@ -11742,6 +11692,8 @@ export interface ApiKeyStatus {
   /** Auth failures in a row since the last success or clear. */
   consecutiveAuthFailures: number;
   lastAuthFailureAt: string | null;
+  lastValidationError: string | null;
+  lastValidationFailedAt: string | null;
   createdAt: string;
   updatedAt: string;
 }
@@ -11767,6 +11719,7 @@ function rowToApiKeyStatus(row: ApiKeyStatusRow): ApiKeyStatus {
 
 export interface AvailableKeyIndicesResult {
   availableIndices: number[];
+  rejectedIndices: number[];
   /** Indices excluded only by an active model-scoped window block (not key-wide). */
   modelBlockedIndices: number[];
   /** ISO of the earliest resetsAt among modelBlockedIndices, or null when none. */
@@ -11787,6 +11740,14 @@ export interface AvailableKeyIndicesResult {
  * whose `plan` cannot run the family (`planAllowsModelFamily`) is excluded
  * and reported in `seatBlockedIndices`, also when it is key-wide or
  * model-window blocked; a seat block has no reset time.
+ *
+ * With `keySuffixes` (the caller's own pool, in order), rows are matched by
+ * key suffix instead of the stored `keyIndex`, across every scope: a key has
+ * one state wherever it is pooled, and a key at the same index in another
+ * agent's pool is never blocked by it. A suffix listed more than once is read
+ * by `keyIndex` in `scope`/`scopeId`, as without the list. Returned indices
+ * are positions in `keySuffixes`. Without it, rows of `scope`/`scopeId` are
+ * read by `keyIndex`.
  */
 export async function getAvailableKeyIndices(
   keyType: string,
@@ -11794,36 +11755,60 @@ export async function getAvailableKeyIndices(
   scope = "global",
   scopeId: string | null = null,
   modelFamily?: ModelFamily,
+  keySuffixes?: string[],
 ): Promise<AvailableKeyIndicesResult> {
   const now = new Date().toISOString();
   const nowMs = Date.now();
   const client = getDbClient();
   const effectiveScopeId = scopeId ?? "";
+  const suffixes = keySuffixes?.length ? keySuffixes : undefined;
+  const rowFilter = suffixes
+    ? `keyType = ? AND keySuffix IN (${suffixes.map(() => "?").join(", ")})`
+    : "keyType = ? AND scope = ? AND scopeId = ?";
+  const rowParams = suffixes ? [keyType, ...suffixes] : [keyType, scope, effectiveScopeId];
 
   // Auto-clear expired rate limits. Never an auth bench: only `clearAuthBench` lifts it.
   await client.run(
     `UPDATE api_key_status
      SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
-     WHERE keyType = ? AND scope = ? AND scopeId = ?
+     WHERE ${rowFilter}
        AND status = 'rate_limited' AND rateLimitedUntil <= ? AND consecutiveAuthFailures < ?`,
-    [now, keyType, scope, effectiveScopeId, now, CODEX_AUTH_FAILURE_BENCH_THRESHOLD],
+    [now, ...rowParams, now, CODEX_AUTH_FAILURE_BENCH_THRESHOLD],
   );
 
   const rows = await client.query<{
     keyIndex: number;
+    keySuffix: string;
+    scope: string;
+    scopeId: string;
     status: string;
     rateLimitWindows: string | null;
     plan: string | null;
     authFailureSeq: number;
+    lastValidationFailedAt: string | null;
   }>(
-    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq FROM api_key_status
-       WHERE keyType = ? AND scope = ? AND scopeId = ?`,
-    [keyType, scope, effectiveScopeId],
-  );
+    `SELECT keyIndex, keySuffix, scope, scopeId, status, rateLimitWindows, plan, authFailureSeq, lastValidationFailedAt FROM api_key_status
+       WHERE ${rowFilter}`,
+    rowParams,
+  );
+  // Pool positions a row applies to: the position holding its suffix, or its stored index.
+  // A suffix held by several pool members cannot tell them apart, so it falls back to
+  // the index read: only a row of the requested scope, at its stored index.
+  const positionsOf = (row: (typeof rows)[number]): number[] => {
+    if (!suffixes) return [row.keyIndex];
+    const positions = suffixes.flatMap((suffix, i) => (suffix === row.keySuffix ? [i] : []));
+    if (positions.length < 2) return positions;
+    const sameScope = row.scope === scope && row.scopeId === effectiveScopeId;
+    return sameScope && positions.includes(row.keyIndex) ? [row.keyIndex] : [];
+  };
 
   const blockedIndices = new Set(
-    rows.filter((r) => r.status === "rate_limited").map((r) => r.keyIndex),
+    rows.filter((r) => r.status === "rate_limited").flatMap(positionsOf),
   );
+  const rejectedIndices = [
+    ...new Set(rows.filter((row) => row.lastValidationFailedAt !== null).flatMap(positionsOf)),
+  ];
+  for (const index of rejectedIndices) blockedIndices.add(index);
 
   const modelBlockedIndices: number[] = [];
   let earliestModelResetsAtSec: number | undefined;
@@ -11832,14 +11817,17 @@ export async function getAvailableKeyIndices(
       // A key already blocked key-wide doesn't need a separate model-block
       // entry — it's excluded from availableIndices either way, and
       // modelBlockedIndices means "excluded only by a model-scoped block".
-      if (blockedIndices.has(row.keyIndex)) continue;
+      const positions = positionsOf(row).filter(
+        (i) => !blockedIndices.has(i) && !modelBlockedIndices.includes(i),
+      );
+      if (positions.length === 0) continue;
       const block = activeModelBlock(
         parseRateLimitWindowsJson(row.rateLimitWindows),
         modelFamily,
         nowMs,
       );
       if (!block) continue;
-      modelBlockedIndices.push(row.keyIndex);
+      modelBlockedIndices.push(...positions);
       if (earliestModelResetsAtSec === undefined || block.resetsAt < earliestModelResetsAtSec) {
         earliestModelResetsAtSec = block.resetsAt;
       }
@@ -11853,7 +11841,10 @@ export async function getAvailableKeyIndices(
   const seatBlockedIndices: number[] = [];
   if (modelFamily) {
     for (const row of rows) {
-      if (!planAllowsModelFamily(row.plan, modelFamily)) seatBlockedIndices.push(row.keyIndex);
+      if (planAllowsModelFamily(row.plan, modelFamily)) continue;
+      for (const i of positionsOf(row)) {
+        if (!seatBlockedIndices.includes(i)) seatBlockedIndices.push(i);
+      }
     }
   }
   const seatBlockedSet = new Set(seatBlockedIndices);
@@ -11866,6 +11857,7 @@ export async function getAvailableKeyIndices(
 
   return {
     availableIndices,
+    rejectedIndices,
     modelBlockedIndices,
     seatBlockedIndices,
     earliestModelResetAt:
@@ -12201,22 +12193,30 @@ export async function clearKeyRateLimit(
   const effectiveScopeId = scopeId ?? "";
   const { keyIndex = -1, authFence = null } = opts;
   const clearAuthBench = opts.clearAuthBench === true && authFence !== null;
-  const key = [keyType, keySuffix, scope, effectiveScopeId];
   return await getDbClient().transaction(async (tx) => {
+    // The key's rows in every scope: `getAvailableKeyIndices` reads them all by suffix.
     const result = await tx.run(
       `UPDATE api_key_status
          SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
-         WHERE keyType = ? AND keySuffix = ? AND scope = ? AND scopeId = ?
+         WHERE keyType = ? AND keySuffix = ?
            AND status = 'rate_limited' AND ((? AND authFailureSeq <= ?) OR consecutiveAuthFailures < ?)`,
-      [now, ...key, clearAuthBench ? 1 : 0, authFence, CODEX_AUTH_FAILURE_BENCH_THRESHOLD],
+      [
+        now,
+        keyType,
+        keySuffix,
+        clearAuthBench ? 1 : 0,
+        authFence,
+        CODEX_AUTH_FAILURE_BENCH_THRESHOLD,
+      ],
     );
     const cleared = result.changes > 0;
     if (!clearAuthBench) return cleared;
     const lifted = await tx.query<{ keySuffix: string }>(
       `UPDATE api_key_status SET consecutiveAuthFailures = 0, updatedAt = ?1,
            status = CASE WHEN keySuffix = ?2 THEN status ELSE 'available' END, rateLimitedUntil = CASE WHEN keySuffix = ?2 THEN rateLimitedUntil END
-         WHERE keyType = ?3 AND scope = ?4 AND scopeId = ?5 AND (consecutiveAuthFailures > 0 OR status = 'rate_limited')
-           AND ((keySuffix = ?2 AND authFailureSeq <= ?7) OR (keySuffix != ?2 AND keyIndex = ?6)) RETURNING keySuffix`,
+         WHERE keyType = ?3 AND (consecutiveAuthFailures > 0 OR status = 'rate_limited')
+           AND ((keySuffix = ?2 AND authFailureSeq <= ?7)
+             OR (keySuffix != ?2 AND keyIndex = ?6 AND scope = ?4 AND scopeId = ?5)) RETURNING keySuffix`,
       [now, keySuffix, keyType, scope, effectiveScopeId, keyIndex, authFence],
     );
     const retired = lifted.filter((row) => row.keySuffix !== keySuffix);
@@ -12997,6 +12997,7 @@ export async function listRecentSessions(
   opts: ListRecentSessionsOpts = {},
 ): Promise<SessionListItem[] | SessionListItemSummary[]> {
   const query = sessionSummaryQuery(opts);
+  const threadRoots = threadRootsSql();
   const rootRows = await getDbClient().query<
     AgentTaskRow & {
       __chainCount: number;
@@ -13005,15 +13006,17 @@ export async function listRecentSessions(
       __readState: SessionReadState | null;
       __snoozeUntil: string | null;
       __needsYou: number;
+      __threadRoots: number;
     }
   >(
     `SELECT r.*, a.chainCount AS __chainCount, a.lastActivityAt AS __lastActivityAt,
       a.latestStatus AS __latestStatus, a.readState AS __readState,
-      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou
+      a.snoozeUntil AS __snoozeUntil, a.needsYou AS __needsYou,
+      ${threadRoots.sql} AS __threadRoots
     FROM (${query.sql}) a CROSS JOIN agent_tasks r ON r.id = a.root_id
     ORDER BY ${opts.triage === "later" ? "a.snoozeUntil ASC," : ""}
       a.lastActivityAt DESC, r.createdAt DESC, r.id DESC LIMIT ? OFFSET ?`,
-    [...query.params, opts.limit ?? 25, opts.offset ?? 0],
+    [...threadRoots.params, ...query.params, opts.limit ?? 25, opts.offset ?? 0],
   );
   const fields = await sessionPageFields(
     rootRows.map((row) => ({
@@ -13029,7 +13032,7 @@ export async function listRecentSessions(
     ...(row.__readState ? { readState: row.__readState } : {}),
     snoozeUntil: row.__snoozeUntil,
     needsYou: !!row.__needsYou,
-    groupKey: sessionGroupKey(row.key),
+    groupKey: sessionThreadGroupKey(row.key, row.__threadRoots),
     ...fields.get(row.id),
   }));
 }

package/src/be/memory/providers/openai-embedding.ts

@@ -1,4 +1,5 @@
 import OpenAI from "openai";
+import { selectCredential } from "../../../utils/credentials";
 import { getOpenRouterAttributionHeaders } from "../../../utils/openrouter-base-url";
 import { DEFAULT_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS } from "../constants";
 import type { EmbeddingProvider } from "../types";
@@ -46,7 +47,10 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
   private readonly apiKey: string | undefined;
 
   constructor(config?: OpenAIEmbeddingConfig) {
-    this.apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    const apiKey = config?.apiKey ?? process.env.EMBEDDING_API_KEY ?? process.env.OPENAI_API_KEY;
+    this.apiKey = apiKey
+      ? selectCredential(apiKey, undefined, "EMBEDDING_API_KEY").selected
+      : undefined;
 
     this.model = config?.model ?? process.env.EMBEDDING_MODEL ?? "text-embedding-3-small";
 

package/src/be/memory/task-learning.ts

@@ -0,0 +1,155 @@
+/**
+ * Learnings captured through `store-progress`: one swarm-scope `manual` memory
+ * per learning, keyed `/learnings/<workstream>/<slug>` and tied to its task by
+ * `sourceTaskId`, so "what did we learn doing X" is one `keyPrefix` search.
+ *
+ * Source stays `manual` so nightly curation (memtax B1 reads only manual and
+ * file_index rows) picks learnings up without a new source value.
+ */
+import { getDbClient } from "@/be/db";
+import { indexMemoryContent } from "@/be/memory/index-content";
+import { contentSha256 } from "@/commands/profile-sync";
+import type { AgentTask } from "@/types";
+import { scrubSecrets } from "@/utils/secret-scrubber";
+
+/** ~500 tokens at ceil(chars / 4); also keeps a learning to one memory chunk. */
+export const LEARNING_MAX_CHARS = 2000;
+export const LEARNING_MAX_TAGS = 5;
+const LEARNINGS_ROOT = "/learnings";
+const SEGMENT_MAX_CHARS = 60;
+const KEY_ATTEMPTS = 4;
+
+export type TaskLearningInput = string | { text: string; workstream?: string; tags?: string[] };
+
+export type TaskLearningResult =
+  | { status: "stored"; memoryId: string; key: string; workstream: string }
+  | { status: "replayed"; memoryId: string; key: string }
+  | { status: "failed"; error: string };
+
+/** Lowercase kebab slug: `Agent Swarm!` -> `agent-swarm`. Empty when nothing usable is left. */
+export function toKebab(value: string, maxChars = SEGMENT_MAX_CHARS): string {
+  return value
+    .normalize("NFKD")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+/, "")
+    .slice(0, maxChars)
+    .replace(/-+$/, "");
+}
+
+/** Explicit workstream, else the repo name from `vcsRepo` (`owner/name` or a URL), else `general`. */
+export function resolveLearningWorkstream(
+  explicit: string | undefined,
+  vcsRepo: string | undefined,
+): string {
+  const fromExplicit = explicit ? toKebab(explicit) : "";
+  if (fromExplicit) return fromExplicit;
+  const repoName = vcsRepo
+    ?.trim()
+    .replace(/\/+$/, "")
+    .split("/")
+    .pop()
+    ?.replace(/\.git$/, "");
+  const fromRepo = repoName ? toKebab(repoName) : "";
+  return fromRepo || "general";
+}
+
+export async function storeTaskLearning(params: {
+  agentId: string;
+  isLead: boolean;
+  task: Pick<AgentTask, "id" | "vcsRepo">;
+  learning: TaskLearningInput;
+}): Promise<TaskLearningResult> {
+  const { agentId, isLead, task } = params;
+  const input = typeof params.learning === "string" ? { text: params.learning } : params.learning;
+  // Scrub before anything is derived: a token in the title, workstream or a tag
+  // would otherwise survive (kebab-cased) in the key and tags, and the replay
+  // hash must match the hash of the scrubbed content the store persists.
+  const text = scrubSecrets(input.text).trim();
+  if (!text) return { status: "failed", error: "learning text is empty" };
+  if (text.length > LEARNING_MAX_CHARS) {
+    return {
+      status: "failed",
+      error: `learning text is over ~500 tokens (${LEARNING_MAX_CHARS} characters); shorten it`,
+    };
+  }
+  const tags = [
+    ...new Set((input.tags ?? []).map((tag) => toKebab(scrubSecrets(tag))).filter(Boolean)),
+  ].filter((tag) => tag !== "learning");
+  if (tags.length > LEARNING_MAX_TAGS) {
+    return { status: "failed", error: `learning takes at most ${LEARNING_MAX_TAGS} tags` };
+  }
+
+  const workstream = resolveLearningWorkstream(
+    input.workstream === undefined ? undefined : scrubSecrets(input.workstream),
+    task.vcsRepo === undefined ? undefined : scrubSecrets(task.vcsRepo),
+  );
+
+  try {
+    // One write transaction for the replay check and the insert, so two
+    // overlapping calls with the same learning store it once.
+    return await getDbClient().transaction(() =>
+      insertLearning({ agentId, isLead, taskId: task.id, text, tags, workstream }),
+    );
+  } catch (err) {
+    return {
+      status: "failed",
+      error: scrubSecrets(err instanceof Error ? err.message : String(err)),
+    };
+  }
+}
+
+async function insertLearning(params: {
+  agentId: string;
+  isLead: boolean;
+  taskId: string;
+  text: string;
+  tags: string[];
+  workstream: string;
+}): Promise<TaskLearningResult> {
+  const { agentId, isLead, taskId, text, tags, workstream } = params;
+  // indexMemoryContent stores text this short as one trimmed chunk, so this
+  // hash is the row's contentHash: a replayed call finds its own earlier write.
+  const contentHash = contentSha256(text);
+  const replay = await getDbClient().get<{ id: string; key: string }>(
+    `SELECT id, key FROM agent_memory
+     WHERE sourceTaskId = ? AND contentHash = ? AND key LIKE '${LEARNINGS_ROOT}/%'
+     LIMIT 1`,
+    [taskId, contentHash],
+  );
+  if (replay) return { status: "replayed", memoryId: replay.id, key: replay.key };
+
+  const title = text.split(/\r?\n/, 1)[0]!.trim().slice(0, 200);
+  const baseKey = `${LEARNINGS_ROOT}/${workstream}/${toKebab(title) || "learning"}`;
+
+  // Keys are unique per owner and a reused key fails rather than upserts.
+  // Keep the path unique across the swarm: on a taken key, add a short suffix.
+  for (let attempt = 0; attempt < KEY_ATTEMPTS; attempt++) {
+    const key = attempt === 0 ? baseKey : `${baseKey}-${crypto.randomUUID().slice(0, 6)}`;
+    const taken = await getDbClient().get<{ id: string }>(
+      "SELECT id FROM agent_memory WHERE scope = 'swarm' AND key = ? LIMIT 1",
+      [key],
+    );
+    if (taken) continue;
+    try {
+      const result = await indexMemoryContent({
+        agentId,
+        content: text,
+        name: title,
+        scope: "swarm",
+        source: "manual",
+        sourceTaskId: taskId,
+        tags: ["learning", ...tags],
+        intent: "store-progress learning",
+        key,
+        writer: { principal: { kind: "agent", agentId, isLead }, source: "mcp" },
+      });
+      return { status: "stored", memoryId: result.memoryIds[0]!, key, workstream };
+    } catch (err) {
+      // The key is taken after all: try a suffixed one.
+      if (/UNIQUE constraint failed/i.test((err as Error).message)) continue;
+      throw err;
+    }
+  }
+  return { status: "failed", error: `no free key under ${baseKey}` };
+}

package/src/be/migrations/244_api_key_validation.sql

@@ -0,0 +1,4 @@
+-- Last provider rejection of a pooled key by a worker live test (HTTP 401).
+-- A key with lastValidationFailedAt set is left out of /api/keys/available.
+ALTER TABLE api_key_status ADD COLUMN lastValidationError TEXT;
+ALTER TABLE api_key_status ADD COLUMN lastValidationFailedAt TEXT;