@aui.io/apollo 0.1.49 → 0.1.52

package/node_modules/@aui.io/apollo-core/dist/src/services/mockdb.js

@@ -1,9 +1,38 @@
+/**
+ * Mock DB service layer.
+ *
+ * Drives the `mock-db-service` management + runtime planes on behalf of the
+ * `apollo mockdb` commands and the TUI. Pure logic — no terminal, no
+ * process.exit.
+ *
+ * ─── Identity & auth ────────────────────────────────────────────────────
+ *
+ * The agent comes from the checkout (`.apollorc`) or `--agent`; the Apollo
+ * token from the active profile. The token is sent to the mock-db service
+ * FIRST-PARTY and only where the service demands it (provision / status /
+ * delete / minting a management key). Every other management call prefers
+ * the per-agent `managementKey` (`X-Mgmt-Key`) — no Apollo token transmitted
+ * — and runtime calls use the per-agent `runtimeKey`. Keys are persisted
+ * under `~/.apollo/mockdb/<agentId>.json` (outside the checkout).
+ *
+ * ─── Resilience ─────────────────────────────────────────────────────────
+ *
+ * The shared backend occasionally answers a 2xx with a BLANK body under
+ * load. Reads retry that with bounded backoff and then fail loudly (never a
+ * silent `{}` that reads as an empty DB); writes surface it as a verify-first
+ * error (a repeat could duplicate rows); key rotation retries until a key
+ * actually comes back. Runtime calls also ride out read-after-write replica
+ * lag (422 no_results / transient 5xx) where a retry can never duplicate a
+ * write.
+ */
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import path from "node:path";
-import { MockDbClient, } from "../api/mock-db.js";
+import { isMockDbTransportError, MockDbClient, mockDbErrorCode, } from "../api/mock-db.js";
 import { clearMockDbKeys, keyPrefixes, loadMockDbKeys, saveMockDbKeys, } from "../config/mockdb-keys.js";
-import { AuthError, ConfigError, ValidationError } from "../errors/index.js";
+import { ApiError, AuthError, CliError, ConfigError, ValidationError, } from "../errors/index.js";
+/** Env var the wired `connections.yaml` reads the runtime key from by default. */
+export const DEFAULT_MOCKDB_TOKEN_ENV = "MOCKDB_TOKEN";
 function accountIdFromToken(token) {
     try {
         const payload = token.split(".")[1];
@@ -64,111 +93,584 @@ export function resolveMockDb(context, options = {}) {
         client,
         agentId,
         ...(context.agentRoot ? { agentRoot: context.agentRoot } : {}),
+        accountId,
         tokenAuth,
         homeDir,
+        tokenEnv: options.tokenEnv?.trim() || DEFAULT_MOCKDB_TOKEN_ENV,
     };
 }
-async function persistProvisionKeys(agentId, baseUrl, resp, homeDir) {
+// ─── Blank-body guards (no silent empty-success) ────────────────────────
+/** A read result is "empty" only when the HTTP body was blank → `request()`
+ * returned a zero-key `{}`. A real read (even of an empty DB) returns a keyed
+ * object like `{ collections: [] }`, so this never flags a legitimately-empty
+ * DB — only a genuinely blank body. */
+function isEmptyReadObject(resp) {
+    return (resp != null &&
+        typeof resp === "object" &&
+        !Array.isArray(resp) &&
+        Object.keys(resp).length === 0);
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** True inside the unit-test harness (skip real backoff sleeps). */
+function inTestEnv() {
+    return !!process.env.VITEST || process.env.NODE_ENV === "test";
+}
+/** Bounded exponential backoff (+ jitter) between blank-body retries. */
+const BLANK_RETRY_ATTEMPTS = 5;
+function backoffMs(attempt) {
+    const base = Math.min(250 * 2 ** (attempt - 1), 2000);
+    return base + Math.floor(Math.random() * 100);
+}
+/**
+ * Re-run `fn` while `retry(result)` is true, up to `BLANK_RETRY_ATTEMPTS`,
+ * with bounded backoff. Only ever wrap calls that are SAFE to repeat
+ * (idempotent reads, or key rotation which mints fresh each time); never
+ * naive writes (insert/seed) where a repeat could duplicate.
+ */
+async function retryWhile(retry, fn) {
+    const inTest = inTestEnv();
+    let result = await fn();
+    for (let i = 1; i < BLANK_RETRY_ATTEMPTS && retry(result); i++) {
+        if (!inTest)
+            await sleep(backoffMs(i));
+        result = await fn();
+    }
+    return result;
+}
+/**
+ * Throw a precise, non-destructive error for a persistently-blank 2xx
+ * response. Worded per operation kind so the caller takes the SAFE next step
+ * (never "just retry" a write that may have landed; never "delete the DB" on
+ * a transient blip).
+ */
+function blankBodyError(resolved, op, kind) {
+    const head = `mockdb ${op}: the mock-db service (${resolved.context.baseUrls.mockDb}) returned HTTP 2xx but a blank body for agent ${resolved.agentId}` +
+        (kind === "write"
+            ? ". The write may have already landed — NOT auto-retried, to avoid duplicate rows."
+            : ", even after retries.");
+    const common = "This is a TRANSIENT transport blank on the shared backend under load (an empty 2xx body), not data loss. ";
+    let tail;
+    if (kind === "read") {
+        tail =
+            "Your data is most likely intact — wait a moment and retry. Do NOT delete or re-provision the mock DB based on this alone.";
+    }
+    else if (kind === "mint") {
+        tail =
+            "No key was stored, so nothing changed and it is safe to re-run. If it persists, the backend key-mint path is failing — check `apollo mockdb status` and report it.";
+    }
+    else {
+        tail =
+            "The write MAY have already landed server-side, so do NOT blindly retry (re-running an insert/seed can DUPLICATE rows). Verify first — `apollo mockdb describe` / `collections list` for base writes, `apollo mockdb session get --key <session>` for runtime writes — then retry only what's actually missing.";
+    }
+    throw new CliError(head, { code: "API_SERVER_ERROR", suggestion: common + tail });
+}
+/** Run an idempotent read, retrying a blank body, then failing loudly if it stays blank. */
+async function readWithRetry(resolved, op, fn) {
+    const result = await retryWhile(isEmptyReadObject, fn);
+    if (isEmptyReadObject(result))
+        blankBodyError(resolved, op, "read");
+    return result;
+}
+function isAuthRejection(error) {
+    return error instanceof ApiError && (error.status === 401 || error.status === 403);
+}
+// ─── Key persistence ────────────────────────────────────────────────────
+async function persistReturnedKeys(resolved, resp) {
     const patch = {
-        mockDbUrl: baseUrl,
+        mockDbUrl: resolved.context.baseUrls.mockDb,
         backend: "aui-managed",
     };
-    if (typeof resp.managementKey === "string")
+    if (typeof resp.managementKey === "string" && resp.managementKey) {
         patch.managementKey = resp.managementKey;
-    if (typeof resp.runtimeKey === "string")
+    }
+    if (typeof resp.runtimeKey === "string" && resp.runtimeKey) {
         patch.runtimeKey = resp.runtimeKey;
+    }
     if (patch.managementKey || patch.runtimeKey) {
-        await saveMockDbKeys(agentId, patch, homeDir);
+        await saveMockDbKeys(resolved.agentId, patch, resolved.homeDir);
+    }
+}
+function responseHasKey(resp, field) {
+    return typeof resp[field] === "string" && resp[field].length > 0;
+}
+/**
+ * Mint-and-retry for key rotation. Under load the service can drop the body
+ * AFTER minting; returning `{}` then persists nothing — a false success that
+ * strands the caller with no usable key. Rotation is safe to repeat (each
+ * call mints fresh; the last one returned wins), so retry until the expected
+ * key comes back and persist it, else fail LOUDLY.
+ */
+async function rotateWithRetry(resolved, field, op, fn) {
+    const resp = await retryWhile((r) => !responseHasKey(r, field), fn);
+    if (!responseHasKey(resp, field))
+        blankBodyError(resolved, op, "mint");
+    await persistReturnedKeys(resolved, resp);
+    return resp;
+}
+// ─── Management auth: mgmt key first, token as the safety net ───────────
+/**
+ * Mint + cache a fresh management key via the token-auth `rotate-mgmt-key`
+ * path. Returns the minted key, or `undefined` when minting isn't possible
+ * (agent not provisioned, no account, backend mint path failing) so callers
+ * fall back to token auth. Never throws.
+ */
+async function tryMintManagementKey(resolved) {
+    try {
+        const resp = await rotateWithRetry(resolved, "managementKey", "rotate mgmt", () => resolved.client.rotateMgmtKey(resolved.agentId, resolved.tokenAuth));
+        return typeof resp.managementKey === "string" ? resp.managementKey : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Run a management call. The management key is sourced from the cache first,
+ * and a fresh one is MINTED (+cached) when the cache has none or the stored
+ * one is rejected (401/403). The Apollo token is the safety net under every
+ * branch, so a missing/dead management key never fails a command.
+ */
+async function withManage(resolved, fn) {
+    let managementKey = (await loadMockDbKeys(resolved.agentId, resolved.homeDir))
+        .managementKey;
+    if (!managementKey) {
+        managementKey = await tryMintManagementKey(resolved);
+        if (!managementKey)
+            return fn(resolved.tokenAuth);
+    }
+    try {
+        return await fn({ mode: "mgmt", managementKey });
+    }
+    catch (error) {
+        if (!isAuthRejection(error))
+            throw error;
+        const fresh = await tryMintManagementKey(resolved);
+        if (fresh) {
+            try {
+                return await fn({ mode: "mgmt", managementKey: fresh });
+            }
+            catch (retryError) {
+                if (isAuthRejection(retryError))
+                    return fn(resolved.tokenAuth);
+                throw retryError;
+            }
+        }
+        return fn(resolved.tokenAuth);
     }
 }
 export async function provisionMockDb(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    const resp = await resolved.client.provision(resolved.agentId, resolved.tokenAuth);
-    await persistProvisionKeys(resolved.agentId, context.baseUrls.mockDb, resp, resolved.homeDir);
-    const stored = keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir));
-    return { ...resp, stored };
+    // Retry ONLY a pure transport blank. A real `created:false` (the DB already
+    // exists — keys not re-shown) is a legitimate, keyed response and must NOT
+    // be retried. Provision is idempotent, so retrying a blank is safe.
+    const resp = await retryWhile(isEmptyReadObject, () => resolved.client.provision(resolved.agentId, resolved.tokenAuth));
+    if (isEmptyReadObject(resp)) {
+        throw new CliError(`mockdb provision: the mock-db service (${context.baseUrls.mockDb}) returned HTTP 2xx but a blank body for agent ${resolved.agentId}, even after retries.`, {
+            code: "API_SERVER_ERROR",
+            suggestion: "Transient backend overload (an empty 2xx body). The DB may or may not have been created — wait a moment, then run `apollo mockdb status`; if it exists, mint keys with `apollo mockdb rotate runtime` / `rotate mgmt`. Do NOT blindly re-provision.",
+        });
+    }
+    await persistReturnedKeys(resolved, resp);
+    const keys = await loadMockDbKeys(resolved.agentId, resolved.homeDir);
+    const stored = keyPrefixes(keys);
+    // Loud-fail the silent "provisioned but no usable keys" dead-end: when the
+    // DB already existed the service returns created:false with NO keys.
+    // Without a runtime key, `execute` (and the deployed agent) can't call it.
+    if (!keys.runtimeKey) {
+        return {
+            ...resp,
+            stored,
+            keysAvailable: false,
+            keysHint: "No runtime key is stored for this agent (provision returned none — the DB already existed). Run `apollo mockdb rotate runtime` to mint one, then `apollo mockdb wire` and update the connection's token env. `execute` self-heals by minting on demand.",
+        };
+    }
+    return { ...resp, stored, keysAvailable: true };
 }
 export async function describeMockDb(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.describe(resolved.agentId, resolved.tokenAuth);
+    return readWithRetry(resolved, "describe", () => withManage(resolved, (auth) => resolved.client.describe(resolved.agentId, auth)));
 }
 export async function statusMockDb(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    const remote = await resolved.client.status(resolved.agentId, resolved.tokenAuth);
-    const keys = await loadMockDbKeys(resolved.agentId, resolved.homeDir);
+    const keys = keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir));
+    const hasLocalKeys = !!(keys.managementKey || keys.runtimeKey);
+    let remote = null;
+    let error;
+    try {
+        const resp = await retryWhile(isEmptyReadObject, () => resolved.client.status(resolved.agentId, resolved.tokenAuth));
+        // A blank `{}` (transport blank under load) is NOT "the service doesn't
+        // own this agent" — treat it as unreachable → `unknown`, never `new`.
+        if (isEmptyReadObject(resp)) {
+            throw new Error("/status returned a blank body (transport blank under load)");
+        }
+        remote = resp;
+    }
+    catch (probeError) {
+        error = probeError instanceof Error ? probeError.message : String(probeError);
+    }
+    let verdict;
+    let message;
+    if (!remote) {
+        verdict = "unknown";
+        message =
+            "The mock-db /status check was unreachable — nothing was changed. Retry, or run `apollo mockdb describe` to confirm.";
+    }
+    else if (remote.exists && remote.hasData !== false) {
+        verdict = "ok";
+        message =
+            "The mock-db service owns this agent's mock DB" +
+                (remote.hasData ? " and it has data." : ".");
+    }
+    else if (remote.exists) {
+        verdict = "empty";
+        message =
+            "The mock-db service owns this agent's record but it is EMPTY (no collections or endpoints). Seed it with `apollo mockdb collections create` / `seed` / `endpoint create`.";
+    }
+    else {
+        verdict = "new";
+        message = hasLocalKeys
+            ? "The mock-db service does NOT own this agent (no mock DB), yet local keys exist — they belong to a DB that was deleted or lives on another deployment. Run `apollo mockdb provision` to create one here; the stale keys are replaced on provision."
+            : "No mock DB for this agent yet. Run `apollo mockdb provision` to create one.";
+    }
     return {
         agentId: resolved.agentId,
+        backend: "aui-managed",
         baseUrl: context.baseUrls.mockDb,
         remote,
-        keys: keyPrefixes(keys),
+        keys,
+        verdict,
+        message,
+        ...(error === undefined ? {} : { error }),
     };
 }
 export async function createCollection(context, name, schema, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.createCollection(resolved.agentId, resolved.tokenAuth, {
-        name,
-        schema,
-    });
+    // No auto-retry on a write: a blank 2xx may mean the create already landed.
+    const resp = await withManage(resolved, (auth) => resolved.client.createCollection(resolved.agentId, auth, { name, schema }));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, "collections create", "write");
+    return resp;
 }
 export async function listCollections(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.listCollections(resolved.agentId, resolved.tokenAuth);
+    return readWithRetry(resolved, "collections list", () => withManage(resolved, (auth) => resolved.client.listCollections(resolved.agentId, auth)));
 }
 export async function updateCollection(context, name, addColumns, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.updateCollection(resolved.agentId, resolved.tokenAuth, name, {
-        addColumns,
-    });
+    const resp = await withManage(resolved, (auth) => resolved.client.updateCollection(resolved.agentId, auth, name, { addColumns }));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, "collections update", "write");
+    return resp;
 }
 export async function seedRows(context, collection, rows, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.seedRows(resolved.agentId, resolved.tokenAuth, collection, rows);
+    // Seeding APPENDS rows — never auto-retry a blank (a repeat could duplicate).
+    const resp = await withManage(resolved, (auth) => resolved.client.seedRows(resolved.agentId, auth, collection, rows));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, "seed", "write");
+    return resp;
 }
 export async function createEndpoint(context, body, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.createEndpoint(resolved.agentId, resolved.tokenAuth, body);
+    const resp = await withManage(resolved, (auth) => resolved.client.createEndpoint(resolved.agentId, auth, body));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, "endpoint create", "write");
+    return resp;
 }
 export async function listEndpoints(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.listEndpoints(resolved.agentId, resolved.tokenAuth);
+    return readWithRetry(resolved, "endpoint list", () => withManage(resolved, (auth) => resolved.client.listEndpoints(resolved.agentId, auth)));
 }
 export async function getEndpoint(context, slug, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.getEndpoint(resolved.agentId, resolved.tokenAuth, slug);
+    return readWithRetry(resolved, `endpoint get ${slug}`, () => withManage(resolved, (auth) => resolved.client.getEndpoint(resolved.agentId, auth, slug)));
 }
 export async function updateEndpoint(context, slug, body, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.updateEndpoint(resolved.agentId, resolved.tokenAuth, slug, body);
+    const resp = await withManage(resolved, (auth) => resolved.client.updateEndpoint(resolved.agentId, auth, slug, body));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, `endpoint update ${slug}`, "write");
+    return resp;
 }
 export async function deleteEndpoint(context, slug, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.deleteEndpoint(resolved.agentId, resolved.tokenAuth, slug);
+    const resp = await withManage(resolved, (auth) => resolved.client.deleteEndpoint(resolved.agentId, auth, slug));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, `endpoint delete ${slug}`, "write");
+    return resp;
+}
+/**
+ * Ordered runtime keys to try. The wired key (`$MOCKDB_TOKEN`, or the
+ * connection's `token_env`) is what a local `chat --local` run resolves, so
+ * it goes first; the cache holds the last key the console minted/stored and
+ * is the fallback. Duplicates collapse. `executeEndpoint` tries these in
+ * order, falls through on a 401/403, and mints only when none authenticate.
+ */
+async function runtimeKeyCandidates(resolved) {
+    const envKey = process.env[resolved.tokenEnv]?.trim();
+    const cacheKey = (await loadMockDbKeys(resolved.agentId, resolved.homeDir)).runtimeKey;
+    const candidates = [];
+    if (envKey)
+        candidates.push({ key: envKey, source: "env" });
+    if (cacheKey && cacheKey !== envKey)
+        candidates.push({ key: cacheKey, source: "cache" });
+    return candidates;
+}
+/**
+ * When `execute` had to mint a fresh key — or the wired env key was rejected
+ * and a cached one worked — the key the deployed agent / local runtime uses
+ * is stale. The console doesn't edit secrets itself, so it attaches a loud,
+ * actionable `runtimeKeyResync` block carrying the working key + the exact
+ * re-wire instruction. The command layer echoes it; tooling reads the JSON.
+ */
+function attachRuntimeKeyResync(resolved, resp, source, runtimeKey) {
+    const reason = source === "cache"
+        ? `the runtime key in $${resolved.tokenEnv} was rejected (HTTP 401); a working key was recovered from the local cache`
+        : "no valid runtime key was wired or cached; a fresh runtime key was minted (any previously wired key is now invalid)";
+    if (resp && typeof resp === "object" && !Array.isArray(resp)) {
+        const block = {
+            required: true,
+            source,
+            reason,
+            runtimeKey,
+            action: `Export ${resolved.tokenEnv}=${runtimeKey} (and update the vault secret / secrets.yaml the connection resolves it from), ` +
+                "then re-run `apollo mockdb wire` if connections.yaml changed and `apollo push`. Until then the DEPLOYED agent keeps getting HTTP 401 on its runtime calls.",
+        };
+        resp.runtimeKeyResync = block;
+    }
+    return resp;
+}
+// Read-after-write lag: a write commits, but the next read can be served by a
+// not-yet-caught-up replica — a 422 no_results (row not visible yet) or a
+// transient 5xx. Retried with bounded backoff where a retry can NEVER
+// duplicate a write:
+//   • 422 no_results ⇒ the service matched 0 rows ⇒ NOTHING was mutated
+//     (an insert always returns count 1, so it never 422s). Always safe.
+//   • a transient 5xx / dropped connection is ambiguous for a write, so it is
+//     retried ONLY for non-insert endpoints (read/update/delete are idempotent);
+//     an insert — or an endpoint whose kind can't be confirmed — surfaces at once.
+const NO_RESULTS_MAX_RETRIES = 2;
+const TRANSIENT_MAX_RETRIES = 4;
+const LAG_BACKOFF_MS = [300, 800, 1800, 3000];
+function lagBackoffMs(attempt) {
+    const base = LAG_BACKOFF_MS[Math.min(attempt, LAG_BACKOFF_MS.length) - 1] ?? 3000;
+    return base + Math.floor(Math.random() * 150);
+}
+const endpointKindCache = new Map();
+/** Best-effort endpoint kind via the management plane (cached per process).
+ *  Unknown ⇒ callers MUST treat as "not safe to retry a 5xx". */
+async function resolveEndpointKind(resolved, slug) {
+    const cacheKey = `${resolved.agentId}:${slug}`;
+    const cached = endpointKindCache.get(cacheKey);
+    if (cached)
+        return cached;
+    try {
+        const resp = await withManage(resolved, (auth) => resolved.client.getEndpoint(resolved.agentId, auth, slug));
+        const kind = resp?.kind;
+        if (kind === "read" || kind === "insert" || kind === "update" || kind === "delete") {
+            endpointKindCache.set(cacheKey, kind);
+            return kind;
+        }
+    }
+    catch {
+        // unknown → unsafe for a 5xx retry (the 422 retry is still allowed)
+    }
+    return undefined;
+}
+function isNoResultsError(error) {
+    return (error instanceof ApiError && error.status === 422 && mockDbErrorCode(error) === "no_results");
+}
+function isTransientServerError(error) {
+    if (error instanceof ApiError)
+        return error.status >= 500 && error.status <= 599;
+    return isMockDbTransportError(error);
+}
+/** Mint + cache a fresh runtime key (loud-fails on a keyless mint). */
+async function mintRuntimeKey(resolved) {
+    const resp = await rotateWithRetry(resolved, "runtimeKey", "rotate runtime", () => withManage(resolved, (auth) => resolved.client.rotateRuntimeKey(resolved.agentId, auth)));
+    // rotateWithRetry loud-fails unless the key came back, so it is present here.
+    return resp.runtimeKey;
 }
 export async function executeEndpoint(context, slug, input = {}, options = {}) {
     const resolved = resolveMockDb(context, options);
-    const keys = await loadMockDbKeys(resolved.agentId, resolved.homeDir);
-    if (!keys.runtimeKey) {
-        throw new ConfigError("No runtime key stored for this agent.", "Run `apollo mockdb provision` or `apollo mockdb rotate runtime`, then `apollo mockdb wire`.");
-    }
-    const auth = {
-        mode: "runtime",
-        runtimeKey: keys.runtimeKey,
-        ...(input.session ? { sessionKey: input.session } : {}),
+    const sessionKey = input.session?.trim() || undefined;
+    const candidates = await runtimeKeyCandidates(resolved);
+    const runOnce = (key) => {
+        const auth = {
+            mode: "runtime",
+            runtimeKey: key,
+            ...(sessionKey ? { sessionKey } : {}),
+        };
+        return resolved.client.execute(resolved.agentId, auth, slug, input.body);
+    };
+    // A WRITE always requires a session (the service rejects a write with no
+    // session key), so an execute WITHOUT a session is guaranteed a read and is
+    // safe to retry on a transient blank / 5xx; WITH a session the kind gates it.
+    let transientRetrySafe;
+    const transientIsSafe = async () => {
+        if (sessionKey === undefined)
+            return true;
+        if (transientRetrySafe === undefined) {
+            const kind = await resolveEndpointKind(resolved, slug);
+            transientRetrySafe = kind !== undefined && kind !== "insert";
+        }
+        return transientRetrySafe;
     };
-    return resolved.client.execute(resolved.agentId, auth, slug, input.body);
+    const inTest = inTestEnv();
+    const runWithRetries = async (key) => {
+        const attempt = () => sessionKey === undefined
+            ? retryWhile(isEmptyReadObject, () => runOnce(key))
+            : runOnce(key);
+        let noResultsTries = 0;
+        let transientTries = 0;
+        for (;;) {
+            try {
+                return await attempt();
+            }
+            catch (error) {
+                if (isAuthRejection(error))
+                    throw error; // next key source / mint
+                if (isNoResultsError(error) && noResultsTries < NO_RESULTS_MAX_RETRIES) {
+                    if (!inTest)
+                        await sleep(lagBackoffMs(++noResultsTries));
+                    continue;
+                }
+                if (isTransientServerError(error) &&
+                    transientTries < TRANSIENT_MAX_RETRIES &&
+                    (await transientIsSafe())) {
+                    if (!inTest)
+                        await sleep(lagBackoffMs(++transientTries));
+                    continue;
+                }
+                throw error;
+            }
+        }
+    };
+    let resp;
+    let usedSource;
+    let usedKey;
+    let envRejected = false;
+    for (const candidate of candidates) {
+        try {
+            resp = await runWithRetries(candidate.key);
+            usedSource = candidate.source;
+            usedKey = candidate.key;
+            break;
+        }
+        catch (error) {
+            if (isAuthRejection(error)) {
+                if (candidate.source === "env")
+                    envRejected = true;
+                continue;
+            }
+            throw error;
+        }
+    }
+    if (usedSource === undefined) {
+        const minted = await mintRuntimeKey(resolved);
+        try {
+            resp = await runWithRetries(minted);
+            usedSource = "minted";
+            usedKey = minted;
+        }
+        catch (error) {
+            if (isAuthRejection(error)) {
+                throw new CliError(`mockdb execute ${slug}: the runtime key is still invalid for agent ${resolved.agentId} even after minting a fresh one (HTTP ${error.status}).`, {
+                    code: "API_CLIENT_ERROR",
+                    suggestion: "Confirm the agent is provisioned with `apollo mockdb status`; the backend may be rejecting freshly-minted keys (report it).",
+                });
+            }
+            throw error;
+        }
+    }
+    if (isEmptyReadObject(resp)) {
+        blankBodyError(resolved, `execute ${slug}`, sessionKey === undefined ? "read" : "write");
+    }
+    // A minted key, or a cached key that had to replace a rejected wired key,
+    // means whatever the runtime resolves is stale ⇒ surface the re-wire step.
+    if (usedKey && (usedSource === "minted" || (usedSource === "cache" && envRejected))) {
+        resp = attachRuntimeKeyResync(resolved, resp, usedSource, usedKey);
+    }
+    return resp;
+}
+// ─── Identities (persona sample data for the playground simulator) ──────
+export async function getIdentities(context, options = {}) {
+    const resolved = resolveMockDb(context, options);
+    // An agent with no personas returns a keyed `{ sample_data: { personas: [] } }`
+    // (the feature is simply off), never `{}` — so a blank body really is a
+    // transport blank and is retried like every other read.
+    return readWithRetry(resolved, "identities get", () => withManage(resolved, (auth) => resolved.client.getIdentities(resolved.agentId, auth)));
+}
+export async function setIdentities(context, sampleData, options = {}) {
+    const resolved = resolveMockDb(context, options);
+    const resp = await withManage(resolved, (auth) => resolved.client.setIdentities(resolved.agentId, auth, sampleData));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, "identities set", "write");
+    return resp;
+}
+/**
+ * Normalize a persona payload into the `sample_data` object the service
+ * stores. Accepts the bare object (`{ personas: [...] }`) or the full request
+ * body (`{ sample_data: { personas: [...] } }`), so a payload copied straight
+ * from the API docs works as-is. Only the outer shape is checked here — the
+ * field-level rules belong to the service, which reports them as precise
+ * 400s naming the offending `personas[i].fields[j]`.
+ */
+export function normalizeSampleData(raw) {
+    const wrapped = raw && typeof raw === "object" && !Array.isArray(raw)
+        ? raw.sample_data
+        : undefined;
+    const payload = wrapped !== undefined ? wrapped : raw;
+    if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+        throw new ValidationError("The persona payload must be a JSON object (optionally wrapped in `sample_data`).", 'e.g. --sample-data \'{"personas":[{"label":"Jane Doe (verified)","state":"verified","default":true,"fields":[]}]}\'');
+    }
+    if (!Array.isArray(payload.personas)) {
+        throw new ValidationError("The persona payload needs a `personas` array (pass an empty array to turn the persona picker off).", 'Each persona needs `label`, `state`, and `fields`; exactly one must have `default: true`. e.g. \'{"personas":[{"label":"Jane Doe (verified)","state":"verified","default":true,"fields":[{"key":"user_id","label":"User ID","value":"U1001","group":"who_you_are"}]}]}\'');
+    }
+    return payload;
+}
+/**
+ * Read the persona payload from an inline JSON string or a file. A persona
+ * payload is the largest JSON any mockdb command takes and its values
+ * routinely contain apostrophes, so the file path avoids shell quoting.
+ */
+export async function readSampleDataInput(input) {
+    const inline = input.sampleData?.trim();
+    const file = input.file?.trim();
+    if (inline && file) {
+        throw new ValidationError("Pass either --sample-data or --file, not both.", "Use --file <path.json> for a large payload, --sample-data '<json>' for a small inline one.");
+    }
+    if (file) {
+        const resolvedPath = path.resolve(file);
+        let raw;
+        try {
+            raw = await readFile(resolvedPath, "utf8");
+        }
+        catch (error) {
+            throw new ValidationError(`--file could not be read: ${error instanceof Error ? error.message : String(error)}`, `Check that ${resolvedPath} exists and is readable JSON.`);
+        }
+        return parseJsonFlag(`--file (${resolvedPath})`, raw);
+    }
+    if (!inline) {
+        throw new ValidationError("Missing required option: --sample-data.", "Pass --sample-data '{\"personas\":[...]}' (or --file <path.json>).");
+    }
+    return parseJsonFlag("--sample-data", inline);
 }
+// ─── Sessions / keys / export / delete ──────────────────────────────────
 export async function inspectSession(context, sessionKey, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.inspectSession(resolved.agentId, resolved.tokenAuth, sessionKey);
+    // A real empty overlay returns a keyed `{ sessionKey, totalRecords:0, … }`,
+    // never `{}` — a blank is a transport blip, not an empty session.
+    return readWithRetry(resolved, `session get ${sessionKey}`, () => withManage(resolved, (auth) => resolved.client.inspectSession(resolved.agentId, auth, sessionKey)));
 }
 export async function resetSession(context, sessionKey, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.resetSession(resolved.agentId, resolved.tokenAuth, sessionKey);
+    return withManage(resolved, (auth) => resolved.client.resetSession(resolved.agentId, auth, sessionKey));
 }
 export async function rotateMgmtKey(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    const resp = await resolved.client.rotateMgmtKey(resolved.agentId, resolved.tokenAuth);
-    await persistProvisionKeys(resolved.agentId, context.baseUrls.mockDb, resp, resolved.homeDir);
+    // The one rotation that always rides the Apollo token — it is also how a
+    // pre-existing agent obtains its FIRST management key.
+    const resp = await rotateWithRetry(resolved, "managementKey", "rotate mgmt", () => resolved.client.rotateMgmtKey(resolved.agentId, resolved.tokenAuth));
     return {
         ...resp,
         stored: keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir)),
@@ -176,8 +678,7 @@ export async function rotateMgmtKey(context, options = {}) {
 }
 export async function rotateRuntimeKey(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    const resp = await resolved.client.rotateRuntimeKey(resolved.agentId, resolved.tokenAuth);
-    await persistProvisionKeys(resolved.agentId, context.baseUrls.mockDb, resp, resolved.homeDir);
+    const resp = await rotateWithRetry(resolved, "runtimeKey", "rotate runtime", () => withManage(resolved, (auth) => resolved.client.rotateRuntimeKey(resolved.agentId, auth)));
     return {
         ...resp,
         stored: keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir)),
@@ -186,10 +687,11 @@ export async function rotateRuntimeKey(context, options = {}) {
 }
 export async function exportMockDb(context, options = {}) {
     const resolved = resolveMockDb(context, options);
-    return resolved.client.exportAgent(resolved.agentId, resolved.tokenAuth);
+    return readWithRetry(resolved, "export", () => withManage(resolved, (auth) => resolved.client.exportAgent(resolved.agentId, auth)));
 }
 export async function deleteMockDb(context, options = {}) {
     const resolved = resolveMockDb(context, options);
+    // DELETE is strict on the service — always the Apollo token, never a mgmt key.
     const resp = await resolved.client.deleteAgent(resolved.agentId, resolved.tokenAuth);
     await clearMockDbKeys(resolved.agentId, resolved.homeDir);
     return resp;
@@ -201,6 +703,32 @@ export async function storedKeys(context, options = {}) {
         ...keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir)),
     };
 }
+/**
+ * Clone another agent's mock DB (collections + base rows + endpoints +
+ * personas) into THIS agent. The target must be empty — the service refuses
+ * (400) to overwrite an existing mock DB. Both agents are authorized on the
+ * account plane (the caller must be a member of the account owning each).
+ * Keys come back only when the target was provisioned by this call; they are
+ * persisted like a provision.
+ */
+export async function cloneMockDb(context, options) {
+    const resolved = resolveMockDb(context, options);
+    const from = options.from.trim();
+    if (!from) {
+        throw new ValidationError("Missing clone source.", "Pass --from <sourceAgentId>.");
+    }
+    if (from === resolved.agentId) {
+        throw new ValidationError("The clone source and target are the same agent.", "Pass --from <a different agent id> (e.g. the template agent this one was created from).");
+    }
+    const resp = await retryWhile(isEmptyReadObject, () => resolved.client.cloneAgent(resolved.agentId, from, resolved.tokenAuth));
+    if (isEmptyReadObject(resp))
+        blankBodyError(resolved, `clone from ${from}`, "write");
+    await persistReturnedKeys(resolved, resp);
+    return {
+        ...resp,
+        stored: keyPrefixes(await loadMockDbKeys(resolved.agentId, resolved.homeDir)),
+    };
+}
 /**
  * Ensure the bundle's `connections.yaml` has a mock-db connection pointing at
  * this agent's execute base, reading the runtime key from MOCKDB_TOKEN (vault
@@ -220,10 +748,10 @@ export async function wireMockDbConnection(context, options = {}) {
     }
     const keys = await loadMockDbKeys(resolved.agentId, resolved.homeDir);
     if (!keys.runtimeKey) {
-        throw new ConfigError("No runtime key to wire.", "Run `apollo mockdb provision` first.");
+        throw new ConfigError("No runtime key to wire.", "Run `apollo mockdb provision` first (or `apollo mockdb rotate runtime` for an existing DB).");
     }
     const connectionId = options.connectionId ?? "mock-db";
-    const tokenEnv = options.tokenEnv ?? "MOCKDB_TOKEN";
+    const tokenEnv = options.tokenEnv ?? DEFAULT_MOCKDB_TOKEN_ENV;
     const baseUrl = `${context.baseUrls.mockDb.replace(/\/+$/, "")}/api/agents/${resolved.agentId}/apis`;
     const sourceRoot = path.join(agentRoot, "bundle", "src");
     const file = path.join(sourceRoot, "connections.yaml");
@@ -287,6 +815,7 @@ export async function wireMockDbConnection(context, options = {}) {
         notes,
     };
 }
+// ─── Flag parsing helpers (shared by the CLI and the TUI) ───────────────
 export function parseJsonFlag(flag, raw) {
     try {
         return JSON.parse(raw);
@@ -303,4 +832,180 @@ export function normalizeSchema(raw) {
     }
     throw new ValidationError("--schema must be a JSON array of columns, or { \"columns\": [...] }.");
 }
+/**
+ * Normalize the `--add-columns` value into a column list. Accepts a bare array
+ * of column defs, or a `{ columns: [...] }` / `{ addColumns: [...] }` wrapper
+ * so the shape matches `collections create --schema`.
+ */
+export function normalizeAddColumns(raw) {
+    const arr = Array.isArray(raw)
+        ? raw
+        : raw && typeof raw === "object"
+            ? (raw.addColumns ??
+                raw.columns)
+            : undefined;
+    if (!Array.isArray(arr) || arr.length === 0) {
+        throw new ValidationError("--add-columns must be a non-empty JSON array of column definitions (or an object with a `columns` array).", 'e.g. --add-columns \'[{"name":"status","type":"text"}]\'. Added columns are nullable; primary keys cannot be added to an existing collection.');
+    }
+    return arr;
+}
+// ─── Guide ──────────────────────────────────────────────────────────────
+/** The text emitted by `apollo mockdb guide` — the how-to for driving the mock DB. */
+export const MOCKDB_GUIDE = `# Mock DB — Apollo CLI Guide
+
+A per-agent **mock database** for building and testing agents that have no
+real backend (or no test environment) yet. You define collections (tables),
+seed realistic data, and declare named read/write endpoints; the agent's
+program sources then call those endpoints at runtime through a
+\`connections.yaml\` connection. It supports relative date/time fields
+(\`now-30d\`) that stay fresh, and per-conversation copy-on-write keyed by the
+session id, so test runs are isolated and resettable.
+
+All \`apollo mockdb\` commands are **non-interactive**. Pass the global
+\`--json\` for the machine envelope (\`{ "success": true, "data": ... }\`).
+
+## How auth works (and why it's safe)
+
+- The agent comes from the checkout you are standing in (\`.apollorc\`) or
+  \`--agent <id>\`; the account from the active project (\`--account <id>\` or
+  \`APOLLO_ACCOUNT_ID\` to override).
+- Your Apollo token is sent **first-party from the CLI** only where the
+  service demands it: \`provision\`, \`status\`, \`delete\`, and minting a
+  management key. The service verifies it against Apollo (you must be a
+  member of the agent's account).
+- Provision returns two per-agent keys, stored under \`~/.apollo/mockdb/\`:
+  - **managementKey** — used for ALL other management calls (\`X-Mgmt-Key\`;
+    no Apollo token transmitted). Minted on demand if missing/rejected.
+  - **runtimeKey** — what the agent's connection uses at runtime; also what
+    \`apollo mockdb execute\` uses to test endpoints.
+
+## Typical workflow
+
+\`\`\`bash
+# 1. Provision (stores the keys) and wire bundle/src/connections.yaml
+apollo mockdb provision --wire
+
+# 2. Create collections (parents before children — FKs need the parent first)
+apollo mockdb collections create --name users \\
+  --schema '{"columns":[{"name":"id","type":"text","primaryKey":true},{"name":"name","type":"text"}]}'
+apollo mockdb collections create --name accounts \\
+  --schema '{"columns":[{"name":"id","type":"text","primaryKey":true},{"name":"user_id","type":"text","references":"users.id"},{"name":"balance","type":"number"},{"name":"opened_at","type":"reldate"}]}'
+
+# 3. Seed data (relative date tokens allowed)
+apollo mockdb seed --collection users --rows '[{"id":"U1","name":"Ada Lovelace"}]'
+apollo mockdb seed --collection accounts --rows '[{"id":"A1","user_id":"U1","balance":4200.5,"opened_at":"now-400d"}]'
+
+# 4. Declare the endpoints the agent will call
+apollo mockdb endpoint create --slug get-accounts \\
+  --param-schema '{"user_id":{"type":"string","required":true}}' \\
+  --spec '{"kind":"read","collection":"accounts","where":[{"column":"user_id","op":"eq","param":"user_id"}]}'
+
+# 5. Test an endpoint under a session (runtime key + session = a conversation)
+apollo mockdb execute --slug get-accounts --session test-1 --body '{"user_id":"U1"}'
+
+# 6. Inspect / reset that session's overlay
+apollo mockdb session get --key test-1
+apollo mockdb session reset --key test-1
+\`\`\`
+
+Then map each program source that names \`connection: mock-db\` to an endpoint
+under \`operations\` in \`bundle/src/connections.yaml\`, export the runtime key as
+\`MOCKDB_TOKEN\` (or the connection's \`token_env\`), and \`apollo validate &&
+apollo push\`.
+
+## Personas for the playground simulator (\`identities\`)
+
+Once the mock DB has data, the playground can offer a **persona picker** before
+the first message so a tester speaks as a real seeded user. The payload is
+authored here — the schema and seed values are already in hand:
+
+\`\`\`bash
+# Read what's stored (no personas ⇒ { "personas": [] } — the picker is off).
+apollo mockdb identities get
+
+# Upsert the payload (large payloads are easier from a file).
+apollo mockdb identities set --file personas.json
+apollo mockdb identities set --sample-data '{
+  "instructions": "Pick a persona to simulate as.",
+  "personas": [
+    {
+      "label": "Jane Doe (verified)",
+      "state": "verified",
+      "default": true,
+      "fields": [
+        { "key": "user_id", "label": "User ID", "value": "U1001", "group": "who_you_are" },
+        { "key": "pin", "label": "PIN", "value": "4321", "group": "verify", "role": "secret" }
+      ]
+    }
+  ]
+}'
+\`\`\`
+
+Rules the service enforces (a 400 names the exact \`personas[i].fields[j]\`):
+
+- Each persona needs a non-empty \`label\` and \`state\`; each field needs
+  \`key\`, \`label\`, \`value\`, and a \`group\` of \`who_you_are\` | \`verify\` |
+  \`good_to_know\` | \`records\`.
+- Exactly **one** persona carries \`default: true\` (when there is at least one).
+- Optional per-field extras: \`role\` (\`key\` | \`secret\`), \`audience\`
+  (\`user\` | \`simulator\` | \`both\`), \`required_to_start\`, \`match\`, \`maps_to\`.
+- Optional per-persona \`intents\` (\`{ title, message }\`) seed suggested openers.
+
+\`set\` is a full **REPLACE**, not a merge — \`{"personas":[]}\` turns the picker
+off. Seed the values first: every persona value must exist in the mock DB.
+Personas travel with the DB — \`clone\` and \`export\` carry them.
+
+## Cloning another agent's mock DB
+
+\`\`\`bash
+# Copy a template agent's collections + rows + endpoints + personas into THIS
+# agent (the target must be empty; your account must own both agents):
+apollo mockdb clone --from <templateAgentId>
+\`\`\`
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| \`mockdb provision [--wire]\` | Create the agent's mock DB; store runtime + management keys |
+| \`mockdb status\` | Ownership + local key prefixes + verdict (ok / empty / new / unknown) |
+| \`mockdb describe\` | Schema, relationships, relative-field flags, endpoints, row counts |
+| \`mockdb collections create --name <n> --schema <json>\` | Create a collection (table) |
+| \`mockdb collections update --name <n> --add-columns <json>\` | Add nullable columns (no rename/type-change/drop) |
+| \`mockdb collections list\` | List collections |
+| \`mockdb seed --collection <n> --rows <json-array>\` | Seed base rows |
+| \`mockdb endpoint create --slug <s> --spec <json> [--param-schema <json>] [--description <t>]\` | Declare a read/insert/update/delete endpoint |
+| \`mockdb endpoint list\` / \`get <slug>\` | List / inspect endpoints |
+| \`mockdb endpoint update <slug> [--spec] [--param-schema] [--description]\` | PATCH-merge an endpoint |
+| \`mockdb endpoint delete <slug>\` | Delete an endpoint (re-map operations + push after) |
+| \`mockdb execute --slug <s> [--session <key>] [--body <json>]\` | Run an endpoint with the runtime key |
+| \`mockdb identities get\` / \`set --sample-data <json> | --file <path>\` | Read / replace the persona payload |
+| \`mockdb session get --key <key>\` / \`reset --key <key>\` | Inspect / reset a session overlay |
+| \`mockdb rotate mgmt\` / \`rotate runtime\` | Mint fresh keys (re-wire after a runtime rotation) |
+| \`mockdb clone --from <id>\` | Clone another agent's mock DB into this one (target must be empty) |
+| \`mockdb export\` | Export schema + base data + endpoints + personas |
+| \`mockdb keys\` | Show locally stored key prefixes (no secrets) |
+| \`mockdb wire [--connection-id] [--token-env]\` | Write bundle/src/connections.yaml for the runtime key |
+| \`mockdb delete --force\` | Permanently delete the mock DB (Apollo token only) |
+| \`mockdb guide [--output <file>]\` | Write this guide to a file (default GUIDE.md) |
+
+Common flags: \`--agent <id>\`, \`--account <id>\`, and the global \`--json\`.
+
+## Notes
+
+- Create collections **parent-first** (a \`references\` FK needs the parent).
+- **Writes need \`--session\`**; reads may omit it to peek at pristine base.
+- Endpoint \`where\`/value params must be declared in \`--param-schema\`.
+- \`execute\` resolves the runtime key from \`$MOCKDB_TOKEN\` → the local cache →
+  a fresh mint (then reports \`runtimeKeyResync\` so you re-wire).
+- A blank 2xx from the service is a transient transport blip: reads retry it,
+  writes stop and tell you to verify before retrying (never duplicate rows).
+- Override the service URL with \`APOLLO_MOCKDB_URL\`.
+`;
+/** Write the guide to `outputPath` (default `GUIDE.md` in the cwd). */
+export async function writeMockDbGuide(outputPath) {
+    const resolvedPath = path.resolve(outputPath?.trim() || "GUIDE.md");
+    await writeFile(resolvedPath, MOCKDB_GUIDE, "utf8");
+    return { written: true, path: resolvedPath, bytes: Buffer.byteLength(MOCKDB_GUIDE) };
+}
 //# sourceMappingURL=mockdb.js.map