codex-multi-auth 2.20.0 → 2.22.0

package/dist/lib/runtime/model-discovery-status.js

@@ -1,3 +1,4 @@
+import { enrichPersistedApiCapabilities } from "./persisted-api-capabilities.js";
 import { withFileTransactionLock } from "../storage/file-lock.js";
 import { styleReportText as paint } from "../ui/format.js";
 import { withCheckProgress } from "../ui/check-progress.js";
@@ -26,7 +27,7 @@ const printable = z
     .max(256)
     .regex(/^[^\x00-\x1f\x7f]*$/);
 const entitlementSchema = z.object({
-    model: printable, checkedAt: z.number().optional(),
+    model: printable, checkedAt: z.number().optional(), programCheckedAt: z.number().finite().nonnegative().max(8.64e15).optional(),
     probes: z.record(printable, z.enum(["verified", "unsupported", "unverified", "downgraded"])).optional(),
     serviceTiers: z.array(z.object({ id: printable, name: printable })).max(100),
     reasoningLevels: z.array(printable).max(100), accessPrograms: z.array(printable).max(100),
@@ -55,24 +56,6 @@ const schema = z.object({
         .optional(),
     entries: z.array(entrySchema).max(3000),
 });
-const probeCacheSchema = z.object({
-    version: z.literal(1),
-    entries: z.array(z.tuple([
-        z.string().regex(/^[0-9a-f]{64}$/),
-        z.object({ at: z.number(), levels: z.array(printable).max(100), tiers: z.array(printable).max(100), status: z.record(printable, printable) }),
-    ])).max(1000),
-});
-function probeCachePath() {
-    return join(getCodexMultiAuthDir(), "api-capability-probes.json");
-}
-async function loadProbeCache() {
-    try {
-        return probeCacheSchema.parse(JSON.parse(await fs.readFile(probeCachePath(), "utf8"))).entries;
-    }
-    catch {
-        return [];
-    }
-}
 export async function discoverModelInventory(storage, routes, options = {}) {
     const fetcher = options.fetchImpl ?? fetch, now = options.now ?? Date.now;
     const manager = new AccountManager(undefined, storage ?? {
@@ -132,28 +115,13 @@ export async function discoverModelInventory(storage, routes, options = {}) {
     finally {
         await manager.flushPendingSave();
     }
-    const capabilities = new ApiModelCapabilities(fetcher, now);
-    if (options.persistProbes)
-        capabilities.importProbes(await loadProbeCache());
+    const capabilities = new ApiModelCapabilities(fetcher, now, undefined, undefined, options.programRefreshBefore);
     const api = new ApiModelRuntime(fetcher, now, options.updateApiDiscovery, options.updateApiDiscovery
-        ? (models, refresh, route, forceProbes) => capabilities.enrich(models, refresh, route, forceProbes)
+        ? (models, refresh, route, forceProbes) => options.persistProbes
+            ? enrichPersistedApiCapabilities(models, refresh, route, forceProbes, { fetchImpl: fetcher, now, requireStoredRoute: true, signal: options.signal, programRefreshBefore: options.programRefreshBefore })
+            : capabilities.enrich(models, refresh, route, forceProbes)
         : undefined);
     await api.catalogs(routes, true, options.forceProbes === true);
-    if (options.persistProbes) {
-        const path = probeCachePath();
-        await withFileTransactionLock(path, async () => {
-            // Merge with the latest file: a concurrent check may have probed other
-            // credentials since this one loaded. The newer result wins per key.
-            const merged = new Map(await loadProbeCache());
-            for (const [key, value] of capabilities.exportProbes()) {
-                const disk = merged.get(key);
-                if (!disk || disk.at <= value.at)
-                    merged.set(key, value);
-            }
-            const entries = [...merged].sort((x, y) => x[1].at - y[1].at).slice(-1000);
-            await writeFileAtomic(path, JSON.stringify(probeCacheSchema.parse({ version: 1, entries })) + "\n");
-        }).catch(() => undefined);
-    }
     entries.push(...api.statuses(routes).map((s, i) => ({
         id: modelScopeId(s.kind, s.id),
         label: s.label,
@@ -194,29 +162,76 @@ export async function loadModelInventory() {
     }
 }
 let inventoryWrites = Promise.resolve();
-export function saveModelInventory(value) {
-    const path = inventoryPath();
-    const pending = inventoryWrites.then(() => withFileTransactionLock(path, () => saveModelInventorySerial(value, path)));
+/** Drain queued writes before resetting process-local test state. */
+export async function resetModelInventoryWritesForTests() {
+    for (;;) {
+        const pending = inventoryWrites;
+        await pending;
+        if (inventoryWrites === pending) {
+            inventoryWrites = Promise.resolve();
+            return;
+        }
+    }
+}
+export function saveModelInventory(value, options = {}) {
+    const path = inventoryPath(), signal = options.signal;
+    if (signal?.aborted)
+        return Promise.reject(signal.reason);
+    let stopWaiting;
+    const pending = inventoryWrites.then(() => {
+        // Once started, drain our own cancellable lock/write and cleanup.
+        stopWaiting?.();
+        signal?.throwIfAborted();
+        return withFileTransactionLock(path, () => saveModelInventorySerial(value, path, options.mergeEntries === true, signal), { signal });
+    });
+    // Keep the underlying queue intact even if a waiting caller cancels.
     inventoryWrites = pending.catch(() => undefined);
-    return pending;
+    if (!signal)
+        return pending;
+    return new Promise((resolve, reject) => {
+        const abort = () => { stopWaiting?.(); reject(signal.reason); };
+        stopWaiting = () => signal.removeEventListener("abort", abort);
+        signal.addEventListener("abort", abort, { once: true });
+        if (signal.aborted)
+            abort();
+        void pending.then(resolve, reject).finally(() => stopWaiting?.());
+    });
 }
-async function saveModelInventorySerial(value, path) {
-    const data = schema.parse(withInventoryChanges(value, await loadModelInventory()));
+async function saveModelInventorySerial(value, path, mergeEntries = false, signal) {
+    signal?.throwIfAborted();
+    const previous = await loadModelInventory();
+    if (mergeEntries && previous) {
+        // Clock rollback must not make old observations permanently newer.
+        const now = Date.now();
+        const entries = [...previous.entries];
+        for (const entry of value.entries) {
+            const index = entries.findIndex(old => old.id === entry.id && old.kind === entry.kind);
+            if (index < 0)
+                entries.push(entry);
+            else if ((entries[index]?.checkedAt ?? -Infinity) > now || (entries[index]?.checkedAt ?? -Infinity) <= entry.checkedAt)
+                entries[index] = entry;
+        }
+        value = { ...previous, ...value, clientVersion: value.clientVersion ?? previous.clientVersion,
+            checkedAt: previous.checkedAt > now ? value.checkedAt : Math.max(previous.checkedAt, value.checkedAt), entries };
+    }
+    const data = schema.parse(withInventoryChanges(value, previous));
     const serialized = JSON.stringify(data) + "\n";
     if (Buffer.byteLength(serialized) > 4 * 1024 * 1024)
         throw Error("Model discovery status exceeds size limit");
-    await writeFileAtomic(path, serialized);
+    await writeFileAtomic(path, serialized, signal);
 }
-async function writeFileAtomic(path, serialized) {
+async function writeFileAtomic(path, serialized, signal) {
+    signal?.throwIfAborted();
     const temp = tempPathFor(path);
     await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
     const retry = { maxAttempts: 6, backoffMs: 25 };
     try {
+        signal?.throwIfAborted();
         await fs.writeFile(temp, serialized, {
             mode: 0o600,
             flag: "wx",
         });
-        await withRetry(() => fs.rename(temp, path), retry);
+        await withRetry(() => { signal?.throwIfAborted(); return fs.rename(temp, path); }, retry);
     }
     finally {
         await withRetry(() => fs.unlink(temp), retry).catch((e) => {
@@ -234,10 +249,10 @@ export function formatModelInventory(inventory, now = Date.now()) {
         ...inventory.entries.flatMap((e) => [
             `  ${paint(e.label, "heading")}${e.routable === true ? " [routing eligible]" : e.routable === false ? " [discovery only]" : ""}${e.bound ? " [stored binding]" : ""}${e.selected ? " [preferred workspace]" : ""} [${e.kind.toUpperCase()}]: ${!e.enabled ? paint("disabled", "muted") : e.error ? paint("discovery failed", "danger") : paint(`${e.models.length} available`, "success")}; ${e.checkedAt ? new Date(e.checkedAt).toISOString() : "never checked"}${now - e.checkedAt > 60000 ? paint(" (stale; run check)", "warning") : ""}`,
             `    ${paint("Available:", "muted")} ${e.models.join(", ") || "none"}`,
-            ...(e.entitlements ?? []).map((m) => `    ${paint(m.model, "accent")}: speeds=${m.serviceTiers.map((t) => `${t.name} (${t.id})`).join(", ") || "not advertised"}; reasoning=${m.reasoningLevels.join(", ") || "not advertised"}; programs=${m.accessPrograms.join(", ") || "not advertised"}${m.probes
+            ...(e.entitlements ?? []).map((m) => `    ${paint(m.model, "accent")}: speeds=${m.serviceTiers.map((t) => `${t.name} (${t.id})`).join(", ") || "not advertised"}; reasoning=${m.reasoningLevels.join(", ") || "not advertised"}; ${e.kind === "oauth" ? "programs" : "configured programs"}=${m.accessPrograms.join(", ") || (e.kind === "oauth" ? "not advertised" : "none")}${m.probes
                 ? `; probes: ${Object.entries(m.probes)
                     .map(([k, v]) => `${k}=${paint(v, v === "verified" ? "success" : v === "unsupported" || v === "downgraded" ? "warning" : "muted")}`)
-                    .join(", ")} (${new Date(m.checkedAt ?? 0).toISOString()})`
+                    .join(", ")} (${new Date(m.checkedAt ?? 0).toISOString()})${m.programCheckedAt !== undefined ? `; programs checked ${new Date(m.programCheckedAt).toISOString()}` : ""}`
                 : ""}`),
             ...(e.kind !== "oauth"
                 ? [

package/dist/lib/runtime/persisted-api-capabilities.d.ts

@@ -0,0 +1,20 @@
+import { type ApiRouteCredential } from "../api-route-store.js";
+import type { RouteModel } from "../model-route-policy.js";
+import { ApiModelCapabilities } from "./api-model-capabilities.js";
+export declare function persistedProbeSlotsForTests(): {
+    active: number;
+    waiting: number;
+};
+/** Reject resets during work so live waiters cannot lose their concurrency slots. */
+export declare function resetPersistedApiCapabilitiesForTests(): void;
+/** Reload under a credential/model lease so independent consumers share paid evidence. */
+declare function enrichPersistedApiCapabilitiesImpl(models: RouteModel[], refresh: boolean, route: ApiRouteCredential, force: boolean, options?: {
+    fetchImpl?: typeof fetch;
+    now?: () => number;
+    metadata?: ApiModelCapabilities;
+    signal?: AbortSignal;
+    programRefreshBefore?: number;
+    requireStoredRoute?: boolean;
+}): Promise<RouteModel[]>;
+export declare function enrichPersistedApiCapabilities(...args: Parameters<typeof enrichPersistedApiCapabilitiesImpl>): Promise<RouteModel[]>;
+export {};

package/dist/lib/runtime/persisted-api-capabilities.js

@@ -0,0 +1,387 @@
+import { createHash } from "node:crypto";
+import { promises as fs } from "node:fs";
+import { dirname, join } from "node:path";
+import { z } from "zod";
+import { logWarn } from "../logger.js";
+import { getCodexMultiAuthDir } from "../runtime-paths.js";
+import { withFileTransactionLock } from "../storage/file-lock.js";
+import { withRetry } from "../fs-retry.js";
+import { tempPathFor } from "../temp-path.js";
+import { mapWithConcurrency } from "../concurrency.js";
+import { getApiRoutesPath, loadApiRoutes, ensureApiRouteCapabilityEpoch } from "../api-route-store.js";
+import { ApiModelCapabilities } from "./api-model-capabilities.js";
+const printable = z.string().max(256).regex(/^[^\x00-\x1f\x7f]*$/);
+const probeCacheSchema = z.object({
+    version: z.literal(1),
+    entries: z.array(z.tuple([
+        z.string().regex(/^[0-9a-f]{64}$/),
+        z.object({
+            ownerEpoch: z.string().uuid().optional(),
+            programRetry: z.boolean().optional(),
+            programCheckedAt: z.number().finite().nonnegative().max(8.64e15).optional(),
+            at: z.number(),
+            levels: z.array(printable).max(100),
+            tiers: z.array(printable).max(100),
+            status: z.record(printable, printable),
+        }),
+    ])).max(1000),
+});
+let activeProbes = 0;
+let activeEnrichments = 0;
+const retainedProbes = new Map();
+const retainedOwners = new Map();
+const probeWaiters = [];
+const unavailableLeaseCodes = new Set([
+    "ELOCKED", "EACCES", "EPERM", "EROFS", "ENOSPC", "EDQUOT", "EMFILE", "ENFILE",
+    "EBUSY", "EIO", "ENOENT", "ENOTDIR", "ESTALE",
+]);
+export function persistedProbeSlotsForTests() {
+    return { active: activeProbes, waiting: probeWaiters.length };
+}
+/** Reject resets during work so live waiters cannot lose their concurrency slots. */
+export function resetPersistedApiCapabilitiesForTests() {
+    if (activeEnrichments || activeProbes || probeWaiters.length) {
+        throw Error("Cannot reset active API capability checks");
+    }
+    retainedProbes.clear();
+    retainedOwners.clear();
+}
+async function withSharedProbeSlot(operation, signal) {
+    signal?.throwIfAborted();
+    if (activeProbes >= 4)
+        await new Promise((resolve, reject) => {
+            const grant = () => {
+                signal?.removeEventListener("abort", abort);
+                resolve();
+            };
+            const abort = () => {
+                const index = probeWaiters.indexOf(grant);
+                if (index < 0)
+                    return; // A granted permit is released by the finally below.
+                probeWaiters.splice(index, 1);
+                signal?.removeEventListener("abort", abort);
+                reject(signal?.reason);
+            };
+            probeWaiters.push(grant);
+            signal?.addEventListener("abort", abort, { once: true });
+            if (signal?.aborted)
+                abort();
+        });
+    else
+        activeProbes++;
+    try {
+        // Cancellation can happen after grant but before this continuation runs.
+        signal?.throwIfAborted();
+        return await operation();
+    }
+    finally {
+        const next = probeWaiters.shift();
+        if (next)
+            next();
+        else
+            activeProbes--;
+    }
+}
+function probeCachePath() {
+    return join(getCodexMultiAuthDir(), "api-capability-probes.json");
+}
+async function loadProbeCache() {
+    try {
+        const text = await withRetry(() => fs.readFile(probeCachePath(), "utf8"), {
+            maxAttempts: 6,
+            backoffMs: 25,
+        });
+        return probeCacheSchema.parse(JSON.parse(text)).entries;
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+        return [];
+    }
+}
+function invalidProbeCache(error) {
+    return error instanceof SyntaxError || error instanceof z.ZodError;
+}
+/** Explicit checks may repair corrupt JSON; access failures never authorize a reset. */
+async function loadOrRepairProbeCache(force, ownsRoute, signal) {
+    try {
+        return await loadProbeCache();
+    }
+    catch (error) {
+        if (!force || !invalidProbeCache(error))
+            throw error;
+    }
+    const path = probeCachePath();
+    return withFileTransactionLock(path, async () => {
+        const repair = async () => {
+            signal?.throwIfAborted();
+            if (ownsRoute && !await ownsRoute())
+                throw Error("API route ownership changed");
+            // A concurrent writer may have recovered it while we waited.
+            try {
+                return await loadProbeCache();
+            }
+            catch (error) {
+                if (!invalidProbeCache(error))
+                    throw error;
+            }
+            const temp = tempPathFor(path), retry = { maxAttempts: 6, backoffMs: 25 };
+            try {
+                await fs.writeFile(temp, JSON.stringify({ version: 1, entries: [] }) + "\n", { mode: 0o600, flag: "wx" });
+                signal?.throwIfAborted();
+                await withRetry(() => fs.rename(temp, path), retry);
+            }
+            finally {
+                await withRetry(() => fs.rm(temp, { force: true }), retry);
+            }
+            return [];
+        };
+        return ownsRoute ? withFileTransactionLock(getApiRoutesPath(), repair, { signal }) : repair();
+    }, { signal });
+}
+async function storedGeneration(route, model) {
+    const path = getApiRoutesPath();
+    const before = await fs.stat(path).catch(() => undefined);
+    if (!before)
+        return undefined;
+    const current = (await loadApiRoutes()).find(value => value.id === route.id && value.kind === route.kind && value.apiKey === route.apiKey &&
+        value.enabled && value.probeCapabilities && value.visibleModels?.includes(model));
+    if (!current || JSON.stringify(current.accessPrograms) !== JSON.stringify(route.accessPrograms) || JSON.stringify(current.modelAccessPrograms) !== JSON.stringify(route.modelAccessPrograms))
+        return undefined;
+    if (route.capabilityEpoch && route.capabilityEpoch !== current.capabilityEpoch)
+        return undefined;
+    if (current.capabilityEpoch)
+        return current.capabilityEpoch;
+    const after = await fs.stat(path).catch(() => undefined);
+    const version = (value) => `${value.dev}:${value.ino}:${value.mtimeMs}:${value.ctimeMs}:${value.size}`;
+    return after && version(before) === version(after) ? version(after) : undefined;
+}
+function dropRetained(key) {
+    retainedProbes.delete(key);
+    retainedOwners.delete(key);
+}
+function rememberProbe(key, value, owner) {
+    if (retainedProbes.size >= 1000)
+        dropRetained(retainedProbes.keys().next().value ?? "");
+    retainedProbes.set(key, value);
+    if (owner)
+        retainedOwners.set(key, owner.generation);
+    else
+        retainedOwners.delete(key);
+}
+async function persistProbe(key, value, now, signal, owner, memoryKey) {
+    if (!value)
+        return true;
+    const path = probeCachePath();
+    return withFileTransactionLock(path, async () => {
+        const persist = async () => {
+            signal?.throwIfAborted();
+            const merged = new Map(await loadProbeCache());
+            const disk = merged.get(key);
+            if (!disk || disk.ownerEpoch !== value.ownerEpoch || disk.at > now() || disk.at <= value.at)
+                merged.set(key, value);
+            const entries = [...merged].sort((a, b) => a[1].at - b[1].at).slice(-1000);
+            const temp = tempPathFor(path);
+            const retry = { maxAttempts: 6, backoffMs: 25 };
+            try {
+                await fs.writeFile(temp, JSON.stringify(probeCacheSchema.parse({ version: 1, entries })) + "\n", { mode: 0o600, flag: "wx" });
+                await withRetry(() => fs.rename(temp, path), retry);
+            }
+            finally {
+                await withRetry(() => fs.rm(temp, { force: true }), retry);
+            }
+            return true;
+        };
+        if (!owner)
+            return persist();
+        return withFileTransactionLock(getApiRoutesPath(), async () => {
+            if (await storedGeneration(owner.route, owner.model) !== owner.generation) {
+                if (memoryKey)
+                    dropRetained(memoryKey);
+                return false;
+            }
+            try {
+                return await persist();
+            }
+            catch (error) {
+                signal?.throwIfAborted();
+                // Keep the ownership lock through the memory decision, too.
+                if (memoryKey)
+                    rememberProbe(memoryKey, value, owner);
+                throw error;
+            }
+        }, { signal });
+    }, { signal });
+}
+function latestProbe(key, retained, entries, now) {
+    const disk = entries.find(([id]) => id === key)?.[1];
+    const validDisk = disk && disk.at <= now ? disk : undefined;
+    return retained && retained.at <= now && (!validDisk || retained.at > validDisk.at) ? retained : validDisk;
+}
+/** Reload under a credential/model lease so independent consumers share paid evidence. */
+async function enrichPersistedApiCapabilitiesImpl(models, refresh, route, force, options = {}) {
+    options.signal?.throwIfAborted();
+    const fetcher = options.fetchImpl ?? fetch;
+    const now = options.now ?? Date.now;
+    const metadata = options.metadata ?? new ApiModelCapabilities(fetcher, now);
+    // Inference metadata must not wait for another consumer's paid probes.
+    if (!route.enabled || !route.probeCapabilities) {
+        return metadata.enrich(models, refresh, route, force);
+    }
+    return mapWithConcurrency(models, 3, async (model) => {
+        const path = probeCachePath();
+        const key = createHash("sha256").update(route.id).update("\0").update(route.apiKey).update("\0").update(model.slug).digest("hex");
+        const memoryKey = `${path}\0${key}`;
+        let ownershipRoute = route;
+        let capturedOwnership = false;
+        let readOnlyOwner = false;
+        let generation;
+        if (options.requireStoredRoute) {
+            try {
+                const migrated = await ensureApiRouteCapabilityEpoch(route, options.signal);
+                if (migrated) {
+                    capturedOwnership = true;
+                    ownershipRoute = { ...route, capabilityEpoch: migrated.capabilityEpoch, legacyCapabilityCache: migrated.legacyCapabilityCache };
+                    generation = await withFileTransactionLock(getApiRoutesPath(), () => storedGeneration(ownershipRoute, model.slug), { signal: options.signal });
+                }
+            }
+            catch {
+                options.signal?.throwIfAborted();
+                readOnlyOwner = true;
+                if (!capturedOwnership) {
+                    const current = (await loadApiRoutes().catch(() => [])).find(value => value.id === route.id && value.kind === route.kind && value.apiKey === route.apiKey &&
+                        (!route.capabilityEpoch || route.capabilityEpoch === value.capabilityEpoch));
+                    if (current) {
+                        ownershipRoute = { ...route, capabilityEpoch: current.capabilityEpoch, legacyCapabilityCache: current.legacyCapabilityCache };
+                        capturedOwnership = true;
+                    }
+                }
+                if (capturedOwnership)
+                    generation = await storedGeneration(ownershipRoute, model.slug).catch(() => undefined);
+            }
+        }
+        const owner = generation ? { route: ownershipRoute, model: model.slug, generation } : undefined;
+        const ownsRoute = async () => !options.requireStoredRoute || Boolean(owner && await storedGeneration(owner.route, model.slug).catch(() => undefined) === owner.generation);
+        if (options.requireStoredRoute && (!owner || retainedOwners.get(memoryKey) !== owner.generation))
+            dropRetained(memoryKey);
+        const ownedEntries = (entries) => entries.filter(([, value]) => !options.requireStoredRoute || (value.ownerEpoch
+            ? value.ownerEpoch === owner?.generation
+            : ownershipRoute.legacyCapabilityCache === true || !ownershipRoute.capabilityEpoch));
+        const emptyEvidence = async () => {
+            options.signal?.throwIfAborted();
+            const [result] = await metadata.enrich([model], refresh, route, false, []);
+            options.signal?.throwIfAborted();
+            return result ?? model;
+        };
+        if (!await ownsRoute())
+            return emptyEvidence();
+        const fallbackWithoutProbes = async (readDisk = true) => {
+            options.signal?.throwIfAborted();
+            let entries = [];
+            // Atomic cache reads are safe without the paid-probe lease; never write here.
+            if (readDisk) {
+                try {
+                    entries = await loadProbeCache();
+                }
+                catch { /* Retained evidence may still be usable. */ }
+            }
+            options.signal?.throwIfAborted();
+            if (!await ownsRoute()) {
+                dropRetained(memoryKey);
+                return emptyEvidence();
+            }
+            const retained = retainedProbes.get(memoryKey);
+            const value = latestProbe(key, retained, ownedEntries(entries), now());
+            const [result] = await metadata.enrich([model], refresh, route, false, value ? [[key, value]] : []);
+            options.signal?.throwIfAborted();
+            // Documentation may await network I/O while a credential wipe completes.
+            if (!await ownsRoute()) {
+                dropRetained(memoryKey);
+                return emptyEvidence();
+            }
+            return result ?? model;
+        };
+        if (readOnlyOwner)
+            return fallbackWithoutProbes();
+        try {
+            return await withFileTransactionLock(join(dirname(path), "api-probe-locks", key), async () => {
+                options.signal?.throwIfAborted();
+                const guardedFetch = async (input, init) => {
+                    if (init?.method?.toUpperCase() === "POST" && !await ownsRoute())
+                        throw Error("API route ownership changed");
+                    return fetcher(input, init);
+                };
+                const capabilities = new ApiModelCapabilities(guardedFetch, now, operation => withSharedProbeSlot(operation, options.signal), metadata, options.programRefreshBefore);
+                let entries;
+                try {
+                    entries = await loadOrRepairProbeCache(force, options.requireStoredRoute ? ownsRoute : undefined, options.signal);
+                }
+                catch {
+                    logWarn("API capability cache unavailable; skipping paid probes.");
+                    return fallbackWithoutProbes(false);
+                }
+                if (!await ownsRoute()) {
+                    dropRetained(memoryKey);
+                    return emptyEvidence();
+                }
+                const retained = retainedProbes.get(memoryKey);
+                const latest = latestProbe(key, retained, ownedEntries(entries), now());
+                if (latest)
+                    capabilities.importProbes([[key, latest]]);
+                options.signal?.throwIfAborted();
+                const [result] = await capabilities.enrich([model], refresh, route, force);
+                // Imported evidence belongs to other leases; only write our own result.
+                const exported = new Map(capabilities.exportProbes()).get(key);
+                const value = exported && owner ? { ...exported, ownerEpoch: owner.generation } : exported;
+                try {
+                    const saved = await persistProbe(key, value, now, options.signal, owner, memoryKey);
+                    if (!saved)
+                        return emptyEvidence();
+                    dropRetained(memoryKey);
+                }
+                catch {
+                    options.signal?.throwIfAborted();
+                    if (value && !options.requireStoredRoute)
+                        rememberProbe(memoryKey, value);
+                    else if (value && owner) {
+                        // Acquisition can fail before persistProbe enters its callback.
+                        // This independent lock never acquires a cache lock underneath it.
+                        await withFileTransactionLock(getApiRoutesPath(), async () => {
+                            if (await ownsRoute())
+                                rememberProbe(memoryKey, value, owner);
+                            else
+                                dropRetained(memoryKey);
+                        }, { signal: options.signal }).catch(() => undefined);
+                    }
+                    if (!await ownsRoute()) {
+                        dropRetained(memoryKey);
+                        return emptyEvidence();
+                    }
+                    logWarn("API capability results could not be persisted.");
+                }
+                return result ?? model;
+            }, { waitMs: 60_000, signal: options.signal,
+                guardAcquisition: options.requireStoredRoute ? action => withFileTransactionLock(getApiRoutesPath(), async () => {
+                    if (!await ownsRoute())
+                        throw Object.assign(Error("API route ownership changed"), { code: "ESTALE" });
+                    return action();
+                }, { signal: options.signal }) : undefined,
+            });
+        }
+        catch (error) {
+            if (!unavailableLeaseCodes.has(error.code ?? ""))
+                throw error;
+            logWarn("API capability lease unavailable; skipping paid probes.");
+            return fallbackWithoutProbes();
+        }
+    });
+}
+export async function enrichPersistedApiCapabilities(...args) {
+    activeEnrichments++;
+    try {
+        return await enrichPersistedApiCapabilitiesImpl(...args);
+    }
+    finally {
+        activeEnrichments--;
+    }
+}

package/dist/lib/runtime/responses-websocket.d.ts

@@ -1,14 +1,26 @@
 import type { Duplex } from "node:stream";
 import type { IncomingMessage, Server } from "node:http";
+/** Safe operational evidence only: never persist upstream reason text or payloads. */
+export interface WebSocketDisconnectDiagnostic {
+    at: number;
+    cause: "close" | "transport_error" | "invalid_event" | "buffer_limit" | "send_error";
+    stage: "awaiting_first_event" | "streaming";
+    closeCode?: number;
+    routeKind?: "oauth" | "api" | "zdr";
+    connectionAgeMs?: number;
+    eventCount?: number;
+}
+interface WebSocketStats {
+    connections: number;
+    upstreamConnections: number;
+    upstreamRequests: number;
+    lastDisconnect?: WebSocketDisconnectDiagnostic;
+}
 /** Adapts only the transport; authenticated HTTP routing remains the policy authority. */
 export declare class ResponsesWebSocketGateway {
     private readonly upstreamFetch;
     private readonly options;
-    readonly stats: {
-        connections: number;
-        upstreamConnections: number;
-        upstreamRequests: number;
-    };
+    readonly stats: WebSocketStats;
     private readonly contexts;
     private readonly scope;
     private readonly sessions;
@@ -28,3 +40,4 @@ export declare class ResponsesWebSocketGateway {
     private serve;
     close(): void;
 }
+export {};