@cohortapp/agent-sdk 2.3.0

package/lib/model-router/auth-profiles.mjs

@@ -0,0 +1,758 @@
+/**
+ * lib/model-router/auth-profiles.mjs — the OpenClaw auth-profile rotation store.
+ *
+ * A 40-60 agent org runs MULTIPLE keys per provider (several Anthropic / DeepSeek
+ * accounts) for throughput isolation. This module turns that pile of keys into a
+ * self-healing POOLED quota with per-credential circuit-breaker semantics:
+ *
+ *   loadAuthProfiles(cfg)                       → per-provider key lists, drawn from
+ *                                                 config/auth-profiles.yaml AND from
+ *                                                 env lists (ANTHROPIC_API_KEY,
+ *                                                 ANTHROPIC_API_KEY_2, …).
+ *   nextCredential(provider, {now,...})         → round-robin pick that SKIPS keys
+ *                                                 currently cooling down.
+ *   markCredentialCoolingDown(provider,key,…)   → a 429 / auth error cools THAT key,
+ *                                                 NOT the whole provider (the
+ *                                                 provider-level breaker is
+ *                                                 rate-guard.mjs / health.mjs's job).
+ *   available(provider, {now})                  → how many keys can serve right now.
+ *
+ * This is the per-CREDENTIAL layer; it composes UNDER the per-BACKEND breaker
+ * (health.mjs) and the shared 429 breaker (rate-guard.mjs). One Anthropic key
+ * hitting a 429 cools only that key — the pool keeps serving on the siblings, and
+ * only when EVERY key in the pool is cooling does the backend look exhausted.
+ *
+ * State lives at `state/rate-limits/auth-profiles.json` (same dir as the breakers
+ * so `maestro doctor` dumps them together):
+ *   {
+ *     "<provider>": {
+ *       "rrIndex": <number — round-robin cursor>,
+ *       "creds": {
+ *         "<fingerprint>": { "coolUntil": <epoch ms>, "reason": <str>, "strikes": <n>, "updatedAt": <epoch ms> }
+ *       }
+ *     }
+ *   }
+ *
+ * Credentials are keyed by a STABLE FINGERPRINT (sha256[:16] of the plaintext),
+ * never the plaintext itself — the cooldown ledger NEVER persists a secret to
+ * disk. The fingerprint is deterministic across processes so two agents (or two
+ * lanes) cooling the same key converge on the same ledger entry.
+ *
+ * Concurrency: read-modify-write goes through the same O_EXCL lock discipline as
+ * health.mjs (openSync(..,"wx"), spin-with-stale-break, atomic tmp+rename). Two
+ * concurrent marks on the same key converge to ONE coherent file with monotonic
+ * `strikes` instead of clobbering each other.
+ *
+ * House style (CLAUDE.md): ESM .mjs, Node 20, near-zero-dep (js-yaml is the one
+ * parse dep, imported defensively). EVERYTHING injectable so tests are hermetic —
+ * `deps.now`, `deps.stateDir`, `deps.agentRoot`, `deps.env` all bypass the real
+ * clock/fs/env. NEVER throws on the read path: a missing/corrupt ledger degrades
+ * to "no cooldowns" (fail-open-for-work — better one extra request on a key we
+ * can't prove is cooling than blocking the whole pool).
+ *
+ * @module lib/model-router/auth-profiles
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  renameSync,
+  unlinkSync,
+  openSync,
+  closeSync,
+} from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import { createHash, randomBytes } from "node:crypto";
+
+import { sanitizeProvider } from "../rate-guard.mjs";
+
+// js-yaml is a hard dep of @cohortapp/agent-sdk, imported defensively so the read
+// path degrades rather than crashes if the dep tree is broken on a machine.
+let yamlParse = null;
+try {
+  const y = await import("js-yaml");
+  yamlParse = (text) => y.load(text);
+} catch {
+  yamlParse = null;
+}
+
+const SECOND = 1000;
+const MINUTE = 60 * SECOND;
+
+/** Stepped cooldown ladder for a cooled credential: 30s → 1m → 5m, then sticks. */
+const COOLDOWN_LADDER_MS = Object.freeze([30 * SECOND, MINUTE, 5 * MINUTE]);
+/** Default cooldown when a caller passes no explicit ms (uses the ladder). */
+function ladderMsForStrike(strike) {
+  const i = Math.max(0, Math.min(COOLDOWN_LADDER_MS.length - 1, strike));
+  return COOLDOWN_LADDER_MS[i];
+}
+
+/** How long we busy-spin trying to grab the O_EXCL lock before giving up. */
+const LOCK_TIMEOUT_MS = 2 * SECOND;
+const LOCK_SPIN_MS = 5;
+
+// ---------------------------------------------------------------------------
+// Injected primitives + paths
+// ---------------------------------------------------------------------------
+
+function clock(deps) {
+  return deps && typeof deps.now === "function" ? deps.now : Date.now;
+}
+
+function envOf(deps) {
+  return (deps && deps.env) || process.env || {};
+}
+
+function resolveStateDir(deps) {
+  if (deps && deps.stateDir) return resolve(deps.stateDir);
+  const root =
+    (deps && deps.agentRoot) ||
+    process.env.AGENT_ROOT ||
+    process.env.AGENT_DIR ||
+    process.cwd();
+  return join(resolve(root), "state", "rate-limits");
+}
+
+function ledgerPath(deps) {
+  return join(resolveStateDir(deps), "auth-profiles.json");
+}
+
+function agentRootOf(deps) {
+  return resolve(
+    (deps && deps.agentRoot) ||
+      process.env.AGENT_ROOT ||
+      process.env.AGENT_DIR ||
+      process.cwd()
+  );
+}
+
+function authProfilesConfigPath(root) {
+  return join(root, "config", "auth-profiles.yaml");
+}
+
+// ---------------------------------------------------------------------------
+// Fingerprinting (never persist plaintext)
+// ---------------------------------------------------------------------------
+
+/**
+ * Stable, deterministic fingerprint of a credential value. We hash the plaintext
+ * so the cooldown ledger references a key WITHOUT storing it, and so two
+ * processes cooling the same key converge on the same ledger entry. 16 hex chars
+ * (64 bits) is collision-resistant for a per-provider key pool.
+ *
+ * @param {string} value
+ * @returns {string}
+ */
+export function fingerprint(value) {
+  if (typeof value !== "string" || value === "") return "";
+  return createHash("sha256").update(value, "utf8").digest("hex").slice(0, 16);
+}
+
+// ---------------------------------------------------------------------------
+// loadAuthProfiles — per-provider key list from config + env
+// ---------------------------------------------------------------------------
+
+function readTextSafe(path) {
+  try {
+    if (!existsSync(path)) return null;
+    return readFileSync(path, "utf-8");
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Parse the config/auth-profiles.yaml document into a provider → key[] map. The
+ * file shape is intentionally simple and forgiving:
+ *
+ *   providers:
+ *     deepseek:
+ *       keys: ["sk-a", "sk-b"]            # explicit pool
+ *     anthropic:
+ *       env: [ANTHROPIC_API_KEY, ANTHROPIC_API_KEY_2]   # names of env vars to pool
+ *
+ * Either `keys` (literal values) or `env` (env var NAMES to read) may appear; both
+ * merge. Bare `provider: [k1, k2]` and `provider: "k1"` are also accepted.
+ *
+ * @param {object|string|null} doc  parsed YAML doc (or raw string, or null)
+ * @param {object} env
+ * @returns {Map<string,string[]>}
+ */
+function profilesFromConfigDoc(doc, env) {
+  const out = new Map();
+  if (typeof doc === "string") {
+    if (!yamlParse) return out;
+    try {
+      doc = yamlParse(doc);
+    } catch {
+      return out;
+    }
+  }
+  if (!doc || typeof doc !== "object") return out;
+
+  const providers = doc.providers && typeof doc.providers === "object" ? doc.providers : doc;
+  if (!providers || typeof providers !== "object") return out;
+
+  for (const [rawName, spec] of Object.entries(providers)) {
+    const provider = sanitizeProvider(rawName);
+    const keys = [];
+    if (Array.isArray(spec)) {
+      for (const v of spec) if (typeof v === "string" && v.trim()) keys.push(v.trim());
+    } else if (typeof spec === "string") {
+      if (spec.trim()) keys.push(spec.trim());
+    } else if (spec && typeof spec === "object") {
+      const kk = spec.keys;
+      if (Array.isArray(kk)) {
+        for (const v of kk) if (typeof v === "string" && v.trim()) keys.push(v.trim());
+      } else if (typeof kk === "string" && kk.trim()) {
+        keys.push(kk.trim());
+      }
+      const ee = spec.env;
+      const envNames = Array.isArray(ee) ? ee : typeof ee === "string" ? [ee] : [];
+      for (const name of envNames) {
+        if (typeof name !== "string") continue;
+        const v = env[name];
+        if (typeof v === "string" && v.trim()) keys.push(v.trim());
+      }
+    }
+    if (keys.length) mergeKeys(out, provider, keys);
+  }
+  return out;
+}
+
+/**
+ * Pool keys from env-var lists. Convention (OpenClaw api-key-rotation): a base
+ * var plus numbered siblings — ANTHROPIC_API_KEY, ANTHROPIC_API_KEY_2,
+ * ANTHROPIC_API_KEY_3, … and also ANTHROPIC_API_KEYS="k1,k2,k3" (comma list).
+ *
+ * The mapping provider → base env var name comes from the catalog's auth_env
+ * (passed via opts.authEnvByProvider) so we don't hardcode provider names. We
+ * also accept an explicit opts.envBases map. As a convenience, the well-known
+ * defaults below are always probed.
+ *
+ * @param {object} env
+ * @param {Map<string,string>} authEnvByProvider  provider -> base env var name
+ * @returns {Map<string,string[]>}
+ */
+function profilesFromEnv(env, authEnvByProvider) {
+  const out = new Map();
+  for (const [rawProvider, baseVar] of authEnvByProvider) {
+    if (typeof baseVar !== "string" || !baseVar) continue;
+    const provider = sanitizeProvider(rawProvider);
+    const keys = [];
+
+    // Comma-list form: <BASE>S="k1,k2" (e.g. ANTHROPIC_API_KEYS).
+    const listVar = env[`${baseVar}S`] ?? env[`${baseVar}_LIST`];
+    if (typeof listVar === "string" && listVar.includes(",")) {
+      for (const part of listVar.split(",")) {
+        const v = part.trim();
+        if (v) keys.push(v);
+      }
+    }
+
+    // Base var.
+    const base = env[baseVar];
+    if (typeof base === "string" && base.trim()) keys.push(base.trim());
+
+    // Numbered siblings: <BASE>_2, <BASE>_3, … (stop at the first gap, cap 32).
+    for (let n = 2; n <= 32; n++) {
+      const v = env[`${baseVar}_${n}`];
+      if (typeof v === "string" && v.trim()) keys.push(v.trim());
+      else break;
+    }
+
+    if (keys.length) mergeKeys(out, provider, keys);
+  }
+  return out;
+}
+
+/** Append keys to a provider's pool, de-duped, preserving first-seen order. */
+function mergeKeys(map, provider, keys) {
+  const list = map.get(provider) || [];
+  const seen = new Set(list);
+  for (const k of keys) {
+    if (typeof k !== "string" || !k.trim()) continue;
+    const v = k.trim();
+    if (seen.has(v)) continue;
+    seen.add(v);
+    list.push(v);
+  }
+  map.set(provider, list);
+}
+
+/**
+ * @typedef {object} AuthProfiles
+ * @property {Map<string,string[]>} byProvider  provider -> ordered key pool (plaintext)
+ * @property {(provider:string)=>string[]} keysFor
+ * @property {(provider:string)=>boolean} isPooled  true ⇒ >1 key (rotation matters)
+ * @property {string[]} providers
+ */
+
+/**
+ * Load the auth-profile key pools. Merges config/auth-profiles.yaml with env-var
+ * lists. NEVER throws — a missing config or env yields an empty pool map.
+ *
+ * @param {object} [cfg]  optional agent config (currently informational; the
+ *                        pools come from the dedicated auth-profiles.yaml + env)
+ * @param {object} [opts]
+ *   @param {object}  [opts.env]                env to read (default process.env)
+ *   @param {string}  [opts.agentRoot]
+ *   @param {object|string} [opts.configDoc]    bypass-FS config doc (tests)
+ *   @param {Map<string,string>|object} [opts.authEnvByProvider]  provider -> base env var
+ * @returns {AuthProfiles}
+ */
+export function loadAuthProfiles(cfg, opts = {}) {
+  const env = envOf(opts);
+  const root = agentRootOf(opts);
+
+  // 1. config/auth-profiles.yaml (or injected configDoc).
+  let configDoc = opts.configDoc;
+  if (configDoc === undefined) {
+    const text = readTextSafe(authProfilesConfigPath(root));
+    configDoc = text;
+  }
+  const fromConfig = profilesFromConfigDoc(configDoc, env);
+
+  // 2. env-var lists, keyed by the catalog's provider → auth_env mapping.
+  const authEnvByProvider = normaliseAuthEnvMap(opts.authEnvByProvider);
+  const fromEnv = profilesFromEnv(env, authEnvByProvider);
+
+  // Merge: config keys first (explicit intent wins ordering), then env siblings.
+  const byProvider = new Map();
+  for (const [provider, keys] of fromConfig) mergeKeys(byProvider, provider, keys);
+  for (const [provider, keys] of fromEnv) mergeKeys(byProvider, provider, keys);
+
+  return {
+    byProvider,
+    keysFor(provider) {
+      return byProvider.get(sanitizeProvider(provider)) || [];
+    },
+    isPooled(provider) {
+      const k = byProvider.get(sanitizeProvider(provider));
+      return Array.isArray(k) && k.length > 1;
+    },
+    get providers() {
+      return [...byProvider.keys()];
+    },
+  };
+}
+
+/** Coerce a Map | plain object | undefined into a Map<provider, baseVar>. */
+function normaliseAuthEnvMap(m) {
+  const out = new Map();
+  if (m instanceof Map) {
+    for (const [p, v] of m) if (typeof v === "string") out.set(sanitizeProvider(p), v);
+  } else if (m && typeof m === "object") {
+    for (const [p, v] of Object.entries(m)) if (typeof v === "string") out.set(sanitizeProvider(p), v);
+  }
+  return out;
+}
+
+/**
+ * Convenience: build the provider → auth_env map loadAuthProfiles wants from a
+ * loaded catalog (its `providers` Map / `models` array each carry auth_env).
+ * NEVER throws.
+ *
+ * @param {object} catalog  loadCatalog result OR bare catalog
+ * @returns {Map<string,string>}
+ */
+export function authEnvMapFromCatalog(catalog) {
+  const out = new Map();
+  let cat = catalog;
+  if (catalog && typeof catalog === "object" && !catalog.providers && !Array.isArray(catalog.models) && catalog.catalog) {
+    cat = catalog.catalog;
+  }
+  const consider = (provider, authEnv) => {
+    const p = typeof provider === "string" ? provider.trim() : "";
+    const a = typeof authEnv === "string" ? authEnv.trim() : "";
+    if (!p || !a) return;
+    if (!out.has(sanitizeProvider(p))) out.set(sanitizeProvider(p), a);
+  };
+  const providers = cat && cat.providers;
+  if (providers && typeof providers.forEach === "function" && !(providers instanceof Array)) {
+    providers.forEach((meta, provider) => consider(provider, meta && meta.auth_env));
+  } else if (Array.isArray(providers)) {
+    for (const meta of providers) consider(meta && meta.provider, meta && meta.auth_env);
+  }
+  const models = (cat && cat.models) || [];
+  if (Array.isArray(models)) {
+    for (const row of models) consider(row && row.provider, row && row.auth_env);
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Cooldown ledger (read/write under O_EXCL lock)
+// ---------------------------------------------------------------------------
+
+function readLedgerSafe(deps) {
+  const p = ledgerPath(deps);
+  if (!existsSync(p)) return {};
+  try {
+    const raw = JSON.parse(readFileSync(p, "utf-8"));
+    return raw && typeof raw === "object" ? raw : {};
+  } catch {
+    return {};
+  }
+}
+
+function providerEntry(ledger, provider) {
+  const e = ledger[provider];
+  if (e && typeof e === "object") {
+    return {
+      rrIndex: Number.isFinite(Number(e.rrIndex)) ? Number(e.rrIndex) : 0,
+      creds: e.creds && typeof e.creds === "object" ? e.creds : {},
+    };
+  }
+  return { rrIndex: 0, creds: {} };
+}
+
+function credState(entry, fp) {
+  const c = entry.creds[fp];
+  if (c && typeof c === "object") {
+    return {
+      coolUntil: Number.isFinite(Number(c.coolUntil)) ? Number(c.coolUntil) : 0,
+      reason: typeof c.reason === "string" ? c.reason : null,
+      strikes: Number.isFinite(Number(c.strikes)) ? Number(c.strikes) : 0,
+      updatedAt: Number.isFinite(Number(c.updatedAt)) ? Number(c.updatedAt) : 0,
+    };
+  }
+  return { coolUntil: 0, reason: null, strikes: 0, updatedAt: 0 };
+}
+
+function isCooling(entry, fp, now) {
+  return credState(entry, fp).coolUntil > now;
+}
+
+// O_EXCL lock — mirrors health.mjs's withLock so concurrent marks converge.
+
+function acquireLock(lockPath, deps) {
+  const now = clock(deps);
+  const deadline = now() + LOCK_TIMEOUT_MS;
+  for (;;) {
+    try {
+      return openSync(lockPath, "wx"); // O_CREAT | O_EXCL | O_WRONLY
+    } catch (err) {
+      if (err && err.code === "EEXIST") {
+        try {
+          const raw = readFileSync(lockPath, "utf-8");
+          const heldAt = Number(JSON.parse(raw).at);
+          if (Number.isFinite(heldAt) && now() - heldAt > LOCK_TIMEOUT_MS) {
+            try { unlinkSync(lockPath); } catch { /* */ }
+            continue;
+          }
+        } catch { /* unreadable lock — spin/timeout */ }
+        if (now() >= deadline) return null;
+        busyWaitMs(LOCK_SPIN_MS);
+        continue;
+      }
+      return undefined; // e.g. ENOENT on missing dir — caller mkdir+retries
+    }
+  }
+}
+
+function busyWaitMs(ms) {
+  const end = Date.now() + ms;
+  while (Date.now() < end) { /* spin */ }
+}
+
+/**
+ * Run `mutator(ledger) → ledger` under an exclusive lock over the whole ledger
+ * file and atomically persist it. Returns the persisted ledger. On unrecoverable
+ * I/O failure applies the mutation in-memory and returns it (fail-open).
+ */
+function withLock(deps, mutator) {
+  const file = ledgerPath(deps);
+  const lockPath = `${file}.lock`;
+  const now = clock(deps);
+  try { mkdirSync(dirname(file), { recursive: true }); } catch { /* */ }
+
+  let fd = acquireLock(lockPath, deps);
+  if (fd === undefined) {
+    try { mkdirSync(dirname(file), { recursive: true }); } catch { /* */ }
+    fd = acquireLock(lockPath, deps);
+  }
+  if (fd == null) {
+    const cur = readLedgerSafe(deps);
+    const next = mutator(cur) || cur;
+    bestEffortWrite(file, next, deps);
+    return next;
+  }
+  try {
+    try { writeFileSync(fd, JSON.stringify({ at: now(), pid: process.pid })); } catch { /* */ }
+    const cur = readLedgerSafe(deps);
+    const next = mutator(cur) || cur;
+    bestEffortWrite(file, next, deps);
+    return next;
+  } finally {
+    try { closeSync(fd); } catch { /* */ }
+    try { unlinkSync(lockPath); } catch { /* */ }
+  }
+}
+
+function bestEffortWrite(file, ledger, deps) {
+  try {
+    mkdirSync(dirname(file), { recursive: true });
+    const tmp = `${file}.tmp.${process.pid}.${Date.now()}.${randomBytes(2).toString("hex")}`;
+    writeFileSync(tmp, JSON.stringify(ledger, null, 2) + "\n");
+    try {
+      renameSync(tmp, file);
+    } catch (err) {
+      try { unlinkSync(tmp); } catch { /* */ }
+      throw err;
+    }
+  } catch {
+    /* fail-open: in-memory only */
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API: nextCredential / markCredentialCoolingDown / available
+// ---------------------------------------------------------------------------
+
+/**
+ * Pick the next credential for `provider` round-robin, SKIPPING any key whose
+ * cooldown has not expired. Advances and persists the round-robin cursor under
+ * the lock so concurrent pickers fairly share the pool. NEVER throws.
+ *
+ * The key pool comes from `opts.profiles` (a loadAuthProfiles result) or
+ * `opts.keys` (an explicit string[]). When the pool has 0 or 1 usable key, the
+ * single-key behavior is preserved exactly (returns that key, or null).
+ *
+ * @param {string} provider
+ * @param {object} [opts]
+ *   @param {AuthProfiles} [opts.profiles]   from loadAuthProfiles
+ *   @param {string[]}     [opts.keys]       explicit pool (overrides profiles)
+ *   @param {number}       [opts.now]
+ *   @param {object}       [opts.deps]       { now?, stateDir?, agentRoot? } for the ledger
+ * @returns {{ key:string|null, fingerprint:string|null, index:number,
+ *             poolSize:number, available:number, allCooling:boolean,
+ *             soonestCoolUntil:number|null }}
+ */
+export function nextCredential(provider, opts = {}) {
+  const prov = sanitizeProvider(provider);
+  const deps = opts.deps || opts; // allow flat opts.now / opts.stateDir too
+  const now = Number.isFinite(opts.now) ? opts.now : clock(deps)();
+
+  const keys = poolKeys(prov, opts);
+  const poolSize = keys.length;
+
+  // Single-key (or empty) pool: preserve today's behavior exactly. No ledger
+  // write, no rotation — just return the one key (respecting its cooldown only
+  // insofar as we still surface allCooling so the caller can skip if it wants).
+  if (poolSize === 0) {
+    return { key: null, fingerprint: null, index: -1, poolSize: 0, available: 0, allCooling: false, soonestCoolUntil: null };
+  }
+  if (poolSize === 1) {
+    const key = keys[0];
+    const fp = fingerprint(key);
+    const led = readLedgerSafe(deps);
+    const entry = providerEntry(led, prov);
+    const cooling = isCooling(entry, fp, now);
+    const cs = credState(entry, fp);
+    return {
+      key,
+      fingerprint: fp,
+      index: 0,
+      poolSize: 1,
+      available: cooling ? 0 : 1,
+      allCooling: cooling,
+      soonestCoolUntil: cooling ? cs.coolUntil : null,
+    };
+  }
+
+  // Multi-key pool: round-robin over the keys, skipping cooled ones. The cursor
+  // is advanced + persisted under the lock so two concurrent pickers don't both
+  // hand out the same key.
+  let chosenIdx = -1;
+  let chosenKey = null;
+  let chosenFp = null;
+  let availableCount = 0;
+  let soonest = null;
+
+  withLock(deps, (ledger) => {
+    const entry = providerEntry(ledger, prov);
+    const start = ((entry.rrIndex % poolSize) + poolSize) % poolSize;
+    // First pass: find the next non-cooling key starting at the cursor.
+    for (let step = 0; step < poolSize; step++) {
+      const idx = (start + step) % poolSize;
+      const key = keys[idx];
+      const fp = fingerprint(key);
+      if (!isCooling(entry, fp, now)) {
+        if (chosenIdx === -1) {
+          chosenIdx = idx;
+          chosenKey = key;
+          chosenFp = fp;
+        }
+      }
+    }
+    // Count availability + soonest cooldown across the whole pool (for the caller).
+    for (let i = 0; i < poolSize; i++) {
+      const fp = fingerprint(keys[i]);
+      const cs = credState(entry, fp);
+      if (cs.coolUntil > now) {
+        if (soonest == null || cs.coolUntil < soonest) soonest = cs.coolUntil;
+      } else {
+        availableCount++;
+      }
+    }
+    // Advance the cursor to just past the chosen key so the NEXT call rotates on.
+    // If everything is cooling, advance by one so we don't pin a single key.
+    const advanceTo = chosenIdx === -1 ? (start + 1) % poolSize : (chosenIdx + 1) % poolSize;
+    ledger[prov] = { ...entry, rrIndex: advanceTo };
+    return ledger;
+  });
+
+  return {
+    key: chosenKey,
+    fingerprint: chosenFp,
+    index: chosenIdx,
+    poolSize,
+    available: availableCount,
+    allCooling: chosenIdx === -1,
+    soonestCoolUntil: soonest,
+  };
+}
+
+/**
+ * Cool down ONE credential (the one that just 429'd / auth-failed), NOT the whole
+ * provider. Stepped backoff via the ladder unless an explicit `ms` is given.
+ * Locked read-modify-write so concurrent marks converge with monotonic strikes
+ * and the longer cooldown wins. NEVER throws.
+ *
+ * The credential may be passed as the plaintext `key` OR a precomputed
+ * `{ fingerprint }`; either way only the fingerprint is persisted.
+ *
+ * @param {string} provider
+ * @param {string} key       the plaintext credential (fingerprinted internally)
+ * @param {string} [reason]  FailoverReason-ish label (e.g. "rate_limit", "auth")
+ * @param {number|null} [ms] explicit cooldown ms; null/undefined → ladder
+ * @param {object} [opts]
+ *   @param {number} [opts.now]
+ *   @param {object} [opts.deps]
+ * @returns {{ fingerprint:string, coolUntil:number, strikes:number, reason:string|null }}
+ */
+export function markCredentialCoolingDown(provider, key, reason, ms, opts = {}) {
+  const prov = sanitizeProvider(provider);
+  const deps = opts.deps || opts;
+  const now = Number.isFinite(opts.now) ? opts.now : clock(deps)();
+  const fp = typeof key === "string" ? fingerprint(key) : (key && key.fingerprint) || "";
+  if (!fp) {
+    return { fingerprint: "", coolUntil: 0, strikes: 0, reason: null };
+  }
+
+  let result = { fingerprint: fp, coolUntil: 0, strikes: 0, reason: reason || null };
+  withLock(deps, (ledger) => {
+    const entry = providerEntry(ledger, prov);
+    const cur = credState(entry, fp);
+    const strikes = cur.strikes + 1;
+    // strike-1 indexes the ladder from 0 (first hit → 30s).
+    const cooldownMs = ms == null ? ladderMsForStrike(strikes - 1) : Math.max(0, Number(ms) || 0);
+    const until = cooldownMs === Infinity ? Infinity : now + cooldownMs;
+    // Never shorten an existing longer cooldown (a racing shorter mark can't undo it).
+    const coolUntil =
+      cur.coolUntil === Infinity ? Infinity : Math.max(cur.coolUntil, until);
+    const next = {
+      coolUntil,
+      reason: reason || cur.reason || null,
+      strikes,
+      updatedAt: now,
+    };
+    entry.creds[fp] = next;
+    ledger[prov] = entry;
+    result = { fingerprint: fp, coolUntil, strikes, reason: next.reason };
+    return ledger;
+  });
+  return result;
+}
+
+/**
+ * Clear a credential's cooldown (clean success). Idempotent + best-effort.
+ * NEVER throws.
+ *
+ * @param {string} provider
+ * @param {string} key
+ * @param {object} [opts]
+ */
+export function markCredentialHealthy(provider, key, opts = {}) {
+  const prov = sanitizeProvider(provider);
+  const deps = opts.deps || opts;
+  const fp = typeof key === "string" ? fingerprint(key) : (key && key.fingerprint) || "";
+  if (!fp) return;
+  const led = readLedgerSafe(deps);
+  const entry = providerEntry(led, prov);
+  // No-op if there's nothing to clear (don't churn the file on every success).
+  if (!entry.creds[fp]) return;
+  withLock(deps, (ledger) => {
+    const e = providerEntry(ledger, prov);
+    if (e.creds[fp]) delete e.creds[fp];
+    ledger[prov] = e;
+    return ledger;
+  });
+}
+
+/**
+ * How many credentials in `provider`'s pool can serve right now (not cooling).
+ * NEVER throws.
+ *
+ * @param {string} provider
+ * @param {object} [opts]  same pool/clock/deps shape as nextCredential
+ * @returns {{ count:number, poolSize:number, soonestCoolUntil:number|null }}
+ */
+export function available(provider, opts = {}) {
+  const prov = sanitizeProvider(provider);
+  const deps = opts.deps || opts;
+  const now = Number.isFinite(opts.now) ? opts.now : clock(deps)();
+  const keys = poolKeys(prov, opts);
+  const poolSize = keys.length;
+  if (poolSize === 0) return { count: 0, poolSize: 0, soonestCoolUntil: null };
+  const led = readLedgerSafe(deps);
+  const entry = providerEntry(led, prov);
+  let count = 0;
+  let soonest = null;
+  for (const k of keys) {
+    const cs = credState(entry, fingerprint(k));
+    if (cs.coolUntil > now) {
+      if (soonest == null || cs.coolUntil < soonest) soonest = cs.coolUntil;
+    } else {
+      count++;
+    }
+  }
+  return { count, poolSize, soonestCoolUntil: soonest };
+}
+
+/** Resolve the candidate key pool for a provider from opts.keys | opts.profiles. */
+function poolKeys(provider, opts) {
+  if (Array.isArray(opts.keys)) {
+    return opts.keys.filter((k) => typeof k === "string" && k.trim()).map((k) => k.trim());
+  }
+  if (opts.profiles && typeof opts.profiles.keysFor === "function") {
+    return opts.profiles.keysFor(provider);
+  }
+  return [];
+}
+
+export const _internals = {
+  profilesFromConfigDoc,
+  profilesFromEnv,
+  mergeKeys,
+  ladderMsForStrike,
+  COOLDOWN_LADDER_MS,
+  readLedgerSafe,
+  providerEntry,
+  credState,
+  poolKeys,
+  ledgerPath,
+};
+
+export default {
+  loadAuthProfiles,
+  nextCredential,
+  markCredentialCoolingDown,
+  markCredentialHealthy,
+  available,
+  fingerprint,
+  authEnvMapFromCatalog,
+};