@cohortapp/agent-sdk 2.3.0

package/lib/model-router/spawn.mjs

@@ -0,0 +1,497 @@
+/**
+ * lib/model-router/spawn.mjs — the router EXECUTION layer for SESSION work.
+ *
+ * `spawnRouted(request, opts)` is what the daemon calls INSTEAD of hand-building
+ * `claude -p` spawn args. It ties the three substrate layers together:
+ *
+ *   1. resolveChain(request)         → a RouteDecision (which model/harness, the
+ *                                       session-retarget env, spawn knobs, the
+ *                                       ordered failover chain).
+ *   2. build argv + child env        → from decision.spawnArgs + decision.chosen
+ *                                       (and each failover candidate's own
+ *                                       envForSpawn), with §7.3 child-env
+ *                                       scrubbing (no foreign *_API_KEY leaks).
+ *   3. failover.stepChain(...)       → walk the chain, applying the 16-reason
+ *                                       DISPOSITION: retry-same where allowed,
+ *                                       fail over to the next candidate on a
+ *                                       failover-class reason, open breakers per
+ *                                       the table's cooldown scope, and STOP
+ *                                       (no failover) for context_overflow /
+ *                                       payload_too_large / content_policy /
+ *                                       network exactly as the table says.
+ *
+ * The actual child process is NOT launched here — the daemon injects a
+ * `spawnImpl(candidate, { args, env, decision }) -> { ok, reason?, result? }`.
+ * This keeps spawn.mjs pure orchestration (no child_process, no fs, no clock of
+ * its own) so it is hermetically testable, mirroring the DI discipline of
+ * failover.mjs / resource-governor.mjs.
+ *
+ * ROUTING IS A DISPATCH-TIME DECISION (SPEC §6.2). spawnRouted owns the fast-fail
+ * failover loop BEFORE real work begins; it never re-routes a live session (cache
+ * economics — a mid-session switch silently re-bills the entire prefix). The
+ * kill switch (MAESTRO_ROUTER_FORCE_ANTHROPIC) is honored inside resolveChain,
+ * which collapses the chain to an Anthropic-only member, so spawnRouted naturally
+ * short-circuits to stock Anthropic with zero third-party candidates to walk.
+ *
+ * House style: ESM .mjs, Node built-ins only, never throws on the orchestration
+ * path (a spawnImpl that throws is caught + classified `unknown`).
+ *
+ * @module lib/model-router/spawn
+ */
+
+import { resolveChain } from "./resolve.mjs";
+import { stepChain } from "./failover.mjs";
+import { keyFor, isOpen as healthIsOpen, open as healthOpen, close as healthClose } from "./health.mjs";
+import { lookupModel } from "./catalog.mjs";
+import {
+  nextCredential as authNextCredential,
+  markCredentialCoolingDown as authMarkCooling,
+} from "./auth-profiles.mjs";
+
+// Failover reasons that indicate the CREDENTIAL itself is the problem (a 429 /
+// auth failure on this key) — these cool THAT credential in the pool so the next
+// attempt rotates to a sibling. A backend/model-class reason (overloaded,
+// server_error, …) cools the breaker, not the key.
+const CRED_COOL_REASONS = new Set(["rate_limit", "auth", "auth_permanent", "billing"]);
+
+// Env keys that the session-retarget set is allowed to introduce. Everything
+// else credential-shaped is scrubbed (§7.3) before injection.
+const RETARGET_KEYS = new Set([
+  "ANTHROPIC_BASE_URL",
+  "ANTHROPIC_AUTH_TOKEN",
+  "ANTHROPIC_API_KEY",
+  "ANTHROPIC_MODEL",
+  "ANTHROPIC_SMALL_FAST_MODEL",
+  "CLAUDE_CODE_ATTRIBUTION_HEADER",
+  "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS",
+]);
+
+// ---------------------------------------------------------------------------
+// Child-env scrubbing (graft #3 / §7.3)
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the child env by allowlist: start from a scrubbed copy of `baseEnv`
+ * with every credential-shaped var (`*_API_KEY` / `*_AUTH_TOKEN`, including the
+ * break-glass key) removed, then inject ONLY the resolved candidate's
+ * session-retarget pair (decision.envForSpawn). A session routed to a third
+ * party never sees Anthropic credentials, and vice versa.
+ *
+ * The Anthropic SESSION path (envForSpawn === {}) rides the stock CLI / keychain
+ * OAuth or a Console key the spawner owns; we keep an explicitly-injected
+ * ANTHROPIC_API_KEY from the caller's base env in that case ONLY if the caller
+ * opted in via opts.keepAnthropicApiKey (default: scrub for hygiene). When we do
+ * retarget (envForSpawn has ANTHROPIC_BASE_URL), ANTHROPIC_API_KEY="" is part of
+ * the retarget set and is injected verbatim (empty-string discipline, W:
+ * keychain fallthrough).
+ *
+ * @param {object} baseEnv      usually process.env
+ * @param {object} envForSpawn  decision.envForSpawn (the retarget pair) or {}
+ * @param {object} [opts]
+ *   @param {boolean} [opts.keepAnthropicApiKey=false]
+ * @returns {object} the child env
+ */
+export function buildChildEnv(baseEnv, envForSpawn = {}, opts = {}) {
+  const out = {};
+  const src = baseEnv && typeof baseEnv === "object" ? baseEnv : {};
+  for (const [k, v] of Object.entries(src)) {
+    if (isCredentialKey(k)) continue; // scrub all credential-shaped vars
+    out[k] = v;
+  }
+  // For a retarget (third-party session), force the empty-string ANTHROPIC_API_KEY
+  // so Claude Code does NOT fall back to keychain OAuth.
+  const retargeting = !!(envForSpawn && envForSpawn.ANTHROPIC_BASE_URL);
+  if (!retargeting && opts.keepAnthropicApiKey && typeof src.ANTHROPIC_API_KEY === "string") {
+    out.ANTHROPIC_API_KEY = src.ANTHROPIC_API_KEY;
+  }
+  // Inject only the retarget set (whitelisted keys).
+  for (const [k, v] of Object.entries(envForSpawn || {})) {
+    if (RETARGET_KEYS.has(k)) out[k] = v;
+  }
+  return out;
+}
+
+function isCredentialKey(k) {
+  return /(_API_KEY|_AUTH_TOKEN|_SECRET|_ACCESS_KEY|_PRIVATE_KEY)$/i.test(k) || k === "ANTHROPIC_API_KEY";
+}
+
+// ---------------------------------------------------------------------------
+// argv construction
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the `claude -p` argv from a RouteDecision's spawnArgs + a concrete
+ * candidate ref (which carries the modelFlag for this chain member). The base
+ * spawnArgs come from the decision (maxTurns/effort/agentsJson/bare); the
+ * per-candidate modelFlag overrides decision.spawnArgs.modelFlag when failing
+ * over to a different chain member.
+ *
+ * Argv order is deterministic so tests can assert it exactly:
+ *   [--print, --bare?, --output-format json, --model <flag>, --max-turns <n>,
+ *    --effort <e>?, --agents <json>?, ...extraArgs]
+ *
+ * @param {object} spawnArgs   decision.spawnArgs
+ * @param {object} [over]
+ *   @param {string} [over.modelFlag]   per-candidate model flag override
+ *   @param {string[]} [over.extraArgs] appended verbatim (prompt file, etc.)
+ *   @param {string} [over.outputFormat="json"]  --output-format value
+ * @returns {string[]}
+ */
+export function buildSpawnArgs(spawnArgs = {}, over = {}) {
+  const args = ["--print"];
+  if (spawnArgs.bare !== false) args.push("--bare");
+  const outputFormat = over.outputFormat || "json";
+  args.push("--output-format", outputFormat);
+
+  const modelFlag = over.modelFlag || spawnArgs.modelFlag;
+  if (modelFlag) args.push("--model", String(modelFlag));
+
+  if (spawnArgs.maxTurns != null && Number.isFinite(Number(spawnArgs.maxTurns))) {
+    args.push("--max-turns", String(spawnArgs.maxTurns));
+  }
+  if (spawnArgs.effort) args.push("--effort", String(spawnArgs.effort));
+  if (spawnArgs.maxBudgetUsd != null && Number.isFinite(Number(spawnArgs.maxBudgetUsd))) {
+    args.push("--max-budget-usd", String(spawnArgs.maxBudgetUsd));
+  }
+  if (spawnArgs.agentsJson) {
+    const json =
+      typeof spawnArgs.agentsJson === "string"
+        ? spawnArgs.agentsJson
+        : JSON.stringify(spawnArgs.agentsJson);
+    args.push("--agents", json);
+  }
+  if (Array.isArray(over.extraArgs) && over.extraArgs.length) {
+    for (const a of over.extraArgs) args.push(String(a));
+  }
+  return args;
+}
+
+// ---------------------------------------------------------------------------
+// Per-candidate retarget env (failover may switch backends)
+// ---------------------------------------------------------------------------
+
+/**
+ * Derive the session-retarget env for a chain candidate that is NOT the decision's
+ * chosen row (a failover target). The decision only carries envForSpawn for its
+ * chosen row, so when we fail over to a different ref we re-derive the retarget
+ * pair from that ref's catalog row. Mirrors resolve.buildEnvForSpawn for the
+ * session harness.
+ */
+function envForCandidate(catalog, ref, harness, baseEnv, decision, authCtx) {
+  // The chosen ref reuses the decision's already-built envForSpawn verbatim ONLY
+  // when there's no pool to rotate over — a pooled set must re-pick per attempt so
+  // a failover that cooled the first key rotates to a sibling.
+  const isChosen = decision && decision.chosen && ref === decision.chosen.catalogRow?.ref;
+  const row = catalog ? lookupModel(catalog, ref) : null;
+  const provider = row ? row.provider : refProvider(ref);
+  const pooled =
+    authCtx && authCtx.profiles && typeof authCtx.profiles.isPooled === "function"
+      ? authCtx.profiles.isPooled(provider)
+      : false;
+  if (isChosen && !pooled) {
+    return decision.envForSpawn || {};
+  }
+  if (!row || harness !== "session") return isChosen ? decision.envForSpawn || {} : {};
+  if (row.provider === "anthropic") return isChosen ? decision.envForSpawn || {} : {};
+  const baseUrl = row.endpoints && row.endpoints.anthropic;
+  if (!baseUrl) return isChosen ? decision.envForSpawn || {} : {};
+  const out = {
+    ANTHROPIC_BASE_URL: baseUrl,
+    ANTHROPIC_API_KEY: "",
+    ANTHROPIC_MODEL: row.id,
+    ANTHROPIC_SMALL_FAST_MODEL: row.id,
+    CLAUDE_CODE_ATTRIBUTION_HEADER: "0",
+    CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS: "1",
+  };
+  // Pooled credential rotation (additive): pick the next non-cooled key, else
+  // fall back to the single env var exactly as before.
+  let tok;
+  let fp = null;
+  if (pooled && row.auth_env) {
+    const pick = authNextCredential(provider, {
+      profiles: authCtx.profiles,
+      now: authCtx.now,
+      deps: authCtx.deps,
+    });
+    if (pick && pick.key) {
+      tok = pick.key;
+      fp = pick.fingerprint;
+    }
+  }
+  if (typeof tok !== "string") {
+    tok = baseEnv && row.auth_env ? baseEnv[row.auth_env] : undefined;
+  }
+  if (typeof tok === "string") {
+    out.ANTHROPIC_AUTH_TOKEN = tok;
+    out.__credFingerprint = fp; // internal-only; scrubbed before injection
+  }
+  return out;
+}
+
+/** Provider segment of a "provider/model" ref. */
+function refProvider(ref) {
+  return typeof ref === "string" && ref.includes("/") ? ref.split("/")[0] : ref;
+}
+
+/** The model flag for a chain candidate ref (the bare id, mirrors resolve). */
+function modelFlagForRef(catalog, ref, fallbackFlag) {
+  const row = catalog ? lookupModel(catalog, ref) : null;
+  if (row && row.id) return row.id;
+  // ref is "provider/model"; the model segment is the flag if no row.
+  if (typeof ref === "string" && ref.includes("/")) return ref.split("/").slice(1).join("/");
+  return fallbackFlag || ref;
+}
+
+/** Build the breaker keys (backend + backend:model) for a chain candidate ref. */
+function breakerKeysForRef(ref) {
+  if (typeof ref !== "string" || !ref.includes("/")) {
+    return { breakerKey: keyFor(ref || "unknown"), backendKey: keyFor(ref || "unknown") };
+  }
+  const [provider, ...rest] = ref.split("/");
+  const model = rest.join("/");
+  return { breakerKey: keyFor(provider, model), backendKey: keyFor(provider) };
+}
+
+// ---------------------------------------------------------------------------
+// spawnRouted — the public entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve a route and EXECUTE the chosen candidate via an injected spawnImpl,
+ * failing over along the chain per the disposition table. Returns the decision,
+ * the chosen candidate, the argv + child env actually built for the successful
+ * (or last) attempt, the failover outcome, and the per-candidate tried[].
+ *
+ * NEVER throws.
+ *
+ * @param {object} request  RouteRequest (§4.3)
+ * @param {object} [opts]
+ *   @param {(candidate, ctx) => ({ok, reason?, result?}|Promise<...>)} opts.spawnImpl
+ *       REQUIRED. ctx = { args, env, decision, candidate, attemptIndex }.
+ *       Returns { ok:true, result } on success, or { ok:false, reason } where
+ *       reason is a FailoverReason (taxonomy.mjs). A throw is caught → unknown.
+ *   @param {object} [opts.decision]    inject a pre-resolved RouteDecision (tests);
+ *                                      otherwise resolveChain(request, opts) is called.
+ *   @param {object} [opts.config]      routing config (passed to resolveChain)
+ *   @param {object} [opts.catalog]     catalog (passed to resolveChain + retarget)
+ *   @param {object} [opts.env]         base env (default process.env)
+ *   @param {()=>number} [opts.now]
+ *   @param {(n)=>number} [opts.rng]
+ *   @param {object} [opts.breaker]     { isOpen, open, close } — defaults to health.mjs
+ *   @param {object} [opts.breakerDeps] deps for the default health breaker (agentRoot, stateDir, now)
+ *   @param {boolean} [opts.allowContentReroute=false]  one content_policy re-route permitted
+ *   @param {string[]} [opts.extraArgs] appended to every candidate's argv
+ *   @param {string} [opts.outputFormat="json"]
+ *   @param {boolean} [opts.keepAnthropicApiKey=false]
+ *   @param {(rec)=>void} [opts.onAttempt]  side-channel per attempt
+ * @returns {Promise<{
+ *   decision, chosen, args, env, outcome,
+ *   tried: Array<{ref, reason, retry?, meta?}>,
+ *   ok: boolean
+ * }>}
+ */
+export async function spawnRouted(request = {}, opts = {}) {
+  const env = opts.env || process.env || {};
+  const decision = opts.decision || resolveChain(request, opts);
+
+  // A no-route or rules decision has no model to spawn — surface it without
+  // touching spawnImpl. The caller's deterministic fallback handles "rules".
+  if (!decision || !decision.chosen || decision.chosen.provider === "rules" || !decision.chain || !decision.chain.length) {
+    return {
+      decision,
+      chosen: decision ? decision.chosen : null,
+      args: null,
+      env: null,
+      outcome: {
+        ok: false,
+        reason: decision && decision.chosen && decision.chosen.provider === "rules" ? "rules" : "no_route",
+        tried: [],
+        soonestCooldownExpiry: null,
+        attempts: 0,
+      },
+      tried: [],
+      ok: false,
+    };
+  }
+
+  const spawnImpl = typeof opts.spawnImpl === "function" ? opts.spawnImpl : null;
+  if (!spawnImpl) {
+    return {
+      decision,
+      chosen: decision.chosen,
+      args: null,
+      env: null,
+      outcome: { ok: false, reason: "unknown", tried: [{ ref: decision.chosen.catalogRow?.ref ?? null, reason: "no_spawn_impl" }], soonestCooldownExpiry: null, attempts: 0 },
+      tried: [{ ref: decision.chosen.catalogRow?.ref ?? null, reason: "no_spawn_impl" }],
+      ok: false,
+    };
+  }
+
+  const catalog = resolveCatalogFromOpts(opts, decision);
+  const harness = decision.chosen.harness || "session";
+
+  // Build the candidate chain from decision.chain. Each candidate carries the
+  // breaker keys (so stepChain skips open keys + opens the right scope) and the
+  // per-candidate argv/env materials.
+  const candidates = decision.chain
+    .filter((c) => c && c.ref && c.transport !== "rules")
+    .map((c) => {
+      const keys = breakerKeysForRef(c.ref);
+      return {
+        ref: c.ref,
+        harness: c.harness || harness,
+        transport: c.transport,
+        breakerKey: keys.breakerKey,
+        backendKey: keys.backendKey,
+      };
+    });
+
+  if (!candidates.length) {
+    return {
+      decision,
+      chosen: decision.chosen,
+      args: null,
+      env: null,
+      outcome: { ok: false, reason: "no_route", tried: [], soonestCooldownExpiry: null, attempts: 0 },
+      tried: [],
+      ok: false,
+    };
+  }
+
+  // The breaker the failover engine drives. Default: the on-disk health.mjs.
+  const breaker = opts.breaker || defaultBreaker(opts);
+
+  // Auth-profile rotation context (additive; absent ⇒ single-key behavior).
+  // `profiles` is a loadAuthProfiles result; `deps` is the ledger I/O context.
+  const authCtx = opts.authProfiles
+    ? { profiles: opts.authProfiles, now: opts.now ? opts.now() : undefined, deps: opts.authProfileDeps }
+    : null;
+
+  // Capture the argv/env actually built for the LAST attempt (for the return +
+  // the daemon's ledger row). Updated inside the attempt fn.
+  let lastArgs = null;
+  let lastEnv = null;
+  let chosenCandidate = null;
+
+  const outcome = await stepChain(candidates, {
+    now: opts.now,
+    breaker,
+    allowContentReroute: opts.allowContentReroute === true,
+    onAttempt: opts.onAttempt,
+    attempt: async (candidate, ctx) => {
+      const modelFlag = modelFlagForRef(catalog, candidate.ref, decision.spawnArgs?.modelFlag);
+      const args = buildSpawnArgs(decision.spawnArgs, {
+        modelFlag,
+        extraArgs: opts.extraArgs,
+        outputFormat: opts.outputFormat,
+      });
+      // Re-derive the per-attempt clock for the auth context so a rotated pick
+      // sees the current time (the ctx.now from stepChain is the attempt time).
+      const aCtx = authCtx ? { ...authCtx, now: ctx.now } : null;
+      const retarget = envForCandidate(catalog, candidate.ref, candidate.harness, env, decision, aCtx);
+      // The fingerprint of the credential this attempt used (pooled rotation only);
+      // strip it from the retarget set so it can never leak into the child env.
+      const credFingerprint = retarget.__credFingerprint || null;
+      if (retarget.__credFingerprint !== undefined) delete retarget.__credFingerprint;
+      const childEnv = buildChildEnv(env, retarget, { keepAnthropicApiKey: opts.keepAnthropicApiKey });
+      lastArgs = args;
+      lastEnv = childEnv;
+
+      let res;
+      try {
+        res = await spawnImpl(candidate, {
+          args,
+          env: childEnv,
+          decision,
+          candidate,
+          attemptIndex: ctx.attemptIndex,
+        });
+      } catch (err) {
+        return { ok: false, reason: "unknown", meta: { thrown: String((err && err.message) || err) } };
+      }
+      res = res || { ok: false, reason: "unknown" };
+      if (res.ok) {
+        chosenCandidate = candidate;
+        return { ok: true, value: res.result, meta: res.meta };
+      }
+      // On a credential-class failure, cool THAT credential in the pool (not the
+      // whole provider — the breaker, driven by stepChain, owns backend cooldown).
+      // The next attempt's nextCredential() then rotates to a sibling key.
+      const reason = res.reason || "unknown";
+      if (authCtx && credFingerprint && CRED_COOL_REASONS.has(reason)) {
+        try {
+          authMarkCooling(refProvider(candidate.ref), { fingerprint: credFingerprint }, reason, null, {
+            now: ctx.now,
+            deps: authCtx.deps,
+          });
+        } catch { /* fail-open: a cooldown we can't persist degrades, never blocks */ }
+      }
+      return { ok: false, reason, meta: res.meta };
+    },
+  });
+
+  // On success, stepChain.chosen is the winning candidate; reflect it.
+  if (outcome.ok && outcome.chosen) chosenCandidate = outcome.chosen;
+
+  return {
+    decision,
+    chosen: chosenCandidate
+      ? { ...decision.chosen, ...candidateChosenView(catalog, chosenCandidate) }
+      : decision.chosen,
+    args: lastArgs,
+    env: lastEnv,
+    outcome,
+    tried: outcome.tried || [],
+    ok: outcome.ok === true,
+  };
+}
+
+/** A `chosen`-shaped view of a winning failover candidate (for the return). */
+function candidateChosenView(catalog, candidate) {
+  const row = catalog ? lookupModel(catalog, candidate.ref) : null;
+  if (!row) return { harness: candidate.harness, transport: candidate.transport };
+  return {
+    provider: row.provider,
+    model: row.id,
+    harness: candidate.harness,
+    transport: candidate.transport,
+    catalogRow: row,
+  };
+}
+
+function resolveCatalogFromOpts(opts, decision) {
+  if (opts && opts.catalog) {
+    return opts.catalog.byRef ? opts.catalog : opts.catalog.catalog || opts.catalog;
+  }
+  // The decision's chosen catalogRow can seed a minimal lookup for the chosen ref,
+  // but failover targets need the full catalog; without it, modelFlag falls back
+  // to the ref's model segment and retarget env is skipped (Anthropic-safe).
+  return null;
+}
+
+/**
+ * The default breaker: a thin adapter over the on-disk health.mjs API, shaped
+ * the way failover.mjs expects (isOpen/open/close). Injectable for tests.
+ */
+function defaultBreaker(opts) {
+  const deps = opts.breakerDeps || {};
+  return {
+    isOpen: (key, now) => healthIsOpen(key, now, deps),
+    open: (key, reason, ms, now) => healthOpen(key, reason, ms, now, deps),
+    close: (key, now) => healthClose(key, now, deps),
+  };
+}
+
+export const _internals = {
+  buildChildEnv,
+  buildSpawnArgs,
+  envForCandidate,
+  modelFlagForRef,
+  breakerKeysForRef,
+  isCredentialKey,
+  refProvider,
+  CRED_COOL_REASONS,
+  RETARGET_KEYS,
+};
+
+export default { spawnRouted, buildChildEnv, buildSpawnArgs };