@cohortapp/agent-sdk 2.17.0 → 2.18.4

package/scripts/eval/probe-gateway.mjs

@@ -0,0 +1,635 @@
+/**
+ * scripts/eval/probe-gateway.mjs — the §8.4.3 provider probes, against a LIVE
+ * cohort-llm gateway.
+ *
+ * Design §8.4 leg 3 ("probe runbook") asks, for each tier a seat could be
+ * flipped onto, what the provider BEHIND the gateway actually accepts and
+ * returns: tool fidelity, the reasoning round-trip, the streaming shape, and
+ * what usage (cache classes included) is reported. Until 2026-09-21 that leg
+ * was blocked on a deployed gateway; it is deployed, so this measures it.
+ *
+ * ── WHY THIS SENDS THE ENGINE'S OWN BODIES ───────────────────────────────────
+ * A probe that hand-rolls a request measures the provider. We need to measure
+ * what the ENGINE will meet, so every request here is built by the engine's own
+ * `buildAnthropicRequest` / `buildOpenAIRequest` and — in the second pass —
+ * driven through the engine's own `callAnthropicMessages` / `callOpenAIChat`.
+ * A wire-shape difference that the engine's accumulator papers over is not a
+ * parity risk; one it cannot is. Only this arrangement can tell them apart.
+ *
+ * Two passes per (tier, wire), because they answer different questions:
+ *   · `raw`    — the engine's body, sent with `fetch`, read frame by frame with
+ *                the engine's `readSse`. Records the ORDERED event names, the
+ *                verbatim `usage` objects, every `x-cohort-*` header, and
+ *                whether the terminal `event: cohort` frame arrives. The engine
+ *                client consumes that frame and never surfaces it, so a raw
+ *                pass is the only way to prove it was on the wire.
+ *   · `engine` — the same tier through the engine client, with a tool offered,
+ *                so tool-call fidelity and the normalised usage record are
+ *                measured through the code a seat would really run.
+ *
+ * ── CF-155: max_tokens IS NOT OPTIONAL HERE ──────────────────────────────────
+ * These tiers are served by models that spend the output budget on reasoning
+ * before they emit a token of content, and the gateway does not raise a small
+ * budget for them (hq `packages/llm/src/capabilities.ts` knows the floor as
+ * `minOutputTokens: 512`; the gateway does not yet apply it). A small
+ * `max_tokens` therefore returns HTTP 200 with EMPTY content and no error.
+ * Every request here asks for at least {@link MIN_OUTPUT_TOKENS}, and an empty
+ * 200 is recorded as `cf155_empty_content` — a KNOWN GATEWAY DEFECT, never a
+ * parity failure. Reading it as the latter is exactly the mistake this comment
+ * exists to prevent.
+ *
+ * ── SPEND ────────────────────────────────────────────────────────────────────
+ * The gateway bills Cohort's own DeepInfra account. Requests are counted and
+ * capped at {@link DEFAULT_MAX_REQUESTS}; the run stops early rather than
+ * exceeding it, and the report states the count. Prompts are short and the
+ * budgets are 512–1024 output tokens.
+ *
+ * ── CREDENTIALS ──────────────────────────────────────────────────────────────
+ * The seat key is read from a `.env` at run time and exchanged for a short-lived
+ * `cst_` seat token at `POST <base>/cohort/v1/tokens`. NEITHER IS EVER PRINTED,
+ * logged or written to the report: the token goes to a 0600 file for a sibling
+ * process (the replay eval) to read, and everything this script prints passes
+ * through {@link redactSecrets}.
+ *
+ * Output is written under `state/` (gitignored), never committed.
+ *
+ * Usage:
+ *   node scripts/eval/probe-gateway.mjs mint  [--env-file <path>] [--base-url <url>] --token-file <path>
+ *   node scripts/eval/probe-gateway.mjs probe [--token-file <path>] [--base-url <url>]
+ *                                             [--tiers a,b] [--wires anthropic,openai]
+ *                                             [--max-requests N] [--out <dir>]
+ *
+ * @module scripts/eval/probe-gateway
+ */
+
+"use strict";
+
+import { readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { buildAnthropicRequest, callAnthropicMessages, ANTHROPIC_MESSAGES_ROUTE, ANTHROPIC_VERSION } from "../../lib/engine/wire/anthropic-messages.mjs";
+import { buildOpenAIRequest, callOpenAIChat, OPENAI_CHAT_ROUTE } from "../../lib/engine/wire/openai-chat.mjs";
+import { joinUrl } from "../../lib/engine/wire/http.mjs";
+import { readSse } from "../../lib/engine/wire/sse.mjs";
+import { readCohortHeaders, readCohortFrame, COHORT_FRAME_EVENT } from "../../lib/engine/wire/errors.mjs";
+import { normaliseAnthropicUsage, normaliseOpenAIUsage } from "../../lib/engine/wire/usage.mjs";
+import { redactSecrets, DEFAULT_LLM_BASE_URL } from "../../lib/org/llm-token.mjs";
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+/** The tiers a seat can be flipped onto (design §4.2; adapter DEFAULT_COHORT_TIER_MAP). */
+export const TIERS = Object.freeze(["cohort-agentic", "cohort-reason", "cohort-extract", "cohort-fast"]);
+
+/** Both wires the gateway serves. The engine speaks Anthropic; the app speaks OpenAI. */
+export const WIRES = Object.freeze(["anthropic", "openai"]);
+
+/**
+ * The CF-155 floor. Never send less: below it these tiers answer 200 with empty
+ * content, which reads exactly like a capability failure and is not one.
+ */
+export const MIN_OUTPUT_TOKENS = 512;
+
+/** Output budget for the tool-fidelity pass — a tool call plus a sentence. */
+export const TOOL_OUTPUT_TOKENS = 1024;
+
+/** Hard spend cap. The run stops before exceeding it and says so in the report. */
+export const DEFAULT_MAX_REQUESTS = 24;
+
+/** How long one probe may take before it is recorded as a timeout. */
+export const PROBE_TIMEOUT_MS = 120_000;
+
+/** How long a minted seat token lives. Stated in the usage string, not guessed at. */
+export const TOKEN_TTL_SECONDS = 3600;
+
+/**
+ * Where a minted `cst_` token rests — OUTSIDE the repository, deliberately.
+ *
+ * It used to default under `state/`. That is gitignored, so it never risked a
+ * commit, but it still put a LIVE credential inside a git worktree — and these
+ * worktrees are copied, archived and deleted by hand all day. A token that
+ * outlives the run it was minted for should not be sitting in one. `forget`
+ * removes it; it expires on its own after {@link TOKEN_TTL_SECONDS}.
+ */
+export const DEFAULT_TOKEN_FILE = path.join(os.tmpdir(), "cohort-probe-seat-token");
+
+/** A short, cheap prompt: enough to force real content, not enough to cost. */
+const TEXT_PROMPT = "In one short sentence, say what a load balancer does.";
+
+/** A tool the model must call to answer — tool fidelity, not tool availability. */
+const PROBE_TOOL = Object.freeze({
+  name: "lookup_seat",
+  description: "Look up a workspace seat by its id. Returns the seat's band and status.",
+  input_schema: {
+    type: "object",
+    properties: { seat_id: { type: "string", description: "The seat id, for example A016" } },
+    required: ["seat_id"],
+    additionalProperties: false,
+  },
+});
+
+const TOOL_PROMPT = "Look up seat A016 using the lookup_seat tool. Call the tool; do not answer from memory.";
+
+/**
+ * The seat key, read from a `.env` at run time.
+ *
+ * Deliberately NOT taken from `process.env`: this script is run by hand beside
+ * a live seat, and reading the file keeps the key out of the shell history, the
+ * process table and any transcript of the invocation.
+ *
+ * @param {string} envFile
+ * @returns {{ok:true, key:string} | {ok:false, error:string}}
+ */
+export function readSeatKey(envFile) {
+  if (!existsSync(envFile)) return { ok: false, error: `no env file at ${envFile}` };
+  let text;
+  try {
+    text = readFileSync(envFile, "utf8");
+  } catch (e) {
+    return { ok: false, error: `could not read ${envFile}: ${e.message}` };
+  }
+  for (const name of ["COHORT_API_TOKEN", "COHORT_TOKEN", "COHORT_API_KEY"]) {
+    const m = text.match(new RegExp(`^${name}=(.+)$`, "m"));
+    if (m) {
+      const key = m[1].trim().replace(/^['"]|['"]$/g, "");
+      if (key) return { ok: true, key };
+    }
+  }
+  return { ok: false, error: `no seat key (COHORT_API_KEY) in ${envFile}` };
+}
+
+/**
+ * Exchange the seat key for a short-lived `cst_` token (design §4.3 lane 2).
+ *
+ * Returns the token so the caller can write it somewhere 0600. The token is
+ * never returned through a log line, and the error path carries only the
+ * gateway's code and status — never a body that might echo the credential.
+ *
+ * @param {{baseUrl:string, key:string, fetchImpl?:typeof fetch, ttlSeconds?:number}} o
+ * @returns {Promise<{ok:true, token:string, expiresAt:string|null, memberId:string|null, orgId:string|null}
+ *                  | {ok:false, error:string, status:number|null}>}
+ */
+export async function mintSeatToken({ baseUrl, key, fetchImpl = globalThis.fetch, ttlSeconds = 3600 }) {
+  let res;
+  try {
+    res = await fetchImpl(`${baseUrl.replace(/\/+$/, "")}/cohort/v1/tokens`, {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
+      body: JSON.stringify({ purpose: "llm", ttlSeconds }),
+      signal: AbortSignal.timeout(20_000),
+    });
+  } catch (e) {
+    return { ok: false, error: `token exchange failed: ${redactSecrets(String(e.message), [key])}`, status: null };
+  }
+  let body = null;
+  try {
+    body = await res.json();
+  } catch {
+    /* an unparseable body is reported by status alone */
+  }
+  if (!res.ok) {
+    const code = body && typeof body === "object" ? (body.error?.code ?? body.code ?? null) : null;
+    return { ok: false, error: `token exchange refused (${res.status}${code ? ` ${code}` : ""})`, status: res.status };
+  }
+  const token = body && typeof body.token === "string" ? body.token : "";
+  if (!token) return { ok: false, error: "token exchange returned no token", status: res.status };
+  return {
+    ok: true,
+    token,
+    expiresAt: typeof body.expiresAt === "string" ? body.expiresAt : null,
+    memberId: typeof body.memberId === "string" ? body.memberId : null,
+    orgId: typeof body.orgId === "string" ? body.orgId : null,
+  };
+}
+
+/**
+ * Build the engine's own request body for a (tier, wire).
+ *
+ * Exported and pure so a test can pin the SHAPE this probe sends without a
+ * gateway: a probe whose body drifted from the engine's would measure nothing.
+ *
+ * @param {{tier:string, wire:string, withTool:boolean}} o
+ */
+export function probeBody({ tier, wire, withTool }) {
+  const common = {
+    model: tier,
+    system: "You are a terse assistant used to probe a model gateway.",
+    messages: [{ role: "user", content: [{ type: "text", text: withTool ? TOOL_PROMPT : TEXT_PROMPT }] }],
+    tools: withTool ? [PROBE_TOOL] : [],
+    maxTokens: withTool ? TOOL_OUTPUT_TOKENS : MIN_OUTPUT_TOKENS,
+  };
+  return wire === "anthropic" ? buildAnthropicRequest(common) : buildOpenAIRequest(common);
+}
+
+/** The route a wire's request goes to. */
+export function probeUrl(baseUrl, wire) {
+  return joinUrl(baseUrl, wire === "anthropic" ? ANTHROPIC_MESSAGES_ROUTE : OPENAI_CHAT_ROUTE);
+}
+
+/**
+ * Every `x-cohort-*` response header, verbatim, for the report.
+ * @param {Headers} headers
+ */
+function cohortHeaderDump(headers) {
+  /** @type {Record<string,string>} */
+  const out = {};
+  headers.forEach((v, k) => {
+    if (k.toLowerCase().startsWith("x-cohort-")) out[k.toLowerCase()] = v;
+  });
+  return out;
+}
+
+/**
+ * PASS 1 — the engine's body, sent raw, read frame by frame.
+ *
+ * Answers the §8.4.3 questions the engine client cannot: the ordered SSE event
+ * names, the verbatim usage objects (so cache fields can be checked for
+ * PRESENCE rather than inferred from a normaliser that defaults them to 0), and
+ * whether the terminal `event: cohort` frame actually arrives.
+ *
+ * @param {{baseUrl:string, token:string, tier:string, wire:string, fetchImpl?:typeof fetch}} o
+ */
+export async function rawProbe({ baseUrl, token, tier, wire, fetchImpl = globalThis.fetch }) {
+  const body = probeBody({ tier, wire, withTool: false });
+  const url = probeUrl(baseUrl, wire);
+  const started = Date.now();
+  /** @type {any} */
+  const rec = {
+    pass: "raw",
+    tier,
+    wire,
+    url,
+    requestBodyKeys: Object.keys(body).sort(),
+    maxTokensSent: body.max_tokens ?? null,
+    ok: false,
+  };
+  let res;
+  try {
+    res = await fetchImpl(url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        accept: "text/event-stream",
+        authorization: `Bearer ${token}`,
+        ...(wire === "anthropic" ? { "anthropic-version": ANTHROPIC_VERSION } : {}),
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+    });
+  } catch (e) {
+    rec.error = { kind: "transport", message: redactSecrets(String(e.message), [token]) };
+    rec.totalMs = Date.now() - started;
+    return rec;
+  }
+  rec.status = res.status;
+  rec.headers = cohortHeaderDump(res.headers);
+  rec.requestId = rec.headers["x-cohort-request-id"] ?? null;
+  rec.contentType = res.headers.get("content-type");
+  if (!res.ok || !res.body) {
+    let text = "";
+    try {
+      text = await res.text();
+    } catch {
+      /* advisory */
+    }
+    // The refusal object, verbatim (capped) — never a guess at the cause.
+    rec.error = { kind: "http", status: res.status, body: redactSecrets(text.slice(0, 2000), [token]) };
+    rec.totalMs = Date.now() - started;
+    return rec;
+  }
+
+  /** @type {string[]} */
+  const events = [];
+  /** @type {any[]} */
+  const usageSeen = [];
+  let firstFrameMs = null;
+  let cohortFrame = null;
+  let text = "";
+  let stopReason = null;
+  let upstreamModel = null;
+  try {
+    for await (const frame of readSse(res.body)) {
+      if (firstFrameMs === null) firstFrameMs = Date.now() - started;
+      const name = frame.event || "(data)";
+      events.push(name);
+      if (name === COHORT_FRAME_EVENT) {
+        cohortFrame = { raw: safeParse(frame.data), parsed: readCohortFrame(frame.data) };
+        continue;
+      }
+      const d = safeParse(frame.data);
+      if (!d || typeof d !== "object") continue;
+      if (wire === "anthropic") {
+        if (d.type === "message_start") {
+          if (d.message?.usage) usageSeen.push({ at: "message_start", usage: d.message.usage });
+          if (d.message?.model) upstreamModel = d.message.model;
+        }
+        if (d.type === "message_delta") {
+          if (d.usage) usageSeen.push({ at: "message_delta", usage: d.usage });
+          if (d.delta?.stop_reason) stopReason = d.delta.stop_reason;
+        }
+        if (d.type === "content_block_delta" && d.delta?.type === "text_delta") text += d.delta.text ?? "";
+      } else {
+        if (d.model) upstreamModel = d.model;
+        if (d.usage) usageSeen.push({ at: "chunk", usage: d.usage });
+        const choice = Array.isArray(d.choices) ? d.choices[0] : null;
+        if (choice?.delta?.content) text += choice.delta.content;
+        if (choice?.finish_reason) stopReason = choice.finish_reason;
+      }
+    }
+    rec.ok = true;
+  } catch (e) {
+    rec.error = { kind: "stream", message: redactSecrets(String(e.message), [token]) };
+  }
+  rec.totalMs = Date.now() - started;
+  rec.ttfbMs = firstFrameMs;
+  rec.eventNames = dedupeOrdered(events);
+  rec.eventCount = events.length;
+  rec.cohortFrame = cohortFrame;
+  rec.cohortFramePresent = cohortFrame !== null;
+  rec.usageSeen = usageSeen;
+  rec.usageNormalised = normaliseUsageFor(wire, usageSeen);
+  rec.upstreamModel = upstreamModel;
+  rec.stopReason = stopReason;
+  rec.contentChars = text.length;
+  rec.contentSample = text.slice(0, 160);
+  // CF-155: a 200 whose content never arrived is the known gateway defect.
+  rec.cf155EmptyContent = rec.ok && text.trim() === "";
+  return rec;
+}
+
+/**
+ * Read an accumulated turn — pure, exported, and the ONLY place that decides
+ * what a turn contained.
+ *
+ * ── WHY THIS IS A FUNCTION AND NOT FOUR LINES INSIDE `engineProbe` ───────────
+ * The first run of this probe reported "200 but EMPTY (CF-155)" for all eight
+ * engine cells. Every one of those responses had in fact carried a valid tool
+ * call: the reader looked for `turn.content`, the accumulator nests blocks
+ * under `turn.message.content`, so the reader found nothing and the CF-155
+ * label — which means "the GATEWAY returned an empty 200" — was attached to a
+ * defect in this file. That artifact is still on disk
+ * (`state/eval/probes/2026-09-21T12-27-56-884Z/`) as the worked example.
+ *
+ * Misreporting a local bug as a gateway defect is the exact conflation this
+ * lane was told not to make, so the two are now decided together, once:
+ * `readerShapeMismatch` is computed FIRST and SUPPRESSES `cf155EmptyContent`.
+ * An empty turn is only ever CF-155 when the accumulator also agrees the model
+ * stopped without producing anything.
+ *
+ * @param {any} turn the accumulator's value (`{message:{role,content}, stopReason, usage}`)
+ */
+export function readTurn(turn) {
+  const t = turn ?? {};
+  const content = Array.isArray(t.message?.content) ? t.message.content : [];
+  const toolUses = content.filter((b) => b && b.type === "tool_use");
+  const stopReason = t.stopReason ?? t.stop_reason ?? null;
+  const toolCalls = toolUses.map((b) => ({
+    name: b.name,
+    // The arguments' SHAPE, never a recorded value beyond the probe's own.
+    inputKeys: b.input && typeof b.input === "object" ? Object.keys(b.input).sort() : [],
+    inputValid: Boolean(b.input && typeof b.input === "object" && typeof b.input.seat_id === "string"),
+    parseError: Boolean(b.inputError),
+  }));
+  const textOut = content.filter((b) => b && b.type === "text").map((b) => b.text ?? "").join("");
+  /**
+   * The accumulator sets `stopReason: "tool_use"` ONLY when it built at least
+   * one tool_use block. So a turn that says `tool_use` while this reader found
+   * none means THE READER is wrong, never the model and never the gateway.
+   */
+  const readerShapeMismatch = stopReason === "tool_use" && toolUses.length === 0;
+  return {
+    stopReason,
+    blockTypes: dedupeOrdered(content.map((b) => b?.type)),
+    toolCallCount: toolUses.length,
+    toolCalls,
+    toolFidelity: toolUses.length > 0 && toolUses.every((b) => b.name === PROBE_TOOL.name) && toolCalls.every((c) => c.inputValid),
+    contentChars: textOut.length,
+    hasThinking: content.some((b) => b && b.type === "thinking"),
+    readerShapeMismatch,
+    // CF-155 is a GATEWAY defect: a 200 that carried nothing. A turn this
+    // reader merely failed to parse is never it — see the guard above.
+    cf155EmptyContent: textOut.trim() === "" && toolUses.length === 0 && !readerShapeMismatch,
+  };
+}
+
+/**
+ * PASS 2 — the same tier through the engine's own client, with a tool offered.
+ *
+ * This is the parity measurement proper: it runs the code a flipped seat runs,
+ * so a wire quirk the accumulator handles shows up as a pass and one it cannot
+ * shows up as the engine's own typed error.
+ *
+ * @param {{baseUrl:string, token:string, tier:string, wire:string, fetchImpl?:typeof fetch}} o
+ */
+export async function engineProbe({ baseUrl, token, tier, wire, fetchImpl }) {
+  const started = Date.now();
+  let ttfbMs = null;
+  const call = {
+    baseUrl,
+    token,
+    model: tier,
+    system: "You are a terse assistant used to probe a model gateway.",
+    messages: [{ role: "user", content: [{ type: "text", text: TOOL_PROMPT }] }],
+    tools: [PROBE_TOOL],
+    maxTokens: TOOL_OUTPUT_TOKENS,
+    maxRetries: 0,
+    signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+    ...(fetchImpl ? { fetchImpl } : {}),
+    onFrame: () => {
+      if (ttfbMs === null) ttfbMs = Date.now() - started;
+    },
+  };
+  const result = wire === "anthropic" ? await callAnthropicMessages(call) : await callOpenAIChat(call);
+  /** @type {any} */
+  const rec = { pass: "engine", tier, wire, ok: result.ok, attempts: result.attempts, apiMs: result.apiMs, ttfbMs, totalMs: Date.now() - started };
+  rec.requestId = result.cohort?.requestId ?? null;
+  rec.modelTier = result.cohort?.modelTier ?? null;
+  rec.costMicros = result.cohort?.costMicros ?? null;
+  rec.quota = result.cohort?.quota ?? null;
+  if (!result.ok) {
+    // The refusal object as the engine classified it — the thing hq needs.
+    rec.error = JSON.parse(redactSecrets(JSON.stringify(result.error ?? null), [token]));
+    return rec;
+  }
+  const turn = result.value ?? {};
+  rec.usage = turn.usage ?? null;
+  // One reader, shared with the tests: it decides tool fidelity, and it decides
+  // whether an empty turn is CF-155 or this file's own bug. See {@link readTurn}.
+  Object.assign(rec, readTurn(turn));
+  return rec;
+}
+
+/** Normalise the LAST usage object a stream reported, through the engine's own normaliser. */
+function normaliseUsageFor(wire, usageSeen) {
+  if (usageSeen.length === 0) return null;
+  const last = usageSeen[usageSeen.length - 1].usage;
+  try {
+    return wire === "anthropic" ? normaliseAnthropicUsage(last) : normaliseOpenAIUsage(last);
+  } catch {
+    return null;
+  }
+}
+
+function safeParse(s) {
+  try {
+    return JSON.parse(s);
+  } catch {
+    return null;
+  }
+}
+
+/** Ordered unique values — an event-name sequence without the per-token noise. */
+function dedupeOrdered(xs) {
+  const seen = new Set();
+  const out = [];
+  for (const x of xs) {
+    if (seen.has(x)) continue;
+    seen.add(x);
+    out.push(x);
+  }
+  return out;
+}
+
+/**
+ * Run the whole matrix, stopping before the request cap is exceeded.
+ * @param {{baseUrl:string, token:string, tiers:string[], wires:string[], maxRequests:number, passes?:string[], log?:(s:string)=>void}} o
+ */
+export async function runMatrix({ baseUrl, token, tiers, wires, maxRequests, passes = ["raw", "engine"], log = () => {} }) {
+  /** @type {any[]} */
+  const records = [];
+  let requests = 0;
+  let stoppedEarly = false;
+  outer: for (const tier of tiers) {
+    for (const wire of wires) {
+      for (const pass of passes) {
+        if (requests >= maxRequests) {
+          stoppedEarly = true;
+          break outer;
+        }
+        requests++;
+        log(`  ${String(requests).padStart(2)} ${pass.padEnd(6)} ${tier.padEnd(15)} ${wire}`);
+        const rec =
+          pass === "raw"
+            ? await rawProbe({ baseUrl, token, tier, wire })
+            : await engineProbe({ baseUrl, token, tier, wire });
+        records.push(rec);
+        // A reader bug is never reported as a gateway defect: it is named as
+        // OURS, loudly, so a run can never again print CF-155 at its own fault.
+        const verdict = !rec.ok
+          ? `FAIL ${rec.error?.status ?? rec.error?.code ?? rec.error?.kind ?? "?"}`
+          : rec.readerShapeMismatch
+            ? "READER SHAPE MISMATCH (this probe's bug, NOT the gateway)"
+            : rec.cf155EmptyContent
+              ? "200 but EMPTY (CF-155)"
+              : "ok";
+        log(`     → ${verdict}${rec.ttfbMs != null ? ` ttfb ${rec.ttfbMs}ms` : ""}${rec.totalMs != null ? ` total ${rec.totalMs}ms` : ""}${rec.requestId ? ` req ${rec.requestId}` : ""}`);
+      }
+    }
+  }
+  return { records, requests, stoppedEarly };
+}
+
+/** @param {string[]} argv */
+export async function main(argv) {
+  const value = (f, d = undefined) => {
+    const i = argv.indexOf(f);
+    return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : d;
+  };
+  const cmd = argv[0];
+  const baseUrl = value("--base-url", process.env.COHORT_LLM_BASE_URL || DEFAULT_LLM_BASE_URL);
+  const tokenFile = value("--token-file", DEFAULT_TOKEN_FILE);
+
+  if (cmd === "forget") {
+    if (!existsSync(tokenFile)) {
+      process.stdout.write(`probe: no seat token at ${tokenFile}\n`);
+      return 0;
+    }
+    unlinkSync(tokenFile);
+    process.stdout.write(`probe: seat token deleted (${tokenFile})\n`);
+    return 0;
+  }
+
+  if (cmd === "mint") {
+    const envFile = value("--env-file", path.join(process.env.HOME || "", "isla", ".env"));
+    const key = readSeatKey(envFile);
+    if (!key.ok) {
+      process.stderr.write(`probe: ${key.error}\n`);
+      return 2;
+    }
+    const minted = await mintSeatToken({ baseUrl, key: key.key });
+    if (!minted.ok) {
+      process.stderr.write(`probe: ${minted.error}\n`);
+      return 1;
+    }
+    mkdirSync(path.dirname(tokenFile), { recursive: true });
+    writeFileSync(tokenFile, minted.token, { mode: 0o600 });
+    // The token itself is NEVER printed — only that one exists and when it dies.
+    process.stdout.write(`probe: seat token minted for member ${minted.memberId ?? "?"} in org ${minted.orgId ?? "?"}, expires ${minted.expiresAt ?? "?"} → ${tokenFile} (0600)\n`);
+    process.stdout.write("probe: run `probe-gateway.mjs forget` when done — do not leave it on disk for its whole TTL\n");
+    return 0;
+  }
+
+  if (cmd !== "probe") {
+    process.stderr.write(
+      `usage: probe-gateway.mjs mint | probe | forget\n` +
+        `  mint   [--env-file <path>] [--base-url <url>] [--token-file <path>]\n` +
+        `         exchanges the seat key for a cst_ token (TTL ${TOKEN_TTL_SECONDS}s), written 0600 to\n` +
+        `         ${DEFAULT_TOKEN_FILE} by default — outside the repo, never printed.\n` +
+        `  probe  [--token-file <path>] [--tiers a,b] [--wires anthropic,openai]\n` +
+        `         [--passes raw,engine] [--max-requests N] [--out <dir>]\n` +
+        `  forget [--token-file <path>]   deletes the minted token. Run it when the run ends.\n`,
+    );
+    return 2;
+  }
+
+  if (!existsSync(tokenFile)) {
+    process.stderr.write(`probe: no seat token at ${tokenFile} — run \`mint\` first\n`);
+    return 2;
+  }
+  const token = readFileSync(tokenFile, "utf8").trim();
+  if (!token) {
+    process.stderr.write(`probe: empty seat token at ${tokenFile}\n`);
+    return 2;
+  }
+  const tiers = String(value("--tiers", TIERS.join(","))).split(",").map((s) => s.trim()).filter(Boolean);
+  const wires = String(value("--wires", WIRES.join(","))).split(",").map((s) => s.trim()).filter(Boolean);
+  const maxRequests = Math.max(1, Number(value("--max-requests", String(DEFAULT_MAX_REQUESTS))) || DEFAULT_MAX_REQUESTS);
+  // `--passes engine` re-runs one pass without paying for the other again —
+  // spend discipline, and how a reader fix is re-measured without a full matrix.
+  const passes = String(value("--passes", "raw,engine")).split(",").map((s) => s.trim()).filter(Boolean);
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const outDir = path.resolve(value("--out", path.join(REPO, "state", "eval", "probes", stamp)));
+
+  process.stdout.write(`probe: ${tiers.length} tier(s) × ${wires.length} wire(s) × ${passes.length} pass(es) [${passes.join(", ")}] against ${baseUrl}, cap ${maxRequests} requests\n`);
+  const { records, requests, stoppedEarly } = await runMatrix({
+    baseUrl,
+    token,
+    tiers,
+    wires,
+    maxRequests,
+    passes,
+    log: (s) => process.stdout.write(`${s}\n`),
+  });
+
+  mkdirSync(outDir, { recursive: true });
+  const report = {
+    startedAt: new Date().toISOString(),
+    baseUrl,
+    tiers,
+    wires,
+    requests,
+    maxRequests,
+    stoppedEarly,
+    minOutputTokens: MIN_OUTPUT_TOKENS,
+    records,
+  };
+  const file = path.join(outDir, "probe.json");
+  writeFileSync(file, `${JSON.stringify(report, null, 2)}\n`);
+  process.stdout.write(`\nprobe: ${requests} request(s)${stoppedEarly ? " (STOPPED EARLY at the cap)" : ""} → ${file}\n`);
+  return 0;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+  process.exitCode = await main(process.argv.slice(2));
+}