@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/lib/engine/wire/errors.mjs

@@ -0,0 +1,496 @@
+/**
+ * lib/engine/wire/errors.mjs — typed wire errors and cohort response headers (pure).
+ *
+ * The gateway's error contract (hq docs/cohort-llm/design.md §4.6) separates a
+ * HARD REFUSAL — a seat window exhausted, credits gone, a spend limit hit, an
+ * unpriced model — from provider overload. A refusal must never be retried:
+ * retrying a 429 `cohort_limit_exceeded` ten times is exactly the failure the
+ * stock binary shows (design C6). So classification here is the one place that
+ * decides `retryable`, and the client obeys it.
+ *
+ * Precedence for `retryable`:
+ *   1. a known non-retryable cohort code          → false (whatever the headers say)
+ *   2. `x-should-retry: false|true`                → that
+ *   3. HTTP class: 408/409/429/5xx/529 retryable; other 4xx not
+ *
+ * Errors are VALUES: `{ kind, code, status, retryable, message, ... }`. Nothing
+ * here throws.
+ *
+ * @module lib/engine/wire/errors
+ */
+
+import { readFrameUsage } from "./usage.mjs";
+
+/** Codes the gateway emits that must never be retried. */
+export const NON_RETRYABLE_CODES = new Set([
+  "cohort_limit_exceeded",
+  "cohort_credits_exhausted",
+  "cohort_spend_limit_reached",
+  "cohort_unpriced_model",
+  "cohort_seat_suspended",
+]);
+
+/**
+ * @typedef {Object} WireError
+ * @property {'refused'|'auth'|'invalid_request'|'rate_limited'|'overloaded'|'server'|'timeout'|'network'|'aborted'|'protocol'} kind
+ * @property {string} code            stable machine code (cohort code when present)
+ * @property {number|null} status     HTTP status, null for transport faults
+ * @property {boolean} retryable
+ * @property {string} message
+ * @property {number|null} retryAfterMs
+ * @property {string|null} [window]   '5h' | '7d' | '30d' for cohort_limit_exceeded
+ * @property {string|null} [resetsAt] ISO timestamp when the refusal lifts
+ * @property {string|null} [scope]    spend-limit scope
+ * @property {string|null} [requestId]
+ */
+
+/** @param {any} headers @param {string} name @returns {string|null} */
+export function header(headers, name) {
+  if (!headers) return null;
+  if (typeof headers.get === "function") return headers.get(name);
+  const key = Object.keys(headers).find((k) => k.toLowerCase() === name);
+  if (key === undefined) return null;
+  const v = headers[key];
+  return Array.isArray(v) ? v.join(", ") : v == null ? null : String(v);
+}
+
+/**
+ * Parse `retry-after` (seconds or HTTP-date) to milliseconds.
+ * @param {string|null} v @param {number} nowMs @returns {number|null}
+ */
+export function parseRetryAfter(v, nowMs) {
+  if (v == null || v === "") return null;
+  const secs = Number(v);
+  if (Number.isFinite(secs)) return Math.max(0, Math.round(secs * 1000));
+  const at = Date.parse(v);
+  return Number.isFinite(at) ? Math.max(0, at - nowMs) : null;
+}
+
+/**
+ * Classify a non-2xx HTTP response.
+ *
+ * @param {number} status
+ * @param {any} headers   WHATWG Headers or a plain object
+ * @param {string} bodyText
+ * @param {number} nowMs  for HTTP-date retry-after
+ * @returns {WireError}
+ */
+export function classifyHttpError(status, headers, bodyText, nowMs) {
+  let body = null;
+  try {
+    body = bodyText ? JSON.parse(bodyText) : null;
+  } catch {
+    /* a non-JSON error body (an HTML proxy page) still classifies by status */
+  }
+  const err = body && typeof body === "object" ? body.error ?? body : {};
+  const cohort = body && typeof body === "object" && body.cohort && typeof body.cohort === "object" ? body.cohort : {};
+  const type = typeof err.type === "string" ? err.type : null;
+  const rawCode =
+    (typeof err.code === "string" && err.code) || (typeof cohort.code === "string" && cohort.code) || null;
+  const message =
+    (typeof err.message === "string" && err.message) ||
+    (typeof bodyText === "string" && bodyText.slice(0, 300)) ||
+    `HTTP ${status}`;
+
+  const shouldRetry = header(headers, "x-should-retry");
+  const retryAfterMs = parseRetryAfter(header(headers, "retry-after"), nowMs);
+
+  let kind;
+  if (rawCode && NON_RETRYABLE_CODES.has(rawCode)) kind = "refused";
+  else if (status === 401 || status === 403) kind = "auth";
+  else if (status === 429) kind = "rate_limited";
+  else if (status === 529 || type === "overloaded_error") kind = "overloaded";
+  else if (status >= 500) kind = "server";
+  else if (status === 402) kind = "refused";
+  else if (status === 408) kind = "timeout"; // the gateway's body idle timeout: transient, the body never arrived
+  else kind = "invalid_request";
+
+  let retryable;
+  if (rawCode && NON_RETRYABLE_CODES.has(rawCode)) retryable = false;
+  else if (shouldRetry === "false") retryable = false;
+  else if (shouldRetry === "true") retryable = true;
+  else retryable = status === 408 || status === 409 || status === 429 || status >= 500;
+
+  const code = rawCode || type || `http_${status}`;
+  return {
+    kind,
+    code,
+    status,
+    retryable,
+    message,
+    retryAfterMs,
+    window: pick(err.window, cohort.window),
+    // §4.8 names it resetsAt; §4.6 and the earliest gateway bodies say
+    // resets_at. Both are accepted; the error carries resetsAt.
+    resetsAt: pick(err.resetsAt, err.resets_at, cohort.resetsAt, cohort.resets_at),
+    scope: pick(err.scope, cohort.scope),
+    requestId: header(headers, "x-cohort-request-id") ?? pick(cohort.requestId, cohort.request_id),
+  };
+}
+
+/**
+ * Transient codes the gateway uses for ITS OWN capacity or availability, not
+ * the tenant's credential or quota (hq apps/llm-gateway README, "Pre-admission
+ * limits and provider faults"; design §4.6):
+ *   cohort_concurrency_exceeded  429, retry-after: 2 — the per-principal
+ *                                in-flight cap or the replica's body budget
+ *   cohort_unavailable           503, x-should-retry: true — Cohort's own
+ *                                provider key was refused or unfunded upstream
+ *   cohort_upstream_overloaded   529 — every leg saturated
+ */
+export const GATEWAY_TRANSIENT_CODES = new Set(["cohort_concurrency_exceeded", "cohort_unavailable", "cohort_upstream_overloaded"]);
+
+/**
+ * @typedef {Object} GatewayDisposition
+ * @property {'retry'|'refresh_token'|'fail'} action
+ *   retry          back off (retry-after when given, else the backoff ladder) and send again
+ *   refresh_token  ask the token function for a replacement of the rejected token (a 401)
+ *   fail           hand the error back
+ * @property {'refusal'|'concurrency'|'unavailable'|'overloaded'|'timeout'|'auth'|'server'|'network'|'aborted'|'protocol'|'invalid'} class
+ * @property {boolean} honourRetryAfter  sleep the server's retry-after before the retry
+ * @property {boolean} remint            may mint a new seat token (a 401 only; never a 429/503)
+ * @property {boolean} coolsCredential   always false: no gateway answer proves the seat's credential bad
+ * @property {boolean} failover          always false: the gateway is the only route; nothing to fail over to
+ */
+
+/**
+ * What a gateway client does with a wire error. PURE and total: every
+ * WireError (and null) maps to exactly one row. The single decision point for
+ * wire/http.mjs, so a new gateway code is classified here or nowhere.
+ *
+ * The load-bearing rows:
+ *   429 cohort_concurrency_exceeded   → retry, honouring retry-after; no mint, no cooling
+ *   503 cohort_unavailable (retryable) → retry with backoff; NOT an auth failure, no mint
+ *   408                                → retry (transient: the request body stalled)
+ *   402/429 hard refusals              → fail on the first response, whatever retry-after says
+ *   401                                → refresh_token (the caller bounds it: once per request,
+ *                                        and the token provider caps re-mints per window)
+ *
+ * @param {WireError|null|undefined} e
+ * @returns {GatewayDisposition}
+ */
+export function gatewayDisposition(e) {
+  const row = (action, cls) => ({
+    action,
+    class: cls,
+    honourRetryAfter: action === "retry",
+    remint: action === "refresh_token",
+    coolsCredential: false,
+    failover: false,
+  });
+  if (!e || typeof e !== "object") return row("fail", "invalid");
+  const retry = e.retryable === true ? "retry" : "fail";
+  if (e.kind === "refused" || (typeof e.code === "string" && NON_RETRYABLE_CODES.has(e.code))) return row("fail", "refusal");
+  if (e.kind === "aborted") return row("fail", "aborted");
+  if (e.kind === "protocol") return row("fail", "protocol");
+  if (e.kind === "network") return row(retry, "network");
+  if (e.status === 401) return row("refresh_token", "auth");
+  if (e.code === "cohort_concurrency_exceeded") return row(retry, "concurrency");
+  if (e.code === "cohort_unavailable") return row(retry, "unavailable");
+  if (e.kind === "overloaded" || e.code === "cohort_upstream_overloaded") return row(retry, "overloaded");
+  if (e.status === 408 || e.kind === "timeout") return row(retry, "timeout");
+  if (e.kind === "auth") return row("fail", "auth");
+  if (typeof e.status === "number" && e.status >= 500) return row(retry, "server");
+  if (e.kind === "rate_limited") return row(retry, "concurrency");
+  return row(retry, "invalid");
+}
+
+/**
+ * The §4.8 refusal object for a wire error that carries a cohort code, or null.
+ * This is what the headless result emits as `cohort.refusal`, so maestro's
+ * rate-guard (`classifyCohortRefusal`) reads one shape from the engine and
+ * from a gateway body alike.
+ * @param {WireError|null|undefined} e
+ * @returns {{code:string, window:string|null, scope:string|null, resetsAt:string|null, retryable:boolean, requestId:string|null}|null}
+ */
+export function refusalFromWireError(e) {
+  if (!e || typeof e !== "object" || typeof e.code !== "string" || !e.code.startsWith("cohort_")) return null;
+  return {
+    code: e.code,
+    window: e.window ?? null,
+    scope: e.scope ?? null,
+    resetsAt: e.resetsAt ?? null,
+    retryable: e.retryable === true,
+    requestId: e.requestId ?? null,
+  };
+}
+
+function pick(...vals) {
+  for (const v of vals) if (typeof v === "string" && v !== "") return v;
+  return null;
+}
+
+/**
+ * A token function that could not produce a gateway token. PURE.
+ *
+ * Normally a configuration failure (no key, a refused key, the re-mint cap):
+ * `token_unavailable`, kind `auth`, not retryable. But when the token helper
+ * reports that the gateway's token endpoint was itself unavailable (maestro's
+ * api-key-helper prints `…: cohort_unavailable: …` after its own bounded
+ * retries, CF-143), the seat's credential is fine and the gateway is down: that
+ * is `cohort_unavailable`, kind `server`, retryable. The engine does not resend
+ * it (the helper already retried, and a resend would run the helper again); the
+ * run ends with a `cohort_unavailable` refusal that maestro holds briefly
+ * instead of reading it as a credential failure.
+ *
+ * @param {unknown} e @returns {WireError}
+ */
+export function tokenFailureError(e) {
+  const message = e instanceof Error ? e.message : String(e);
+  const outage = /(^|:\s*)cohort_unavailable:/.test(message);
+  return {
+    kind: outage ? "server" : "auth",
+    code: outage ? "cohort_unavailable" : "token_unavailable",
+    status: null,
+    retryable: outage,
+    message,
+    retryAfterMs: null,
+    window: null,
+    resetsAt: null,
+    scope: null,
+    requestId: null,
+  };
+}
+
+/**
+ * A transport fault (DNS, reset socket) or an abort.
+ * @param {unknown} e @param {AbortSignal} [signal] @returns {WireError}
+ */
+export function classifyTransportError(e, signal) {
+  const aborted = Boolean(signal?.aborted) || (e && typeof e === "object" && /** @type any */ (e).name === "AbortError");
+  const message = e instanceof Error ? e.message : String(e);
+  return {
+    kind: aborted ? "aborted" : "network",
+    code: aborted ? "aborted" : "network_error",
+    status: null,
+    retryable: !aborted,
+    message,
+    retryAfterMs: null,
+    window: null,
+    resetsAt: null,
+    scope: null,
+    requestId: null,
+  };
+}
+
+/** Code for a stream the client gave up on because it went silent (CF-156). */
+export const STREAM_IDLE_TIMEOUT_CODE = "stream_idle_timeout";
+
+/**
+ * A gateway stream that was accepted and then went silent past the client's
+ * inter-event budget (`wire/stall.mjs`). W13 measured this twice in 25
+ * requests on `cohort-reason` and could not diagnose it; the engine now ends
+ * such a turn itself instead of spending a whole task budget on it (the
+ * tightest this repo owns is the replay eval's 180 s — NOT the dispatcher's,
+ * whose own timeouts run from 10 min to 12 h; see `wire/stall.mjs`).
+ *
+ * TRANSIENT, deliberately — kind `timeout`, so `gatewayDisposition` routes it
+ * to `retry`/`timeout`: no re-mint, no cooled credential, no failover. A
+ * stalled stream is not a credential problem and not a refusal.
+ *
+ * The code carries NO `cohort_` prefix, and that is load-bearing: this is the
+ * CLIENT's verdict, not an answer the gateway gave. `refusalFromWireError`
+ * therefore returns null for it and maestro's `classifyCohortRefusal` (which
+ * only looks at text containing `cohort_`) never reads it as a gateway refusal
+ * that would hold the seat.
+ *
+ * `mayHaveBilled` says WHICH phase went quiet. After a 2xx it is true — the
+ * request may already have been billed even though no completion was
+ * delivered — and the engine still never resends it, because after acceptance
+ * `postStream` has no retry path at all, which is what keeps a possibly-billed
+ * turn from being billed twice. Before any response headers it is false:
+ * nothing was accepted, so nothing was billed. Either way the flag is advice
+ * to a caller, not something acted on here.
+ *
+ * @param {{ idleMs:number, requestId?:string|null, message?:string, mayHaveBilled?:boolean }} p
+ * @returns {WireError & { mayHaveBilled: boolean }}
+ */
+export function streamIdleTimeoutError({ idleMs, requestId = null, message, mayHaveBilled = true }) {
+  return {
+    kind: "timeout",
+    code: STREAM_IDLE_TIMEOUT_CODE,
+    status: null,
+    retryable: true,
+    message: message ?? `the gateway stream sent no event for ${idleMs}ms`,
+    retryAfterMs: null,
+    window: null,
+    resetsAt: null,
+    scope: null,
+    requestId,
+    mayHaveBilled,
+  };
+}
+
+/**
+ * A malformed or truncated stream (the gateway spoke, but not the protocol).
+ * @param {string} message @param {string|null} [requestId] @returns {WireError}
+ */
+export function protocolError(message, requestId = null) {
+  return {
+    kind: "protocol",
+    code: "stream_protocol_error",
+    status: null,
+    retryable: false,
+    message,
+    retryAfterMs: null,
+    window: null,
+    resetsAt: null,
+    scope: null,
+    requestId,
+  };
+}
+
+/**
+ * An error event sent INSIDE a 200 stream (Anthropic `event: error`, or an
+ * OpenAI `{error:{…}}` data frame). Classified by its type/code like an HTTP one.
+ * @param {any} payload @param {string|null} requestId @param {number} nowMs @returns {WireError}
+ */
+export function classifyStreamError(payload, requestId, nowMs) {
+  const err = payload && typeof payload === "object" ? payload.error ?? payload : {};
+  const type = typeof err.type === "string" ? err.type : "";
+  const status = type === "overloaded_error" ? 529 : type === "rate_limit_error" ? 429 : 500;
+  const e = classifyHttpError(status, null, JSON.stringify(payload ?? {}), nowMs);
+  return { ...e, requestId };
+}
+
+/**
+ * Read the cohort-llm response headers (design §4.4/§4.6). Cost is kept as a
+ * decimal string of integer micro-USD so it survives JSON and never touches a
+ * float until the result is rendered.
+ *
+ * @param {any} headers
+ */
+export function readCohortHeaders(headers) {
+  const windows = {};
+  for (const w of ["5h", "7d", "30d"]) {
+    const utilization = header(headers, `x-cohort-quota-${w}-utilization`);
+    const reset = header(headers, `x-cohort-quota-${w}-reset`);
+    if (utilization != null || reset != null) {
+      const u = utilization == null ? null : Number(utilization);
+      windows[w] = { utilization: Number.isFinite(u) ? u : null, resetsAt: reset };
+    }
+  }
+  const costRaw = header(headers, "x-cohort-cost-micros");
+  const credits = header(headers, "x-cohort-credits-remaining-micros");
+  return {
+    requestId: header(headers, "x-cohort-request-id"),
+    modelTier: header(headers, "x-cohort-model-tier"),
+    costMicros: micros(costRaw, false),
+    // Headers never carry usage — only the terminal frame does (CF-158). The key
+    // is present so the merged shape is uniform whether a frame arrived or not.
+    usage: null,
+    quota: {
+      windows,
+      creditsRemainingMicros: micros(credits, true),
+      funding: header(headers, "x-cohort-funding"),
+      limitKind: header(headers, "x-cohort-limit-kind"),
+    },
+  };
+}
+
+/**
+ * Integer micro-USD as a decimal string, or null. Accepts a digit string or a
+ * safe integer number (a JSON frame may send either).
+ * @param {unknown} v @param {boolean} signed
+ */
+function micros(v, signed) {
+  if (typeof v === "number") return Number.isSafeInteger(v) && (signed || v >= 0) ? String(v) : null;
+  if (typeof v !== "string") return null;
+  const s = v.trim();
+  return (signed ? /^-?\d+$/ : /^\d+$/).test(s) ? s : null;
+}
+
+/** SSE event name of the gateway's terminal cost/quota frame. */
+export const COHORT_FRAME_EVENT = "cohort";
+
+/**
+ * Parse the terminal `event: cohort` SSE frame. Response headers leave before a
+ * streamed completion exists, so they cannot carry its final cost; the gateway
+ * sends this frame after settling and before the wire's terminator
+ * (`data: [DONE]` / `message_stop`). The contract (design §4.8):
+ *
+ *   event: cohort
+ *   data: {"request_id":"…","model_tier":"…","cost_micros":"4321",
+ *          "quota":{"5h":{"utilization":0.25,"resetsAt":"…"}},
+ *          "credits_remaining_micros":"…","funding":"included","limit_kind":null}
+ *
+ * A window's reset is read from `resetsAt` or the older `resets_at`, and kept
+ * as `resetsAt`. Every field is optional. An unparseable frame is advisory and yields an
+ * empty patch — it must never fail a completion that streamed correctly.
+ *
+ * **CF-158 — the frame also carries `usage`, which design §4.8 does not list.**
+ * Measured live on 2026-09-22 (`req_bf94feaab6405214071532a5`): a `usage` object
+ * in the engine's own normalised shape rides every terminal frame. It used to be
+ * dropped here. It is now read through `readFrameUsage` (defensive: unknown
+ * fields ignored, a missing or malformed object yields `null`, never zeros) and
+ * carried on the wire result as `res.cohort.usage`, so a caller can tell an
+ * unreported turn from a free one.
+ *
+ * It reaches a run's JSON result under the DISTINCT key `cohort.gatewayUsage`
+ * — never as `cohort.usage`, which is the engine's own accumulated total for
+ * the whole run, a near-equal but different number. Promising the gateway's
+ * figure under the accumulator's name would reproduce CF-158's own failure
+ * (a real-but-different number read under the expected name), one layer up.
+ * Because it is undocumented upstream it is treated as advisory —
+ * nothing fails when it is absent. See docs/engine/eval.md §3; the fix for the
+ * contract itself belongs to hq, which this repo does not edit.
+ *
+ * @param {string} data
+ * @returns {Partial<ReturnType<typeof readCohortHeaders>> & { quota?: any }}
+ */
+export function readCohortFrame(data) {
+  let o;
+  try {
+    o = JSON.parse(data);
+  } catch {
+    return {};
+  }
+  if (!o || typeof o !== "object") return {};
+  /** @type {any} */
+  const patch = { quota: { windows: {} } };
+  if (typeof o.request_id === "string" && o.request_id) patch.requestId = o.request_id;
+  if (typeof o.model_tier === "string" && o.model_tier) patch.modelTier = o.model_tier;
+  const cost = micros(o.cost_micros, false);
+  if (cost !== null) patch.costMicros = cost;
+  const q = o.quota && typeof o.quota === "object" ? o.quota : {};
+  for (const w of ["5h", "7d", "30d"]) {
+    const win = q[w];
+    if (!win || typeof win !== "object") continue;
+    const u = Number(win.utilization);
+    patch.quota.windows[w] = {
+      utilization: win.utilization != null && Number.isFinite(u) ? u : null,
+      resetsAt: pick(win.resetsAt, win.resets_at),
+    };
+  }
+  const credits = micros(o.credits_remaining_micros, true);
+  if (credits !== null) patch.quota.creditsRemainingMicros = credits;
+  if (typeof o.funding === "string") patch.quota.funding = o.funding;
+  if (typeof o.limit_kind === "string") patch.quota.limitKind = o.limit_kind;
+  // CF-158: undocumented upstream, so absence is normal and never an error.
+  const usage = readFrameUsage(o.usage);
+  if (usage) patch.usage = usage;
+  return patch;
+}
+
+/**
+ * Overlay a frame patch on the header reading: a value the frame states wins,
+ * anything it omits keeps the header's value.
+ * @param {ReturnType<typeof readCohortHeaders>} base
+ * @param {ReturnType<typeof readCohortFrame>} patch
+ * @returns {ReturnType<typeof readCohortHeaders>}
+ */
+export function mergeCohort(base, patch) {
+  const pq = patch.quota || {};
+  return {
+    requestId: patch.requestId ?? base.requestId,
+    modelTier: patch.modelTier ?? base.modelTier,
+    costMicros: patch.costMicros ?? base.costMicros,
+    usage: patch.usage ?? base.usage ?? null,
+    quota: {
+      windows: { ...base.quota.windows, ...(pq.windows || {}) },
+      creditsRemainingMicros: pq.creditsRemainingMicros ?? base.quota.creditsRemainingMicros,
+      funding: pq.funding ?? base.quota.funding,
+      limitKind: pq.limitKind ?? base.quota.limitKind,
+    },
+  };
+}