@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/engine/wire/http.mjs

@@ -0,0 +1,441 @@
+/**
+ * lib/engine/wire/http.mjs — the one HTTP edge both gateway wires share.
+ *
+ * Everything that touches the network for a model call lives here: URL
+ * joining, auth + attribution headers, the retry policy, and the hand-off of a
+ * 2xx streaming body to a wire-specific accumulator. The wires themselves
+ * (`openai-chat.mjs`, `anthropic-messages.mjs`) are pure request builders and
+ * stream accumulators; they never call `fetch`.
+ *
+ * Retry policy (design §4.6, constraint C6):
+ *   · only BEFORE a streaming body has been accepted — re-sending a request
+ *     whose completion was half-relayed would bill twice;
+ *   · only when `gatewayDisposition` (errors.mjs) says `retry`, so a
+ *     `cohort_limit_exceeded` 429 or a `cohort_credits_exhausted` 402 is
+ *     returned on the first attempt, whatever `retry-after` says, while a 429
+ *     `cohort_concurrency_exceeded` (retry-after honoured), a 503
+ *     `cohort_unavailable` and a 408 are retried with backoff — never through
+ *     the token function, so they never mint a token;
+ *   · a 401 gets exactly one retry with a refreshed token when the caller gave
+ *     a token FUNCTION (seat tokens are short-lived and refreshable); only that
+ *     one attempt asks for `{refresh:true, rejected:<the 401'd token>}`, so a
+ *     provider that was already refreshed by a concurrent request can hand
+ *     back the replacement instead of minting again;
+ *   · a `retry-after` longer than `maxRetryAfterMs` (default 30s) is not slept
+ *     through: the retryable error is returned with its `retryAfterMs`, so the
+ *     caller reschedules instead of a run blocking for an hour.
+ *
+ * Cost and quota. The `x-cohort-*` response headers are read on acceptance, but
+ * a streamed completion's cost is not known until it ends, so the gateway may
+ * also send a terminal `event: cohort` SSE frame (before the wire's own
+ * terminator) whose fields override the headers — see `readCohortFrame`. The
+ * frame is consumed here and never reaches a wire accumulator.
+ *
+ * Stalls (CF-156). One silence budget (`wire/stall.mjs`, default 60 s) covers
+ * both phases of a call — the wait for response headers, which `fetch` does
+ * not bound on its own, and then the gap between SSE events — rather than
+ * letting either consume a whole task budget (the tightest this repo owns is
+ * the replay eval's 180 s, `scripts/eval/replay/lib/config.mjs`).
+ *
+ * It is spent per CALL, not per attempt, and that is ENFORCED rather than
+ * asserted: the header wait is armed with what REMAINS of the budget, and a
+ * retry that could not start inside what remains is not started at all — the
+ * error is already retryable, so it goes back for the caller to reschedule,
+ * the same answer an over-long `retry-after` gets. A stalled call therefore
+ * ends within 2 × the budget however often the gateway 503s first, instead of
+ * re-arming a full budget per attempt and spending 3 × 60 s plus backoff.
+ *
+ * Either phase comes back as a TRANSIENT `stream_idle_timeout` — no re-mint,
+ * no cooled credential, no failover — carrying `mayHaveBilled` to say whether
+ * a 2xx had already been given. A stalled BODY is never resent here, because
+ * after acceptance this function has no retry path at all, which is what keeps
+ * a possibly-billed turn from being billed twice.
+ *
+ * The same seam records a diagnosis (request id, TTFB, silence, last event
+ * type, whether the terminal `cohort` frame had arrived) on a stall, on a
+ * mid-stream fault and on a slow stream, and nothing on an ordinary turn.
+ *
+ * A failed result carries `accepted`: true when the gateway had already
+ * returned 2xx, i.e. the request may have been billed.
+ *
+ * Every expected failure is a returned `{ ok:false, error: WireError }`.
+ *
+ * @module lib/engine/wire/http
+ */
+
+import {
+  classifyHttpError,
+  gatewayDisposition,
+  classifyTransportError,
+  tokenFailureError,
+  protocolError,
+  readCohortHeaders,
+  readCohortFrame,
+  mergeCohort,
+  streamIdleTimeoutError,
+  COHORT_FRAME_EVENT,
+} from "./errors.mjs";
+import { readSse } from "./sse.mjs";
+import {
+  withIdleTimeout,
+  createStallWatch,
+  StreamIdleTimeoutSignal,
+  DEFAULT_STREAM_IDLE_TIMEOUT_MS,
+  DEFAULT_SLOW_STREAM_MS,
+} from "./stall.mjs";
+
+/** Longest `retry-after` the client will sleep through before handing the error back. */
+export const DEFAULT_MAX_RETRY_AFTER_MS = 30_000;
+
+/**
+ * Join a base URL and a `/v1/...` route. Accepts a base with or without a
+ * trailing slash or a trailing `/v1` (both spellings appear in the wild:
+ * OpenAI SDKs want `…/v1`, Anthropic SDKs want the bare host).
+ * @param {string} baseUrl @param {string} route e.g. "/v1/messages"
+ * @returns {string}
+ */
+export function joinUrl(baseUrl, route) {
+  let base = String(baseUrl || "").trim().replace(/\/+$/, "");
+  if (base.endsWith("/v1") && route.startsWith("/v1/")) base = base.slice(0, -3);
+  return base + route;
+}
+
+/** @param {number} ms @param {AbortSignal} [signal] */
+export function defaultSleep(ms, signal) {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const t = setTimeout(done, ms);
+    function done() {
+      clearTimeout(t);
+      signal?.removeEventListener?.("abort", done);
+      resolve();
+    }
+    signal?.addEventListener?.("abort", done, { once: true });
+  });
+}
+
+/**
+ * Backoff for attempt n (0-based) when the server gave no retry-after:
+ * 500ms, 1s, 2s … capped. Deterministic — no jitter — so tests are exact; the
+ * gateway's per-seat concurrency cap, not client jitter, is what spreads load.
+ * @param {number} attempt @param {number} [capMs]
+ */
+export function backoffMs(attempt, capMs = 8000) {
+  return Math.min(capMs, 500 * 2 ** attempt);
+}
+
+/**
+ * One signal that aborts when any of `signals` does.
+ *
+ * `AbortSignal.any` is the obvious implementation and is what runs in practice
+ * — but it landed in Node **20.3**, and this package declares
+ * `engines.node: ">=20.0.0"`. The composed path is not a corner: the engine's
+ * loop passes a run signal on every turn, so a consumer on Node 20.0–20.2 would
+ * hit `AbortSignal.any is not a function` on its first model call. CI cannot
+ * catch that — `node-version: "20.x"` always resolves to the newest 20 — so the
+ * declared floor is kept honest here instead, by composing the signals by hand
+ * when `any` is missing.
+ *
+ * The fallback listens on signals that outlive a single call (a run's signal
+ * lives for the whole run), so it returns a `dispose` that detaches them. Call
+ * it exactly once, when the call is over.
+ *
+ * @param {Array<AbortSignal|undefined|null>} signals
+ * @param {{ any?: ((s:AbortSignal[]) => AbortSignal)|null }} [deps] test seam for the fallback
+ * @returns {{ signal: AbortSignal|undefined, dispose: () => void }}
+ */
+export function composeSignals(signals, deps = {}) {
+  const list = /** @type {AbortSignal[]} */ (signals.filter(Boolean));
+  const noop = () => {};
+  if (list.length < 2) return { signal: list[0], dispose: noop };
+  const any = "any" in deps ? deps.any : typeof AbortSignal !== "undefined" && AbortSignal.any ? AbortSignal.any.bind(AbortSignal) : null;
+  if (typeof any === "function") return { signal: any(list), dispose: noop };
+  const ac = new AbortController();
+  const already = list.find((s) => s.aborted);
+  if (already) {
+    ac.abort(already.reason);
+    return { signal: ac.signal, dispose: noop };
+  }
+  /** @type {Array<[AbortSignal, () => void]>} */
+  const links = list.map((s) => {
+    const onAbort = () => ac.abort(s.reason);
+    s.addEventListener("abort", onAbort, { once: true });
+    return [s, onAbort];
+  });
+  return {
+    signal: ac.signal,
+    dispose: () => {
+      for (const [s, fn] of links) s.removeEventListener?.("abort", fn);
+    },
+  };
+}
+
+/**
+ * @typedef {Object} StreamCall
+ * @property {string} url
+ * @property {Record<string,string>} headers       wire-specific headers (no auth)
+ * @property {unknown} body                        JSON body
+ * @property {string|((o:{refresh:boolean})=>string|Promise<string>)} token
+ * @property {() => Accumulator} createAccumulator  fresh wire accumulator per attempt
+ * @property {(frame: import('./sse.mjs').SseFrame) => void} [onFrame]  sees each accepted wire frame (not the cohort frame)
+ * @property {AbortSignal} [signal]
+ * @property {typeof fetch} [fetchImpl]
+ * @property {number} [maxRetries]                 default 2
+ * @property {number} [maxRetryAfterMs]            default 30000
+ * @property {() => number} [now]
+ * @property {(ms:number, signal?:AbortSignal) => Promise<void>} [sleep]
+ * @property {number} [idleTimeoutMs]   silence budget between SSE events (stall.mjs); 0 disables
+ * @property {number} [slowStreamMs]    a completed stream slower than this is recorded
+ * @property {string} [wire]            'openai' | 'anthropic', for diagnostics
+ * @property {string} [tier]            the model tier asked for, for diagnostics
+ * @property {(d: import('./stall.mjs').StreamDiagnostic) => void} [onStreamDiagnostic]
+ *   called on a stall, a mid-stream fault and a slow stream — never on an ordinary turn
+ */
+
+/**
+ * @typedef {Object} Accumulator
+ * @property {(frame: import('./sse.mjs').SseFrame) => ({ ok:true, done?:boolean } | { ok:false, error:any })} onFrame
+ * @property {() => ({ ok:true, value:any } | { ok:false, error:any })} finish
+ */
+
+/**
+ * POST a streaming request and drive the accumulator over its SSE frames.
+ *
+ * `createAccumulator` is called before every attempt so a retried request
+ * starts from an empty accumulator.
+ *
+ * @param {StreamCall} call
+ * @returns {Promise<{ ok:true, value:any, cohort: ReturnType<typeof readCohortHeaders>, attempts:number, apiMs:number }
+ *                  | { ok:false, error:any, cohort: ReturnType<typeof readCohortHeaders>|null, attempts:number, apiMs:number }>}
+ */
+export async function postStream(call) {
+  const fetchImpl = call.fetchImpl || globalThis.fetch;
+  const now = call.now || Date.now;
+  const sleep = call.sleep || defaultSleep;
+  const maxRetries = call.maxRetries ?? 2;
+  const maxRetryAfterMs = call.maxRetryAfterMs ?? DEFAULT_MAX_RETRY_AFTER_MS;
+  const started = now();
+  const idleTimeoutMs = call.idleTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT_MS;
+  const slowStreamMs = call.slowStreamMs ?? DEFAULT_SLOW_STREAM_MS;
+  // ONE silence budget for the whole CALL (see the header): the deadline is
+  // fixed here, so every attempt and every backoff sleep spends the same one.
+  const callDeadline = idleTimeoutMs > 0 ? started + idleTimeoutMs : Infinity;
+  const remainingBudgetMs = () => callDeadline - now();
+  /** Would sleeping `waitMs` leave nothing for another attempt? */
+  const budgetSpentAfter = (/** @type {number} */ waitMs) => remainingBudgetMs() - waitMs <= 0;
+  // One controller for the whole call: when the budget is spent there is no
+  // next attempt, so ending the call is exactly right. Composed with the
+  // caller's signal ONCE — an interrupt still aborts, and no listener is added
+  // per attempt to a signal that outlives the call.
+  const callAbort = idleTimeoutMs > 0 ? new AbortController() : null;
+  const linked = composeSignals([call.signal, callAbort?.signal]);
+  const signal = linked.signal;
+  /** Hand a diagnostic to the observer; an observer never breaks a call. */
+  const diagnose = (/** @type {import('./stall.mjs').StreamDiagnostic} */ d) => {
+    if (!call.onStreamDiagnostic) return;
+    try {
+      call.onStreamDiagnostic(d);
+    } catch {
+      /* ignored: an observer must not fail a model call */
+    }
+  };
+  let refreshed = false;
+  let refreshNext = false;
+  /** @type {string|undefined} the token the gateway answered 401 to */
+  let rejected;
+  let accepted = false;
+  let attempt = 0;
+
+  for (;;) {
+    const acc = call.createAccumulator();
+    // Per attempt: a retried request's timings describe the attempt that ran,
+    // not the ladder before it — `callMs` and `attempts` carry the ladder, so
+    // a record can never be read as describing more than it does.
+    const attemptStartedAt = now();
+    const watch = createStallWatch({
+      now,
+      startedAt: attemptStartedAt,
+      callStartedAt: started,
+      attempts: attempt + 1,
+      wire: call.wire ?? null,
+      tier: call.tier ?? null,
+    });
+    let token;
+    try {
+      token = typeof call.token === "function" ? await call.token(refreshNext ? { refresh: true, rejected } : { refresh: false }) : call.token;
+    } catch (e) {
+      // A token provider that cannot mint is a configuration failure, not a
+      // defect — or, when the token endpoint itself was down, a retryable
+      // cohort_unavailable (tokenFailureError). Never resent here: the helper
+      // already retried, and a resend would run it again.
+      return fail(tokenFailureError(e), null);
+    }
+    refreshNext = false;
+
+    // What REMAINS of the silence budget bounds the wait for response headers.
+    // `fetch` has no timeout of its own, so a gateway that accepts the
+    // connection and never answers would otherwise hang with nothing to stop
+    // it — the phase W13 could not rule out for its 120 s hang. Armed with the
+    // remainder, not a fresh budget, so retries cannot multiply it; cleared
+    // once headers arrive, after which only the inter-event timer governs.
+    let res;
+    let headersTimedOut = false;
+    const headersTimer = callAbort
+      ? setTimeout(
+          () => {
+            headersTimedOut = true;
+            callAbort.abort();
+          },
+          Math.max(0, remainingBudgetMs()),
+        )
+      : null;
+    try {
+      res = await fetchImpl(call.url, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          accept: "text/event-stream",
+          authorization: `Bearer ${token}`,
+          ...call.headers,
+        },
+        body: JSON.stringify(call.body),
+        signal,
+      });
+    } catch (e) {
+      // A gateway that never answered is transient — and is NOT resent here:
+      // one budget per call, not per attempt, or three silent attempts would
+      // spend 3 × the budget and blow the task budget this guard protects.
+      if (headersTimedOut && !call.signal?.aborted) {
+        const err = streamIdleTimeoutError({
+          idleMs: idleTimeoutMs,
+          message: `the gateway sent no response headers within ${idleTimeoutMs}ms`,
+          mayHaveBilled: false,
+        });
+        diagnose(watch.report({ outcome: "stalled", requestId: null, accepted: false, error: err }));
+        return fail(err, null);
+      }
+      const err = classifyTransportError(e, call.signal);
+      // Retry only while the call's silence budget can still hold an attempt.
+      if (err.retryable && attempt < maxRetries && !budgetSpentAfter(backoffMs(attempt))) {
+        await sleep(backoffMs(attempt), call.signal);
+        attempt++;
+        continue;
+      }
+      return fail(err, null);
+    } finally {
+      // Headers are in (or the attempt failed): only the inter-event timer
+      // governs from here. The caller's signal stays attached to the body.
+      if (headersTimer) clearTimeout(headersTimer);
+    }
+
+    let cohort = readCohortHeaders(res.headers);
+    if (!res.ok) {
+      let text = "";
+      try {
+        text = await res.text();
+      } catch {
+        /* the error body is advisory; classification falls back to the status */
+      }
+      const err = classifyHttpError(res.status, res.headers, text, now());
+      const d = gatewayDisposition(err);
+      if (d.action === "refresh_token" && typeof call.token === "function" && !refreshed) {
+        refreshed = true;
+        refreshNext = true;
+        rejected = token;
+        continue;
+      }
+      // A 429 concurrency cap, a 503 provider-key outage and a 408 retry here and
+      // nowhere else: none of them ever reaches the token function (no mint).
+      const waitMs = d.honourRetryAfter && err.retryAfterMs != null ? err.retryAfterMs : backoffMs(attempt);
+      // …and only while the CALL's budget can still hold another attempt. When
+      // it cannot, the retryable error is handed back for the caller to
+      // reschedule — the same answer an over-long `retry-after` already gets,
+      // and the reason a 503 burst cannot multiply the silence budget.
+      if (d.action === "retry" && attempt < maxRetries && (err.retryAfterMs == null || err.retryAfterMs <= maxRetryAfterMs) && !budgetSpentAfter(waitMs)) {
+        await sleep(waitMs, call.signal);
+        attempt++;
+        continue;
+      }
+      return fail(err, cohort);
+    }
+
+    // From here the request is accepted and possibly billed: no more retries.
+    accepted = true;
+    watch.accepted();
+    if (!res.body) return fail(protocolError("response had no body", cohort.requestId), cohort);
+    try {
+      for await (const frame of withIdleTimeout(readSse(res.body), idleTimeoutMs)) {
+        watch.event(frame);
+        if (frame.event === COHORT_FRAME_EVENT) {
+          watch.cohortFrame();
+          cohort = mergeCohort(cohort, readCohortFrame(frame.data));
+          continue;
+        }
+        if (call.onFrame) {
+          try {
+            call.onFrame(frame); // stream-json partial messages; an observer never breaks a stream
+          } catch {
+            /* ignored: see above */
+          }
+        }
+        const r = acc.onFrame(frame);
+        if (!r.ok) {
+          try {
+            await res.body.cancel?.();
+          } catch {
+            /* the stream may already be closed; nothing more to release */
+          }
+          const err = { ...r.error, requestId: r.error.requestId ?? cohort.requestId };
+          diagnose(watch.report({ outcome: "failed", requestId: err.requestId, error: err }));
+          return fail(err, cohort);
+        }
+        if (r.done) break;
+      }
+    } catch (e) {
+      // A stall is its own verdict: the stream was accepted and then went
+      // silent. It is handed back TRANSIENT (kind `timeout`) so the taxonomy
+      // holds the task briefly instead of reading it as a credential fault —
+      // and it is still not resent here, because after acceptance this loop
+      // has no retry path at all, which is what keeps it from billing twice.
+      if (e instanceof StreamIdleTimeoutSignal) {
+        // Release the socket nobody is reading any more. Not awaited: a cancel
+        // on a stalled body can itself wait on the chunk that never lands.
+        try {
+          void Promise.resolve(res.body.cancel?.()).catch(() => {});
+        } catch {
+          /* already torn down */
+        }
+        const err = streamIdleTimeoutError({ idleMs: e.idleMs, requestId: cohort.requestId });
+        diagnose(watch.report({ outcome: "stalled", requestId: err.requestId, error: err }));
+        return fail(err, cohort);
+      }
+      const err = classifyTransportError(e, call.signal);
+      // Mid-stream: never retryable (see header).
+      const failed = { ...err, retryable: false, requestId: cohort.requestId };
+      diagnose(watch.report({ outcome: "failed", requestId: failed.requestId, error: failed }));
+      return fail(failed, cohort);
+    }
+    const out = acc.finish();
+    if (!out.ok) {
+      const err = { ...out.error, requestId: out.error.requestId ?? cohort.requestId };
+      diagnose(watch.report({ outcome: "failed", requestId: err.requestId, error: err }));
+      return fail(err, cohort);
+    }
+    const apiMs = now() - started;
+    // The happy path builds no record unless the turn was slow enough to be
+    // worth one: on an ordinary turn nothing above runs but three counters.
+    // Measured on THIS ATTEMPT's stream, which is what the record describes —
+    // triggering on the whole call's elapsed time (retries and backoff sleeps
+    // included) produced records labelled `slow` that described a 3 ms stream.
+    if (slowStreamMs > 0 && now() - attemptStartedAt >= slowStreamMs) diagnose(watch.report({ outcome: "slow", requestId: cohort.requestId }));
+    linked.dispose();
+    return { ok: true, value: out.value, cohort, attempts: attempt + 1, apiMs };
+  }
+
+  function fail(error, cohort) {
+    // Every failing exit goes through here, so this is where the composed
+    // signal's listeners come off the caller's long-lived signal.
+    linked.dispose();
+    return { ok: false, error, cohort, accepted, attempts: attempt + 1, apiMs: now() - started };
+  }
+}

package/lib/engine/wire/index.mjs

@@ -0,0 +1,76 @@
+/**
+ * lib/engine/wire/index.mjs — pick a wire by name and bind its connection.
+ *
+ * The loop never knows which wire it is talking: it receives one
+ * `callModel({system, messages, tools, signal})` function. This module is the
+ * single place a wire name becomes that function.
+ *
+ * @module lib/engine/wire/index
+ */
+
+import { callOpenAIChat } from "./openai-chat.mjs";
+import { callAnthropicMessages } from "./anthropic-messages.mjs";
+import { observingFetch } from "./prompt-cache.mjs";
+
+/** @typedef {'openai' | 'anthropic'} WireName */
+
+export const WIRES = Object.freeze(["openai", "anthropic"]);
+
+/** @param {unknown} v @returns {v is WireName} */
+export function isWireName(v) {
+  return v === "openai" || v === "anthropic";
+}
+
+/**
+ * @param {object} cfg
+ * @param {WireName} cfg.wire
+ * @param {string} cfg.baseUrl
+ * @param {string|((o:{refresh:boolean})=>string|Promise<string>)} cfg.token
+ * @param {string} cfg.model
+ * @param {number} [cfg.maxTokens]
+ * @param {Record<string,string>} [cfg.headers]
+ * @param {boolean} [cfg.cacheBreakpoints]
+ * @param {typeof fetch} [cfg.fetchImpl]
+ * @param {number} [cfg.maxRetries]
+ * @param {() => number} [cfg.now]
+ * @param {(ms:number, s?:AbortSignal)=>Promise<void>} [cfg.sleep]
+ * @param {'low'|'medium'|'high'|null} [cfg.reasoningEffort]  OpenAI wire only (wire/effort.mjs)
+ * @param {(frame:any)=>void} [cfg.onFrame]  observer of each stream frame
+ * @param {number} [cfg.idleTimeoutMs]  silence budget between SSE events (CF-156); 0 disables
+ * @param {number} [cfg.slowStreamMs]   a completed stream slower than this is recorded
+ * @param {(d: import('./stall.mjs').StreamDiagnostic) => void} [cfg.onStreamDiagnostic]
+ *   stall / mid-stream fault / slow stream, never an ordinary turn
+ * @param {boolean} [cfg.images]   send tool-result images (wire default when unset)
+ * @param {ReturnType<typeof import('./prompt-cache.mjs').createPromptCacheKeyState>|null} [cfg.promptCache]
+ *   OpenAI wire: sends `prompt_cache_key` once the state allows it
+ */
+export function createModelCaller(cfg) {
+  const call = cfg.wire === "anthropic" ? callAnthropicMessages : callOpenAIChat;
+  const promptCache = cfg.wire === "openai" ? cfg.promptCache ?? null : null;
+  const fetchImpl = promptCache ? observingFetch(cfg.fetchImpl, promptCache) : cfg.fetchImpl;
+  /** @param {{system:string, messages:any[], tools:any[], signal?:AbortSignal, maxTokens?:number}} turn */
+  return (turn) =>
+    call({
+      baseUrl: cfg.baseUrl,
+      token: cfg.token,
+      model: cfg.model,
+      maxTokens: turn.maxTokens ?? cfg.maxTokens,
+      headers: cfg.headers,
+      cacheBreakpoints: cfg.cacheBreakpoints,
+      images: cfg.images,
+      promptCacheKey: promptCache ? promptCache.current() : null,
+      fetchImpl,
+      maxRetries: cfg.maxRetries,
+      now: cfg.now,
+      sleep: cfg.sleep,
+      reasoningEffort: cfg.reasoningEffort,
+      onFrame: cfg.onFrame,
+      idleTimeoutMs: cfg.idleTimeoutMs,
+      slowStreamMs: cfg.slowStreamMs,
+      onStreamDiagnostic: cfg.onStreamDiagnostic,
+      system: turn.system,
+      messages: turn.messages,
+      tools: turn.tools,
+      signal: turn.signal,
+    });
+}