@tangle-network/agent-app 0.44.3 → 0.44.5

package/dist/chat-routes/index.js

@@ -87,7 +87,7 @@ import {
   buildModelChain,
   isUpstreamUnavailable,
   runWithModelFailover
-} from "../chunk-CHLOH4DG.js";
+} from "../chunk-DEXBRUZR.js";
 import {
   flattenHistory,
   readSandboxBinaryBytes,

package/dist/{chunk-CHLOH4DG.js → chunk-DEXBRUZR.js}

@@ -140,4 +140,4 @@ export {
   runWithModelFailover,
   buildModelChain
 };
-//# sourceMappingURL=chunk-CHLOH4DG.js.map
+//# sourceMappingURL=chunk-DEXBRUZR.js.map

package/dist/chunk-DEXBRUZR.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/model-resolution/failover.ts"],"sourcesContent":["/**\n * Model failover — the answer to \"one upstream hit a quota wall and the customer\n * got a Bad Gateway even though the router had abundant healthy capacity\".\n *\n * This is deliberately NOT a health probe. Probing before every turn buys a\n * round-trip of latency on the happy path and still races the outage (a model\n * healthy at probe time can 502 a second later). Failover is REACTIVE: run the\n * preferred model, and on an upstream-outage signal move to the next model in\n * the chain. Zero added latency when the preferred model works.\n *\n * Two facts drive the design, both measured against a live box during the\n * 2026-07-25 Anthropic/DeepSeek outage:\n *\n * 1. An outage is NOT always a thrown error. The sandbox resolves with a\n *    payload — `{ success: false, errorCode: 'provider_inference_unavailable' }`\n *    — so a classifier that only inspects `catch` misses the customer-visible\n *    case entirely. `isUpstreamUnavailable` inspects resolved values too.\n * 2. Catalog membership is NOT liveness. The router's `/v1/models` still listed\n *    every dead model during the outage, so `validateChatModelId` admitted\n *    `claude-sonnet-4-6` while every call to it returned 502. Nothing upstream\n *    of the actual call can be trusted to tell you a model is serving.\n *\n * Substrate-free: the caller injects `run`, so this composes with a sandbox\n * turn, a router chat completion, or a test fake without importing any of them.\n */\n\n/**\n * Error codes that mean \"this model's upstream is unavailable — a different\n * model may still work\". Deliberately excludes codes that would fail identically\n * on every model (bad request, auth, content filter): retrying those down a\n * chain burns latency and money to reach the same failure.\n */\nexport const UPSTREAM_UNAVAILABLE_CODES: readonly string[] = [\n  'provider_inference_unavailable',\n  'upstream_unavailable',\n  'insufficient_quota',\n  'model_not_available',\n  'server_error',\n  'bad_gateway',\n  'service_unavailable',\n]\n\n/** HTTP statuses that indicate an upstream capacity/availability problem. */\nexport const UPSTREAM_UNAVAILABLE_STATUSES: readonly number[] = [429, 500, 502, 503, 504]\n\n/**\n * Statuses that mean the REQUEST is wrong, so every model in the chain would\n * fail on it identically. Listed explicitly rather than as \"any 4xx\" because\n * 404 is NOT one of them: a model the endpoint cannot find is model-scoped, and\n * the next model in the chain may well be there. 429 is absent for the same\n * reason in reverse — it is a capacity signal and lives in the list above.\n */\nconst REQUEST_ERROR_STATUSES: readonly number[] = [400, 401, 403, 405, 413, 422]\n\n/**\n * Message fragments emitted by real upstreams during this class of outage.\n * Matched case-insensitively as a last resort, after code and status.\n */\nconst UPSTREAM_UNAVAILABLE_MESSAGES: readonly string[] = [\n  'bad gateway',\n  'service unavailable',\n  'inference temporarily unavailable',\n  'provider inference is unavailable',\n  'insufficient balance',\n  'usage limits',\n  'quota exceeded',\n  'rate limit',\n  'overloaded',\n  'temporarily unavailable',\n  // Model-SCOPED unavailability. A different model in the chain can still\n  // serve, so this belongs here and not with the client errors. Measured on a\n  // real box 2026-07-27: the sandbox now reports an unservable model as a\n  // terminal `error` whose data is `{\"message\":\"Session error:\n  // {\\\"error\\\":{\\\"name\\\":\\\"UnknownError\\\",\\\"data\\\":{\\\"message\\\":\\\"Model not\n  // found: openai-compat/zai/glm-4.7.\\\"}}}\"}` — no `code`, no numeric status,\n  // and none of the fragments above. The shipped live product-path proof went\n  // red on exactly this: `usedFallback:false`, `failed:true`, an error row to\n  // the customer where a fallback was available and would have answered.\n  'model not found',\n  'is not currently available',\n]\n\n/**\n * Textual forms an HTTP status arrives in when NO numeric field carries it.\n *\n * The case that forced this is the opaque Cloudflare 502. Captured verbatim\n * from `router.tangle.tools` on 2026-07-27 17:48:12 GMT (cf-ray\n * `a21d793899f6cef3-DEN`): `HTTP/2 502`, `content-type: text/plain;\n * charset=UTF-8`, a 16-byte body reading `error code: 502\\n`, `server:\n * cloudflare`, and **zero `x-tangle-*` headers** where a healthy 200 from the\n * same endpoint carries nine of them.\n *\n * That capture was originally read as \"zero headers means the origin never\n * executed\". It does not, and the correction matters more than the original\n * guess did. Matching client-observed cf-rays against the origin's own access\n * log (tangle-router #307) showed all 62 of those responses WERE produced by\n * the router — `content-type: application/json`, an `X-Generation-Id`, and a\n * routing trace showing the upstream had returned 200 — and Cloudflare\n * replaced the body and every response header with its own error page, because\n * an origin 502 reads to the edge as the origin having failed.\n *\n * The classifier's BEHAVIOR here was right for the wrong reason, and stays\n * exactly as it is. What changes is why: the rescue is needed not because the\n * origin is unreachable, but because a response the origin carefully\n * structured arrives at the client as sixteen unlabelled bytes. The router now\n * emits 503 instead of 502 on those paths so the body survives, but this\n * pattern must remain — it is the client's only handle on any origin 5xx the\n * edge decides to rewrite, including from a router that has not deployed the\n * fix yet.\n *\n * Every pattern below matches a string a real producer emits:\n * - `error code: 502` — Cloudflare's edge body (the capture above).\n * - `(HTTP 502)` — `src/runtime/openai-stream.ts` wrapping a non-ok response.\n * - `returned 502` — `src/runtime/model-catalog.ts`.\n * - `failed with status 502` — `src/turn-stream/adapters.ts`.\n * - `502 Bad Gateway` — an HTTP/1.1 status line. HTTP/2 carries no reason\n *   phrase at all, which is precisely why matching the phrase alone (the\n *   `'bad gateway'` fragment below) read the capture as a non-outage.\n */\nconst HTTP_STATUS_HINT_PATTERNS: readonly RegExp[] = [\n  /\\berror\\s+code:?\\s*([1-5]\\d{2})\\b/i,\n  /\\bhttp(?:\\/[\\d.]+)?[\\s:]\\s*([1-5]\\d{2})\\b/i,\n  /\\bstatus(?:\\s*code)?[\\s:=]\\s*([1-5]\\d{2})\\b/i,\n  /\\b(?:returned|responded(?:\\s+with)?)\\s+([1-5]\\d{2})\\b/i,\n  /\\b([1-5]\\d{2})\\s+(?:bad\\s+gateway|service\\s+unavailable|gateway\\s+time-?out|internal\\s+server\\s+error|too\\s+many\\s+requests)\\b/i,\n]\n\n/**\n * The HTTP status a message states in prose, or `undefined` when it states\n * none. Deliberately NOT \"any three digits in the string\" — an unanchored digit\n * scan would read a token count or a duration as a status.\n */\nexport function readHttpStatusHint(text: string): number | undefined {\n  for (const pattern of HTTP_STATUS_HINT_PATTERNS) {\n    const match = pattern.exec(text)\n    if (match?.[1]) return Number(match[1])\n  }\n  return undefined\n}\n\nfunction readString(source: Record<string, unknown>, key: string): string | undefined {\n  const value = source[key]\n  return typeof value === 'string' && value.trim().length > 0 ? value : undefined\n}\n\n/**\n * True when `signal` — a thrown error OR a resolved result payload — indicates\n * the model's upstream is unavailable and another model is worth trying.\n *\n * Checked in order of decreasing confidence: explicit code, HTTP status, then\n * message text. A resolved payload only counts as a failure when it carries an\n * explicit failure marker (`success: false`, or an `error`/`errorCode` field) —\n * a successful result is never misread as an outage.\n */\nexport function isUpstreamUnavailable(signal: unknown): boolean {\n  if (signal === null || typeof signal !== 'object') return false\n  const record = signal as Record<string, unknown>\n\n  // A resolved payload that explicitly reports success is never an outage.\n  if (record.success === true) return false\n\n  const nested = record.error\n  const nestedRecord = nested !== null && typeof nested === 'object' ? (nested as Record<string, unknown>) : undefined\n\n  const code =\n    readString(record, 'errorCode') ??\n    readString(record, 'code') ??\n    (nestedRecord ? (readString(nestedRecord, 'code') ?? readString(nestedRecord, 'type')) : undefined)\n  if (code && UPSTREAM_UNAVAILABLE_CODES.includes(code)) return true\n\n  for (const key of ['status', 'statusCode', 'httpStatus']) {\n    const value = record[key]\n    if (typeof value === 'number' && UPSTREAM_UNAVAILABLE_STATUSES.includes(value)) return true\n  }\n\n  const message =\n    readString(record, 'message') ??\n    readString(record, 'error') ??\n    (nestedRecord ? readString(nestedRecord, 'message') : undefined)\n  if (!message) return false\n\n  // A status carried as PROSE, checked before the fragment list because it is\n  // the stronger signal and it cuts BOTH ways. An edge 502 names no exception\n  // type and sets no numeric field, so without this the shipped classifier read\n  // `error code: 502` as a non-outage and handed the customer a Bad Gateway\n  // instead of trying the next model — the exact failure the router cannot\n  // rescue from inside its own origin.\n  const hinted = readHttpStatusHint(message)\n  if (hinted !== undefined) {\n    if (UPSTREAM_UNAVAILABLE_STATUSES.includes(hinted)) return true\n    // An explicit REQUEST-error status is decisive the OTHER way. A 400\n    // carrying a validation message must surface rather than walk the chain,\n    // even when its body happens to contain a word from the fragment list —\n    // the router's own `Function tools with reasoning_effort are not supported`\n    // 400 is a request-shaping bug that fails identically on every model.\n    if (REQUEST_ERROR_STATUSES.includes(hinted)) return false\n  }\n\n  const lowered = message.toLowerCase()\n  return UPSTREAM_UNAVAILABLE_MESSAGES.some((fragment) => lowered.includes(fragment))\n}\n\n/** One model tried, and how it went. */\nexport interface ModelFailoverAttempt {\n  model: string\n  ok: boolean\n  /** Why this model was abandoned. Absent when `ok`. */\n  reason?: string\n}\n\n/** The outcome of a failover run: the value plus the full attempt trail. */\nexport interface ModelFailoverResult<T> {\n  value: T\n  /** The model that actually produced `value`. */\n  model: string\n  attempts: ModelFailoverAttempt[]\n  /** True when the preferred (first) model did not serve the request. */\n  usedFallback: boolean\n}\n\n/** Every model in the chain failed; carries the trail for logging. */\nexport class ModelFailoverExhaustedError extends Error {\n  readonly attempts: ModelFailoverAttempt[]\n  constructor(attempts: ModelFailoverAttempt[]) {\n    const trail = attempts.map((a) => `${a.model}: ${a.reason ?? 'failed'}`).join(' | ')\n    super(`All ${attempts.length} model(s) failed. ${trail}`)\n    this.name = 'ModelFailoverExhaustedError'\n    this.attempts = attempts\n  }\n}\n\n/** Inputs to {@link runWithModelFailover}. */\nexport interface RunWithModelFailoverInput<T> {\n  /** Preferred model first, then fallbacks in descending preference. */\n  models: readonly string[]\n  /** Executes one turn with the given model. */\n  run: (model: string) => Promise<T>\n  /**\n   * Classifies a resolved result as an upstream outage. Defaults to\n   * {@link isUpstreamUnavailable}, which understands the sandbox's\n   * `{ success: false, errorCode }` payload.\n   */\n  isUnavailableResult?: (result: T) => boolean\n  /** Classifies a thrown error. Defaults to {@link isUpstreamUnavailable}. */\n  isUnavailableError?: (error: unknown) => boolean\n  /** Observability hook fired each time a model is abandoned. */\n  onFallback?: (attempt: ModelFailoverAttempt, nextModel: string) => void\n}\n\nfunction describe(signal: unknown): string {\n  if (signal instanceof Error) return signal.message\n  if (signal !== null && typeof signal === 'object') {\n    const record = signal as Record<string, unknown>\n    const message = readString(record, 'error') ?? readString(record, 'message') ?? readString(record, 'errorCode')\n    if (message) return message\n  }\n  return String(signal)\n}\n\n/**\n * Run `run` against the first model in `models` that does not report an upstream\n * outage, falling through the chain in order.\n *\n * A non-outage failure (bad request, auth, content filter) is re-thrown\n * immediately rather than retried down the chain — those fail identically on\n * every model, so walking the chain would only multiply latency and spend.\n *\n * @throws ModelFailoverExhaustedError when every model reports an outage.\n */\nexport async function runWithModelFailover<T>(\n  input: RunWithModelFailoverInput<T>,\n): Promise<ModelFailoverResult<T>> {\n  const models = input.models.map((m) => m.trim()).filter((m) => m.length > 0)\n  if (models.length === 0) throw new Error('runWithModelFailover requires at least one model')\n\n  const isUnavailableResult = input.isUnavailableResult ?? ((r: T) => isUpstreamUnavailable(r))\n  const isUnavailableError = input.isUnavailableError ?? isUpstreamUnavailable\n  const attempts: ModelFailoverAttempt[] = []\n\n  for (let index = 0; index < models.length; index += 1) {\n    const model = models[index]!\n    let result: T\n    try {\n      result = await input.run(model)\n    } catch (error) {\n      if (!isUnavailableError(error)) throw error\n      const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(error) }\n      attempts.push(attempt)\n      const next = models[index + 1]\n      if (next) input.onFallback?.(attempt, next)\n      continue\n    }\n\n    if (isUnavailableResult(result)) {\n      const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(result) }\n      attempts.push(attempt)\n      const next = models[index + 1]\n      if (next) input.onFallback?.(attempt, next)\n      continue\n    }\n\n    attempts.push({ model, ok: true })\n    return { value: result, model, attempts, usedFallback: index > 0 }\n  }\n\n  throw new ModelFailoverExhaustedError(attempts)\n}\n\n/**\n * Build a failover chain: the preferred model first, then `fallbacks`, with\n * duplicates removed so a model is never retried twice in one turn.\n */\nexport function buildModelChain(preferred: string, fallbacks: readonly string[]): string[] {\n  const chain: string[] = []\n  for (const model of [preferred, ...fallbacks]) {\n    const cleaned = typeof model === 'string' ? model.trim() : ''\n    if (cleaned.length > 0 && !chain.includes(cleaned)) chain.push(cleaned)\n  }\n  return chain\n}\n"],"mappings":";AAgCO,IAAM,6BAAgD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGO,IAAM,gCAAmD,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG;AASxF,IAAM,yBAA4C,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG;AAM/E,IAAM,gCAAmD;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA;AAAA,EACA;AACF;AAuCA,IAAM,4BAA+C;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAOO,SAAS,mBAAmB,MAAkC;AACnE,aAAW,WAAW,2BAA2B;AAC/C,UAAM,QAAQ,QAAQ,KAAK,IAAI;AAC/B,QAAI,QAAQ,CAAC,EAAG,QAAO,OAAO,MAAM,CAAC,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,QAAiC,KAAiC;AACpF,QAAM,QAAQ,OAAO,GAAG;AACxB,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAWO,SAAS,sBAAsB,QAA0B;AAC9D,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,SAAS;AAGf,MAAI,OAAO,YAAY,KAAM,QAAO;AAEpC,QAAM,SAAS,OAAO;AACtB,QAAM,eAAe,WAAW,QAAQ,OAAO,WAAW,WAAY,SAAqC;AAE3G,QAAM,OACJ,WAAW,QAAQ,WAAW,KAC9B,WAAW,QAAQ,MAAM,MACxB,eAAgB,WAAW,cAAc,MAAM,KAAK,WAAW,cAAc,MAAM,IAAK;AAC3F,MAAI,QAAQ,2BAA2B,SAAS,IAAI,EAAG,QAAO;AAE9D,aAAW,OAAO,CAAC,UAAU,cAAc,YAAY,GAAG;AACxD,UAAM,QAAQ,OAAO,GAAG;AACxB,QAAI,OAAO,UAAU,YAAY,8BAA8B,SAAS,KAAK,EAAG,QAAO;AAAA,EACzF;AAEA,QAAM,UACJ,WAAW,QAAQ,SAAS,KAC5B,WAAW,QAAQ,OAAO,MACzB,eAAe,WAAW,cAAc,SAAS,IAAI;AACxD,MAAI,CAAC,QAAS,QAAO;AAQrB,QAAM,SAAS,mBAAmB,OAAO;AACzC,MAAI,WAAW,QAAW;AACxB,QAAI,8BAA8B,SAAS,MAAM,EAAG,QAAO;AAM3D,QAAI,uBAAuB,SAAS,MAAM,EAAG,QAAO;AAAA,EACtD;AAEA,QAAM,UAAU,QAAQ,YAAY;AACpC,SAAO,8BAA8B,KAAK,CAAC,aAAa,QAAQ,SAAS,QAAQ,CAAC;AACpF;AAqBO,IAAM,8BAAN,cAA0C,MAAM;AAAA,EAC5C;AAAA,EACT,YAAY,UAAkC;AAC5C,UAAM,QAAQ,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,KAAK,EAAE,UAAU,QAAQ,EAAE,EAAE,KAAK,KAAK;AACnF,UAAM,OAAO,SAAS,MAAM,qBAAqB,KAAK,EAAE;AACxD,SAAK,OAAO;AACZ,SAAK,WAAW;AAAA,EAClB;AACF;AAoBA,SAAS,SAAS,QAAyB;AACzC,MAAI,kBAAkB,MAAO,QAAO,OAAO;AAC3C,MAAI,WAAW,QAAQ,OAAO,WAAW,UAAU;AACjD,UAAM,SAAS;AACf,UAAM,UAAU,WAAW,QAAQ,OAAO,KAAK,WAAW,QAAQ,SAAS,KAAK,WAAW,QAAQ,WAAW;AAC9G,QAAI,QAAS,QAAO;AAAA,EACtB;AACA,SAAO,OAAO,MAAM;AACtB;AAYA,eAAsB,qBACpB,OACiC;AACjC,QAAM,SAAS,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC3E,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,kDAAkD;AAE3F,QAAM,sBAAsB,MAAM,wBAAwB,CAAC,MAAS,sBAAsB,CAAC;AAC3F,QAAM,qBAAqB,MAAM,sBAAsB;AACvD,QAAM,WAAmC,CAAC;AAE1C,WAAS,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;AACrD,UAAM,QAAQ,OAAO,KAAK;AAC1B,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,MAAM,IAAI,KAAK;AAAA,IAChC,SAAS,OAAO;AACd,UAAI,CAAC,mBAAmB,KAAK,EAAG,OAAM;AACtC,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,KAAK,EAAE;AAClF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,QAAI,oBAAoB,MAAM,GAAG;AAC/B,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,MAAM,EAAE;AACnF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,aAAS,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC;AACjC,WAAO,EAAE,OAAO,QAAQ,OAAO,UAAU,cAAc,QAAQ,EAAE;AAAA,EACnE;AAEA,QAAM,IAAI,4BAA4B,QAAQ;AAChD;AAMO,SAAS,gBAAgB,WAAmB,WAAwC;AACzF,QAAM,QAAkB,CAAC;AACzB,aAAW,SAAS,CAAC,WAAW,GAAG,SAAS,GAAG;AAC7C,UAAM,UAAU,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI;AAC3D,QAAI,QAAQ,SAAS,KAAK,CAAC,MAAM,SAAS,OAAO,EAAG,OAAM,KAAK,OAAO;AAAA,EACxE;AACA,SAAO;AACT;","names":[]}

package/dist/design-canvas-react/index.js

@@ -83,15 +83,15 @@ import {
   setPagePropsCommand,
   ungroupElementCommand
 } from "../chunk-VTWKH4YU.js";
-import {
-  DesignCanvasChromeLazy,
-  DesignCanvasLazy
-} from "../chunk-4TI7QWLW.js";
 import {
   assertSceneMediaSrc,
   bleedAwareExportBounds,
   scaleForPreset
 } from "../chunk-VWOPOLVO.js";
+import {
+  DesignCanvasChromeLazy,
+  DesignCanvasLazy
+} from "../chunk-4TI7QWLW.js";
 import "../chunk-PTUMBMVH.js";
 
 // src/design-canvas-react/components/CanvasInsertPanel.tsx

package/dist/model-resolution/index.js

@@ -6,7 +6,7 @@ import {
   isUpstreamUnavailable,
   readHttpStatusHint,
   runWithModelFailover
-} from "../chunk-CHLOH4DG.js";
+} from "../chunk-DEXBRUZR.js";
 
 // src/model-resolution/index.ts
 function canonicalModelId(model) {

package/dist/turn-health/index.d.ts

@@ -0,0 +1,398 @@
+/**
+ * The classifier for turns that FAIL BY RETURNING SUCCESS.
+ *
+ * Every failure this module names shipped to a customer with HTTP 200, no
+ * thrown error, and no log line anyone read. Three were measured in production
+ * in a single week:
+ *
+ *   - a turn settled `{"outcome":{"type":"completed"},"finalText":"",
+ *     "tokenUsage":{"outputTokens":0}}` — the customer saw a blank bubble;
+ *   - six `submit_proposal` tool calls collapsed into ONE whose arguments were
+ *     a 1,652-character non-JSON string, so zero proposals persisted and
+ *     nothing errored (agent-runtime #626);
+ *   - a thread took 255 user messages over 17 days and produced 2 replies,
+ *     both of them error text.
+ *
+ * A conventional health check cannot see any of these, because it probes
+ * DEPENDENCIES (is the sandbox reachable, is the router up) and every one of
+ * these failures happens with all dependencies green. This classifier probes
+ * the OUTCOME instead.
+ *
+ * It is deliberately pure and structural: it reads a settled turn's own
+ * projection, so the SAME function judges a live turn through the
+ * `/chat-routes` lifecycle seam and a historical row read back out of the
+ * store during a sweep. One definition of "silently broken", two call sites.
+ */
+/** How loudly a reason should be routed. `critical` means a customer got
+ *  nothing usable; `warning` means the turn degraded but still produced
+ *  something a human could read. */
+type TurnHealthSeverity = 'critical' | 'warning';
+/** One specific way a turn returned success while failing.
+ *
+ *  Each variant carries the evidence that identified it, so an alert can name
+ *  the offending value instead of asserting a verdict the reader has to take
+ *  on faith. */
+type TurnHealthReason = 
+/** Settled without error and produced nothing a user can read: no text, and
+ *  no artifact part (file/image/work-product/plan/interaction). This is the
+ *  verbatim blank-completion capture. */
+{
+    kind: 'empty_completion';
+    outputTokens: number | null;
+    partCount: number;
+    durationMs?: number;
+}
+/** A tool call whose arguments never parsed. The engine surfaces unparseable
+ *  arguments as a RAW STRING rather than throwing, so the call is neither
+ *  dropped nor errored — it silently does nothing. Detecting a string-typed
+ *  tool input that fails `JSON.parse` is the exact fingerprint of the
+ *  index-less parallel-tool-call collapse. */
+ | {
+    kind: 'malformed_tool_call';
+    tool: string;
+    inputLength: number;
+    /** Leading characters of the offending input, for the alert body. */
+    sample: string;
+}
+/** A tool call that never reached a terminal state carrying output. The call
+ *  was issued and then simply produced no effect. */
+ | {
+    kind: 'tool_call_no_effect';
+    tool: string;
+    status: string;
+}
+/** The turn failed outright. Not silent by itself — but it becomes silent
+ *  the moment nothing is watching, which is how 16 days of
+ *  `TANGLE_HUB_URL is required` reached customers unnoticed. */
+ | {
+    kind: 'turn_failed';
+    reason: string;
+};
+/** A settled turn, in the narrowest shape both call sites can supply.
+ *
+ *  Structural on purpose: the lifecycle seam supplies `finalText`/`usage`, a
+ *  store sweep supplies `content`/`parts` read back from a row, and neither
+ *  has to import the other's types. */
+interface TurnOutcomeInput {
+    /** The turn's final assistant text. */
+    finalText?: string | null;
+    /** The persisted assistant parts. Untyped by design — a sweep reads these
+     *  out of a JSON column and must not be forced to validate them first. */
+    parts?: readonly unknown[] | null;
+    /** Output tokens, when the caller has usage. `null`/absent is unknown, which
+     *  is NOT the same as zero and is never treated as evidence. */
+    outputTokens?: number | null;
+    /** Set when the turn surfaced a terminal error event. */
+    failed?: boolean;
+    failureReason?: string | null;
+    durationMs?: number;
+}
+/** The verdict for one turn. `healthy` is exactly `reasons.length === 0`, kept
+ *  as a field so callers read intent rather than an array length. */
+interface TurnHealthVerdict {
+    healthy: boolean;
+    severity: TurnHealthSeverity | null;
+    reasons: TurnHealthReason[];
+}
+/**
+ * Judge one settled turn.
+ *
+ * Never throws: a malformed `parts` blob is a thing this function REPORTS on,
+ * so it must not be a thing it dies on. Telemetry that can crash the turn it
+ * measures is worse than no telemetry.
+ */
+declare function classifyTurnOutcome(input: TurnOutcomeInput): TurnHealthVerdict;
+/** One-line human summary of a reason, for an alert body. */
+declare function describeReason(reason: TurnHealthReason): string;
+
+/**
+ * Where a silent-failure verdict GOES.
+ *
+ * The detection half is worthless without this half. Every failure this module
+ * finds was already visible in the database the whole time — 255 unanswered
+ * messages sat in a table for 17 days. What was missing was not the data, it
+ * was delivery to a human who had not thought to look.
+ *
+ * So the sink is a seam, not a channel: the product supplies the transport,
+ * and agent-app ships the two shapes the fleet already has credentials for
+ * (an ops webhook, and stderr). No new channel is invented here.
+ */
+
+/** One deliverable alert. */
+interface TurnHealthAlert {
+    /** Which product raised it (`projectId`). Alerts from four products land in
+     *  one channel, so this is what makes the message actionable. */
+    product: string;
+    severity: TurnHealthSeverity;
+    /** Stable grouping key. Throttling is keyed on this, so it must NOT contain
+     *  a turn id or a timestamp or every alert is unique and nothing dedupes. */
+    key: string;
+    title: string;
+    /** Human-readable lines. */
+    details: string[];
+    /** Structured payload for a machine consumer. */
+    data?: Record<string, unknown>;
+    at: number;
+}
+/** Deliver an alert. Implementations MUST NOT throw — see
+ *  {@link createGuardedAlertSink}. */
+interface AlertSink {
+    deliver(alert: TurnHealthAlert): Promise<void>;
+}
+/** Build the alert for a set of reasons found on one turn. */
+declare function turnAlert(input: {
+    product: string;
+    severity: TurnHealthSeverity;
+    reasons: TurnHealthReason[];
+    threadId?: string;
+    turnId?: string;
+    model?: string;
+    at?: number;
+}): TurnHealthAlert;
+/** Minimal structural fetch, so this module has no lib-dom dependency and can
+ *  be driven by a fake in tests. */
+type FetchLike = (url: string, init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    text?(): Promise<string>;
+}>;
+/**
+ * POST to an incoming webhook in the Slack message format.
+ *
+ * Chosen because the org already runs one (`SLACK_OPS_WEBHOOK_URL`) and
+ * gtm-agent's outbound webhook code already speaks this exact shape — the
+ * instruction was to route somewhere humans already look, not to stand up a
+ * new channel. Discord and most log drains accept the same `{text}` body.
+ */
+declare function createWebhookAlertSink(options: {
+    webhookUrl: string;
+    fetchImpl?: FetchLike;
+}): AlertSink;
+/** stderr sink. The zero-config fallback so a product that has not yet been
+ *  given a webhook still emits something a log search can find. */
+declare function createConsoleAlertSink(log?: (message: string) => void): AlertSink;
+/** Fan out to several sinks. One failing transport must not stop the others. */
+declare function createMultiAlertSink(sinks: readonly AlertSink[]): AlertSink;
+/** Records the last time a key was alerted on. A product backs this with KV,
+ *  D1, or a Durable Object; the in-memory default is correct for a sweep that
+ *  runs as a single cron invocation. */
+interface AlertThrottleStore {
+    lastSentAt(key: string): Promise<number | null>;
+    markSent(key: string, at: number): Promise<void>;
+}
+/** Process-local throttle store. */
+declare function createMemoryThrottleStore(): AlertThrottleStore;
+/**
+ * Collapse repeats of the same `key` inside `windowMs`.
+ *
+ * Deliberately re-alerts once per window rather than going silent after the
+ * first: an incident that is still burning must keep saying so. Going quiet
+ * after one message is how a 17-day outage stays invisible after someone
+ * dismisses the first notification.
+ */
+declare function createThrottledAlertSink(inner: AlertSink, options: {
+    windowMs: number;
+    store?: AlertThrottleStore;
+}): AlertSink;
+/**
+ * Swallow transport errors so telemetry can never fail the turn it measures.
+ *
+ * Use this at the LIVE lifecycle call site only. A sweep should let the error
+ * surface, because a sweep that cannot deliver has done nothing at all and its
+ * cron run should go red.
+ */
+declare function createGuardedAlertSink(inner: AlertSink, onError?: (error: unknown) => void): AlertSink;
+
+/**
+ * The LIVE half: judge each turn the moment it settles.
+ *
+ * This adds no control flow. `createChatTurnRoutes` already exposes a
+ * `lifecycle` seam that fires exactly one of `onTurnComplete`/`onTurnError`
+ * after a turn settles, and already swallows hook errors so telemetry cannot
+ * fail a turn. That seam was shipped and then wired by nobody, which is a fair
+ * description of why the outage lasted 17 days. This function fills it.
+ *
+ * The shape is declared structurally rather than imported from
+ * `/chat-routes`, so `/turn-health` stays free of the server chat vertical and
+ * can be used by any turn driver that reports the same three moments.
+ */
+
+/** Structural mirror of `/chat-routes`' `ChatTurnLifecycle` complete payload. */
+interface TurnHealthCompleteInfo {
+    finalText: string;
+    usage?: {
+        outputTokens?: number | null;
+    } | null;
+    durationMs: number;
+    threadId?: string;
+    turnStreamId?: string;
+    executionId?: string;
+}
+/** Structural mirror of the lifecycle error payload. */
+interface TurnHealthErrorInfo {
+    error: unknown;
+    durationMs: number;
+    threadId?: string;
+    turnStreamId?: string;
+    executionId?: string;
+}
+/** What {@link createTurnHealthLifecycle} returns — assignable to
+ *  `createChatTurnRoutes`' `lifecycle` option. */
+interface TurnHealthLifecycle {
+    onTurnComplete(info: TurnHealthCompleteInfo): Promise<void>;
+    onTurnError(info: TurnHealthErrorInfo): Promise<void>;
+}
+interface TurnHealthLifecycleOptions {
+    /** Names the product in every alert. */
+    product: string;
+    sink: AlertSink;
+    /** Called for every verdict, healthy or not — the hook for a counter or a
+     *  metrics push. Alerts are for humans; this is for graphs. */
+    onVerdict?(verdict: {
+        product: string;
+        healthy: boolean;
+        kinds: string[];
+        durationMs: number;
+    }): void;
+}
+/**
+ * Build the lifecycle hooks that page on a turn which succeeded at nothing.
+ *
+ * The live lane sees `finalText` and usage but not the persisted parts, so it
+ * catches the blank-completion and hard-failure shapes immediately. The
+ * parts-dependent shapes (a tool call whose arguments never parsed, a tool
+ * call that left no effect) are caught by {@link sweepSilentFailures}, which
+ * reads what was actually written to the store — the honest place to ask
+ * whether an effect persisted.
+ */
+declare function createTurnHealthLifecycle(options: TurnHealthLifecycleOptions): TurnHealthLifecycle;
+
+/**
+ * The SWEEP half: ask the store what it has been quietly accumulating.
+ *
+ * A live per-turn hook cannot see the failure that matters most, because the
+ * worst outage produced NO turns at all to hook: gtm-agent took 9–21 user
+ * messages a day for sixteen straight days and wrote zero real assistant
+ * replies. Nothing crashed on a schedule; the product simply stopped
+ * answering. The only thing that could have noticed is something that
+ * periodically counts what arrived against what was answered.
+ *
+ * That is this. It runs on a cron, reads the shared `/chat-store` schema, and
+ * pages when the ratio breaks.
+ *
+ * The queries live HERE and not in each product because all four products
+ * (gtm, tax, legal, workcomp) persist to the same `message`/`thread` tables —
+ * four copies of this cron is exactly the duplication the repo's engine/shell
+ * rule exists to prevent.
+ */
+
+/** A thread that has taken user messages with no reply since. */
+interface UnansweredThread {
+    threadId: string;
+    /** User messages newer than the newest real assistant reply. */
+    pendingMessages: number;
+    /** Age of the OLDEST unanswered user message, in ms. */
+    oldestAgeMs: number;
+}
+/** A persisted assistant row, as the sweep needs to judge it. */
+interface PersistedTurnRow {
+    id: string;
+    threadId: string;
+    content: string;
+    /** Raw `parts` column. A JSON string or an already-parsed array; the sweep
+     *  accepts both because D1 drivers differ. */
+    parts: unknown;
+    outputTokens?: number | null;
+    model?: string | null;
+    createdAt: number;
+}
+/** What the sweep needs from a store. A product on a non-standard schema
+ *  implements these two reads; everything else is shared. */
+interface TurnHealthSource {
+    findUnansweredThreads(input: {
+        minAgeMs: number;
+        now: number;
+    }): Promise<UnansweredThread[]>;
+    listRecentAssistantTurns(input: {
+        sinceMs: number;
+        now: number;
+        limit: number;
+    }): Promise<PersistedTurnRow[]>;
+}
+interface SweepOptions {
+    product: string;
+    source: TurnHealthSource;
+    sink: AlertSink;
+    /** A user message must go unanswered this long before it counts. Guards
+     *  against alerting on a turn that is simply still streaming. Default 15 min. */
+    minAgeMs?: number;
+    /** How far back to judge settled turns. Default 24 h. */
+    lookbackMs?: number;
+    /** Cap on rows judged per sweep. Default 500. */
+    limit?: number;
+    /** Fraction of recent turns allowed to be silently broken before paging.
+     *  Default 0.05 — the measured blank-completion rate on the tax tool surface
+     *  was 12.2%, so 5% separates a real regression from noise. */
+    emptyRateThreshold?: number;
+    /** Absolute floor: never page on a rate computed from fewer turns than this. */
+    minTurnsForRate?: number;
+    now?: number;
+}
+/** What the sweep found. Returned as well as alerted, so a cron can log it and
+ *  a test can assert on it. */
+interface SweepResult {
+    product: string;
+    unansweredThreads: number;
+    pendingUserMessages: number;
+    oldestUnansweredMs: number;
+    turnsJudged: number;
+    unhealthyTurns: number;
+    emptyCompletions: number;
+    malformedToolCalls: number;
+    toolCallsWithoutEffect: number;
+    alerts: TurnHealthAlert[];
+}
+/**
+ * Run one sweep and deliver whatever it finds.
+ *
+ * Errors from the sink are NOT swallowed here (unlike the live lane): a sweep
+ * that could not deliver has accomplished nothing, and its cron invocation
+ * should go red rather than report a clean run.
+ */
+declare function sweepSilentFailures(options: SweepOptions): Promise<SweepResult>;
+/** Minimal structural D1 contract (Cloudflare's `D1Database` satisfies it). */
+interface D1LikeForHealth {
+    prepare(sql: string): {
+        bind(...values: unknown[]): {
+            all<T = Record<string, unknown>>(): Promise<{
+                results: T[];
+            }>;
+        };
+    };
+}
+/**
+ * The sweep source for products on the canonical `/chat-store` tables.
+ *
+ * "Answered" deliberately means an assistant row with NON-EMPTY content. A
+ * blank assistant row is what a broken turn writes, so counting it as an
+ * answer would let the exact failure being hunted mark itself resolved. That
+ * single predicate is the difference between this catching the gtm outage and
+ * sleeping through it — during those sixteen days the table was NOT empty.
+ *
+ * The query deliberately does NOT join the thread table. Products do not all
+ * keep one: tax-agent's `thread` table holds zero rows because it groups by
+ * its own `tax_sessions`, and an inner join against it silently reported "0
+ * unanswered threads, healthy" while 18 real messages sat unanswered. A
+ * detector that reports healthy because its join found nothing is the same
+ * bug class it was built to catch.
+ */
+declare function createD1TurnHealthSource(db: D1LikeForHealth, options?: {
+    messageTable?: string;
+    threadTable?: string;
+}): TurnHealthSource;
+
+export { type AlertSink, type AlertThrottleStore, type D1LikeForHealth, type FetchLike, type PersistedTurnRow, type SweepOptions, type SweepResult, type TurnHealthAlert, type TurnHealthCompleteInfo, type TurnHealthErrorInfo, type TurnHealthLifecycle, type TurnHealthLifecycleOptions, type TurnHealthReason, type TurnHealthSeverity, type TurnHealthSource, type TurnHealthVerdict, type TurnOutcomeInput, type UnansweredThread, classifyTurnOutcome, createConsoleAlertSink, createD1TurnHealthSource, createGuardedAlertSink, createMemoryThrottleStore, createMultiAlertSink, createThrottledAlertSink, createTurnHealthLifecycle, createWebhookAlertSink, describeReason, sweepSilentFailures, turnAlert };

package/dist/turn-health/index.js

@@ -0,0 +1,428 @@
+// src/turn-health/classify.ts
+var ARTIFACT_PART_KINDS = /* @__PURE__ */ new Set(["file", "image", "work-product", "plan", "interaction"]);
+var SAMPLE_CHARS = 120;
+function asRecord(value) {
+  return typeof value === "object" && value !== null ? value : null;
+}
+function nonEmptyString(value) {
+  return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+function isUnparseableJson(value) {
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return false;
+  try {
+    JSON.parse(trimmed);
+    return false;
+  } catch {
+    return true;
+  }
+}
+var SETTLED_TOOL_STATUSES = /* @__PURE__ */ new Set(["completed", "complete", "success", "done"]);
+function classifyTurnOutcome(input) {
+  const reasons = [];
+  if (input.failed) {
+    reasons.push({
+      kind: "turn_failed",
+      reason: nonEmptyString(input.failureReason) ?? "unspecified"
+    });
+  }
+  const parts = Array.isArray(input.parts) ? input.parts : [];
+  let hasVisibleText = nonEmptyString(input.finalText) !== null;
+  let artifactCount = 0;
+  for (const raw of parts) {
+    const part = asRecord(raw);
+    if (!part) continue;
+    const type = typeof part.type === "string" ? part.type : "";
+    if (type === "text" && nonEmptyString(part.text) !== null) {
+      hasVisibleText = true;
+      continue;
+    }
+    if (ARTIFACT_PART_KINDS.has(type)) {
+      artifactCount += 1;
+      continue;
+    }
+    if (type !== "tool") continue;
+    const tool = nonEmptyString(part.tool) ?? "unknown";
+    const state = asRecord(part.state);
+    const status = typeof state?.status === "string" ? state.status : "unknown";
+    const toolInput = state?.input;
+    if (typeof toolInput === "string" && isUnparseableJson(toolInput)) {
+      reasons.push({
+        kind: "malformed_tool_call",
+        tool,
+        inputLength: toolInput.length,
+        sample: toolInput.slice(0, SAMPLE_CHARS)
+      });
+      continue;
+    }
+    if (!SETTLED_TOOL_STATUSES.has(status)) {
+      reasons.push({ kind: "tool_call_no_effect", tool, status });
+    }
+  }
+  if (!input.failed && !hasVisibleText && artifactCount === 0) {
+    reasons.push({
+      kind: "empty_completion",
+      outputTokens: input.outputTokens ?? null,
+      partCount: parts.length,
+      ...input.durationMs !== void 0 ? { durationMs: input.durationMs } : {}
+    });
+  }
+  return {
+    healthy: reasons.length === 0,
+    severity: severityOf(reasons),
+    reasons
+  };
+}
+function severityOf(reasons) {
+  if (reasons.length === 0) return null;
+  const critical = reasons.some((r) => r.kind === "empty_completion" || r.kind === "turn_failed");
+  return critical ? "critical" : "warning";
+}
+function describeReason(reason) {
+  switch (reason.kind) {
+    case "empty_completion":
+      return `completed with NO output (${reason.partCount} parts, outputTokens=${reason.outputTokens ?? "unknown"})`;
+    case "malformed_tool_call":
+      return `tool \`${reason.tool}\` arguments did not parse (${reason.inputLength} chars): ${reason.sample}`;
+    case "tool_call_no_effect":
+      return `tool \`${reason.tool}\` left no effect (status=${reason.status})`;
+    case "turn_failed":
+      return `turn failed: ${reason.reason}`;
+  }
+}
+
+// src/turn-health/sink.ts
+function turnAlert(input) {
+  const kinds = [...new Set(input.reasons.map((r) => r.kind))].sort();
+  return {
+    product: input.product,
+    severity: input.severity,
+    // Keyed by product + reason kinds ONLY. A blank-completion storm across
+    // 200 turns is one incident, not 200 pages.
+    key: `turn:${input.product}:${kinds.join("+")}`,
+    title: `${input.product}: turn completed but delivered nothing (${kinds.join(", ")})`,
+    details: input.reasons.map(describeReason),
+    data: {
+      kinds,
+      ...input.threadId ? { threadId: input.threadId } : {},
+      ...input.turnId ? { turnId: input.turnId } : {},
+      ...input.model ? { model: input.model } : {}
+    },
+    at: input.at ?? Date.now()
+  };
+}
+function createWebhookAlertSink(options) {
+  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+  return {
+    async deliver(alert) {
+      const icon = alert.severity === "critical" ? ":rotating_light:" : ":warning:";
+      const lines = [
+        `${icon} *${alert.title}*`,
+        ...alert.details.map((d) => `\u2022 ${d}`),
+        `_${new Date(alert.at).toISOString()}_`
+      ];
+      const response = await fetchImpl(options.webhookUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ text: lines.join("\n") })
+      });
+      if (!response.ok) {
+        throw new Error(`alert webhook responded ${response.status}`);
+      }
+    }
+  };
+}
+function createConsoleAlertSink(log = console.error) {
+  return {
+    async deliver(alert) {
+      log(
+        `[turn-health] ${alert.severity.toUpperCase()} ${alert.title} :: ${alert.details.join(" | ")}`
+      );
+    }
+  };
+}
+function createMultiAlertSink(sinks) {
+  return {
+    async deliver(alert) {
+      const settled = await Promise.allSettled(sinks.map((s) => s.deliver(alert)));
+      const failures = settled.filter((r) => r.status === "rejected");
+      if (failures.length === sinks.length && sinks.length > 0) {
+        throw new Error("every alert sink failed");
+      }
+    }
+  };
+}
+function createMemoryThrottleStore() {
+  const seen = /* @__PURE__ */ new Map();
+  return {
+    async lastSentAt(key) {
+      return seen.get(key) ?? null;
+    },
+    async markSent(key, at) {
+      seen.set(key, at);
+    }
+  };
+}
+function createThrottledAlertSink(inner, options) {
+  const store = options.store ?? createMemoryThrottleStore();
+  return {
+    async deliver(alert) {
+      const last = await store.lastSentAt(alert.key);
+      if (last !== null && alert.at - last < options.windowMs) return;
+      await inner.deliver(alert);
+      await store.markSent(alert.key, alert.at);
+    }
+  };
+}
+function createGuardedAlertSink(inner, onError = (e) => console.error("[turn-health] alert delivery failed", e)) {
+  return {
+    async deliver(alert) {
+      try {
+        await inner.deliver(alert);
+      } catch (error) {
+        onError(error);
+      }
+    }
+  };
+}
+
+// src/turn-health/lifecycle.ts
+function errorText(error) {
+  if (error instanceof Error) return error.message;
+  if (typeof error === "string") return error;
+  return String(error);
+}
+function createTurnHealthLifecycle(options) {
+  const sink = createGuardedAlertSink(options.sink);
+  return {
+    async onTurnComplete(info) {
+      const verdict = classifyTurnOutcome({
+        finalText: info.finalText,
+        outputTokens: info.usage?.outputTokens ?? null,
+        durationMs: info.durationMs
+      });
+      options.onVerdict?.({
+        product: options.product,
+        healthy: verdict.healthy,
+        kinds: verdict.reasons.map((r) => r.kind),
+        durationMs: info.durationMs
+      });
+      if (verdict.healthy || verdict.severity === null) return;
+      await sink.deliver(
+        turnAlert({
+          product: options.product,
+          severity: verdict.severity,
+          reasons: verdict.reasons,
+          ...info.threadId ? { threadId: info.threadId } : {},
+          ...info.executionId ? { turnId: info.executionId } : {}
+        })
+      );
+    },
+    async onTurnError(info) {
+      const verdict = classifyTurnOutcome({
+        failed: true,
+        failureReason: errorText(info.error),
+        durationMs: info.durationMs
+      });
+      options.onVerdict?.({
+        product: options.product,
+        healthy: false,
+        kinds: verdict.reasons.map((r) => r.kind),
+        durationMs: info.durationMs
+      });
+      await sink.deliver(
+        turnAlert({
+          product: options.product,
+          severity: "critical",
+          reasons: verdict.reasons,
+          ...info.threadId ? { threadId: info.threadId } : {},
+          ...info.executionId ? { turnId: info.executionId } : {}
+        })
+      );
+    }
+  };
+}
+
+// src/turn-health/sweep.ts
+function parseParts(raw) {
+  if (Array.isArray(raw)) return raw;
+  if (typeof raw !== "string" || raw.trim().length === 0) return [];
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+var HOUR_MS = 36e5;
+async function sweepSilentFailures(options) {
+  const now = options.now ?? Date.now();
+  const minAgeMs = options.minAgeMs ?? 15 * 6e4;
+  const lookbackMs = options.lookbackMs ?? 24 * HOUR_MS;
+  const limit = options.limit ?? 500;
+  const emptyRateThreshold = options.emptyRateThreshold ?? 0.05;
+  const minTurnsForRate = options.minTurnsForRate ?? 10;
+  const [unanswered, turns] = await Promise.all([
+    options.source.findUnansweredThreads({ minAgeMs, now }),
+    options.source.listRecentAssistantTurns({ sinceMs: now - lookbackMs, now, limit })
+  ]);
+  const alerts = [];
+  const pendingUserMessages = unanswered.reduce((sum, t) => sum + t.pendingMessages, 0);
+  const oldestUnansweredMs = unanswered.reduce((max, t) => Math.max(max, t.oldestAgeMs), 0);
+  if (unanswered.length > 0) {
+    const hours = (oldestUnansweredMs / HOUR_MS).toFixed(1);
+    alerts.push({
+      product: options.product,
+      // A day of total silence is not a warning.
+      severity: oldestUnansweredMs >= 24 * HOUR_MS ? "critical" : "warning",
+      key: `sweep:${options.product}:unanswered_threads`,
+      title: `${options.product}: ${pendingUserMessages} user message(s) unanswered across ${unanswered.length} thread(s)`,
+      details: [
+        `oldest unanswered message: ${hours}h`,
+        ...unanswered.slice(0, 5).map(
+          (t) => `thread ${t.threadId}: ${t.pendingMessages} pending, oldest ${(t.oldestAgeMs / HOUR_MS).toFixed(1)}h`
+        )
+      ],
+      data: {
+        unansweredThreads: unanswered.length,
+        pendingUserMessages,
+        oldestUnansweredMs
+      },
+      at: now
+    });
+  }
+  let emptyCompletions = 0;
+  let malformedToolCalls = 0;
+  let toolCallsWithoutEffect = 0;
+  let unhealthyTurns = 0;
+  const malformedSamples = [];
+  for (const row of turns) {
+    const verdict = classifyTurnOutcome({
+      finalText: row.content,
+      parts: parseParts(row.parts),
+      outputTokens: row.outputTokens ?? null
+    });
+    if (verdict.healthy) continue;
+    unhealthyTurns += 1;
+    for (const reason of verdict.reasons) {
+      if (reason.kind === "empty_completion") emptyCompletions += 1;
+      if (reason.kind === "malformed_tool_call") {
+        malformedToolCalls += 1;
+        if (malformedSamples.length < 3) malformedSamples.push(reason);
+      }
+      if (reason.kind === "tool_call_no_effect") toolCallsWithoutEffect += 1;
+    }
+  }
+  if (malformedToolCalls > 0) {
+    alerts.push({
+      product: options.product,
+      severity: "critical",
+      key: `sweep:${options.product}:malformed_tool_call`,
+      title: `${options.product}: ${malformedToolCalls} tool call(s) had unparseable arguments \u2014 deliverables silently dropped`,
+      details: malformedSamples.map(describeReason),
+      data: { malformedToolCalls, turnsJudged: turns.length },
+      at: now
+    });
+  }
+  if (turns.length >= minTurnsForRate) {
+    const rate = emptyCompletions / turns.length;
+    if (rate > emptyRateThreshold) {
+      alerts.push({
+        product: options.product,
+        severity: "critical",
+        key: `sweep:${options.product}:empty_completion_rate`,
+        title: `${options.product}: ${(rate * 100).toFixed(1)}% of turns completed with no output`,
+        details: [
+          `${emptyCompletions} of ${turns.length} settled turns delivered nothing`,
+          `threshold ${(emptyRateThreshold * 100).toFixed(1)}%`
+        ],
+        data: { emptyCompletions, turnsJudged: turns.length, rate },
+        at: now
+      });
+    }
+  }
+  for (const alert of alerts) await options.sink.deliver(alert);
+  return {
+    product: options.product,
+    unansweredThreads: unanswered.length,
+    pendingUserMessages,
+    oldestUnansweredMs,
+    turnsJudged: turns.length,
+    unhealthyTurns,
+    emptyCompletions,
+    malformedToolCalls,
+    toolCallsWithoutEffect,
+    alerts
+  };
+}
+function createD1TurnHealthSource(db, options = {}) {
+  const message = safeIdentifier(options.messageTable ?? "message");
+  const thread = safeIdentifier(options.threadTable ?? "thread");
+  return {
+    async findUnansweredThreads({ minAgeMs, now }) {
+      const cutoffSeconds = Math.floor((now - minAgeMs) / 1e3);
+      const { results } = await db.prepare(
+        `SELECT m.thread_id AS threadId,
+                  COUNT(*) AS pendingMessages,
+                  MIN(m.created_at) AS oldestCreatedAt
+             FROM ${message} m
+            WHERE m.role = 'user'
+              AND m.created_at <= ?1
+              AND m.created_at > COALESCE(
+                    (SELECT MAX(a.created_at)
+                       FROM ${message} a
+                      WHERE a.thread_id = m.thread_id
+                        AND a.role = 'assistant'
+                        AND length(trim(a.content)) > 0), 0)
+            GROUP BY m.thread_id
+            ORDER BY oldestCreatedAt ASC`
+      ).bind(cutoffSeconds).all();
+      return results.map((row) => ({
+        threadId: row.threadId,
+        pendingMessages: Number(row.pendingMessages),
+        oldestAgeMs: now - Number(row.oldestCreatedAt) * 1e3
+      }));
+    },
+    async listRecentAssistantTurns({ sinceMs, limit }) {
+      const sinceSeconds = Math.floor(sinceMs / 1e3);
+      const { results } = await db.prepare(
+        `SELECT id, thread_id AS threadId, content, parts,
+                  output_tokens AS outputTokens, model, created_at AS createdAt
+             FROM ${message}
+            WHERE role = 'assistant' AND created_at >= ?1
+            ORDER BY created_at DESC
+            LIMIT ?2`
+      ).bind(sinceSeconds, limit).all();
+      return results.map((row) => ({
+        id: String(row.id),
+        threadId: String(row.threadId),
+        content: typeof row.content === "string" ? row.content : "",
+        parts: row.parts,
+        outputTokens: row.outputTokens === null ? null : Number(row.outputTokens),
+        model: row.model ?? null,
+        createdAt: Number(row.createdAt) * 1e3
+      }));
+    }
+  };
+  function safeIdentifier(name) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+      throw new Error(`unsafe table identifier: ${name}`);
+    }
+    return name;
+  }
+}
+export {
+  classifyTurnOutcome,
+  createConsoleAlertSink,
+  createD1TurnHealthSource,
+  createGuardedAlertSink,
+  createMemoryThrottleStore,
+  createMultiAlertSink,
+  createThrottledAlertSink,
+  createTurnHealthLifecycle,
+  createWebhookAlertSink,
+  describeReason,
+  sweepSilentFailures,
+  turnAlert
+};
+//# sourceMappingURL=index.js.map

package/dist/turn-health/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/turn-health/classify.ts","../../src/turn-health/sink.ts","../../src/turn-health/lifecycle.ts","../../src/turn-health/sweep.ts"],"sourcesContent":["/**\n * The classifier for turns that FAIL BY RETURNING SUCCESS.\n *\n * Every failure this module names shipped to a customer with HTTP 200, no\n * thrown error, and no log line anyone read. Three were measured in production\n * in a single week:\n *\n *   - a turn settled `{\"outcome\":{\"type\":\"completed\"},\"finalText\":\"\",\n *     \"tokenUsage\":{\"outputTokens\":0}}` — the customer saw a blank bubble;\n *   - six `submit_proposal` tool calls collapsed into ONE whose arguments were\n *     a 1,652-character non-JSON string, so zero proposals persisted and\n *     nothing errored (agent-runtime #626);\n *   - a thread took 255 user messages over 17 days and produced 2 replies,\n *     both of them error text.\n *\n * A conventional health check cannot see any of these, because it probes\n * DEPENDENCIES (is the sandbox reachable, is the router up) and every one of\n * these failures happens with all dependencies green. This classifier probes\n * the OUTCOME instead.\n *\n * It is deliberately pure and structural: it reads a settled turn's own\n * projection, so the SAME function judges a live turn through the\n * `/chat-routes` lifecycle seam and a historical row read back out of the\n * store during a sweep. One definition of \"silently broken\", two call sites.\n */\n\n/** How loudly a reason should be routed. `critical` means a customer got\n *  nothing usable; `warning` means the turn degraded but still produced\n *  something a human could read. */\nexport type TurnHealthSeverity = 'critical' | 'warning'\n\n/** One specific way a turn returned success while failing.\n *\n *  Each variant carries the evidence that identified it, so an alert can name\n *  the offending value instead of asserting a verdict the reader has to take\n *  on faith. */\nexport type TurnHealthReason =\n  /** Settled without error and produced nothing a user can read: no text, and\n   *  no artifact part (file/image/work-product/plan/interaction). This is the\n   *  verbatim blank-completion capture. */\n  | {\n      kind: 'empty_completion'\n      outputTokens: number | null\n      partCount: number\n      durationMs?: number\n    }\n  /** A tool call whose arguments never parsed. The engine surfaces unparseable\n   *  arguments as a RAW STRING rather than throwing, so the call is neither\n   *  dropped nor errored — it silently does nothing. Detecting a string-typed\n   *  tool input that fails `JSON.parse` is the exact fingerprint of the\n   *  index-less parallel-tool-call collapse. */\n  | {\n      kind: 'malformed_tool_call'\n      tool: string\n      inputLength: number\n      /** Leading characters of the offending input, for the alert body. */\n      sample: string\n    }\n  /** A tool call that never reached a terminal state carrying output. The call\n   *  was issued and then simply produced no effect. */\n  | {\n      kind: 'tool_call_no_effect'\n      tool: string\n      status: string\n    }\n  /** The turn failed outright. Not silent by itself — but it becomes silent\n   *  the moment nothing is watching, which is how 16 days of\n   *  `TANGLE_HUB_URL is required` reached customers unnoticed. */\n  | { kind: 'turn_failed'; reason: string }\n\n/** A settled turn, in the narrowest shape both call sites can supply.\n *\n *  Structural on purpose: the lifecycle seam supplies `finalText`/`usage`, a\n *  store sweep supplies `content`/`parts` read back from a row, and neither\n *  has to import the other's types. */\nexport interface TurnOutcomeInput {\n  /** The turn's final assistant text. */\n  finalText?: string | null\n  /** The persisted assistant parts. Untyped by design — a sweep reads these\n   *  out of a JSON column and must not be forced to validate them first. */\n  parts?: readonly unknown[] | null\n  /** Output tokens, when the caller has usage. `null`/absent is unknown, which\n   *  is NOT the same as zero and is never treated as evidence. */\n  outputTokens?: number | null\n  /** Set when the turn surfaced a terminal error event. */\n  failed?: boolean\n  failureReason?: string | null\n  durationMs?: number\n}\n\n/** The verdict for one turn. `healthy` is exactly `reasons.length === 0`, kept\n *  as a field so callers read intent rather than an array length. */\nexport interface TurnHealthVerdict {\n  healthy: boolean\n  severity: TurnHealthSeverity | null\n  reasons: TurnHealthReason[]\n}\n\n/** Part kinds that count as something a user actually receives.\n *\n *  A tool part is deliberately NOT here. A turn that ran six tools and said\n *  nothing, with no artifact to show for it, is the malformed-tool-call\n *  disaster — counting a tool chip as output would suppress the very alert\n *  this module exists to raise. */\nconst ARTIFACT_PART_KINDS = new Set(['file', 'image', 'work-product', 'plan', 'interaction'])\n\nconst SAMPLE_CHARS = 120\n\nfunction asRecord(value: unknown): Record<string, unknown> | null {\n  return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null\n}\n\nfunction nonEmptyString(value: unknown): string | null {\n  return typeof value === 'string' && value.trim().length > 0 ? value : null\n}\n\n/** True when a string is not parseable JSON.\n *\n *  Only meaningful for tool INPUT, where the engine's contract is that a\n *  well-formed call carries an object (or a string that parses into one). A\n *  string that fails to parse means the arguments were concatenated or\n *  truncated upstream. */\nfunction isUnparseableJson(value: string): boolean {\n  const trimmed = value.trim()\n  if (trimmed.length === 0) return false\n  try {\n    JSON.parse(trimmed)\n    return false\n  } catch {\n    return true\n  }\n}\n\n/** Tool statuses that mean the call actually landed.\n *\n *  Anything else — `pending`, `running`, `error`, an unknown string — left no\n *  persisted effect by the time the turn settled. */\nconst SETTLED_TOOL_STATUSES = new Set(['completed', 'complete', 'success', 'done'])\n\n/**\n * Judge one settled turn.\n *\n * Never throws: a malformed `parts` blob is a thing this function REPORTS on,\n * so it must not be a thing it dies on. Telemetry that can crash the turn it\n * measures is worse than no telemetry.\n */\nexport function classifyTurnOutcome(input: TurnOutcomeInput): TurnHealthVerdict {\n  const reasons: TurnHealthReason[] = []\n\n  if (input.failed) {\n    reasons.push({\n      kind: 'turn_failed',\n      reason: nonEmptyString(input.failureReason) ?? 'unspecified',\n    })\n  }\n\n  const parts = Array.isArray(input.parts) ? input.parts : []\n\n  let hasVisibleText = nonEmptyString(input.finalText) !== null\n  let artifactCount = 0\n\n  for (const raw of parts) {\n    const part = asRecord(raw)\n    if (!part) continue\n    const type = typeof part.type === 'string' ? part.type : ''\n\n    if (type === 'text' && nonEmptyString(part.text) !== null) {\n      hasVisibleText = true\n      continue\n    }\n    if (ARTIFACT_PART_KINDS.has(type)) {\n      artifactCount += 1\n      continue\n    }\n    if (type !== 'tool') continue\n\n    const tool = nonEmptyString(part.tool) ?? 'unknown'\n    const state = asRecord(part.state)\n    const status = typeof state?.status === 'string' ? state.status : 'unknown'\n\n    // The #626 fingerprint: arguments surfaced as a raw string because they\n    // failed to parse upstream. Checked before the status gate — a malformed\n    // call can still be marked completed, which is precisely why it is silent.\n    const toolInput = state?.input\n    if (typeof toolInput === 'string' && isUnparseableJson(toolInput)) {\n      reasons.push({\n        kind: 'malformed_tool_call',\n        tool,\n        inputLength: toolInput.length,\n        sample: toolInput.slice(0, SAMPLE_CHARS),\n      })\n      continue\n    }\n\n    if (!SETTLED_TOOL_STATUSES.has(status)) {\n      reasons.push({ kind: 'tool_call_no_effect', tool, status })\n    }\n  }\n\n  // `outputTokens` is corroborating evidence, never the trigger: a turn can\n  // spend tokens on reasoning and still deliver nothing, and a turn with\n  // unknown usage can still be perfectly fine.\n  if (!input.failed && !hasVisibleText && artifactCount === 0) {\n    reasons.push({\n      kind: 'empty_completion',\n      outputTokens: input.outputTokens ?? null,\n      partCount: parts.length,\n      ...(input.durationMs !== undefined ? { durationMs: input.durationMs } : {}),\n    })\n  }\n\n  return {\n    healthy: reasons.length === 0,\n    severity: severityOf(reasons),\n    reasons,\n  }\n}\n\n/** `critical` when the customer got nothing usable out of the turn. A\n *  malformed tool call alongside readable text is a `warning` — degraded, but\n *  a human still received an answer. */\nfunction severityOf(reasons: TurnHealthReason[]): TurnHealthSeverity | null {\n  if (reasons.length === 0) return null\n  const critical = reasons.some((r) => r.kind === 'empty_completion' || r.kind === 'turn_failed')\n  return critical ? 'critical' : 'warning'\n}\n\n/** One-line human summary of a reason, for an alert body. */\nexport function describeReason(reason: TurnHealthReason): string {\n  switch (reason.kind) {\n    case 'empty_completion':\n      return `completed with NO output (${reason.partCount} parts, outputTokens=${\n        reason.outputTokens ?? 'unknown'\n      })`\n    case 'malformed_tool_call':\n      return `tool \\`${reason.tool}\\` arguments did not parse (${reason.inputLength} chars): ${reason.sample}`\n    case 'tool_call_no_effect':\n      return `tool \\`${reason.tool}\\` left no effect (status=${reason.status})`\n    case 'turn_failed':\n      return `turn failed: ${reason.reason}`\n  }\n}\n","/**\n * Where a silent-failure verdict GOES.\n *\n * The detection half is worthless without this half. Every failure this module\n * finds was already visible in the database the whole time — 255 unanswered\n * messages sat in a table for 17 days. What was missing was not the data, it\n * was delivery to a human who had not thought to look.\n *\n * So the sink is a seam, not a channel: the product supplies the transport,\n * and agent-app ships the two shapes the fleet already has credentials for\n * (an ops webhook, and stderr). No new channel is invented here.\n */\n\nimport { describeReason, type TurnHealthReason, type TurnHealthSeverity } from './classify.js'\n\n/** One deliverable alert. */\nexport interface TurnHealthAlert {\n  /** Which product raised it (`projectId`). Alerts from four products land in\n   *  one channel, so this is what makes the message actionable. */\n  product: string\n  severity: TurnHealthSeverity\n  /** Stable grouping key. Throttling is keyed on this, so it must NOT contain\n   *  a turn id or a timestamp or every alert is unique and nothing dedupes. */\n  key: string\n  title: string\n  /** Human-readable lines. */\n  details: string[]\n  /** Structured payload for a machine consumer. */\n  data?: Record<string, unknown>\n  at: number\n}\n\n/** Deliver an alert. Implementations MUST NOT throw — see\n *  {@link createGuardedAlertSink}. */\nexport interface AlertSink {\n  deliver(alert: TurnHealthAlert): Promise<void>\n}\n\n/** Build the alert for a set of reasons found on one turn. */\nexport function turnAlert(input: {\n  product: string\n  severity: TurnHealthSeverity\n  reasons: TurnHealthReason[]\n  threadId?: string\n  turnId?: string\n  model?: string\n  at?: number\n}): TurnHealthAlert {\n  const kinds = [...new Set(input.reasons.map((r) => r.kind))].sort()\n  return {\n    product: input.product,\n    severity: input.severity,\n    // Keyed by product + reason kinds ONLY. A blank-completion storm across\n    // 200 turns is one incident, not 200 pages.\n    key: `turn:${input.product}:${kinds.join('+')}`,\n    title: `${input.product}: turn completed but delivered nothing (${kinds.join(', ')})`,\n    details: input.reasons.map(describeReason),\n    data: {\n      kinds,\n      ...(input.threadId ? { threadId: input.threadId } : {}),\n      ...(input.turnId ? { turnId: input.turnId } : {}),\n      ...(input.model ? { model: input.model } : {}),\n    },\n    at: input.at ?? Date.now(),\n  }\n}\n\n// ── transports ────────────────────────────────────────────────────────────\n\n/** Minimal structural fetch, so this module has no lib-dom dependency and can\n *  be driven by a fake in tests. */\nexport type FetchLike = (\n  url: string,\n  init: { method: string; headers: Record<string, string>; body: string },\n) => Promise<{ ok: boolean; status: number; text?(): Promise<string> }>\n\n/**\n * POST to an incoming webhook in the Slack message format.\n *\n * Chosen because the org already runs one (`SLACK_OPS_WEBHOOK_URL`) and\n * gtm-agent's outbound webhook code already speaks this exact shape — the\n * instruction was to route somewhere humans already look, not to stand up a\n * new channel. Discord and most log drains accept the same `{text}` body.\n */\nexport function createWebhookAlertSink(options: {\n  webhookUrl: string\n  fetchImpl?: FetchLike\n}): AlertSink {\n  const fetchImpl = options.fetchImpl ?? (globalThis.fetch as unknown as FetchLike)\n  return {\n    async deliver(alert) {\n      const icon = alert.severity === 'critical' ? ':rotating_light:' : ':warning:'\n      const lines = [\n        `${icon} *${alert.title}*`,\n        ...alert.details.map((d) => `• ${d}`),\n        `_${new Date(alert.at).toISOString()}_`,\n      ]\n      const response = await fetchImpl(options.webhookUrl, {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/json' },\n        body: JSON.stringify({ text: lines.join('\\n') }),\n      })\n      if (!response.ok) {\n        // A dropped alert is a silent failure of the silent-failure detector.\n        // It must be loud in the one place that is still working: the log.\n        throw new Error(`alert webhook responded ${response.status}`)\n      }\n    },\n  }\n}\n\n/** stderr sink. The zero-config fallback so a product that has not yet been\n *  given a webhook still emits something a log search can find. */\nexport function createConsoleAlertSink(log: (message: string) => void = console.error): AlertSink {\n  return {\n    async deliver(alert) {\n      log(\n        `[turn-health] ${alert.severity.toUpperCase()} ${alert.title} :: ${alert.details.join(' | ')}`,\n      )\n    },\n  }\n}\n\n/** Fan out to several sinks. One failing transport must not stop the others. */\nexport function createMultiAlertSink(sinks: readonly AlertSink[]): AlertSink {\n  return {\n    async deliver(alert) {\n      const settled = await Promise.allSettled(sinks.map((s) => s.deliver(alert)))\n      const failures = settled.filter((r) => r.status === 'rejected')\n      if (failures.length === sinks.length && sinks.length > 0) {\n        throw new Error('every alert sink failed')\n      }\n    },\n  }\n}\n\n// ── throttling ────────────────────────────────────────────────────────────\n\n/** Records the last time a key was alerted on. A product backs this with KV,\n *  D1, or a Durable Object; the in-memory default is correct for a sweep that\n *  runs as a single cron invocation. */\nexport interface AlertThrottleStore {\n  lastSentAt(key: string): Promise<number | null>\n  markSent(key: string, at: number): Promise<void>\n}\n\n/** Process-local throttle store. */\nexport function createMemoryThrottleStore(): AlertThrottleStore {\n  const seen = new Map<string, number>()\n  return {\n    async lastSentAt(key) {\n      return seen.get(key) ?? null\n    },\n    async markSent(key, at) {\n      seen.set(key, at)\n    },\n  }\n}\n\n/**\n * Collapse repeats of the same `key` inside `windowMs`.\n *\n * Deliberately re-alerts once per window rather than going silent after the\n * first: an incident that is still burning must keep saying so. Going quiet\n * after one message is how a 17-day outage stays invisible after someone\n * dismisses the first notification.\n */\nexport function createThrottledAlertSink(\n  inner: AlertSink,\n  options: { windowMs: number; store?: AlertThrottleStore },\n): AlertSink {\n  const store = options.store ?? createMemoryThrottleStore()\n  return {\n    async deliver(alert) {\n      const last = await store.lastSentAt(alert.key)\n      if (last !== null && alert.at - last < options.windowMs) return\n      await inner.deliver(alert)\n      await store.markSent(alert.key, alert.at)\n    },\n  }\n}\n\n/**\n * Swallow transport errors so telemetry can never fail the turn it measures.\n *\n * Use this at the LIVE lifecycle call site only. A sweep should let the error\n * surface, because a sweep that cannot deliver has done nothing at all and its\n * cron run should go red.\n */\nexport function createGuardedAlertSink(\n  inner: AlertSink,\n  onError: (error: unknown) => void = (e) => console.error('[turn-health] alert delivery failed', e),\n): AlertSink {\n  return {\n    async deliver(alert) {\n      try {\n        await inner.deliver(alert)\n      } catch (error) {\n        onError(error)\n      }\n    },\n  }\n}\n","/**\n * The LIVE half: judge each turn the moment it settles.\n *\n * This adds no control flow. `createChatTurnRoutes` already exposes a\n * `lifecycle` seam that fires exactly one of `onTurnComplete`/`onTurnError`\n * after a turn settles, and already swallows hook errors so telemetry cannot\n * fail a turn. That seam was shipped and then wired by nobody, which is a fair\n * description of why the outage lasted 17 days. This function fills it.\n *\n * The shape is declared structurally rather than imported from\n * `/chat-routes`, so `/turn-health` stays free of the server chat vertical and\n * can be used by any turn driver that reports the same three moments.\n */\n\nimport { classifyTurnOutcome } from './classify.js'\nimport { type AlertSink, createGuardedAlertSink, turnAlert } from './sink.js'\n\n/** Structural mirror of `/chat-routes`' `ChatTurnLifecycle` complete payload. */\nexport interface TurnHealthCompleteInfo {\n  finalText: string\n  usage?: { outputTokens?: number | null } | null\n  durationMs: number\n  threadId?: string\n  turnStreamId?: string\n  executionId?: string\n}\n\n/** Structural mirror of the lifecycle error payload. */\nexport interface TurnHealthErrorInfo {\n  error: unknown\n  durationMs: number\n  threadId?: string\n  turnStreamId?: string\n  executionId?: string\n}\n\n/** What {@link createTurnHealthLifecycle} returns — assignable to\n *  `createChatTurnRoutes`' `lifecycle` option. */\nexport interface TurnHealthLifecycle {\n  onTurnComplete(info: TurnHealthCompleteInfo): Promise<void>\n  onTurnError(info: TurnHealthErrorInfo): Promise<void>\n}\n\nexport interface TurnHealthLifecycleOptions {\n  /** Names the product in every alert. */\n  product: string\n  sink: AlertSink\n  /** Called for every verdict, healthy or not — the hook for a counter or a\n   *  metrics push. Alerts are for humans; this is for graphs. */\n  onVerdict?(verdict: {\n    product: string\n    healthy: boolean\n    kinds: string[]\n    durationMs: number\n  }): void\n}\n\nfunction errorText(error: unknown): string {\n  if (error instanceof Error) return error.message\n  if (typeof error === 'string') return error\n  return String(error)\n}\n\n/**\n * Build the lifecycle hooks that page on a turn which succeeded at nothing.\n *\n * The live lane sees `finalText` and usage but not the persisted parts, so it\n * catches the blank-completion and hard-failure shapes immediately. The\n * parts-dependent shapes (a tool call whose arguments never parsed, a tool\n * call that left no effect) are caught by {@link sweepSilentFailures}, which\n * reads what was actually written to the store — the honest place to ask\n * whether an effect persisted.\n */\nexport function createTurnHealthLifecycle(\n  options: TurnHealthLifecycleOptions,\n): TurnHealthLifecycle {\n  // Guarded: a paging failure must never take down a customer's turn.\n  const sink = createGuardedAlertSink(options.sink)\n\n  return {\n    async onTurnComplete(info) {\n      const verdict = classifyTurnOutcome({\n        finalText: info.finalText,\n        outputTokens: info.usage?.outputTokens ?? null,\n        durationMs: info.durationMs,\n      })\n      options.onVerdict?.({\n        product: options.product,\n        healthy: verdict.healthy,\n        kinds: verdict.reasons.map((r) => r.kind),\n        durationMs: info.durationMs,\n      })\n      if (verdict.healthy || verdict.severity === null) return\n      await sink.deliver(\n        turnAlert({\n          product: options.product,\n          severity: verdict.severity,\n          reasons: verdict.reasons,\n          ...(info.threadId ? { threadId: info.threadId } : {}),\n          ...(info.executionId ? { turnId: info.executionId } : {}),\n        }),\n      )\n    },\n\n    async onTurnError(info) {\n      const verdict = classifyTurnOutcome({\n        failed: true,\n        failureReason: errorText(info.error),\n        durationMs: info.durationMs,\n      })\n      options.onVerdict?.({\n        product: options.product,\n        healthy: false,\n        kinds: verdict.reasons.map((r) => r.kind),\n        durationMs: info.durationMs,\n      })\n      await sink.deliver(\n        turnAlert({\n          product: options.product,\n          severity: 'critical',\n          reasons: verdict.reasons,\n          ...(info.threadId ? { threadId: info.threadId } : {}),\n          ...(info.executionId ? { turnId: info.executionId } : {}),\n        }),\n      )\n    },\n  }\n}\n","/**\n * The SWEEP half: ask the store what it has been quietly accumulating.\n *\n * A live per-turn hook cannot see the failure that matters most, because the\n * worst outage produced NO turns at all to hook: gtm-agent took 9–21 user\n * messages a day for sixteen straight days and wrote zero real assistant\n * replies. Nothing crashed on a schedule; the product simply stopped\n * answering. The only thing that could have noticed is something that\n * periodically counts what arrived against what was answered.\n *\n * That is this. It runs on a cron, reads the shared `/chat-store` schema, and\n * pages when the ratio breaks.\n *\n * The queries live HERE and not in each product because all four products\n * (gtm, tax, legal, workcomp) persist to the same `message`/`thread` tables —\n * four copies of this cron is exactly the duplication the repo's engine/shell\n * rule exists to prevent.\n */\n\nimport { classifyTurnOutcome, describeReason, type TurnHealthReason } from './classify.js'\nimport type { AlertSink, TurnHealthAlert } from './sink.js'\n\n/** A thread that has taken user messages with no reply since. */\nexport interface UnansweredThread {\n  threadId: string\n  /** User messages newer than the newest real assistant reply. */\n  pendingMessages: number\n  /** Age of the OLDEST unanswered user message, in ms. */\n  oldestAgeMs: number\n}\n\n/** A persisted assistant row, as the sweep needs to judge it. */\nexport interface PersistedTurnRow {\n  id: string\n  threadId: string\n  content: string\n  /** Raw `parts` column. A JSON string or an already-parsed array; the sweep\n   *  accepts both because D1 drivers differ. */\n  parts: unknown\n  outputTokens?: number | null\n  model?: string | null\n  createdAt: number\n}\n\n/** What the sweep needs from a store. A product on a non-standard schema\n *  implements these two reads; everything else is shared. */\nexport interface TurnHealthSource {\n  findUnansweredThreads(input: { minAgeMs: number; now: number }): Promise<UnansweredThread[]>\n  listRecentAssistantTurns(input: { sinceMs: number; now: number; limit: number }): Promise<\n    PersistedTurnRow[]\n  >\n}\n\nexport interface SweepOptions {\n  product: string\n  source: TurnHealthSource\n  sink: AlertSink\n  /** A user message must go unanswered this long before it counts. Guards\n   *  against alerting on a turn that is simply still streaming. Default 15 min. */\n  minAgeMs?: number\n  /** How far back to judge settled turns. Default 24 h. */\n  lookbackMs?: number\n  /** Cap on rows judged per sweep. Default 500. */\n  limit?: number\n  /** Fraction of recent turns allowed to be silently broken before paging.\n   *  Default 0.05 — the measured blank-completion rate on the tax tool surface\n   *  was 12.2%, so 5% separates a real regression from noise. */\n  emptyRateThreshold?: number\n  /** Absolute floor: never page on a rate computed from fewer turns than this. */\n  minTurnsForRate?: number\n  now?: number\n}\n\n/** What the sweep found. Returned as well as alerted, so a cron can log it and\n *  a test can assert on it. */\nexport interface SweepResult {\n  product: string\n  unansweredThreads: number\n  pendingUserMessages: number\n  oldestUnansweredMs: number\n  turnsJudged: number\n  unhealthyTurns: number\n  emptyCompletions: number\n  malformedToolCalls: number\n  toolCallsWithoutEffect: number\n  alerts: TurnHealthAlert[]\n}\n\nfunction parseParts(raw: unknown): unknown[] {\n  if (Array.isArray(raw)) return raw\n  if (typeof raw !== 'string' || raw.trim().length === 0) return []\n  try {\n    const parsed = JSON.parse(raw)\n    return Array.isArray(parsed) ? parsed : []\n  } catch {\n    // A parts column that is not JSON is itself a corruption worth seeing, but\n    // it is not this detector's job — treat as no parts rather than throwing.\n    return []\n  }\n}\n\nconst HOUR_MS = 3_600_000\n\n/**\n * Run one sweep and deliver whatever it finds.\n *\n * Errors from the sink are NOT swallowed here (unlike the live lane): a sweep\n * that could not deliver has accomplished nothing, and its cron invocation\n * should go red rather than report a clean run.\n */\nexport async function sweepSilentFailures(options: SweepOptions): Promise<SweepResult> {\n  const now = options.now ?? Date.now()\n  const minAgeMs = options.minAgeMs ?? 15 * 60_000\n  const lookbackMs = options.lookbackMs ?? 24 * HOUR_MS\n  const limit = options.limit ?? 500\n  const emptyRateThreshold = options.emptyRateThreshold ?? 0.05\n  const minTurnsForRate = options.minTurnsForRate ?? 10\n\n  const [unanswered, turns] = await Promise.all([\n    options.source.findUnansweredThreads({ minAgeMs, now }),\n    options.source.listRecentAssistantTurns({ sinceMs: now - lookbackMs, now, limit }),\n  ])\n\n  const alerts: TurnHealthAlert[] = []\n\n  // ── silence: messages in, nothing out ──────────────────────────────────\n  const pendingUserMessages = unanswered.reduce((sum, t) => sum + t.pendingMessages, 0)\n  const oldestUnansweredMs = unanswered.reduce((max, t) => Math.max(max, t.oldestAgeMs), 0)\n\n  if (unanswered.length > 0) {\n    const hours = (oldestUnansweredMs / HOUR_MS).toFixed(1)\n    alerts.push({\n      product: options.product,\n      // A day of total silence is not a warning.\n      severity: oldestUnansweredMs >= 24 * HOUR_MS ? 'critical' : 'warning',\n      key: `sweep:${options.product}:unanswered_threads`,\n      title: `${options.product}: ${pendingUserMessages} user message(s) unanswered across ${unanswered.length} thread(s)`,\n      details: [\n        `oldest unanswered message: ${hours}h`,\n        ...unanswered\n          .slice(0, 5)\n          .map(\n            (t) =>\n              `thread ${t.threadId}: ${t.pendingMessages} pending, oldest ${(\n                t.oldestAgeMs / HOUR_MS\n              ).toFixed(1)}h`,\n          ),\n      ],\n      data: {\n        unansweredThreads: unanswered.length,\n        pendingUserMessages,\n        oldestUnansweredMs,\n      },\n      at: now,\n    })\n  }\n\n  // ── success that delivered nothing ─────────────────────────────────────\n  let emptyCompletions = 0\n  let malformedToolCalls = 0\n  let toolCallsWithoutEffect = 0\n  let unhealthyTurns = 0\n  const malformedSamples: TurnHealthReason[] = []\n\n  for (const row of turns) {\n    const verdict = classifyTurnOutcome({\n      finalText: row.content,\n      parts: parseParts(row.parts),\n      outputTokens: row.outputTokens ?? null,\n    })\n    if (verdict.healthy) continue\n    unhealthyTurns += 1\n    for (const reason of verdict.reasons) {\n      if (reason.kind === 'empty_completion') emptyCompletions += 1\n      if (reason.kind === 'malformed_tool_call') {\n        malformedToolCalls += 1\n        if (malformedSamples.length < 3) malformedSamples.push(reason)\n      }\n      if (reason.kind === 'tool_call_no_effect') toolCallsWithoutEffect += 1\n    }\n  }\n\n  // A malformed tool call is never acceptable at any rate — it means a\n  // deliverable was requested and silently discarded. Page on the first one.\n  if (malformedToolCalls > 0) {\n    alerts.push({\n      product: options.product,\n      severity: 'critical',\n      key: `sweep:${options.product}:malformed_tool_call`,\n      title: `${options.product}: ${malformedToolCalls} tool call(s) had unparseable arguments — deliverables silently dropped`,\n      details: malformedSamples.map(describeReason),\n      data: { malformedToolCalls, turnsJudged: turns.length },\n      at: now,\n    })\n  }\n\n  if (turns.length >= minTurnsForRate) {\n    const rate = emptyCompletions / turns.length\n    if (rate > emptyRateThreshold) {\n      alerts.push({\n        product: options.product,\n        severity: 'critical',\n        key: `sweep:${options.product}:empty_completion_rate`,\n        title: `${options.product}: ${(rate * 100).toFixed(1)}% of turns completed with no output`,\n        details: [\n          `${emptyCompletions} of ${turns.length} settled turns delivered nothing`,\n          `threshold ${(emptyRateThreshold * 100).toFixed(1)}%`,\n        ],\n        data: { emptyCompletions, turnsJudged: turns.length, rate },\n        at: now,\n      })\n    }\n  }\n\n  for (const alert of alerts) await options.sink.deliver(alert)\n\n  return {\n    product: options.product,\n    unansweredThreads: unanswered.length,\n    pendingUserMessages,\n    oldestUnansweredMs,\n    turnsJudged: turns.length,\n    unhealthyTurns,\n    emptyCompletions,\n    malformedToolCalls,\n    toolCallsWithoutEffect,\n    alerts,\n  }\n}\n\n// ── D1 source for the shared chat-store schema ────────────────────────────\n\n/** Minimal structural D1 contract (Cloudflare's `D1Database` satisfies it). */\nexport interface D1LikeForHealth {\n  prepare(sql: string): {\n    bind(...values: unknown[]): {\n      all<T = Record<string, unknown>>(): Promise<{ results: T[] }>\n    }\n  }\n}\n\n/**\n * The sweep source for products on the canonical `/chat-store` tables.\n *\n * \"Answered\" deliberately means an assistant row with NON-EMPTY content. A\n * blank assistant row is what a broken turn writes, so counting it as an\n * answer would let the exact failure being hunted mark itself resolved. That\n * single predicate is the difference between this catching the gtm outage and\n * sleeping through it — during those sixteen days the table was NOT empty.\n *\n * The query deliberately does NOT join the thread table. Products do not all\n * keep one: tax-agent's `thread` table holds zero rows because it groups by\n * its own `tax_sessions`, and an inner join against it silently reported \"0\n * unanswered threads, healthy\" while 18 real messages sat unanswered. A\n * detector that reports healthy because its join found nothing is the same\n * bug class it was built to catch.\n */\nexport function createD1TurnHealthSource(\n  db: D1LikeForHealth,\n  options: { messageTable?: string; threadTable?: string } = {},\n): TurnHealthSource {\n  // Table names are identifiers and cannot be bound as parameters. They come\n  // from deploy-time product config, never from a request, and are validated\n  // here so this can never become an injection point.\n  const message = safeIdentifier(options.messageTable ?? 'message')\n  const thread = safeIdentifier(options.threadTable ?? 'thread')\n\n  return {\n    async findUnansweredThreads({ minAgeMs, now }) {\n      const cutoffSeconds = Math.floor((now - minAgeMs) / 1000)\n      const { results } = await db\n        .prepare(\n          `SELECT m.thread_id AS threadId,\n                  COUNT(*) AS pendingMessages,\n                  MIN(m.created_at) AS oldestCreatedAt\n             FROM ${message} m\n            WHERE m.role = 'user'\n              AND m.created_at <= ?1\n              AND m.created_at > COALESCE(\n                    (SELECT MAX(a.created_at)\n                       FROM ${message} a\n                      WHERE a.thread_id = m.thread_id\n                        AND a.role = 'assistant'\n                        AND length(trim(a.content)) > 0), 0)\n            GROUP BY m.thread_id\n            ORDER BY oldestCreatedAt ASC`,\n        )\n        .bind(cutoffSeconds)\n        .all<{ threadId: string; pendingMessages: number; oldestCreatedAt: number }>()\n\n      return results.map((row) => ({\n        threadId: row.threadId,\n        pendingMessages: Number(row.pendingMessages),\n        oldestAgeMs: now - Number(row.oldestCreatedAt) * 1000,\n      }))\n    },\n\n    async listRecentAssistantTurns({ sinceMs, limit }) {\n      const sinceSeconds = Math.floor(sinceMs / 1000)\n      const { results } = await db\n        .prepare(\n          `SELECT id, thread_id AS threadId, content, parts,\n                  output_tokens AS outputTokens, model, created_at AS createdAt\n             FROM ${message}\n            WHERE role = 'assistant' AND created_at >= ?1\n            ORDER BY created_at DESC\n            LIMIT ?2`,\n        )\n        .bind(sinceSeconds, limit)\n        .all<Record<string, unknown>>()\n\n      return results.map((row) => ({\n        id: String(row.id),\n        threadId: String(row.threadId),\n        content: typeof row.content === 'string' ? row.content : '',\n        parts: row.parts,\n        outputTokens: row.outputTokens === null ? null : Number(row.outputTokens),\n        model: (row.model as string | null) ?? null,\n        createdAt: Number(row.createdAt) * 1000,\n      }))\n    },\n  }\n\n  function safeIdentifier(name: string): string {\n    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {\n      throw new Error(`unsafe table identifier: ${name}`)\n    }\n    return name\n  }\n}\n"],"mappings":";AAwGA,IAAM,sBAAsB,oBAAI,IAAI,CAAC,QAAQ,SAAS,gBAAgB,QAAQ,aAAa,CAAC;AAE5F,IAAM,eAAe;AAErB,SAAS,SAAS,OAAgD;AAChE,SAAO,OAAO,UAAU,YAAY,UAAU,OAAQ,QAAoC;AAC5F;AAEA,SAAS,eAAe,OAA+B;AACrD,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAQA,SAAS,kBAAkB,OAAwB;AACjD,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,MAAI;AACF,SAAK,MAAM,OAAO;AAClB,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,IAAM,wBAAwB,oBAAI,IAAI,CAAC,aAAa,YAAY,WAAW,MAAM,CAAC;AAS3E,SAAS,oBAAoB,OAA4C;AAC9E,QAAM,UAA8B,CAAC;AAErC,MAAI,MAAM,QAAQ;AAChB,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,QAAQ,eAAe,MAAM,aAAa,KAAK;AAAA,IACjD,CAAC;AAAA,EACH;AAEA,QAAM,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,MAAM,QAAQ,CAAC;AAE1D,MAAI,iBAAiB,eAAe,MAAM,SAAS,MAAM;AACzD,MAAI,gBAAgB;AAEpB,aAAW,OAAO,OAAO;AACvB,UAAM,OAAO,SAAS,GAAG;AACzB,QAAI,CAAC,KAAM;AACX,UAAM,OAAO,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;AAEzD,QAAI,SAAS,UAAU,eAAe,KAAK,IAAI,MAAM,MAAM;AACzD,uBAAiB;AACjB;AAAA,IACF;AACA,QAAI,oBAAoB,IAAI,IAAI,GAAG;AACjC,uBAAiB;AACjB;AAAA,IACF;AACA,QAAI,SAAS,OAAQ;AAErB,UAAM,OAAO,eAAe,KAAK,IAAI,KAAK;AAC1C,UAAM,QAAQ,SAAS,KAAK,KAAK;AACjC,UAAM,SAAS,OAAO,OAAO,WAAW,WAAW,MAAM,SAAS;AAKlE,UAAM,YAAY,OAAO;AACzB,QAAI,OAAO,cAAc,YAAY,kBAAkB,SAAS,GAAG;AACjE,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN;AAAA,QACA,aAAa,UAAU;AAAA,QACvB,QAAQ,UAAU,MAAM,GAAG,YAAY;AAAA,MACzC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,IAAI,MAAM,GAAG;AACtC,cAAQ,KAAK,EAAE,MAAM,uBAAuB,MAAM,OAAO,CAAC;AAAA,IAC5D;AAAA,EACF;AAKA,MAAI,CAAC,MAAM,UAAU,CAAC,kBAAkB,kBAAkB,GAAG;AAC3D,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,cAAc,MAAM,gBAAgB;AAAA,MACpC,WAAW,MAAM;AAAA,MACjB,GAAI,MAAM,eAAe,SAAY,EAAE,YAAY,MAAM,WAAW,IAAI,CAAC;AAAA,IAC3E,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,SAAS,QAAQ,WAAW;AAAA,IAC5B,UAAU,WAAW,OAAO;AAAA,IAC5B;AAAA,EACF;AACF;AAKA,SAAS,WAAW,SAAwD;AAC1E,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,QAAM,WAAW,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,sBAAsB,EAAE,SAAS,aAAa;AAC9F,SAAO,WAAW,aAAa;AACjC;AAGO,SAAS,eAAe,QAAkC;AAC/D,UAAQ,OAAO,MAAM;AAAA,IACnB,KAAK;AACH,aAAO,6BAA6B,OAAO,SAAS,wBAClD,OAAO,gBAAgB,SACzB;AAAA,IACF,KAAK;AACH,aAAO,UAAU,OAAO,IAAI,+BAA+B,OAAO,WAAW,YAAY,OAAO,MAAM;AAAA,IACxG,KAAK;AACH,aAAO,UAAU,OAAO,IAAI,6BAA6B,OAAO,MAAM;AAAA,IACxE,KAAK;AACH,aAAO,gBAAgB,OAAO,MAAM;AAAA,EACxC;AACF;;;AC1MO,SAAS,UAAU,OAQN;AAClB,QAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK;AAClE,SAAO;AAAA,IACL,SAAS,MAAM;AAAA,IACf,UAAU,MAAM;AAAA;AAAA;AAAA,IAGhB,KAAK,QAAQ,MAAM,OAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAAA,IAC7C,OAAO,GAAG,MAAM,OAAO,2CAA2C,MAAM,KAAK,IAAI,CAAC;AAAA,IAClF,SAAS,MAAM,QAAQ,IAAI,cAAc;AAAA,IACzC,MAAM;AAAA,MACJ;AAAA,MACA,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MACrD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;AAAA,MAC/C,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;AAAA,IAC9C;AAAA,IACA,IAAI,MAAM,MAAM,KAAK,IAAI;AAAA,EAC3B;AACF;AAmBO,SAAS,uBAAuB,SAGzB;AACZ,QAAM,YAAY,QAAQ,aAAc,WAAW;AACnD,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,OAAO,MAAM,aAAa,aAAa,qBAAqB;AAClE,YAAM,QAAQ;AAAA,QACZ,GAAG,IAAI,KAAK,MAAM,KAAK;AAAA,QACvB,GAAG,MAAM,QAAQ,IAAI,CAAC,MAAM,UAAK,CAAC,EAAE;AAAA,QACpC,IAAI,IAAI,KAAK,MAAM,EAAE,EAAE,YAAY,CAAC;AAAA,MACtC;AACA,YAAM,WAAW,MAAM,UAAU,QAAQ,YAAY;AAAA,QACnD,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,EAAE,MAAM,MAAM,KAAK,IAAI,EAAE,CAAC;AAAA,MACjD,CAAC;AACD,UAAI,CAAC,SAAS,IAAI;AAGhB,cAAM,IAAI,MAAM,2BAA2B,SAAS,MAAM,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,uBAAuB,MAAiC,QAAQ,OAAkB;AAChG,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB;AAAA,QACE,iBAAiB,MAAM,SAAS,YAAY,CAAC,IAAI,MAAM,KAAK,OAAO,MAAM,QAAQ,KAAK,KAAK,CAAC;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AACF;AAGO,SAAS,qBAAqB,OAAwC;AAC3E,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,UAAU,MAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,CAAC,CAAC;AAC3E,YAAM,WAAW,QAAQ,OAAO,CAAC,MAAM,EAAE,WAAW,UAAU;AAC9D,UAAI,SAAS,WAAW,MAAM,UAAU,MAAM,SAAS,GAAG;AACxD,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAAA,IACF;AAAA,EACF;AACF;AAaO,SAAS,4BAAgD;AAC9D,QAAM,OAAO,oBAAI,IAAoB;AACrC,SAAO;AAAA,IACL,MAAM,WAAW,KAAK;AACpB,aAAO,KAAK,IAAI,GAAG,KAAK;AAAA,IAC1B;AAAA,IACA,MAAM,SAAS,KAAK,IAAI;AACtB,WAAK,IAAI,KAAK,EAAE;AAAA,IAClB;AAAA,EACF;AACF;AAUO,SAAS,yBACd,OACA,SACW;AACX,QAAM,QAAQ,QAAQ,SAAS,0BAA0B;AACzD,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,OAAO,MAAM,MAAM,WAAW,MAAM,GAAG;AAC7C,UAAI,SAAS,QAAQ,MAAM,KAAK,OAAO,QAAQ,SAAU;AACzD,YAAM,MAAM,QAAQ,KAAK;AACzB,YAAM,MAAM,SAAS,MAAM,KAAK,MAAM,EAAE;AAAA,IAC1C;AAAA,EACF;AACF;AASO,SAAS,uBACd,OACA,UAAoC,CAAC,MAAM,QAAQ,MAAM,uCAAuC,CAAC,GACtF;AACX,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,UAAI;AACF,cAAM,MAAM,QAAQ,KAAK;AAAA,MAC3B,SAAS,OAAO;AACd,gBAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAAA,EACF;AACF;;;ACjJA,SAAS,UAAU,OAAwB;AACzC,MAAI,iBAAiB,MAAO,QAAO,MAAM;AACzC,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,SAAO,OAAO,KAAK;AACrB;AAYO,SAAS,0BACd,SACqB;AAErB,QAAM,OAAO,uBAAuB,QAAQ,IAAI;AAEhD,SAAO;AAAA,IACL,MAAM,eAAe,MAAM;AACzB,YAAM,UAAU,oBAAoB;AAAA,QAClC,WAAW,KAAK;AAAA,QAChB,cAAc,KAAK,OAAO,gBAAgB;AAAA,QAC1C,YAAY,KAAK;AAAA,MACnB,CAAC;AACD,cAAQ,YAAY;AAAA,QAClB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,QACjB,OAAO,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,QACxC,YAAY,KAAK;AAAA,MACnB,CAAC;AACD,UAAI,QAAQ,WAAW,QAAQ,aAAa,KAAM;AAClD,YAAM,KAAK;AAAA,QACT,UAAU;AAAA,UACR,SAAS,QAAQ;AAAA,UACjB,UAAU,QAAQ;AAAA,UAClB,SAAS,QAAQ;AAAA,UACjB,GAAI,KAAK,WAAW,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC;AAAA,UACnD,GAAI,KAAK,cAAc,EAAE,QAAQ,KAAK,YAAY,IAAI,CAAC;AAAA,QACzD,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IAEA,MAAM,YAAY,MAAM;AACtB,YAAM,UAAU,oBAAoB;AAAA,QAClC,QAAQ;AAAA,QACR,eAAe,UAAU,KAAK,KAAK;AAAA,QACnC,YAAY,KAAK;AAAA,MACnB,CAAC;AACD,cAAQ,YAAY;AAAA,QAClB,SAAS,QAAQ;AAAA,QACjB,SAAS;AAAA,QACT,OAAO,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,QACxC,YAAY,KAAK;AAAA,MACnB,CAAC;AACD,YAAM,KAAK;AAAA,QACT,UAAU;AAAA,UACR,SAAS,QAAQ;AAAA,UACjB,UAAU;AAAA,UACV,SAAS,QAAQ;AAAA,UACjB,GAAI,KAAK,WAAW,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC;AAAA,UACnD,GAAI,KAAK,cAAc,EAAE,QAAQ,KAAK,YAAY,IAAI,CAAC;AAAA,QACzD,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;;;ACvCA,SAAS,WAAW,KAAyB;AAC3C,MAAI,MAAM,QAAQ,GAAG,EAAG,QAAO;AAC/B,MAAI,OAAO,QAAQ,YAAY,IAAI,KAAK,EAAE,WAAW,EAAG,QAAO,CAAC;AAChE,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,WAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC;AAAA,EAC3C,QAAQ;AAGN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,IAAM,UAAU;AAShB,eAAsB,oBAAoB,SAA6C;AACrF,QAAM,MAAM,QAAQ,OAAO,KAAK,IAAI;AACpC,QAAM,WAAW,QAAQ,YAAY,KAAK;AAC1C,QAAM,aAAa,QAAQ,cAAc,KAAK;AAC9C,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,qBAAqB,QAAQ,sBAAsB;AACzD,QAAM,kBAAkB,QAAQ,mBAAmB;AAEnD,QAAM,CAAC,YAAY,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IAC5C,QAAQ,OAAO,sBAAsB,EAAE,UAAU,IAAI,CAAC;AAAA,IACtD,QAAQ,OAAO,yBAAyB,EAAE,SAAS,MAAM,YAAY,KAAK,MAAM,CAAC;AAAA,EACnF,CAAC;AAED,QAAM,SAA4B,CAAC;AAGnC,QAAM,sBAAsB,WAAW,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,iBAAiB,CAAC;AACpF,QAAM,qBAAqB,WAAW,OAAO,CAAC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,WAAW,GAAG,CAAC;AAExF,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,SAAS,qBAAqB,SAAS,QAAQ,CAAC;AACtD,WAAO,KAAK;AAAA,MACV,SAAS,QAAQ;AAAA;AAAA,MAEjB,UAAU,sBAAsB,KAAK,UAAU,aAAa;AAAA,MAC5D,KAAK,SAAS,QAAQ,OAAO;AAAA,MAC7B,OAAO,GAAG,QAAQ,OAAO,KAAK,mBAAmB,sCAAsC,WAAW,MAAM;AAAA,MACxG,SAAS;AAAA,QACP,8BAA8B,KAAK;AAAA,QACnC,GAAG,WACA,MAAM,GAAG,CAAC,EACV;AAAA,UACC,CAAC,MACC,UAAU,EAAE,QAAQ,KAAK,EAAE,eAAe,qBACxC,EAAE,cAAc,SAChB,QAAQ,CAAC,CAAC;AAAA,QAChB;AAAA,MACJ;AAAA,MACA,MAAM;AAAA,QACJ,mBAAmB,WAAW;AAAA,QAC9B;AAAA,QACA;AAAA,MACF;AAAA,MACA,IAAI;AAAA,IACN,CAAC;AAAA,EACH;AAGA,MAAI,mBAAmB;AACvB,MAAI,qBAAqB;AACzB,MAAI,yBAAyB;AAC7B,MAAI,iBAAiB;AACrB,QAAM,mBAAuC,CAAC;AAE9C,aAAW,OAAO,OAAO;AACvB,UAAM,UAAU,oBAAoB;AAAA,MAClC,WAAW,IAAI;AAAA,MACf,OAAO,WAAW,IAAI,KAAK;AAAA,MAC3B,cAAc,IAAI,gBAAgB;AAAA,IACpC,CAAC;AACD,QAAI,QAAQ,QAAS;AACrB,sBAAkB;AAClB,eAAW,UAAU,QAAQ,SAAS;AACpC,UAAI,OAAO,SAAS,mBAAoB,qBAAoB;AAC5D,UAAI,OAAO,SAAS,uBAAuB;AACzC,8BAAsB;AACtB,YAAI,iBAAiB,SAAS,EAAG,kBAAiB,KAAK,MAAM;AAAA,MAC/D;AACA,UAAI,OAAO,SAAS,sBAAuB,2BAA0B;AAAA,IACvE;AAAA,EACF;AAIA,MAAI,qBAAqB,GAAG;AAC1B,WAAO,KAAK;AAAA,MACV,SAAS,QAAQ;AAAA,MACjB,UAAU;AAAA,MACV,KAAK,SAAS,QAAQ,OAAO;AAAA,MAC7B,OAAO,GAAG,QAAQ,OAAO,KAAK,kBAAkB;AAAA,MAChD,SAAS,iBAAiB,IAAI,cAAc;AAAA,MAC5C,MAAM,EAAE,oBAAoB,aAAa,MAAM,OAAO;AAAA,MACtD,IAAI;AAAA,IACN,CAAC;AAAA,EACH;AAEA,MAAI,MAAM,UAAU,iBAAiB;AACnC,UAAM,OAAO,mBAAmB,MAAM;AACtC,QAAI,OAAO,oBAAoB;AAC7B,aAAO,KAAK;AAAA,QACV,SAAS,QAAQ;AAAA,QACjB,UAAU;AAAA,QACV,KAAK,SAAS,QAAQ,OAAO;AAAA,QAC7B,OAAO,GAAG,QAAQ,OAAO,MAAM,OAAO,KAAK,QAAQ,CAAC,CAAC;AAAA,QACrD,SAAS;AAAA,UACP,GAAG,gBAAgB,OAAO,MAAM,MAAM;AAAA,UACtC,cAAc,qBAAqB,KAAK,QAAQ,CAAC,CAAC;AAAA,QACpD;AAAA,QACA,MAAM,EAAE,kBAAkB,aAAa,MAAM,QAAQ,KAAK;AAAA,QAC1D,IAAI;AAAA,MACN,CAAC;AAAA,IACH;AAAA,EACF;AAEA,aAAW,SAAS,OAAQ,OAAM,QAAQ,KAAK,QAAQ,KAAK;AAE5D,SAAO;AAAA,IACL,SAAS,QAAQ;AAAA,IACjB,mBAAmB,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,IACA,aAAa,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AA6BO,SAAS,yBACd,IACA,UAA2D,CAAC,GAC1C;AAIlB,QAAM,UAAU,eAAe,QAAQ,gBAAgB,SAAS;AAChE,QAAM,SAAS,eAAe,QAAQ,eAAe,QAAQ;AAE7D,SAAO;AAAA,IACL,MAAM,sBAAsB,EAAE,UAAU,IAAI,GAAG;AAC7C,YAAM,gBAAgB,KAAK,OAAO,MAAM,YAAY,GAAI;AACxD,YAAM,EAAE,QAAQ,IAAI,MAAM,GACvB;AAAA,QACC;AAAA;AAAA;AAAA,oBAGU,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,8BAKG,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAM7B,EACC,KAAK,aAAa,EAClB,IAA4E;AAE/E,aAAO,QAAQ,IAAI,CAAC,SAAS;AAAA,QAC3B,UAAU,IAAI;AAAA,QACd,iBAAiB,OAAO,IAAI,eAAe;AAAA,QAC3C,aAAa,MAAM,OAAO,IAAI,eAAe,IAAI;AAAA,MACnD,EAAE;AAAA,IACJ;AAAA,IAEA,MAAM,yBAAyB,EAAE,SAAS,MAAM,GAAG;AACjD,YAAM,eAAe,KAAK,MAAM,UAAU,GAAI;AAC9C,YAAM,EAAE,QAAQ,IAAI,MAAM,GACvB;AAAA,QACC;AAAA;AAAA,oBAEU,OAAO;AAAA;AAAA;AAAA;AAAA,MAInB,EACC,KAAK,cAAc,KAAK,EACxB,IAA6B;AAEhC,aAAO,QAAQ,IAAI,CAAC,SAAS;AAAA,QAC3B,IAAI,OAAO,IAAI,EAAE;AAAA,QACjB,UAAU,OAAO,IAAI,QAAQ;AAAA,QAC7B,SAAS,OAAO,IAAI,YAAY,WAAW,IAAI,UAAU;AAAA,QACzD,OAAO,IAAI;AAAA,QACX,cAAc,IAAI,iBAAiB,OAAO,OAAO,OAAO,IAAI,YAAY;AAAA,QACxE,OAAQ,IAAI,SAA2B;AAAA,QACvC,WAAW,OAAO,IAAI,SAAS,IAAI;AAAA,MACrC,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,eAAe,MAAsB;AAC5C,QAAI,CAAC,2BAA2B,KAAK,IAAI,GAAG;AAC1C,YAAM,IAAI,MAAM,4BAA4B,IAAI,EAAE;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACF;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-app",
-  "version": "0.44.3",
+  "version": "0.44.5",
   "packageManager": "pnpm@10.33.4",
   "description": "Application-shell framework for Tangle agent products: a bounded tool loop, the structured agent→app tool side channel, integration-hub client, per-workspace billing, and crypto — composed over the Tangle agent substrate through typed seams.",
   "keywords": [
@@ -177,6 +177,11 @@
       "import": "./dist/turn-stream/index.js",
       "default": "./dist/turn-stream/index.js"
     },
+    "./turn-health": {
+      "types": "./dist/turn-health/index.d.ts",
+      "import": "./dist/turn-health/index.js",
+      "default": "./dist/turn-health/index.js"
+    },
     "./integrations": {
       "types": "./dist/integrations/index.d.ts",
       "import": "./dist/integrations/index.js",

package/dist/chunk-CHLOH4DG.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/model-resolution/failover.ts"],"sourcesContent":["/**\n * Model failover — the answer to \"one upstream hit a quota wall and the customer\n * got a Bad Gateway even though the router had abundant healthy capacity\".\n *\n * This is deliberately NOT a health probe. Probing before every turn buys a\n * round-trip of latency on the happy path and still races the outage (a model\n * healthy at probe time can 502 a second later). Failover is REACTIVE: run the\n * preferred model, and on an upstream-outage signal move to the next model in\n * the chain. Zero added latency when the preferred model works.\n *\n * Two facts drive the design, both measured against a live box during the\n * 2026-07-25 Anthropic/DeepSeek outage:\n *\n * 1. An outage is NOT always a thrown error. The sandbox resolves with a\n *    payload — `{ success: false, errorCode: 'provider_inference_unavailable' }`\n *    — so a classifier that only inspects `catch` misses the customer-visible\n *    case entirely. `isUpstreamUnavailable` inspects resolved values too.\n * 2. Catalog membership is NOT liveness. The router's `/v1/models` still listed\n *    every dead model during the outage, so `validateChatModelId` admitted\n *    `claude-sonnet-4-6` while every call to it returned 502. Nothing upstream\n *    of the actual call can be trusted to tell you a model is serving.\n *\n * Substrate-free: the caller injects `run`, so this composes with a sandbox\n * turn, a router chat completion, or a test fake without importing any of them.\n */\n\n/**\n * Error codes that mean \"this model's upstream is unavailable — a different\n * model may still work\". Deliberately excludes codes that would fail identically\n * on every model (bad request, auth, content filter): retrying those down a\n * chain burns latency and money to reach the same failure.\n */\nexport const UPSTREAM_UNAVAILABLE_CODES: readonly string[] = [\n  'provider_inference_unavailable',\n  'upstream_unavailable',\n  'insufficient_quota',\n  'model_not_available',\n  'server_error',\n  'bad_gateway',\n  'service_unavailable',\n]\n\n/** HTTP statuses that indicate an upstream capacity/availability problem. */\nexport const UPSTREAM_UNAVAILABLE_STATUSES: readonly number[] = [429, 500, 502, 503, 504]\n\n/**\n * Statuses that mean the REQUEST is wrong, so every model in the chain would\n * fail on it identically. Listed explicitly rather than as \"any 4xx\" because\n * 404 is NOT one of them: a model the endpoint cannot find is model-scoped, and\n * the next model in the chain may well be there. 429 is absent for the same\n * reason in reverse — it is a capacity signal and lives in the list above.\n */\nconst REQUEST_ERROR_STATUSES: readonly number[] = [400, 401, 403, 405, 413, 422]\n\n/**\n * Message fragments emitted by real upstreams during this class of outage.\n * Matched case-insensitively as a last resort, after code and status.\n */\nconst UPSTREAM_UNAVAILABLE_MESSAGES: readonly string[] = [\n  'bad gateway',\n  'service unavailable',\n  'inference temporarily unavailable',\n  'provider inference is unavailable',\n  'insufficient balance',\n  'usage limits',\n  'quota exceeded',\n  'rate limit',\n  'overloaded',\n  'temporarily unavailable',\n  // Model-SCOPED unavailability. A different model in the chain can still\n  // serve, so this belongs here and not with the client errors. Measured on a\n  // real box 2026-07-27: the sandbox now reports an unservable model as a\n  // terminal `error` whose data is `{\"message\":\"Session error:\n  // {\\\"error\\\":{\\\"name\\\":\\\"UnknownError\\\",\\\"data\\\":{\\\"message\\\":\\\"Model not\n  // found: openai-compat/zai/glm-4.7.\\\"}}}\"}` — no `code`, no numeric status,\n  // and none of the fragments above. The shipped live product-path proof went\n  // red on exactly this: `usedFallback:false`, `failed:true`, an error row to\n  // the customer where a fallback was available and would have answered.\n  'model not found',\n  'is not currently available',\n]\n\n/**\n * Textual forms an HTTP status arrives in when NO numeric field carries it.\n *\n * The case that forced this is the Cloudflare EDGE 502. Captured verbatim from\n * `router.tangle.tools` on 2026-07-27 17:48:12 GMT (cf-ray\n * `a21d793899f6cef3-DEN`): `HTTP/2 502`, `content-type: text/plain;\n * charset=UTF-8`, a 16-byte body reading `error code: 502\\n`, `server:\n * cloudflare`, and **zero `x-tangle-*` headers** where a healthy 200 from the\n * same endpoint carries nine of them. Zero means the origin never executed —\n * so the router's own upstream failover, which lives inside the origin,\n * structurally cannot fire. Only a client OUTSIDE the edge can rescue that\n * turn, which is why this classifier has to recognize a failure shape that\n * nobody upstream will ever label for it.\n *\n * Every pattern below matches a string a real producer emits:\n * - `error code: 502` — Cloudflare's edge body (the capture above).\n * - `(HTTP 502)` — `src/runtime/openai-stream.ts` wrapping a non-ok response.\n * - `returned 502` — `src/runtime/model-catalog.ts`.\n * - `failed with status 502` — `src/turn-stream/adapters.ts`.\n * - `502 Bad Gateway` — an HTTP/1.1 status line. HTTP/2 carries no reason\n *   phrase at all, which is precisely why matching the phrase alone (the\n *   `'bad gateway'` fragment below) read the capture as a non-outage.\n */\nconst HTTP_STATUS_HINT_PATTERNS: readonly RegExp[] = [\n  /\\berror\\s+code:?\\s*([1-5]\\d{2})\\b/i,\n  /\\bhttp(?:\\/[\\d.]+)?[\\s:]\\s*([1-5]\\d{2})\\b/i,\n  /\\bstatus(?:\\s*code)?[\\s:=]\\s*([1-5]\\d{2})\\b/i,\n  /\\b(?:returned|responded(?:\\s+with)?)\\s+([1-5]\\d{2})\\b/i,\n  /\\b([1-5]\\d{2})\\s+(?:bad\\s+gateway|service\\s+unavailable|gateway\\s+time-?out|internal\\s+server\\s+error|too\\s+many\\s+requests)\\b/i,\n]\n\n/**\n * The HTTP status a message states in prose, or `undefined` when it states\n * none. Deliberately NOT \"any three digits in the string\" — an unanchored digit\n * scan would read a token count or a duration as a status.\n */\nexport function readHttpStatusHint(text: string): number | undefined {\n  for (const pattern of HTTP_STATUS_HINT_PATTERNS) {\n    const match = pattern.exec(text)\n    if (match?.[1]) return Number(match[1])\n  }\n  return undefined\n}\n\nfunction readString(source: Record<string, unknown>, key: string): string | undefined {\n  const value = source[key]\n  return typeof value === 'string' && value.trim().length > 0 ? value : undefined\n}\n\n/**\n * True when `signal` — a thrown error OR a resolved result payload — indicates\n * the model's upstream is unavailable and another model is worth trying.\n *\n * Checked in order of decreasing confidence: explicit code, HTTP status, then\n * message text. A resolved payload only counts as a failure when it carries an\n * explicit failure marker (`success: false`, or an `error`/`errorCode` field) —\n * a successful result is never misread as an outage.\n */\nexport function isUpstreamUnavailable(signal: unknown): boolean {\n  if (signal === null || typeof signal !== 'object') return false\n  const record = signal as Record<string, unknown>\n\n  // A resolved payload that explicitly reports success is never an outage.\n  if (record.success === true) return false\n\n  const nested = record.error\n  const nestedRecord = nested !== null && typeof nested === 'object' ? (nested as Record<string, unknown>) : undefined\n\n  const code =\n    readString(record, 'errorCode') ??\n    readString(record, 'code') ??\n    (nestedRecord ? (readString(nestedRecord, 'code') ?? readString(nestedRecord, 'type')) : undefined)\n  if (code && UPSTREAM_UNAVAILABLE_CODES.includes(code)) return true\n\n  for (const key of ['status', 'statusCode', 'httpStatus']) {\n    const value = record[key]\n    if (typeof value === 'number' && UPSTREAM_UNAVAILABLE_STATUSES.includes(value)) return true\n  }\n\n  const message =\n    readString(record, 'message') ??\n    readString(record, 'error') ??\n    (nestedRecord ? readString(nestedRecord, 'message') : undefined)\n  if (!message) return false\n\n  // A status carried as PROSE, checked before the fragment list because it is\n  // the stronger signal and it cuts BOTH ways. An edge 502 names no exception\n  // type and sets no numeric field, so without this the shipped classifier read\n  // `error code: 502` as a non-outage and handed the customer a Bad Gateway\n  // instead of trying the next model — the exact failure the router cannot\n  // rescue from inside its own origin.\n  const hinted = readHttpStatusHint(message)\n  if (hinted !== undefined) {\n    if (UPSTREAM_UNAVAILABLE_STATUSES.includes(hinted)) return true\n    // An explicit REQUEST-error status is decisive the OTHER way. A 400\n    // carrying a validation message must surface rather than walk the chain,\n    // even when its body happens to contain a word from the fragment list —\n    // the router's own `Function tools with reasoning_effort are not supported`\n    // 400 is a request-shaping bug that fails identically on every model.\n    if (REQUEST_ERROR_STATUSES.includes(hinted)) return false\n  }\n\n  const lowered = message.toLowerCase()\n  return UPSTREAM_UNAVAILABLE_MESSAGES.some((fragment) => lowered.includes(fragment))\n}\n\n/** One model tried, and how it went. */\nexport interface ModelFailoverAttempt {\n  model: string\n  ok: boolean\n  /** Why this model was abandoned. Absent when `ok`. */\n  reason?: string\n}\n\n/** The outcome of a failover run: the value plus the full attempt trail. */\nexport interface ModelFailoverResult<T> {\n  value: T\n  /** The model that actually produced `value`. */\n  model: string\n  attempts: ModelFailoverAttempt[]\n  /** True when the preferred (first) model did not serve the request. */\n  usedFallback: boolean\n}\n\n/** Every model in the chain failed; carries the trail for logging. */\nexport class ModelFailoverExhaustedError extends Error {\n  readonly attempts: ModelFailoverAttempt[]\n  constructor(attempts: ModelFailoverAttempt[]) {\n    const trail = attempts.map((a) => `${a.model}: ${a.reason ?? 'failed'}`).join(' | ')\n    super(`All ${attempts.length} model(s) failed. ${trail}`)\n    this.name = 'ModelFailoverExhaustedError'\n    this.attempts = attempts\n  }\n}\n\n/** Inputs to {@link runWithModelFailover}. */\nexport interface RunWithModelFailoverInput<T> {\n  /** Preferred model first, then fallbacks in descending preference. */\n  models: readonly string[]\n  /** Executes one turn with the given model. */\n  run: (model: string) => Promise<T>\n  /**\n   * Classifies a resolved result as an upstream outage. Defaults to\n   * {@link isUpstreamUnavailable}, which understands the sandbox's\n   * `{ success: false, errorCode }` payload.\n   */\n  isUnavailableResult?: (result: T) => boolean\n  /** Classifies a thrown error. Defaults to {@link isUpstreamUnavailable}. */\n  isUnavailableError?: (error: unknown) => boolean\n  /** Observability hook fired each time a model is abandoned. */\n  onFallback?: (attempt: ModelFailoverAttempt, nextModel: string) => void\n}\n\nfunction describe(signal: unknown): string {\n  if (signal instanceof Error) return signal.message\n  if (signal !== null && typeof signal === 'object') {\n    const record = signal as Record<string, unknown>\n    const message = readString(record, 'error') ?? readString(record, 'message') ?? readString(record, 'errorCode')\n    if (message) return message\n  }\n  return String(signal)\n}\n\n/**\n * Run `run` against the first model in `models` that does not report an upstream\n * outage, falling through the chain in order.\n *\n * A non-outage failure (bad request, auth, content filter) is re-thrown\n * immediately rather than retried down the chain — those fail identically on\n * every model, so walking the chain would only multiply latency and spend.\n *\n * @throws ModelFailoverExhaustedError when every model reports an outage.\n */\nexport async function runWithModelFailover<T>(\n  input: RunWithModelFailoverInput<T>,\n): Promise<ModelFailoverResult<T>> {\n  const models = input.models.map((m) => m.trim()).filter((m) => m.length > 0)\n  if (models.length === 0) throw new Error('runWithModelFailover requires at least one model')\n\n  const isUnavailableResult = input.isUnavailableResult ?? ((r: T) => isUpstreamUnavailable(r))\n  const isUnavailableError = input.isUnavailableError ?? isUpstreamUnavailable\n  const attempts: ModelFailoverAttempt[] = []\n\n  for (let index = 0; index < models.length; index += 1) {\n    const model = models[index]!\n    let result: T\n    try {\n      result = await input.run(model)\n    } catch (error) {\n      if (!isUnavailableError(error)) throw error\n      const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(error) }\n      attempts.push(attempt)\n      const next = models[index + 1]\n      if (next) input.onFallback?.(attempt, next)\n      continue\n    }\n\n    if (isUnavailableResult(result)) {\n      const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(result) }\n      attempts.push(attempt)\n      const next = models[index + 1]\n      if (next) input.onFallback?.(attempt, next)\n      continue\n    }\n\n    attempts.push({ model, ok: true })\n    return { value: result, model, attempts, usedFallback: index > 0 }\n  }\n\n  throw new ModelFailoverExhaustedError(attempts)\n}\n\n/**\n * Build a failover chain: the preferred model first, then `fallbacks`, with\n * duplicates removed so a model is never retried twice in one turn.\n */\nexport function buildModelChain(preferred: string, fallbacks: readonly string[]): string[] {\n  const chain: string[] = []\n  for (const model of [preferred, ...fallbacks]) {\n    const cleaned = typeof model === 'string' ? model.trim() : ''\n    if (cleaned.length > 0 && !chain.includes(cleaned)) chain.push(cleaned)\n  }\n  return chain\n}\n"],"mappings":";AAgCO,IAAM,6BAAgD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGO,IAAM,gCAAmD,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG;AASxF,IAAM,yBAA4C,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG;AAM/E,IAAM,gCAAmD;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA;AAAA,EACA;AACF;AAyBA,IAAM,4BAA+C;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAOO,SAAS,mBAAmB,MAAkC;AACnE,aAAW,WAAW,2BAA2B;AAC/C,UAAM,QAAQ,QAAQ,KAAK,IAAI;AAC/B,QAAI,QAAQ,CAAC,EAAG,QAAO,OAAO,MAAM,CAAC,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,QAAiC,KAAiC;AACpF,QAAM,QAAQ,OAAO,GAAG;AACxB,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAWO,SAAS,sBAAsB,QAA0B;AAC9D,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,SAAS;AAGf,MAAI,OAAO,YAAY,KAAM,QAAO;AAEpC,QAAM,SAAS,OAAO;AACtB,QAAM,eAAe,WAAW,QAAQ,OAAO,WAAW,WAAY,SAAqC;AAE3G,QAAM,OACJ,WAAW,QAAQ,WAAW,KAC9B,WAAW,QAAQ,MAAM,MACxB,eAAgB,WAAW,cAAc,MAAM,KAAK,WAAW,cAAc,MAAM,IAAK;AAC3F,MAAI,QAAQ,2BAA2B,SAAS,IAAI,EAAG,QAAO;AAE9D,aAAW,OAAO,CAAC,UAAU,cAAc,YAAY,GAAG;AACxD,UAAM,QAAQ,OAAO,GAAG;AACxB,QAAI,OAAO,UAAU,YAAY,8BAA8B,SAAS,KAAK,EAAG,QAAO;AAAA,EACzF;AAEA,QAAM,UACJ,WAAW,QAAQ,SAAS,KAC5B,WAAW,QAAQ,OAAO,MACzB,eAAe,WAAW,cAAc,SAAS,IAAI;AACxD,MAAI,CAAC,QAAS,QAAO;AAQrB,QAAM,SAAS,mBAAmB,OAAO;AACzC,MAAI,WAAW,QAAW;AACxB,QAAI,8BAA8B,SAAS,MAAM,EAAG,QAAO;AAM3D,QAAI,uBAAuB,SAAS,MAAM,EAAG,QAAO;AAAA,EACtD;AAEA,QAAM,UAAU,QAAQ,YAAY;AACpC,SAAO,8BAA8B,KAAK,CAAC,aAAa,QAAQ,SAAS,QAAQ,CAAC;AACpF;AAqBO,IAAM,8BAAN,cAA0C,MAAM;AAAA,EAC5C;AAAA,EACT,YAAY,UAAkC;AAC5C,UAAM,QAAQ,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,KAAK,EAAE,UAAU,QAAQ,EAAE,EAAE,KAAK,KAAK;AACnF,UAAM,OAAO,SAAS,MAAM,qBAAqB,KAAK,EAAE;AACxD,SAAK,OAAO;AACZ,SAAK,WAAW;AAAA,EAClB;AACF;AAoBA,SAAS,SAAS,QAAyB;AACzC,MAAI,kBAAkB,MAAO,QAAO,OAAO;AAC3C,MAAI,WAAW,QAAQ,OAAO,WAAW,UAAU;AACjD,UAAM,SAAS;AACf,UAAM,UAAU,WAAW,QAAQ,OAAO,KAAK,WAAW,QAAQ,SAAS,KAAK,WAAW,QAAQ,WAAW;AAC9G,QAAI,QAAS,QAAO;AAAA,EACtB;AACA,SAAO,OAAO,MAAM;AACtB;AAYA,eAAsB,qBACpB,OACiC;AACjC,QAAM,SAAS,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC3E,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,kDAAkD;AAE3F,QAAM,sBAAsB,MAAM,wBAAwB,CAAC,MAAS,sBAAsB,CAAC;AAC3F,QAAM,qBAAqB,MAAM,sBAAsB;AACvD,QAAM,WAAmC,CAAC;AAE1C,WAAS,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;AACrD,UAAM,QAAQ,OAAO,KAAK;AAC1B,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,MAAM,IAAI,KAAK;AAAA,IAChC,SAAS,OAAO;AACd,UAAI,CAAC,mBAAmB,KAAK,EAAG,OAAM;AACtC,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,KAAK,EAAE;AAClF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,QAAI,oBAAoB,MAAM,GAAG;AAC/B,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,MAAM,EAAE;AACnF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,aAAS,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC;AACjC,WAAO,EAAE,OAAO,QAAQ,OAAO,UAAU,cAAc,QAAQ,EAAE;AAAA,EACnE;AAEA,QAAM,IAAI,4BAA4B,QAAQ;AAChD;AAMO,SAAS,gBAAgB,WAAmB,WAAwC;AACzF,QAAM,QAAkB,CAAC;AACzB,aAAW,SAAS,CAAC,WAAW,GAAG,SAAS,GAAG;AAC7C,UAAM,UAAU,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI;AAC3D,QAAI,QAAQ,SAAS,KAAK,CAAC,MAAM,SAAS,OAAO,EAAG,OAAM,KAAK,OAAO;AAAA,EACxE;AACA,SAAO;AACT;","names":[]}