@tangle-network/agent-app 0.44.4 → 0.44.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/model-resolution/failover.ts"],"sourcesContent":["/**\n * Model failover — the answer to \"one upstream hit a quota wall and the customer\n * got a Bad Gateway even though the router had abundant healthy capacity\".\n *\n * This is deliberately NOT a health probe. Probing before every turn buys a\n * round-trip of latency on the happy path and still races the outage (a model\n * healthy at probe time can 502 a second later). Failover is REACTIVE: run the\n * preferred model, and on an upstream-outage signal move to the next model in\n * the chain. Zero added latency when the preferred model works.\n *\n * Two facts drive the design, both measured against a live box during the\n * 2026-07-25 Anthropic/DeepSeek outage:\n *\n * 1. An outage is NOT always a thrown error. The sandbox resolves with a\n * payload — `{ success: false, errorCode: 'provider_inference_unavailable' }`\n * — so a classifier that only inspects `catch` misses the customer-visible\n * case entirely. `isUpstreamUnavailable` inspects resolved values too.\n * 2. Catalog membership is NOT liveness. The router's `/v1/models` still listed\n * every dead model during the outage, so `validateChatModelId` admitted\n * `claude-sonnet-4-6` while every call to it returned 502. Nothing upstream\n * of the actual call can be trusted to tell you a model is serving.\n *\n * Substrate-free: the caller injects `run`, so this composes with a sandbox\n * turn, a router chat completion, or a test fake without importing any of them.\n */\n\n/**\n * Error codes that mean \"this model's upstream is unavailable — a different\n * model may still work\". Deliberately excludes codes that would fail identically\n * on every model (bad request, auth, content filter): retrying those down a\n * chain burns latency and money to reach the same failure.\n */\nexport const UPSTREAM_UNAVAILABLE_CODES: readonly string[] = [\n 'provider_inference_unavailable',\n 'upstream_unavailable',\n 'insufficient_quota',\n 'model_not_available',\n 'server_error',\n 'bad_gateway',\n 'service_unavailable',\n]\n\n/** HTTP statuses that indicate an upstream capacity/availability problem. */\nexport const UPSTREAM_UNAVAILABLE_STATUSES: readonly number[] = [429, 500, 502, 503, 504]\n\n/**\n * Statuses that mean the REQUEST is wrong, so every model in the chain would\n * fail on it identically. Listed explicitly rather than as \"any 4xx\" because\n * 404 is NOT one of them: a model the endpoint cannot find is model-scoped, and\n * the next model in the chain may well be there. 429 is absent for the same\n * reason in reverse — it is a capacity signal and lives in the list above.\n */\nconst REQUEST_ERROR_STATUSES: readonly number[] = [400, 401, 403, 405, 413, 422]\n\n/**\n * Message fragments emitted by real upstreams during this class of outage.\n * Matched case-insensitively as a last resort, after code and status.\n */\nconst UPSTREAM_UNAVAILABLE_MESSAGES: readonly string[] = [\n 'bad gateway',\n 'service unavailable',\n 'inference temporarily unavailable',\n 'provider inference is unavailable',\n 'insufficient balance',\n 'usage limits',\n 'quota exceeded',\n 'rate limit',\n 'overloaded',\n 'temporarily unavailable',\n // Model-SCOPED unavailability. A different model in the chain can still\n // serve, so this belongs here and not with the client errors. Measured on a\n // real box 2026-07-27: the sandbox now reports an unservable model as a\n // terminal `error` whose data is `{\"message\":\"Session error:\n // {\\\"error\\\":{\\\"name\\\":\\\"UnknownError\\\",\\\"data\\\":{\\\"message\\\":\\\"Model not\n // found: openai-compat/zai/glm-4.7.\\\"}}}\"}` — no `code`, no numeric status,\n // and none of the fragments above. The shipped live product-path proof went\n // red on exactly this: `usedFallback:false`, `failed:true`, an error row to\n // the customer where a fallback was available and would have answered.\n 'model not found',\n 'is not currently available',\n]\n\n/**\n * Textual forms an HTTP status arrives in when NO numeric field carries it.\n *\n * The case that forced this is the opaque Cloudflare 502. Captured verbatim\n * from `router.tangle.tools` on 2026-07-27 17:48:12 GMT (cf-ray\n * `a21d793899f6cef3-DEN`): `HTTP/2 502`, `content-type: text/plain;\n * charset=UTF-8`, a 16-byte body reading `error code: 502\\n`, `server:\n * cloudflare`, and **zero `x-tangle-*` headers** where a healthy 200 from the\n * same endpoint carries nine of them.\n *\n * That capture was originally read as \"zero headers means the origin never\n * executed\". It does not, and the correction matters more than the original\n * guess did. Matching client-observed cf-rays against the origin's own access\n * log (tangle-router #307) showed all 62 of those responses WERE produced by\n * the router — `content-type: application/json`, an `X-Generation-Id`, and a\n * routing trace showing the upstream had returned 200 — and Cloudflare\n * replaced the body and every response header with its own error page, because\n * an origin 502 reads to the edge as the origin having failed.\n *\n * The classifier's BEHAVIOR here was right for the wrong reason, and stays\n * exactly as it is. What changes is why: the rescue is needed not because the\n * origin is unreachable, but because a response the origin carefully\n * structured arrives at the client as sixteen unlabelled bytes. The router now\n * emits 503 instead of 502 on those paths so the body survives, but this\n * pattern must remain — it is the client's only handle on any origin 5xx the\n * edge decides to rewrite, including from a router that has not deployed the\n * fix yet.\n *\n * Every pattern below matches a string a real producer emits:\n * - `error code: 502` — Cloudflare's edge body (the capture above).\n * - `(HTTP 502)` — `src/runtime/openai-stream.ts` wrapping a non-ok response.\n * - `returned 502` — `src/runtime/model-catalog.ts`.\n * - `failed with status 502` — `src/turn-stream/adapters.ts`.\n * - `502 Bad Gateway` — an HTTP/1.1 status line. HTTP/2 carries no reason\n * phrase at all, which is precisely why matching the phrase alone (the\n * `'bad gateway'` fragment below) read the capture as a non-outage.\n */\nconst HTTP_STATUS_HINT_PATTERNS: readonly RegExp[] = [\n /\\berror\\s+code:?\\s*([1-5]\\d{2})\\b/i,\n /\\bhttp(?:\\/[\\d.]+)?[\\s:]\\s*([1-5]\\d{2})\\b/i,\n /\\bstatus(?:\\s*code)?[\\s:=]\\s*([1-5]\\d{2})\\b/i,\n /\\b(?:returned|responded(?:\\s+with)?)\\s+([1-5]\\d{2})\\b/i,\n /\\b([1-5]\\d{2})\\s+(?:bad\\s+gateway|service\\s+unavailable|gateway\\s+time-?out|internal\\s+server\\s+error|too\\s+many\\s+requests)\\b/i,\n]\n\n/**\n * The HTTP status a message states in prose, or `undefined` when it states\n * none. Deliberately NOT \"any three digits in the string\" — an unanchored digit\n * scan would read a token count or a duration as a status.\n */\nexport function readHttpStatusHint(text: string): number | undefined {\n for (const pattern of HTTP_STATUS_HINT_PATTERNS) {\n const match = pattern.exec(text)\n if (match?.[1]) return Number(match[1])\n }\n return undefined\n}\n\nfunction readString(source: Record<string, unknown>, key: string): string | undefined {\n const value = source[key]\n return typeof value === 'string' && value.trim().length > 0 ? value : undefined\n}\n\n/**\n * True when `signal` — a thrown error OR a resolved result payload — indicates\n * the model's upstream is unavailable and another model is worth trying.\n *\n * Checked in order of decreasing confidence: explicit code, HTTP status, then\n * message text. A resolved payload only counts as a failure when it carries an\n * explicit failure marker (`success: false`, or an `error`/`errorCode` field) —\n * a successful result is never misread as an outage.\n */\nexport function isUpstreamUnavailable(signal: unknown): boolean {\n if (signal === null || typeof signal !== 'object') return false\n const record = signal as Record<string, unknown>\n\n // A resolved payload that explicitly reports success is never an outage.\n if (record.success === true) return false\n\n const nested = record.error\n const nestedRecord = nested !== null && typeof nested === 'object' ? (nested as Record<string, unknown>) : undefined\n\n const code =\n readString(record, 'errorCode') ??\n readString(record, 'code') ??\n (nestedRecord ? (readString(nestedRecord, 'code') ?? readString(nestedRecord, 'type')) : undefined)\n if (code && UPSTREAM_UNAVAILABLE_CODES.includes(code)) return true\n\n for (const key of ['status', 'statusCode', 'httpStatus']) {\n const value = record[key]\n if (typeof value === 'number' && UPSTREAM_UNAVAILABLE_STATUSES.includes(value)) return true\n }\n\n const message =\n readString(record, 'message') ??\n readString(record, 'error') ??\n (nestedRecord ? readString(nestedRecord, 'message') : undefined)\n if (!message) return false\n\n // A status carried as PROSE, checked before the fragment list because it is\n // the stronger signal and it cuts BOTH ways. An edge 502 names no exception\n // type and sets no numeric field, so without this the shipped classifier read\n // `error code: 502` as a non-outage and handed the customer a Bad Gateway\n // instead of trying the next model — the exact failure the router cannot\n // rescue from inside its own origin.\n const hinted = readHttpStatusHint(message)\n if (hinted !== undefined) {\n if (UPSTREAM_UNAVAILABLE_STATUSES.includes(hinted)) return true\n // An explicit REQUEST-error status is decisive the OTHER way. A 400\n // carrying a validation message must surface rather than walk the chain,\n // even when its body happens to contain a word from the fragment list —\n // the router's own `Function tools with reasoning_effort are not supported`\n // 400 is a request-shaping bug that fails identically on every model.\n if (REQUEST_ERROR_STATUSES.includes(hinted)) return false\n }\n\n const lowered = message.toLowerCase()\n return UPSTREAM_UNAVAILABLE_MESSAGES.some((fragment) => lowered.includes(fragment))\n}\n\n/** One model tried, and how it went. */\nexport interface ModelFailoverAttempt {\n model: string\n ok: boolean\n /** Why this model was abandoned. Absent when `ok`. */\n reason?: string\n}\n\n/** The outcome of a failover run: the value plus the full attempt trail. */\nexport interface ModelFailoverResult<T> {\n value: T\n /** The model that actually produced `value`. */\n model: string\n attempts: ModelFailoverAttempt[]\n /** True when the preferred (first) model did not serve the request. */\n usedFallback: boolean\n}\n\n/** Every model in the chain failed; carries the trail for logging. */\nexport class ModelFailoverExhaustedError extends Error {\n readonly attempts: ModelFailoverAttempt[]\n constructor(attempts: ModelFailoverAttempt[]) {\n const trail = attempts.map((a) => `${a.model}: ${a.reason ?? 'failed'}`).join(' | ')\n super(`All ${attempts.length} model(s) failed. ${trail}`)\n this.name = 'ModelFailoverExhaustedError'\n this.attempts = attempts\n }\n}\n\n/** Inputs to {@link runWithModelFailover}. */\nexport interface RunWithModelFailoverInput<T> {\n /** Preferred model first, then fallbacks in descending preference. */\n models: readonly string[]\n /** Executes one turn with the given model. */\n run: (model: string) => Promise<T>\n /**\n * Classifies a resolved result as an upstream outage. Defaults to\n * {@link isUpstreamUnavailable}, which understands the sandbox's\n * `{ success: false, errorCode }` payload.\n */\n isUnavailableResult?: (result: T) => boolean\n /** Classifies a thrown error. Defaults to {@link isUpstreamUnavailable}. */\n isUnavailableError?: (error: unknown) => boolean\n /** Observability hook fired each time a model is abandoned. */\n onFallback?: (attempt: ModelFailoverAttempt, nextModel: string) => void\n}\n\nfunction describe(signal: unknown): string {\n if (signal instanceof Error) return signal.message\n if (signal !== null && typeof signal === 'object') {\n const record = signal as Record<string, unknown>\n const message = readString(record, 'error') ?? readString(record, 'message') ?? readString(record, 'errorCode')\n if (message) return message\n }\n return String(signal)\n}\n\n/**\n * Run `run` against the first model in `models` that does not report an upstream\n * outage, falling through the chain in order.\n *\n * A non-outage failure (bad request, auth, content filter) is re-thrown\n * immediately rather than retried down the chain — those fail identically on\n * every model, so walking the chain would only multiply latency and spend.\n *\n * @throws ModelFailoverExhaustedError when every model reports an outage.\n */\nexport async function runWithModelFailover<T>(\n input: RunWithModelFailoverInput<T>,\n): Promise<ModelFailoverResult<T>> {\n const models = input.models.map((m) => m.trim()).filter((m) => m.length > 0)\n if (models.length === 0) throw new Error('runWithModelFailover requires at least one model')\n\n const isUnavailableResult = input.isUnavailableResult ?? ((r: T) => isUpstreamUnavailable(r))\n const isUnavailableError = input.isUnavailableError ?? isUpstreamUnavailable\n const attempts: ModelFailoverAttempt[] = []\n\n for (let index = 0; index < models.length; index += 1) {\n const model = models[index]!\n let result: T\n try {\n result = await input.run(model)\n } catch (error) {\n if (!isUnavailableError(error)) throw error\n const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(error) }\n attempts.push(attempt)\n const next = models[index + 1]\n if (next) input.onFallback?.(attempt, next)\n continue\n }\n\n if (isUnavailableResult(result)) {\n const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(result) }\n attempts.push(attempt)\n const next = models[index + 1]\n if (next) input.onFallback?.(attempt, next)\n continue\n }\n\n attempts.push({ model, ok: true })\n return { value: result, model, attempts, usedFallback: index > 0 }\n }\n\n throw new ModelFailoverExhaustedError(attempts)\n}\n\n/**\n * Build a failover chain: the preferred model first, then `fallbacks`, with\n * duplicates removed so a model is never retried twice in one turn.\n */\nexport function buildModelChain(preferred: string, fallbacks: readonly string[]): string[] {\n const chain: string[] = []\n for (const model of [preferred, ...fallbacks]) {\n const cleaned = typeof model === 'string' ? model.trim() : ''\n if (cleaned.length > 0 && !chain.includes(cleaned)) chain.push(cleaned)\n }\n return chain\n}\n"],"mappings":";AAgCO,IAAM,6BAAgD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGO,IAAM,gCAAmD,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG;AASxF,IAAM,yBAA4C,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG;AAM/E,IAAM,gCAAmD;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA;AAAA,EACA;AACF;AAuCA,IAAM,4BAA+C;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAOO,SAAS,mBAAmB,MAAkC;AACnE,aAAW,WAAW,2BAA2B;AAC/C,UAAM,QAAQ,QAAQ,KAAK,IAAI;AAC/B,QAAI,QAAQ,CAAC,EAAG,QAAO,OAAO,MAAM,CAAC,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,QAAiC,KAAiC;AACpF,QAAM,QAAQ,OAAO,GAAG;AACxB,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAWO,SAAS,sBAAsB,QAA0B;AAC9D,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,SAAS;AAGf,MAAI,OAAO,YAAY,KAAM,QAAO;AAEpC,QAAM,SAAS,OAAO;AACtB,QAAM,eAAe,WAAW,QAAQ,OAAO,WAAW,WAAY,SAAqC;AAE3G,QAAM,OACJ,WAAW,QAAQ,WAAW,KAC9B,WAAW,QAAQ,MAAM,MACxB,eAAgB,WAAW,cAAc,MAAM,KAAK,WAAW,cAAc,MAAM,IAAK;AAC3F,MAAI,QAAQ,2BAA2B,SAAS,IAAI,EAAG,QAAO;AAE9D,aAAW,OAAO,CAAC,UAAU,cAAc,YAAY,GAAG;AACxD,UAAM,QAAQ,OAAO,GAAG;AACxB,QAAI,OAAO,UAAU,YAAY,8BAA8B,SAAS,KAAK,EAAG,QAAO;AAAA,EACzF;AAEA,QAAM,UACJ,WAAW,QAAQ,SAAS,KAC5B,WAAW,QAAQ,OAAO,MACzB,eAAe,WAAW,cAAc,SAAS,IAAI;AACxD,MAAI,CAAC,QAAS,QAAO;AAQrB,QAAM,SAAS,mBAAmB,OAAO;AACzC,MAAI,WAAW,QAAW;AACxB,QAAI,8BAA8B,SAAS,MAAM,EAAG,QAAO;AAM3D,QAAI,uBAAuB,SAAS,MAAM,EAAG,QAAO;AAAA,EACtD;AAEA,QAAM,UAAU,QAAQ,YAAY;AACpC,SAAO,8BAA8B,KAAK,CAAC,aAAa,QAAQ,SAAS,QAAQ,CAAC;AACpF;AAqBO,IAAM,8BAAN,cAA0C,MAAM;AAAA,EAC5C;AAAA,EACT,YAAY,UAAkC;AAC5C,UAAM,QAAQ,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,KAAK,EAAE,UAAU,QAAQ,EAAE,EAAE,KAAK,KAAK;AACnF,UAAM,OAAO,SAAS,MAAM,qBAAqB,KAAK,EAAE;AACxD,SAAK,OAAO;AACZ,SAAK,WAAW;AAAA,EAClB;AACF;AAoBA,SAAS,SAAS,QAAyB;AACzC,MAAI,kBAAkB,MAAO,QAAO,OAAO;AAC3C,MAAI,WAAW,QAAQ,OAAO,WAAW,UAAU;AACjD,UAAM,SAAS;AACf,UAAM,UAAU,WAAW,QAAQ,OAAO,KAAK,WAAW,QAAQ,SAAS,KAAK,WAAW,QAAQ,WAAW;AAC9G,QAAI,QAAS,QAAO;AAAA,EACtB;AACA,SAAO,OAAO,MAAM;AACtB;AAYA,eAAsB,qBACpB,OACiC;AACjC,QAAM,SAAS,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC3E,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,kDAAkD;AAE3F,QAAM,sBAAsB,MAAM,wBAAwB,CAAC,MAAS,sBAAsB,CAAC;AAC3F,QAAM,qBAAqB,MAAM,sBAAsB;AACvD,QAAM,WAAmC,CAAC;AAE1C,WAAS,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;AACrD,UAAM,QAAQ,OAAO,KAAK;AAC1B,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,MAAM,IAAI,KAAK;AAAA,IAChC,SAAS,OAAO;AACd,UAAI,CAAC,mBAAmB,KAAK,EAAG,OAAM;AACtC,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,KAAK,EAAE;AAClF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,QAAI,oBAAoB,MAAM,GAAG;AAC/B,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,MAAM,EAAE;AACnF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,aAAS,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC;AACjC,WAAO,EAAE,OAAO,QAAQ,OAAO,UAAU,cAAc,QAAQ,EAAE;AAAA,EACnE;AAEA,QAAM,IAAI,4BAA4B,QAAQ;AAChD;AAMO,SAAS,gBAAgB,WAAmB,WAAwC;AACzF,QAAM,QAAkB,CAAC;AACzB,aAAW,SAAS,CAAC,WAAW,GAAG,SAAS,GAAG;AAC7C,UAAM,UAAU,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI;AAC3D,QAAI,QAAQ,SAAS,KAAK,CAAC,MAAM,SAAS,OAAO,EAAG,OAAM,KAAK,OAAO;AAAA,EACxE;AACA,SAAO;AACT;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-app",
|
|
3
|
-
"version": "0.44.
|
|
3
|
+
"version": "0.44.5",
|
|
4
4
|
"packageManager": "pnpm@10.33.4",
|
|
5
5
|
"description": "Application-shell framework for Tangle agent products: a bounded tool loop, the structured agent→app tool side channel, integration-hub client, per-workspace billing, and crypto — composed over the Tangle agent substrate through typed seams.",
|
|
6
6
|
"keywords": [
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/model-resolution/failover.ts"],"sourcesContent":["/**\n * Model failover — the answer to \"one upstream hit a quota wall and the customer\n * got a Bad Gateway even though the router had abundant healthy capacity\".\n *\n * This is deliberately NOT a health probe. Probing before every turn buys a\n * round-trip of latency on the happy path and still races the outage (a model\n * healthy at probe time can 502 a second later). Failover is REACTIVE: run the\n * preferred model, and on an upstream-outage signal move to the next model in\n * the chain. Zero added latency when the preferred model works.\n *\n * Two facts drive the design, both measured against a live box during the\n * 2026-07-25 Anthropic/DeepSeek outage:\n *\n * 1. An outage is NOT always a thrown error. The sandbox resolves with a\n * payload — `{ success: false, errorCode: 'provider_inference_unavailable' }`\n * — so a classifier that only inspects `catch` misses the customer-visible\n * case entirely. `isUpstreamUnavailable` inspects resolved values too.\n * 2. Catalog membership is NOT liveness. The router's `/v1/models` still listed\n * every dead model during the outage, so `validateChatModelId` admitted\n * `claude-sonnet-4-6` while every call to it returned 502. Nothing upstream\n * of the actual call can be trusted to tell you a model is serving.\n *\n * Substrate-free: the caller injects `run`, so this composes with a sandbox\n * turn, a router chat completion, or a test fake without importing any of them.\n */\n\n/**\n * Error codes that mean \"this model's upstream is unavailable — a different\n * model may still work\". Deliberately excludes codes that would fail identically\n * on every model (bad request, auth, content filter): retrying those down a\n * chain burns latency and money to reach the same failure.\n */\nexport const UPSTREAM_UNAVAILABLE_CODES: readonly string[] = [\n 'provider_inference_unavailable',\n 'upstream_unavailable',\n 'insufficient_quota',\n 'model_not_available',\n 'server_error',\n 'bad_gateway',\n 'service_unavailable',\n]\n\n/** HTTP statuses that indicate an upstream capacity/availability problem. */\nexport const UPSTREAM_UNAVAILABLE_STATUSES: readonly number[] = [429, 500, 502, 503, 504]\n\n/**\n * Statuses that mean the REQUEST is wrong, so every model in the chain would\n * fail on it identically. Listed explicitly rather than as \"any 4xx\" because\n * 404 is NOT one of them: a model the endpoint cannot find is model-scoped, and\n * the next model in the chain may well be there. 429 is absent for the same\n * reason in reverse — it is a capacity signal and lives in the list above.\n */\nconst REQUEST_ERROR_STATUSES: readonly number[] = [400, 401, 403, 405, 413, 422]\n\n/**\n * Message fragments emitted by real upstreams during this class of outage.\n * Matched case-insensitively as a last resort, after code and status.\n */\nconst UPSTREAM_UNAVAILABLE_MESSAGES: readonly string[] = [\n 'bad gateway',\n 'service unavailable',\n 'inference temporarily unavailable',\n 'provider inference is unavailable',\n 'insufficient balance',\n 'usage limits',\n 'quota exceeded',\n 'rate limit',\n 'overloaded',\n 'temporarily unavailable',\n // Model-SCOPED unavailability. A different model in the chain can still\n // serve, so this belongs here and not with the client errors. Measured on a\n // real box 2026-07-27: the sandbox now reports an unservable model as a\n // terminal `error` whose data is `{\"message\":\"Session error:\n // {\\\"error\\\":{\\\"name\\\":\\\"UnknownError\\\",\\\"data\\\":{\\\"message\\\":\\\"Model not\n // found: openai-compat/zai/glm-4.7.\\\"}}}\"}` — no `code`, no numeric status,\n // and none of the fragments above. The shipped live product-path proof went\n // red on exactly this: `usedFallback:false`, `failed:true`, an error row to\n // the customer where a fallback was available and would have answered.\n 'model not found',\n 'is not currently available',\n]\n\n/**\n * Textual forms an HTTP status arrives in when NO numeric field carries it.\n *\n * The case that forced this is the Cloudflare EDGE 502. Captured verbatim from\n * `router.tangle.tools` on 2026-07-27 17:48:12 GMT (cf-ray\n * `a21d793899f6cef3-DEN`): `HTTP/2 502`, `content-type: text/plain;\n * charset=UTF-8`, a 16-byte body reading `error code: 502\\n`, `server:\n * cloudflare`, and **zero `x-tangle-*` headers** where a healthy 200 from the\n * same endpoint carries nine of them. Zero means the origin never executed —\n * so the router's own upstream failover, which lives inside the origin,\n * structurally cannot fire. Only a client OUTSIDE the edge can rescue that\n * turn, which is why this classifier has to recognize a failure shape that\n * nobody upstream will ever label for it.\n *\n * Every pattern below matches a string a real producer emits:\n * - `error code: 502` — Cloudflare's edge body (the capture above).\n * - `(HTTP 502)` — `src/runtime/openai-stream.ts` wrapping a non-ok response.\n * - `returned 502` — `src/runtime/model-catalog.ts`.\n * - `failed with status 502` — `src/turn-stream/adapters.ts`.\n * - `502 Bad Gateway` — an HTTP/1.1 status line. HTTP/2 carries no reason\n * phrase at all, which is precisely why matching the phrase alone (the\n * `'bad gateway'` fragment below) read the capture as a non-outage.\n */\nconst HTTP_STATUS_HINT_PATTERNS: readonly RegExp[] = [\n /\\berror\\s+code:?\\s*([1-5]\\d{2})\\b/i,\n /\\bhttp(?:\\/[\\d.]+)?[\\s:]\\s*([1-5]\\d{2})\\b/i,\n /\\bstatus(?:\\s*code)?[\\s:=]\\s*([1-5]\\d{2})\\b/i,\n /\\b(?:returned|responded(?:\\s+with)?)\\s+([1-5]\\d{2})\\b/i,\n /\\b([1-5]\\d{2})\\s+(?:bad\\s+gateway|service\\s+unavailable|gateway\\s+time-?out|internal\\s+server\\s+error|too\\s+many\\s+requests)\\b/i,\n]\n\n/**\n * The HTTP status a message states in prose, or `undefined` when it states\n * none. Deliberately NOT \"any three digits in the string\" — an unanchored digit\n * scan would read a token count or a duration as a status.\n */\nexport function readHttpStatusHint(text: string): number | undefined {\n for (const pattern of HTTP_STATUS_HINT_PATTERNS) {\n const match = pattern.exec(text)\n if (match?.[1]) return Number(match[1])\n }\n return undefined\n}\n\nfunction readString(source: Record<string, unknown>, key: string): string | undefined {\n const value = source[key]\n return typeof value === 'string' && value.trim().length > 0 ? value : undefined\n}\n\n/**\n * True when `signal` — a thrown error OR a resolved result payload — indicates\n * the model's upstream is unavailable and another model is worth trying.\n *\n * Checked in order of decreasing confidence: explicit code, HTTP status, then\n * message text. A resolved payload only counts as a failure when it carries an\n * explicit failure marker (`success: false`, or an `error`/`errorCode` field) —\n * a successful result is never misread as an outage.\n */\nexport function isUpstreamUnavailable(signal: unknown): boolean {\n if (signal === null || typeof signal !== 'object') return false\n const record = signal as Record<string, unknown>\n\n // A resolved payload that explicitly reports success is never an outage.\n if (record.success === true) return false\n\n const nested = record.error\n const nestedRecord = nested !== null && typeof nested === 'object' ? (nested as Record<string, unknown>) : undefined\n\n const code =\n readString(record, 'errorCode') ??\n readString(record, 'code') ??\n (nestedRecord ? (readString(nestedRecord, 'code') ?? readString(nestedRecord, 'type')) : undefined)\n if (code && UPSTREAM_UNAVAILABLE_CODES.includes(code)) return true\n\n for (const key of ['status', 'statusCode', 'httpStatus']) {\n const value = record[key]\n if (typeof value === 'number' && UPSTREAM_UNAVAILABLE_STATUSES.includes(value)) return true\n }\n\n const message =\n readString(record, 'message') ??\n readString(record, 'error') ??\n (nestedRecord ? readString(nestedRecord, 'message') : undefined)\n if (!message) return false\n\n // A status carried as PROSE, checked before the fragment list because it is\n // the stronger signal and it cuts BOTH ways. An edge 502 names no exception\n // type and sets no numeric field, so without this the shipped classifier read\n // `error code: 502` as a non-outage and handed the customer a Bad Gateway\n // instead of trying the next model — the exact failure the router cannot\n // rescue from inside its own origin.\n const hinted = readHttpStatusHint(message)\n if (hinted !== undefined) {\n if (UPSTREAM_UNAVAILABLE_STATUSES.includes(hinted)) return true\n // An explicit REQUEST-error status is decisive the OTHER way. A 400\n // carrying a validation message must surface rather than walk the chain,\n // even when its body happens to contain a word from the fragment list —\n // the router's own `Function tools with reasoning_effort are not supported`\n // 400 is a request-shaping bug that fails identically on every model.\n if (REQUEST_ERROR_STATUSES.includes(hinted)) return false\n }\n\n const lowered = message.toLowerCase()\n return UPSTREAM_UNAVAILABLE_MESSAGES.some((fragment) => lowered.includes(fragment))\n}\n\n/** One model tried, and how it went. */\nexport interface ModelFailoverAttempt {\n model: string\n ok: boolean\n /** Why this model was abandoned. Absent when `ok`. */\n reason?: string\n}\n\n/** The outcome of a failover run: the value plus the full attempt trail. */\nexport interface ModelFailoverResult<T> {\n value: T\n /** The model that actually produced `value`. */\n model: string\n attempts: ModelFailoverAttempt[]\n /** True when the preferred (first) model did not serve the request. */\n usedFallback: boolean\n}\n\n/** Every model in the chain failed; carries the trail for logging. */\nexport class ModelFailoverExhaustedError extends Error {\n readonly attempts: ModelFailoverAttempt[]\n constructor(attempts: ModelFailoverAttempt[]) {\n const trail = attempts.map((a) => `${a.model}: ${a.reason ?? 'failed'}`).join(' | ')\n super(`All ${attempts.length} model(s) failed. ${trail}`)\n this.name = 'ModelFailoverExhaustedError'\n this.attempts = attempts\n }\n}\n\n/** Inputs to {@link runWithModelFailover}. */\nexport interface RunWithModelFailoverInput<T> {\n /** Preferred model first, then fallbacks in descending preference. */\n models: readonly string[]\n /** Executes one turn with the given model. */\n run: (model: string) => Promise<T>\n /**\n * Classifies a resolved result as an upstream outage. Defaults to\n * {@link isUpstreamUnavailable}, which understands the sandbox's\n * `{ success: false, errorCode }` payload.\n */\n isUnavailableResult?: (result: T) => boolean\n /** Classifies a thrown error. Defaults to {@link isUpstreamUnavailable}. */\n isUnavailableError?: (error: unknown) => boolean\n /** Observability hook fired each time a model is abandoned. */\n onFallback?: (attempt: ModelFailoverAttempt, nextModel: string) => void\n}\n\nfunction describe(signal: unknown): string {\n if (signal instanceof Error) return signal.message\n if (signal !== null && typeof signal === 'object') {\n const record = signal as Record<string, unknown>\n const message = readString(record, 'error') ?? readString(record, 'message') ?? readString(record, 'errorCode')\n if (message) return message\n }\n return String(signal)\n}\n\n/**\n * Run `run` against the first model in `models` that does not report an upstream\n * outage, falling through the chain in order.\n *\n * A non-outage failure (bad request, auth, content filter) is re-thrown\n * immediately rather than retried down the chain — those fail identically on\n * every model, so walking the chain would only multiply latency and spend.\n *\n * @throws ModelFailoverExhaustedError when every model reports an outage.\n */\nexport async function runWithModelFailover<T>(\n input: RunWithModelFailoverInput<T>,\n): Promise<ModelFailoverResult<T>> {\n const models = input.models.map((m) => m.trim()).filter((m) => m.length > 0)\n if (models.length === 0) throw new Error('runWithModelFailover requires at least one model')\n\n const isUnavailableResult = input.isUnavailableResult ?? ((r: T) => isUpstreamUnavailable(r))\n const isUnavailableError = input.isUnavailableError ?? isUpstreamUnavailable\n const attempts: ModelFailoverAttempt[] = []\n\n for (let index = 0; index < models.length; index += 1) {\n const model = models[index]!\n let result: T\n try {\n result = await input.run(model)\n } catch (error) {\n if (!isUnavailableError(error)) throw error\n const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(error) }\n attempts.push(attempt)\n const next = models[index + 1]\n if (next) input.onFallback?.(attempt, next)\n continue\n }\n\n if (isUnavailableResult(result)) {\n const attempt: ModelFailoverAttempt = { model, ok: false, reason: describe(result) }\n attempts.push(attempt)\n const next = models[index + 1]\n if (next) input.onFallback?.(attempt, next)\n continue\n }\n\n attempts.push({ model, ok: true })\n return { value: result, model, attempts, usedFallback: index > 0 }\n }\n\n throw new ModelFailoverExhaustedError(attempts)\n}\n\n/**\n * Build a failover chain: the preferred model first, then `fallbacks`, with\n * duplicates removed so a model is never retried twice in one turn.\n */\nexport function buildModelChain(preferred: string, fallbacks: readonly string[]): string[] {\n const chain: string[] = []\n for (const model of [preferred, ...fallbacks]) {\n const cleaned = typeof model === 'string' ? model.trim() : ''\n if (cleaned.length > 0 && !chain.includes(cleaned)) chain.push(cleaned)\n }\n return chain\n}\n"],"mappings":";AAgCO,IAAM,6BAAgD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGO,IAAM,gCAAmD,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG;AASxF,IAAM,yBAA4C,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG;AAM/E,IAAM,gCAAmD;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA;AAAA,EACA;AACF;AAyBA,IAAM,4BAA+C;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAOO,SAAS,mBAAmB,MAAkC;AACnE,aAAW,WAAW,2BAA2B;AAC/C,UAAM,QAAQ,QAAQ,KAAK,IAAI;AAC/B,QAAI,QAAQ,CAAC,EAAG,QAAO,OAAO,MAAM,CAAC,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,QAAiC,KAAiC;AACpF,QAAM,QAAQ,OAAO,GAAG;AACxB,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAWO,SAAS,sBAAsB,QAA0B;AAC9D,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,SAAS;AAGf,MAAI,OAAO,YAAY,KAAM,QAAO;AAEpC,QAAM,SAAS,OAAO;AACtB,QAAM,eAAe,WAAW,QAAQ,OAAO,WAAW,WAAY,SAAqC;AAE3G,QAAM,OACJ,WAAW,QAAQ,WAAW,KAC9B,WAAW,QAAQ,MAAM,MACxB,eAAgB,WAAW,cAAc,MAAM,KAAK,WAAW,cAAc,MAAM,IAAK;AAC3F,MAAI,QAAQ,2BAA2B,SAAS,IAAI,EAAG,QAAO;AAE9D,aAAW,OAAO,CAAC,UAAU,cAAc,YAAY,GAAG;AACxD,UAAM,QAAQ,OAAO,GAAG;AACxB,QAAI,OAAO,UAAU,YAAY,8BAA8B,SAAS,KAAK,EAAG,QAAO;AAAA,EACzF;AAEA,QAAM,UACJ,WAAW,QAAQ,SAAS,KAC5B,WAAW,QAAQ,OAAO,MACzB,eAAe,WAAW,cAAc,SAAS,IAAI;AACxD,MAAI,CAAC,QAAS,QAAO;AAQrB,QAAM,SAAS,mBAAmB,OAAO;AACzC,MAAI,WAAW,QAAW;AACxB,QAAI,8BAA8B,SAAS,MAAM,EAAG,QAAO;AAM3D,QAAI,uBAAuB,SAAS,MAAM,EAAG,QAAO;AAAA,EACtD;AAEA,QAAM,UAAU,QAAQ,YAAY;AACpC,SAAO,8BAA8B,KAAK,CAAC,aAAa,QAAQ,SAAS,QAAQ,CAAC;AACpF;AAqBO,IAAM,8BAAN,cAA0C,MAAM;AAAA,EAC5C;AAAA,EACT,YAAY,UAAkC;AAC5C,UAAM,QAAQ,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,KAAK,EAAE,UAAU,QAAQ,EAAE,EAAE,KAAK,KAAK;AACnF,UAAM,OAAO,SAAS,MAAM,qBAAqB,KAAK,EAAE;AACxD,SAAK,OAAO;AACZ,SAAK,WAAW;AAAA,EAClB;AACF;AAoBA,SAAS,SAAS,QAAyB;AACzC,MAAI,kBAAkB,MAAO,QAAO,OAAO;AAC3C,MAAI,WAAW,QAAQ,OAAO,WAAW,UAAU;AACjD,UAAM,SAAS;AACf,UAAM,UAAU,WAAW,QAAQ,OAAO,KAAK,WAAW,QAAQ,SAAS,KAAK,WAAW,QAAQ,WAAW;AAC9G,QAAI,QAAS,QAAO;AAAA,EACtB;AACA,SAAO,OAAO,MAAM;AACtB;AAYA,eAAsB,qBACpB,OACiC;AACjC,QAAM,SAAS,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC3E,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,kDAAkD;AAE3F,QAAM,sBAAsB,MAAM,wBAAwB,CAAC,MAAS,sBAAsB,CAAC;AAC3F,QAAM,qBAAqB,MAAM,sBAAsB;AACvD,QAAM,WAAmC,CAAC;AAE1C,WAAS,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;AACrD,UAAM,QAAQ,OAAO,KAAK;AAC1B,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,MAAM,IAAI,KAAK;AAAA,IAChC,SAAS,OAAO;AACd,UAAI,CAAC,mBAAmB,KAAK,EAAG,OAAM;AACtC,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,KAAK,EAAE;AAClF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,QAAI,oBAAoB,MAAM,GAAG;AAC/B,YAAM,UAAgC,EAAE,OAAO,IAAI,OAAO,QAAQ,SAAS,MAAM,EAAE;AACnF,eAAS,KAAK,OAAO;AACrB,YAAM,OAAO,OAAO,QAAQ,CAAC;AAC7B,UAAI,KAAM,OAAM,aAAa,SAAS,IAAI;AAC1C;AAAA,IACF;AAEA,aAAS,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC;AACjC,WAAO,EAAE,OAAO,QAAQ,OAAO,UAAU,cAAc,QAAQ,EAAE;AAAA,EACnE;AAEA,QAAM,IAAI,4BAA4B,QAAQ;AAChD;AAMO,SAAS,gBAAgB,WAAmB,WAAwC;AACzF,QAAM,QAAkB,CAAC;AACzB,aAAW,SAAS,CAAC,WAAW,GAAG,SAAS,GAAG;AAC7C,UAAM,UAAU,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI;AAC3D,QAAI,QAAQ,SAAS,KAAK,CAAC,MAAM,SAAS,OAAO,EAAG,OAAM,KAAK,OAAO;AAAA,EACxE;AACA,SAAO;AACT;","names":[]}
|