@tangle-network/agent-eval 0.135.0 → 0.135.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/dist/analyst/index.js +3 -3
  3. package/dist/{analyze-runs-DMo3Lb_y.d.ts → analyze-runs-Cda5Xkj1.d.ts} +3 -3
  4. package/dist/{analyze-runs-DMo3Lb_y.d.ts.map → analyze-runs-Cda5Xkj1.d.ts.map} +1 -1
  5. package/dist/{analyze-runs-qk8op0tN.js → analyze-runs-jjCmF8pU.js} +14 -7
  6. package/dist/analyze-runs-jjCmF8pU.js.map +1 -0
  7. package/dist/{baseline-BaPxoROc.js → baseline-BUeFcgrn.js} +2 -2
  8. package/dist/{baseline-BaPxoROc.js.map → baseline-BUeFcgrn.js.map} +1 -1
  9. package/dist/benchmarks/index.d.ts +1 -1
  10. package/dist/benchmarks/index.js +1 -1
  11. package/dist/{benchmarks-Dw1Wv_JQ.js → benchmarks-Mtu251Jz.js} +3 -3
  12. package/dist/{benchmarks-Dw1Wv_JQ.js.map → benchmarks-Mtu251Jz.js.map} +1 -1
  13. package/dist/builder-eval/index.js +1 -1
  14. package/dist/campaign/index.d.ts +2 -2
  15. package/dist/campaign/index.js +2 -2
  16. package/dist/{campaign-B1c1T0kv.js → campaign-RVIqtJh0.js} +7 -7
  17. package/dist/{campaign-B1c1T0kv.js.map → campaign-RVIqtJh0.js.map} +1 -1
  18. package/dist/cli.js +1 -1
  19. package/dist/{client-BIyh1RCr.d.ts → client-DcvgkaZi.d.ts} +13 -4
  20. package/dist/client-DcvgkaZi.d.ts.map +1 -0
  21. package/dist/contract/index.d.ts +3 -3
  22. package/dist/contract/index.d.ts.map +1 -1
  23. package/dist/contract/index.js +19 -7
  24. package/dist/contract/index.js.map +1 -1
  25. package/dist/{cost-ledger-ZAa_P4r0.js → cost-ledger-DHAjwNj7.js} +6 -2
  26. package/dist/{cost-ledger-ZAa_P4r0.js.map → cost-ledger-DHAjwNj7.js.map} +1 -1
  27. package/dist/{default-registry-CFUZyNeZ.js → default-registry-BAhV-lbE.js} +3 -3
  28. package/dist/{default-registry-CFUZyNeZ.js.map → default-registry-BAhV-lbE.js.map} +1 -1
  29. package/dist/{eval-campaign-CHFxPTVl.js → eval-campaign-Cc8WZJ6b.js} +3 -3
  30. package/dist/{eval-campaign-CHFxPTVl.js.map → eval-campaign-Cc8WZJ6b.js.map} +1 -1
  31. package/dist/fuzz.js +1 -1
  32. package/dist/hosted/index.d.ts +1 -1
  33. package/dist/{index-BoJNQR6n.d.ts → index-B4Fjfo5U.d.ts} +93 -15
  34. package/dist/index-B4Fjfo5U.d.ts.map +1 -0
  35. package/dist/{index-C21xKtxu.d.ts → index-CQsJcqch.d.ts} +3 -3
  36. package/dist/{index-C21xKtxu.d.ts.map → index-CQsJcqch.d.ts.map} +1 -1
  37. package/dist/{index-DSC51roc2.d.ts → index-DSC51roc.d.ts} +1 -1
  38. package/dist/index-DSC51roc.d.ts.map +1 -0
  39. package/dist/index.d.ts +41 -10
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +135 -57
  42. package/dist/index.js.map +1 -1
  43. package/dist/{llm-client-BNcP4v08.js → llm-client-DHx8pzyJ.js} +2 -2
  44. package/dist/{llm-client-BNcP4v08.js.map → llm-client-DHx8pzyJ.js.map} +1 -1
  45. package/dist/matrix/index.d.ts +1 -1
  46. package/dist/meta-eval/index.d.ts +1 -2
  47. package/dist/meta-eval/index.d.ts.map +1 -1
  48. package/dist/meta-eval/index.js +2 -2
  49. package/dist/multishot/index.d.ts +1 -1
  50. package/dist/openapi.json +1 -1
  51. package/dist/{paired-arms-CA_8pN01.js → paired-arms-BbFKrAU-.js} +2 -2
  52. package/dist/{paired-arms-CA_8pN01.js.map → paired-arms-BbFKrAU-.js.map} +1 -1
  53. package/dist/pipelines/index.js +2 -2
  54. package/dist/{release-report-BVZBmRZp.js → release-report-DooPguBc.js} +4 -3
  55. package/dist/{release-report-BVZBmRZp.js.map → release-report-DooPguBc.js.map} +1 -1
  56. package/dist/{release-report-CuULWKyk.d.ts → release-report-DpBxGGI1.d.ts} +2 -2
  57. package/dist/{release-report-CuULWKyk.d.ts.map → release-report-DpBxGGI1.d.ts.map} +1 -1
  58. package/dist/reporting.d.ts +3 -3
  59. package/dist/reporting.js +4 -4
  60. package/dist/{researcher-DVtruQ9U.d.ts → researcher-Doo95b50.d.ts} +2 -2
  61. package/dist/{researcher-DVtruQ9U.d.ts.map → researcher-Doo95b50.d.ts.map} +1 -1
  62. package/dist/{reward-hacking-DCdRK9TY.js → reward-hacking-a-kYs0-i.js} +2 -2
  63. package/dist/{reward-hacking-DCdRK9TY.js.map → reward-hacking-a-kYs0-i.js.map} +1 -1
  64. package/dist/rl.d.ts +45 -3
  65. package/dist/rl.d.ts.map +1 -1
  66. package/dist/rl.js +108 -22
  67. package/dist/rl.js.map +1 -1
  68. package/dist/{rubric-predictive-validity-D6Q6n9oq.js → rubric-predictive-validity-BJf-8ejY.js} +2 -2
  69. package/dist/{rubric-predictive-validity-D6Q6n9oq.js.map → rubric-predictive-validity-BJf-8ejY.js.map} +1 -1
  70. package/dist/{semantic-concept-judge-Ca--u10C.js → semantic-concept-judge-Btozx3Vc.js} +3 -3
  71. package/dist/{semantic-concept-judge-Ca--u10C.js.map → semantic-concept-judge-Btozx3Vc.js.map} +1 -1
  72. package/dist/{server-Dc_lsOYd.js → server-Bz3WQJs6.js} +3 -3
  73. package/dist/{server-Dc_lsOYd.js.map → server-Bz3WQJs6.js.map} +1 -1
  74. package/dist/{skillopt-optimization-method-Cl4XPkLC.js → skillopt-optimization-method-0UmPD6aP.js} +342 -56
  75. package/dist/skillopt-optimization-method-0UmPD6aP.js.map +1 -0
  76. package/dist/{skillopt-optimization-method-DJ3l4w8W.d.ts → skillopt-optimization-method-CwSYkv35.d.ts} +39 -9
  77. package/dist/skillopt-optimization-method-CwSYkv35.d.ts.map +1 -0
  78. package/dist/{statistics-D_4Snl-5.d.ts → statistics-CKOqre5S.d.ts} +329 -3
  79. package/dist/statistics-CKOqre5S.d.ts.map +1 -0
  80. package/dist/{statistics-RwRNu2__.js → statistics-CnGCLLqc.js} +315 -2
  81. package/dist/statistics-CnGCLLqc.js.map +1 -0
  82. package/dist/{summary-report-BxtossFi.js → summary-report-BEk8OFLs.js} +11 -6
  83. package/dist/summary-report-BEk8OFLs.js.map +1 -0
  84. package/dist/{summary-report-DGp0-_XO.d.ts → summary-report-CPMINBqs.d.ts} +182 -7
  85. package/dist/summary-report-CPMINBqs.d.ts.map +1 -0
  86. package/dist/wire/index.js +1 -1
  87. package/package.json +1 -1
  88. package/dist/analyze-runs-qk8op0tN.js.map +0 -1
  89. package/dist/client-BIyh1RCr.d.ts.map +0 -1
  90. package/dist/index-BoJNQR6n.d.ts.map +0 -1
  91. package/dist/index-DSC51roc2.d.ts.map +0 -1
  92. package/dist/judge-calibration-DFtEMlde.d.ts +0 -146
  93. package/dist/judge-calibration-DFtEMlde.d.ts.map +0 -1
  94. package/dist/skillopt-optimization-method-Cl4XPkLC.js.map +0 -1
  95. package/dist/skillopt-optimization-method-DJ3l4w8W.d.ts.map +0 -1
  96. package/dist/statistics-D_4Snl-5.d.ts.map +0 -1
  97. package/dist/statistics-RwRNu2__.js.map +0 -1
  98. package/dist/summary-report-BxtossFi.js.map +0 -1
  99. package/dist/summary-report-DGp0-_XO.d.ts.map +0 -1
@@ -1,5 +1,5 @@
1
1
  import { n as CaptureIntegrityError, t as AgentEvalError } from "./errors-8YnH8WlF.js";
2
- import { c as costForTokenPricing } from "./cost-ledger-ZAa_P4r0.js";
2
+ import { c as costForTokenPricing } from "./cost-ledger-DHAjwNj7.js";
3
3
  import { a as providerFromBaseUrl, i as defaultProviderRedactor } from "./raw-provider-sink-BQd7mzyT.js";
4
4
  //#region src/llm-client.ts
5
5
  /**
@@ -684,4 +684,4 @@ var LlmClient = class {
684
684
  //#endregion
685
685
  export { assertLlmRoute as a, callLlmJson as c, isTransientLlmError as d, maximumChargeForLlmRequest as f, LlmRouteAssertionError as i, costReceiptFromLlm as l, stripFencedJson as m, LlmClient as n, backoffMs as o, probeLlm as p, LlmResponseError as r, callLlm as s, LlmCallError as t, costReceiptFromLlmError as u };
686
686
 
687
- //# sourceMappingURL=llm-client-BNcP4v08.js.map
687
+ //# sourceMappingURL=llm-client-DHx8pzyJ.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"llm-client-BNcP4v08.js","names":[],"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n * - One retry at temperature 1 when a model explicitly requires it.\n * - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n * directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n * const { value, result } = await callLlmJson<MyType>(\n * { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n * { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n * )\n *\n * `createChatClient` wraps this implementation for provider-neutral package\n * entry points. Direct callers can use `callLlm` or `callLlmJson`.\n */\n\nimport {\n type CostReceiptInput,\n type CustomTokenPricing,\n costForTokenPricing,\n type MaximumCharge,\n} from './cost-ledger'\nimport { AgentEvalError, CaptureIntegrityError } from './errors'\nimport {\n defaultProviderRedactor,\n type ProviderRedactor,\n providerFromBaseUrl,\n type RawProviderEvent,\n type RawProviderSink,\n} from './trace/raw-provider-sink'\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n role: 'system' | 'user' | 'assistant'\n /**\n * Either a plain text content string OR a multimodal content array\n * (text + image_url parts) for vision-capable models.\n */\n content:\n | string\n | Array<\n | { type: 'text'; text: string }\n | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n >\n}\n\nexport type LlmThinkingMode = 'enabled' | 'disabled'\n\nexport interface LlmCallRequest {\n model: string\n messages: LlmMessage[]\n /** Optional JSON-mode response format (response_format: json_object). */\n jsonMode?: boolean\n /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n jsonSchema?: { name: string; schema: Record<string, unknown> }\n temperature?: number\n maxTokens?: number\n /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */\n thinking?: LlmThinkingMode\n /** Per-call timeout, default 300s. */\n timeoutMs?: number\n}\n\n/** Conservative priced bound for the exact text request sent to a provider.\n * Returns undefined when output or multimodal input is not bounded, causing a\n * capped CostLedger to reject the call before execution. Pass\n * `customTokenPricing` when package pricing does not cover the model or endpoint. */\nexport function maximumChargeForLlmRequest(\n request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'maxTokens' | 'thinking'>,\n options: LlmClientOptions = {},\n): MaximumCharge | undefined {\n if (request.maxTokens === undefined) return undefined\n if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {\n throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`)\n }\n if (\n request.messages.some(\n (message) =>\n Array.isArray(message.content) && message.content.some((part) => part.type === 'image_url'),\n )\n ) {\n return undefined\n }\n\n const attempts = resolveMaximumAttempts(options.maximumAttempts)\n const forceJsonObject = options.jsonSchemaTransport === 'json-object'\n // A byte-level tokenizer cannot emit more input tokens than request bytes.\n // Pricing the complete body also covers role/schema framing omitted from content-only estimates.\n const requestBytes = new TextEncoder().encode(\n JSON.stringify(buildBody(request, forceJsonObject, options.thinking)),\n ).byteLength\n // A rejected response schema can trigger one JSON-mode batch with the same output limit.\n const batches = request.jsonSchema && !forceJsonObject ? 2 : 1\n const usage = {\n inputTokens: requestBytes * attempts * batches,\n outputTokens: request.maxTokens * attempts * batches,\n }\n return options.customTokenPricing\n ? { customTokenPricing: options.customTokenPricing, ...usage }\n : { model: request.model, ...usage }\n}\n\nexport interface LlmUsage {\n promptTokens: number\n completionTokens: number\n totalTokens: number\n /** False when the provider omitted or malformed prompt/completion usage. */\n captured?: boolean\n /** Reasoning-token subset of completionTokens, when reported. */\n reasoningTokens?: number\n /** Proxies populate this when prompt caching is on. */\n cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n /** The text content of the first choice. Empty string if none. */\n content: string\n usage: LlmUsage\n /**\n * Cost in USD. Uses the provider's reported cost when present, otherwise\n * caller-supplied token pricing. `null` when neither is available.\n */\n costUsd: number | null\n /** Model name actually used (echoed from response). */\n model: string\n /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n durationMs: number\n /**\n * `finish_reason` echoed from the first choice (`stop`, `length`,\n * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.\n * Exposed so a free-form `callLlm` caller CAN detect a truncated answer\n * (`length`) instead of treating a cut-off completion as complete. Note:\n * `callLlm` does not itself reject on it — acting on this signal is the\n * caller's responsibility (in-repo free-form drivers do not yet enforce it).\n */\n finishReason?: string | null\n /**\n * True when `content.trim()` is empty. An empty completion is a silent zero\n * for free-form `callLlm` callers; this flag is the signal a caller can\n * inspect to fail loud rather than proceed on an empty string. `callLlm`\n * surfaces it but does not throw on it.\n */\n contentEmpty?: boolean\n /** Raw response body. */\n raw: Record<string, unknown>\n}\n\nexport type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>\n\n/** Convert a provider result into the canonical paid-call receipt input. */\nexport function costReceiptFromLlm(\n result: LlmCallResult,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput {\n const cachedTokens = result.usage.cachedPromptTokens ?? 0\n const inputTokens = Math.max(0, result.usage.promptTokens - cachedTokens)\n const configuredCostUsd =\n result.costUsd === null && customTokenPricing && result.usage.captured !== false\n ? costForTokenPricing(customTokenPricing, {\n inputTokens,\n ...(cachedTokens > 0 ? { cachedTokens } : {}),\n outputTokens: result.usage.completionTokens,\n })\n : undefined\n return {\n model: result.model,\n inputTokens,\n outputTokens: result.usage.completionTokens,\n reasoningTokens: result.usage.reasoningTokens,\n cachedTokens: cachedTokens > 0 ? cachedTokens : undefined,\n actualCostUsd: result.costUsd ?? configuredCostUsd,\n usageUnknown: result.usage.captured === false,\n }\n}\n\n/** Structured-response failures retain their completed provider receipt. */\nexport function costReceiptFromLlmError(\n error: Error,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput | undefined {\n return error instanceof LlmResponseError\n ? costReceiptFromLlm(error.result, customTokenPricing)\n : undefined\n}\n\nexport class LlmCallError extends AgentEvalError {\n constructor(\n message: string,\n public readonly status: number,\n public readonly body: string,\n public readonly model: string,\n ) {\n super('judge', message)\n }\n}\n\n/** A provider response completed and incurred measurable usage, but its content\n * could not satisfy the caller's response contract. The response envelope is\n * retained so accounting can commit the receipt before the error propagates. */\nexport class LlmResponseError extends AgentEvalError {\n constructor(\n message: string,\n public readonly result: LlmCallResult,\n options?: { cause?: unknown },\n ) {\n super('judge', message, options)\n }\n}\n\nexport interface LlmClientOptions {\n /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n baseUrl?: string\n /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n apiKey?: string\n bearer?: string\n /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n authHeader?: { name: string; value: string }\n /** Stable provider idempotency key, reused across retries of this logical call. */\n idempotencyKey?: string\n /** Default timeout in ms. Per-call can override. */\n defaultTimeoutMs?: number\n /**\n * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to\n * each attempt's per-attempt timeout controller, so aborting it cancels\n * the in-flight fetch. A caller abort is FATAL: it is not retried even\n * though an AbortError otherwise matches the transient patterns.\n */\n signal?: AbortSignal\n /**\n * Cross-attempt wall-clock budget in ms, measured from the first attempt.\n * Before launching each attempt the loop checks the remaining budget and\n * stops retrying once it is exhausted, rather than waiting the full\n * per-attempt timeout on every retry. Bounds total time independent of\n * total attempts × `timeoutMs`.\n */\n deadlineMs?: number\n /** Total provider attempts. Default 3. */\n maximumAttempts?: number\n /** Token rates used when the provider omits cost or package pricing does not cover the model. */\n customTokenPricing?: CustomTokenPricing\n /**\n * Transport for requests that declare `jsonSchema`. `native` sends\n * `response_format: json_schema`; `json-object` sends the broadly supported\n * JSON mode and relies on the caller to include the schema in model-visible\n * instructions. Default: `native`.\n */\n jsonSchemaTransport?: 'native' | 'json-object'\n /**\n * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.\n * `exact` requires the complete response content to be one JSON value.\n * Default: `extract`.\n */\n jsonPayloadMode?: 'extract' | 'exact'\n /** Default provider reasoning mode. A per-call request value takes precedence. */\n thinking?: LlmThinkingMode\n /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n fetch?: typeof fetch\n /**\n * Optional raw HTTP capture sink. When provided, every request, response,\n * and error (across all retry attempts) is recorded to the sink, with auth\n * headers and credential-shaped body fields redacted by default. This is\n * the layer-1 forensics primitive: structured `LlmSpan`s record intent,\n * raw events record what actually crossed the wire.\n */\n rawSink?: RawProviderSink\n /**\n * Logical provider id attached to raw events. When omitted, derived from\n * `baseUrl` via `providerFromBaseUrl`.\n */\n provider?: string\n /** Trace context attached to raw events; populated by emitter-aware callers. */\n traceContext?: { runId?: string; spanId?: string }\n /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */\n redactor?: ProviderRedactor\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\n// Flagship / reasoning models routinely take several minutes on large prompts (a\n// reflection over many failures, a long tool transcript). A tight cap aborts a\n// legitimately-slow but healthy call — and because every retry attempt re-uses\n// the same window, such a model aborts on ALL attempts and the loop throws. The\n// default is generous enough to let those complete, bounded enough that a truly\n// hung call still fails over after retries, and tunable per deployment via\n// TANGLE_LLM_TIMEOUT_MS. Per-call `req.timeoutMs` / `opts.defaultTimeoutMs`\n// still win for callers that know their model's latency.\nconst DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 300_000\nconst DEFAULT_MAXIMUM_ATTEMPTS =\n process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS === undefined\n ? 3\n : Number(process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS)\n\nfunction resolveMaximumAttempts(configured: number | undefined): number {\n const attempts = configured ?? DEFAULT_MAXIMUM_ATTEMPTS\n if (!Number.isInteger(attempts) || attempts <= 0) {\n throw new RangeError('LLM maximum attempts must be a positive integer')\n }\n return attempts\n}\n\nfunction providerTokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\n/**\n * Transient transport/network error signatures, matched against an error's\n * name, message, and `code`. Covers fetch/undici network failures, aborts\n * and timeouts, and — critically — HTTP/2 transport faults a keep-alive\n * connection raises mid-response: `terminated`, `NGHTTP2_INTERNAL_ERROR`,\n * `UND_ERR_*`, `other side closed`. Those last ones carry no clean HTTP\n * status; unrecognised, they escape the retry loop and surface as an\n * uncaught rejection.\n */\nconst TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [\n /AbortError/i,\n /TimeoutError/i,\n /this operation was aborted/i,\n /fetch failed/i,\n /ECONNRESET/i,\n /ETIMEDOUT/i,\n /EAI_AGAIN/i,\n /socket hang up/i,\n /stream.*ended.*unexpectedly/i,\n /terminated/i,\n /other side closed/i,\n /NGHTTP2/i,\n /UND_ERR/i,\n]\n\n/**\n * True when an error is a transient transport/network fault worth retrying,\n * as opposed to a deterministic failure (4xx schema reject, JSON parse) that\n * a retry cannot fix. Inspects `LlmCallError.status`, then the error's\n * name/message/code, then recurses into `error.cause` — undici nests the\n * real socket fault one or more levels under `.cause`.\n *\n * This is the retry classifier for the package: `callLlm` and\n * `withJudgeRetry` both route through it, so connection failures are treated\n * consistently across transports.\n */\nexport function isTransientLlmError(err: unknown): boolean {\n return classifyTransient(err, 0)\n}\n\nfunction classifyTransient(err: unknown, depth: number): boolean {\n if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n if (!(err instanceof Error)) return false\n // Foreign transport errors can carry a numeric HTTP status without being an\n // LlmCallError. A retryable status is decisive.\n const status = (err as { status?: unknown }).status\n if (typeof status === 'number' && RETRYABLE_STATUS.has(status)) return true\n const code = (err as { code?: unknown }).code\n const haystack = `${err.name}\\n${err.message}\\n${typeof code === 'string' ? code : ''}`\n if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true\n const cause = (err as { cause?: unknown }).cause\n if (depth < 4 && cause instanceof Error && cause !== err) {\n return classifyTransient(cause, depth + 1)\n }\n return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n const h = headers.get('retry-after')\n if (!h) return null\n const asNumber = Number(h)\n if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n const asDate = Date.parse(h)\n if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n return null\n}\n\n/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */\nexport function backoffMs(attempt: number): number {\n return Math.min(500 * 2 ** attempt, 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n Accept: 'application/json',\n }\n if (opts.authHeader) {\n headers[opts.authHeader.name] = opts.authHeader.value\n } else if (opts.bearer || opts.apiKey) {\n headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n }\n if (opts.idempotencyKey) headers['Idempotency-Key'] = opts.idempotencyKey\n return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n if (status !== 400) return false\n const lower = body.toLowerCase()\n return (\n lower.includes('response_format') ||\n lower.includes('json_schema') ||\n lower.includes('is unavailable') ||\n lower.includes('not supported')\n )\n}\n\nfunction isTemperatureOneRejection(status: number, body: string): boolean {\n if (status !== 400 || !/temperature/i.test(body)) return false\n return (\n /temperature[^.\\n]{0,120}\\b(?:only|must|should|required|requires?)\\b[^.\\n]{0,40}\\b1(?:\\.0+)?\\b/i.test(\n body,\n ) || /\\bonly\\s+1(?:\\.0+)?\\s+is\\s+allowed\\b[^.\\n]{0,120}\\btemperature\\b/i.test(body)\n )\n}\n\nfunction buildBody(\n req: LlmCallRequest,\n forceJsonObject: boolean,\n defaultThinking?: LlmThinkingMode,\n): Record<string, unknown> {\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0,\n }\n if (req.maxTokens != null) {\n if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens\n else body.max_tokens = req.maxTokens\n }\n const thinking = req.thinking ?? defaultThinking\n if (thinking !== undefined) {\n body.thinking = { type: thinking }\n }\n\n if (req.jsonSchema && !forceJsonObject) {\n body.response_format = {\n type: 'json_schema',\n json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n }\n } else if (req.jsonMode || req.jsonSchema) {\n body.response_format = { type: 'json_object' }\n }\n\n return body\n}\n\nfunction usesMaxCompletionTokens(model: string): boolean {\n return /^gpt-5(?:[.-]|$)/i.test(model)\n}\n\nasync function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n/**\n * Combine the per-attempt timeout signal with an optional caller signal into\n * one signal the fetch listens on. Prefers the native `AbortSignal.any`; falls\n * back to manual wiring on runtimes that predate it. The caller signal is also\n * propagated to the timeout controller so aborting it cancels the in-flight\n * fetch immediately.\n */\nfunction linkSignals(timeoutController: AbortController, caller?: AbortSignal): AbortSignal {\n if (!caller) return timeoutController.signal\n if (typeof (AbortSignal as { any?: unknown }).any === 'function') {\n return AbortSignal.any([timeoutController.signal, caller])\n }\n if (caller.aborted) {\n timeoutController.abort()\n } else {\n caller.addEventListener('abort', () => timeoutController.abort(), { once: true })\n }\n return timeoutController.signal\n}\n\n/** True once the cross-attempt wall-clock budget (if any) is exhausted. */\nfunction deadlineExceeded(start: number, deadlineMs: number | undefined): boolean {\n return deadlineMs != null && Date.now() - start >= deadlineMs\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n const trimmed = raw.trim()\n const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n const stripped = stripFencedJson(raw)\n try {\n JSON.parse(stripped)\n return stripped\n } catch {\n // A response that declares a JSON root must parse as that complete root.\n // Scanning onward could turn a truncated object into one of its valid nested\n // arrays or objects and silently change the response schema.\n if (stripped.startsWith('{') || stripped.startsWith('[')) return stripped\n }\n\n // Only prose-leading responses may contain a recoverable JSON payload.\n const starts = [...stripped.matchAll(/[[{]/g)]\n .map((match) => match.index)\n .filter((index) => index != null)\n for (const start of starts) {\n const candidate = extractBalancedJson(stripped, start)\n if (!candidate) continue\n try {\n JSON.parse(candidate)\n return candidate\n } catch {\n // Keep scanning; earlier braces may belong to prose.\n }\n }\n\n return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n const opener = input[start]\n const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n if (!closer) return null\n\n const stack: string[] = [closer]\n let isInString = false\n let isEscaped = false\n\n for (let i = start + 1; i < input.length; i++) {\n const char = input[i]!\n if (isEscaped) {\n isEscaped = false\n continue\n }\n if (char === '\\\\') {\n isEscaped = isInString\n continue\n }\n if (char === '\"') {\n isInString = !isInString\n continue\n }\n if (isInString) continue\n\n if (char === '{') stack.push('}')\n else if (char === '[') stack.push(']')\n else if (char === stack[stack.length - 1]) {\n stack.pop()\n if (stack.length === 0) return input.slice(start, i + 1)\n }\n }\n\n return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n const url = `${baseUrl}/chat/completions`\n const endpoint = '/chat/completions'\n const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n const maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n const fetchFn = opts.fetch ?? globalThis.fetch\n const headers = buildHeaders(opts)\n const provider = opts.provider ?? providerFromBaseUrl(baseUrl)\n const sink = opts.rawSink\n const redactor = opts.redactor ?? defaultProviderRedactor\n const traceContext = opts.traceContext\n const callerSignal = opts.signal\n const deadlineMs = opts.deadlineMs\n const deadlineStart = Date.now()\n if (opts.customTokenPricing) {\n costForTokenPricing(opts.customTokenPricing, { inputTokens: 0, outputTokens: 0 })\n }\n\n let lastErr: unknown\n let effectiveRequest = req\n for (let attempt = 0; attempt < maximumAttempts; attempt++) {\n // A caller cancel is fatal — never retried. Checking before each attempt\n // means an already-aborted signal short-circuits without firing fetch.\n if (callerSignal?.aborted) {\n throw new DOMException('callLlm aborted by caller signal', 'AbortError')\n }\n // Stop retrying once the cross-attempt budget is spent rather than burning\n // a full per-attempt timeout on each remaining retry.\n if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n }\n const controller = new AbortController()\n const attemptSignal = linkSignals(controller, callerSignal)\n const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n const started = Date.now()\n const requestBody = buildBody(\n effectiveRequest,\n opts.jsonSchemaTransport === 'json-object',\n opts.thinking,\n )\n let attemptErrorRecorded = false\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'request',\n timestamp: started,\n requestHeaders: headers,\n requestBody,\n redactedFields: [],\n })\n }\n\n try {\n const res = await fetchFn(url, {\n method: 'POST',\n headers,\n body: JSON.stringify(requestBody),\n signal: attemptSignal,\n })\n clearTimeout(timeoutHandle)\n const responseHeaders = sink ? headersToObject(res.headers) : undefined\n\n if (!res.ok) {\n const body = await res.text()\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: body,\n errorMessage: `HTTP ${res.status}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n const err = new LlmCallError(\n `LLM call ${res.status}: ${body.slice(0, 300)}`,\n res.status,\n body,\n req.model,\n )\n if (\n isTemperatureOneRejection(res.status, body) &&\n effectiveRequest.temperature !== 1 &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n effectiveRequest = { ...effectiveRequest, temperature: 1 }\n continue\n }\n if (\n RETRYABLE_STATUS.has(res.status) &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n const retryAfter = parseRetryAfter(res.headers)\n await sleep(retryAfter ?? backoffMs(attempt))\n continue\n }\n throw err\n }\n\n const text = await res.text()\n let json: Record<string, unknown>\n try {\n json = JSON.parse(text) as Record<string, unknown>\n } catch (parseErr) {\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: text,\n errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n throw parseErr\n }\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'response',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: json,\n redactedFields: [],\n })\n }\n const choice = (\n json.choices as\n | Array<{ message?: { content?: string }; finish_reason?: string | null }>\n | undefined\n )?.[0]\n const usageRaw =\n json.usage && typeof json.usage === 'object' && !Array.isArray(json.usage)\n ? (json.usage as Record<string, unknown>)\n : undefined\n const promptTokens = providerTokenCount(usageRaw?.prompt_tokens)\n const completionTokens = providerTokenCount(usageRaw?.completion_tokens)\n const totalTokens = providerTokenCount(usageRaw?.total_tokens)\n const completionDetails =\n usageRaw?.completion_tokens_details &&\n typeof usageRaw.completion_tokens_details === 'object' &&\n !Array.isArray(usageRaw.completion_tokens_details)\n ? (usageRaw.completion_tokens_details as Record<string, unknown>)\n : undefined\n const reasoningRaw = completionDetails?.reasoning_tokens\n const reasoningTokens =\n reasoningRaw === undefined ? undefined : providerTokenCount(reasoningRaw)\n const cachedRaw =\n usageRaw?.prompt_tokens_details &&\n typeof usageRaw.prompt_tokens_details === 'object' &&\n !Array.isArray(usageRaw.prompt_tokens_details)\n ? (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens\n : undefined\n const cachedPromptTokens = cachedRaw === undefined ? undefined : providerTokenCount(cachedRaw)\n const usageCaptured =\n promptTokens !== undefined &&\n completionTokens !== undefined &&\n (reasoningRaw === undefined ||\n (reasoningTokens !== undefined && reasoningTokens <= completionTokens)) &&\n (cachedRaw === undefined ||\n (cachedPromptTokens !== undefined && cachedPromptTokens <= promptTokens)) &&\n (totalTokens === undefined || totalTokens === promptTokens + completionTokens)\n const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n const content = choice?.message?.content ?? ''\n\n const configuredCost =\n typeof costFromProxy !== 'number' && usageCaptured && opts.customTokenPricing\n ? costForTokenPricing(opts.customTokenPricing, {\n inputTokens: promptTokens! - (cachedPromptTokens ?? 0),\n ...(cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {}),\n outputTokens: completionTokens!,\n })\n : undefined\n\n return {\n content,\n finishReason: choice?.finish_reason ?? null,\n contentEmpty: content.trim().length === 0,\n usage: {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),\n captured: usageCaptured,\n reasoningTokens,\n cachedPromptTokens,\n },\n costUsd: typeof costFromProxy === 'number' ? costFromProxy : (configuredCost ?? null),\n model: (json.model as string) ?? req.model,\n durationMs: Date.now() - started,\n raw: json,\n }\n } catch (err) {\n clearTimeout(timeoutHandle)\n lastErr = err\n // A caller cancel is fatal even though an AbortError matches the\n // transient patterns — a cancelled call must surface immediately, not\n // be retried against the same dead intent.\n if (callerSignal?.aborted) {\n if (sink && !attemptErrorRecorded) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n throw err\n }\n if (sink && !attemptErrorRecorded) {\n // Record only if neither the !res.ok branch nor the JSON.parse catch\n // already produced an error event for this attempt. Covers network\n // failures, timeouts, and aborts.\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n if (\n attempt < maximumAttempts - 1 &&\n isTransientLlmError(err) &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n await sleep(backoffMs(attempt))\n continue\n }\n throw err\n }\n }\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\nasync function recordRaw(\n sink: RawProviderSink,\n redactor: ProviderRedactor,\n event: RawProviderEvent,\n): Promise<void> {\n // Errors from sinks must not crash the LLM call. Forensic capture is\n // best-effort; the structured trace is the system of record.\n try {\n await sink.record(redactor(event))\n } catch {\n // Intentionally swallowed.\n }\n}\n\nfunction headersToObject(h: Headers): Record<string, string> {\n const out: Record<string, string> = {}\n h.forEach((value, key) => {\n out[key] = value\n })\n return out\n}\n\nfunction cryptoEventId(): string {\n if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n const result = await callLlmStructured(req, opts)\n const value = parseJsonResult<T>(result, opts.jsonPayloadMode ?? 'extract')\n return { value, result }\n}\n\n/** Shared schema-to-JSON-mode fallback that preserves the raw result. */\nasync function callLlmStructured(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n try {\n return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n } catch (err) {\n if (\n opts.jsonSchemaTransport !== 'json-object' &&\n err instanceof LlmCallError &&\n isSchemaRejection(err.status, err.body) &&\n req.jsonSchema\n ) {\n const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n return await callLlm(degradedReq, opts)\n }\n throw err\n }\n}\n\nfunction parseJsonResult<T>(\n result: LlmCallResult,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n try {\n if (result.finishReason === 'length') {\n throw new Error(\n `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`,\n )\n }\n return parseJsonSafely<T>(result.content, result.model, jsonPayloadMode)\n } catch (error) {\n if (error instanceof LlmResponseError) throw error\n const cause = error instanceof Error ? error : new Error(String(error))\n throw new LlmResponseError(cause.message, result, { cause })\n }\n}\n\nfunction parseJsonSafely<T>(\n content: string,\n model: string,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n const payload = jsonPayloadMode === 'exact' ? content : extractJsonPayload(content)\n try {\n return JSON.parse(payload) as T\n } catch (err) {\n throw new Error(\n `LLM returned non-JSON content (model=${model}): ${\n err instanceof Error ? err.message : String(err)\n }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n )\n }\n}\n\n// ─── Route assertion ────────────────────────────────────────────────────\n\nexport type LlmRouteAssertionReason =\n | 'no_explicit_base_url'\n | 'base_url_blocked'\n | 'base_url_not_allowed'\n | 'no_auth'\n | 'wrong_provider'\n\nexport class LlmRouteAssertionError extends CaptureIntegrityError {\n constructor(\n message: string,\n public readonly reason: LlmRouteAssertionReason,\n public readonly baseUrl: string,\n ) {\n super(message)\n }\n}\n\nexport interface LlmRouteRequirements {\n /**\n * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to\n * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using\n * the public/free-tier router is a defect — the launch reviewer needs to\n * know exactly which provider answered.\n */\n requireExplicitBaseUrl?: boolean\n /**\n * Allowlist of acceptable base URLs. Strings match by prefix\n * (case-insensitive); RegExps test against the full base URL.\n */\n allowedBaseUrls?: Array<string | RegExp>\n /** Blocklist that takes precedence over `allowedBaseUrls`. */\n blockedBaseUrls?: Array<string | RegExp>\n /** Throw if no auth header / api key is configured. */\n requireAuth?: boolean\n /**\n * Logical provider id the configured `baseUrl` is expected to match (via\n * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.\n */\n expectedProvider?: string\n}\n\n/**\n * Fail-loud assertion that the configured LLM client points at the route\n * the caller intends. Designed for the matrix-runner preflight: invoke\n * once before any LLM call to catch misconfiguration before a sweep burns\n * dollars on the wrong provider.\n *\n * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call\n * from constructors and CI gates.\n */\nexport function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {\n const baseUrlExplicit = opts.baseUrl !== undefined\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n if (req.requireExplicitBaseUrl && !baseUrlExplicit) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,\n 'no_explicit_base_url',\n baseUrl,\n )\n }\n\n if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,\n 'base_url_blocked',\n baseUrl,\n )\n }\n\n if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {\n const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))\n if (!ok) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,\n 'base_url_not_allowed',\n baseUrl,\n )\n }\n }\n\n if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,\n 'no_auth',\n baseUrl,\n )\n }\n\n if (req.expectedProvider) {\n const actual = opts.provider ?? providerFromBaseUrl(baseUrl)\n if (actual !== req.expectedProvider) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,\n 'wrong_provider',\n baseUrl,\n )\n }\n }\n}\n\nfunction matchUrl(url: string, pattern: string | RegExp): boolean {\n if (pattern instanceof RegExp) return pattern.test(url)\n return url.toLowerCase().startsWith(pattern.toLowerCase())\n}\n\nfunction describePattern(p: string | RegExp): string {\n return p instanceof RegExp ? p.source : p\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n model: string,\n opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n const start = Date.now()\n try {\n await callLlm(\n {\n model,\n messages: [{ role: 'user', content: 'ping' }],\n maxTokens: 64,\n timeoutMs: opts.timeoutMs ?? 30_000,\n },\n opts,\n )\n return { ok: true, latencyMs: Date.now() - start, error: null }\n } catch (err) {\n return {\n ok: false,\n latencyMs: Date.now() - start,\n error: err instanceof Error ? err.message : String(err),\n }\n }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n readonly maximumAttempts: number\n private readonly opts: LlmClientOptions\n\n constructor(opts: LlmClientOptions = {}) {\n this.opts = opts\n this.maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n }\n\n call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n const options = { ...this.opts, ...per }\n return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)\n }\n\n callJson<T = unknown>(\n req: LlmCallRequest,\n per?: LlmClientOptions,\n ): Promise<{ value: T; result: LlmCallResult }> {\n return callLlmJson<T>(req, { ...this.opts, ...per })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,SAAgB,2BACd,SACA,UAA4B,CAAC,GACF;CAC3B,IAAI,QAAQ,cAAc,KAAA,GAAW,OAAO,KAAA;CAC5C,IAAI,CAAC,OAAO,UAAU,QAAQ,SAAS,KAAK,QAAQ,aAAa,GAC/D,MAAM,IAAI,WAAW,kEAAkE;CAEzF,IACE,QAAQ,SAAS,MACd,YACC,MAAM,QAAQ,QAAQ,OAAO,KAAK,QAAQ,QAAQ,MAAM,SAAS,KAAK,SAAS,WAAW,CAC9F,GAEA;CAGF,MAAM,WAAW,uBAAuB,QAAQ,eAAe;CAC/D,MAAM,kBAAkB,QAAQ,wBAAwB;CAGxD,MAAM,eAAe,IAAI,YAAY,CAAC,CAAC,OACrC,KAAK,UAAU,UAAU,SAAS,iBAAiB,QAAQ,QAAQ,CAAC,CACtE,CAAC,CAAC;CAEF,MAAM,UAAU,QAAQ,cAAc,CAAC,kBAAkB,IAAI;CAC7D,MAAM,QAAQ;EACZ,aAAa,eAAe,WAAW;EACvC,cAAc,QAAQ,YAAY,WAAW;CAC/C;CACA,OAAO,QAAQ,qBACX;EAAE,oBAAoB,QAAQ;EAAoB,GAAG;CAAM,IAC3D;EAAE,OAAO,QAAQ;EAAO,GAAG;CAAM;AACvC;;AAkDA,SAAgB,mBACd,QACA,oBACkB;CAClB,MAAM,eAAe,OAAO,MAAM,sBAAsB;CACxD,MAAM,cAAc,KAAK,IAAI,GAAG,OAAO,MAAM,eAAe,YAAY;CACxE,MAAM,oBACJ,OAAO,YAAY,QAAQ,sBAAsB,OAAO,MAAM,aAAa,QACvE,oBAAoB,oBAAoB;EACtC;EACA,GAAI,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC3C,cAAc,OAAO,MAAM;CAC7B,CAAC,IACD,KAAA;CACN,OAAO;EACL,OAAO,OAAO;EACd;EACA,cAAc,OAAO,MAAM;EAC3B,iBAAiB,OAAO,MAAM;EAC9B,cAAc,eAAe,IAAI,eAAe,KAAA;EAChD,eAAe,OAAO,WAAW;EACjC,cAAc,OAAO,MAAM,aAAa;CAC1C;AACF;;AAGA,SAAgB,wBACd,OACA,oBAC8B;CAC9B,OAAO,iBAAiB,mBACpB,mBAAmB,MAAM,QAAQ,kBAAkB,IACnD,KAAA;AACN;AAEA,IAAa,eAAb,cAAkC,eAAe;CAG7B;CACA;CACA;CAJlB,YACE,SACA,QACA,MACA,OACA;EACA,MAAM,SAAS,OAAO;EAJN,KAAA,SAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;CAGlB;AACF;;;;AAKA,IAAa,mBAAb,cAAsC,eAAe;CAGjC;CAFlB,YACE,SACA,QACA,SACA;EACA,MAAM,SAAS,SAAS,OAAO;EAHf,KAAA,SAAA;CAIlB;AACF;AAuEA,MAAM,mBAAmB;AASzB,MAAM,qBAAqB,OAAO,QAAQ,IAAI,qBAAqB,KAAK;AACxE,MAAM,2BACJ,QAAQ,IAAI,gCAAgC,KAAA,IACxC,IACA,OAAO,QAAQ,IAAI,2BAA2B;AAEpD,SAAS,uBAAuB,YAAwC;CACtE,MAAM,WAAW,cAAc;CAC/B,IAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,YAAY,GAC7C,MAAM,IAAI,WAAW,iDAAiD;CAExE,OAAO;AACT;AAEA,SAAS,mBAAmB,OAAoC;CAC9D,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;AAEA,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;;;;;;;;;;AAWrD,MAAM,2BAA8C;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;;;;;;AAaA,SAAgB,oBAAoB,KAAuB;CACzD,OAAO,kBAAkB,KAAK,CAAC;AACjC;AAEA,SAAS,kBAAkB,KAAc,OAAwB;CAC/D,IAAI,eAAe,cAAc,OAAO,iBAAiB,IAAI,IAAI,MAAM;CACvE,IAAI,EAAE,eAAe,QAAQ,OAAO;CAGpC,MAAM,SAAU,IAA6B;CAC7C,IAAI,OAAO,WAAW,YAAY,iBAAiB,IAAI,MAAM,GAAG,OAAO;CACvE,MAAM,OAAQ,IAA2B;CACzC,MAAM,WAAW,GAAG,IAAI,KAAK,IAAI,IAAI,QAAQ,IAAI,OAAO,SAAS,WAAW,OAAO;CACnF,IAAI,yBAAyB,MAAM,MAAM,EAAE,KAAK,QAAQ,CAAC,GAAG,OAAO;CACnE,MAAM,QAAS,IAA4B;CAC3C,IAAI,QAAQ,KAAK,iBAAiB,SAAS,UAAU,KACnD,OAAO,kBAAkB,OAAO,QAAQ,CAAC;CAE3C,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;CACxD,MAAM,IAAI,QAAQ,IAAI,aAAa;CACnC,IAAI,CAAC,GAAG,OAAO;CACf,MAAM,WAAW,OAAO,CAAC;CACzB,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAAG,OAAO,WAAW;CACjE,MAAM,SAAS,KAAK,MAAM,CAAC;CAC3B,IAAI,OAAO,SAAS,MAAM,GAAG,OAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;CACnE,OAAO;AACT;;AAGA,SAAgB,UAAU,SAAyB;CACjD,OAAO,KAAK,IAAI,MAAM,KAAK,SAAS,IAAM;AAC5C;AAEA,SAAS,aAAa,MAAgD;CACpE,MAAM,UAAkC;EACtC,gBAAgB;EAChB,QAAQ;CACV;CACA,IAAI,KAAK,YACP,QAAQ,KAAK,WAAW,QAAQ,KAAK,WAAW;MAC3C,IAAI,KAAK,UAAU,KAAK,QAC7B,QAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK;CAExD,IAAI,KAAK,gBAAgB,QAAQ,qBAAqB,KAAK;CAC3D,OAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;CAChE,IAAI,WAAW,KAAK,OAAO;CAC3B,MAAM,QAAQ,KAAK,YAAY;CAC/B,OACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,0BAA0B,QAAgB,MAAuB;CACxE,IAAI,WAAW,OAAO,CAAC,eAAe,KAAK,IAAI,GAAG,OAAO;CACzD,OACE,iGAAiG,KAC/F,IACF,KAAK,oEAAoE,KAAK,IAAI;AAEtF;AAEA,SAAS,UACP,KACA,iBACA,iBACyB;CACzB,MAAM,OAAgC;EACpC,OAAO,IAAI;EACX,UAAU,IAAI;EACd,aAAa,IAAI,eAAe;CAClC;CACA,IAAI,IAAI,aAAa,MACnB,IAAI,wBAAwB,IAAI,KAAK,GAAG,KAAK,wBAAwB,IAAI;MACpE,KAAK,aAAa,IAAI;CAE7B,MAAM,WAAW,IAAI,YAAY;CACjC,IAAI,aAAa,KAAA,GACf,KAAK,WAAW,EAAE,MAAM,SAAS;CAGnC,IAAI,IAAI,cAAc,CAAC,iBACrB,KAAK,kBAAkB;EACrB,MAAM;EACN,aAAa;GAAE,MAAM,IAAI,WAAW;GAAM,QAAQ,IAAI,WAAW;GAAQ,QAAQ;EAAK;CACxF;MACK,IAAI,IAAI,YAAY,IAAI,YAC7B,KAAK,kBAAkB,EAAE,MAAM,cAAc;CAG/C,OAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;CACvD,OAAO,oBAAoB,KAAK,KAAK;AACvC;AAEA,eAAe,MAAM,IAA2B;CAC9C,OAAO,IAAI,SAAS,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;;;;;;;;AASA,SAAS,YAAY,mBAAoC,QAAmC;CAC1F,IAAI,CAAC,QAAQ,OAAO,kBAAkB;CACtC,IAAI,OAAQ,YAAkC,QAAQ,YACpD,OAAO,YAAY,IAAI,CAAC,kBAAkB,QAAQ,MAAM,CAAC;CAE3D,IAAI,OAAO,SACT,kBAAkB,MAAM;MAExB,OAAO,iBAAiB,eAAe,kBAAkB,MAAM,GAAG,EAAE,MAAM,KAAK,CAAC;CAElF,OAAO,kBAAkB;AAC3B;;AAGA,SAAS,iBAAiB,OAAe,YAAyC;CAChF,OAAO,cAAc,QAAQ,KAAK,IAAI,IAAI,SAAS;AACrD;;;;;;AASA,SAAgB,gBAAgB,KAAqB;CACnD,MAAM,UAAU,IAAI,KAAK;CACzB,MAAM,IAAI,QAAQ,MAAM,yCAAyC;CACjE,OAAO,IAAI,EAAE,EAAE,CAAE,KAAK,IAAI;AAC5B;AAEA,SAAgB,mBAAmB,KAAqB;CACtD,MAAM,WAAW,gBAAgB,GAAG;CACpC,IAAI;EACF,KAAK,MAAM,QAAQ;EACnB,OAAO;CACT,QAAQ;EAIN,IAAI,SAAS,WAAW,GAAG,KAAK,SAAS,WAAW,GAAG,GAAG,OAAO;CACnE;CAGA,MAAM,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,CAAC,CAAC,CAC3C,KAAK,UAAU,MAAM,KAAK,CAAC,CAC3B,QAAQ,UAAU,SAAS,IAAI;CAClC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,YAAY,oBAAoB,UAAU,KAAK;EACrD,IAAI,CAAC,WAAW;EAChB,IAAI;GACF,KAAK,MAAM,SAAS;GACpB,OAAO;EACT,QAAQ,CAER;CACF;CAEA,OAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;CACxE,MAAM,SAAS,MAAM;CACrB,MAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;CAC7D,IAAI,CAAC,QAAQ,OAAO;CAEpB,MAAM,QAAkB,CAAC,MAAM;CAC/B,IAAI,aAAa;CACjB,IAAI,YAAY;CAEhB,KAAK,IAAI,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;EAC7C,MAAM,OAAO,MAAM;EACnB,IAAI,WAAW;GACb,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAM;GACjB,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAK;GAChB,aAAa,CAAC;GACd;EACF;EACA,IAAI,YAAY;EAEhB,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAC3B,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAChC,IAAI,SAAS,MAAM,MAAM,SAAS,IAAI;GACzC,MAAM,IAAI;GACV,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;EACzD;CACF;CAEA,OAAO;AACT;;;;;;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;CACxB,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CACrE,MAAM,MAAM,GAAG,QAAQ;CACvB,MAAM,WAAW;CACjB,MAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;CAC5D,MAAM,kBAAkB,uBAAuB,KAAK,eAAe;CACnE,MAAM,UAAU,KAAK,SAAS,WAAW;CACzC,MAAM,UAAU,aAAa,IAAI;CACjC,MAAM,WAAW,KAAK,YAAY,oBAAoB,OAAO;CAC7D,MAAM,OAAO,KAAK;CAClB,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,eAAe,KAAK;CAC1B,MAAM,eAAe,KAAK;CAC1B,MAAM,aAAa,KAAK;CACxB,MAAM,gBAAgB,KAAK,IAAI;CAC/B,IAAI,KAAK,oBACP,oBAAoB,KAAK,oBAAoB;EAAE,aAAa;EAAG,cAAc;CAAE,CAAC;CAGlF,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,IAAI,UAAU,GAAG,UAAU,iBAAiB,WAAW;EAG1D,IAAI,cAAc,SAChB,MAAM,IAAI,aAAa,oCAAoC,YAAY;EAIzE,IAAI,UAAU,KAAK,iBAAiB,eAAe,UAAU,GAC3D,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;EAEtE,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,gBAAgB,YAAY,YAAY,YAAY;EAC1D,MAAM,gBAAgB,iBAAiB,WAAW,MAAM,GAAG,SAAS;EACpE,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,cAAc,UAClB,kBACA,KAAK,wBAAwB,eAC7B,KAAK,QACP;EACA,IAAI,uBAAuB;EAC3B,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;GAC9B,SAAS,cAAc;GACvB,OAAO,cAAc;GACrB,QAAQ,cAAc;GACtB;GACA,OAAO,IAAI;GACX;GACA;GACA,cAAc;GACd,WAAW;GACX,WAAW;GACX,gBAAgB;GAChB;GACA,gBAAgB,CAAC;EACnB,CAAC;EAGH,IAAI;GACF,MAAM,MAAM,MAAM,QAAQ,KAAK;IAC7B,QAAQ;IACR;IACA,MAAM,KAAK,UAAU,WAAW;IAChC,QAAQ;GACV,CAAC;GACD,aAAa,aAAa;GAC1B,MAAM,kBAAkB,OAAO,gBAAgB,IAAI,OAAO,IAAI,KAAA;GAE9D,IAAI,CAAC,IAAI,IAAI;IACX,MAAM,OAAO,MAAM,IAAI,KAAK;IAC5B,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,QAAQ,IAAI;MAC1B,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM,MAAM,IAAI,aACd,YAAY,IAAI,OAAO,IAAI,KAAK,MAAM,GAAG,GAAG,KAC5C,IAAI,QACJ,MACA,IAAI,KACN;IACA,IACE,0BAA0B,IAAI,QAAQ,IAAI,KAC1C,iBAAiB,gBAAgB,KACjC,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KACV,mBAAmB;MAAE,GAAG;MAAkB,aAAa;KAAE;KACzD;IACF;IACA,IACE,iBAAiB,IAAI,IAAI,MAAM,KAC/B,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KAEV,MAAM,MADa,gBAAgB,IAAI,OAClB,KAAK,UAAU,OAAO,CAAC;KAC5C;IACF;IACA,MAAM;GACR;GAEA,MAAM,OAAO,MAAM,IAAI,KAAK;GAC5B,IAAI;GACJ,IAAI;IACF,OAAO,KAAK,MAAM,IAAI;GACxB,SAAS,UAAU;IACjB,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,sBAAsB,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;MAClG,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM;GACR;GACA,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,YAAY,IAAI;IAChB;IACA,cAAc;IACd,gBAAgB,CAAC;GACnB,CAAC;GAEH,MAAM,SACJ,KAAK,UAGH;GACJ,MAAM,WACJ,KAAK,SAAS,OAAO,KAAK,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,KAAK,IACpE,KAAK,QACN,KAAA;GACN,MAAM,eAAe,mBAAmB,UAAU,aAAa;GAC/D,MAAM,mBAAmB,mBAAmB,UAAU,iBAAiB;GACvE,MAAM,cAAc,mBAAmB,UAAU,YAAY;GAO7D,MAAM,gBALJ,UAAU,6BACV,OAAO,SAAS,8BAA8B,YAC9C,CAAC,MAAM,QAAQ,SAAS,yBAAyB,IAC5C,SAAS,4BACV,KAAA,EAAA,EACkC;GACxC,MAAM,kBACJ,iBAAiB,KAAA,IAAY,KAAA,IAAY,mBAAmB,YAAY;GAC1E,MAAM,YACJ,UAAU,yBACV,OAAO,SAAS,0BAA0B,YAC1C,CAAC,MAAM,QAAQ,SAAS,qBAAqB,IACxC,SAAS,sBAAkD,gBAC5D,KAAA;GACN,MAAM,qBAAqB,cAAc,KAAA,IAAY,KAAA,IAAY,mBAAmB,SAAS;GAC7F,MAAM,gBACJ,iBAAiB,KAAA,KACjB,qBAAqB,KAAA,MACpB,iBAAiB,KAAA,KACf,oBAAoB,KAAA,KAAa,mBAAmB,sBACtD,cAAc,KAAA,KACZ,uBAAuB,KAAA,KAAa,sBAAsB,kBAC5D,gBAAgB,KAAA,KAAa,gBAAgB,eAAe;GAC/D,MAAM,gBAAiB,KAAK,kBAAkB,KAAK;GACnD,MAAM,UAAU,QAAQ,SAAS,WAAW;GAE5C,MAAM,iBACJ,OAAO,kBAAkB,YAAY,iBAAiB,KAAK,qBACvD,oBAAoB,KAAK,oBAAoB;IAC3C,aAAa,gBAAiB,sBAAsB;IACpD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;IACjE,cAAc;GAChB,CAAC,IACD,KAAA;GAEN,OAAO;IACL;IACA,cAAc,QAAQ,iBAAiB;IACvC,cAAc,QAAQ,KAAK,CAAC,CAAC,WAAW;IACxC,OAAO;KACL,cAAc,gBAAgB;KAC9B,kBAAkB,oBAAoB;KACtC,aAAa,gBAAgB,gBAAgB,MAAM,oBAAoB;KACvE,UAAU;KACV;KACA;IACF;IACA,SAAS,OAAO,kBAAkB,WAAW,gBAAiB,kBAAkB;IAChF,OAAQ,KAAK,SAAoB,IAAI;IACrC,YAAY,KAAK,IAAI,IAAI;IACzB,KAAK;GACP;EACF,SAAS,KAAK;GACZ,aAAa,aAAa;GAC1B,UAAU;GAIV,IAAI,cAAc,SAAS;IACzB,IAAI,QAAQ,CAAC,sBACX,MAAM,UAAU,MAAM,UAAU;KAC9B,SAAS,cAAc;KACvB,OAAO,cAAc;KACrB,QAAQ,cAAc;KACtB;KACA,OAAO,IAAI;KACX;KACA;KACA,cAAc;KACd,WAAW;KACX,WAAW,KAAK,IAAI;KACpB,YAAY,KAAK,IAAI,IAAI;KACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;KAC7D,gBAAgB,CAAC;IACnB,CAAC;IAEH,MAAM;GACR;GACA,IAAI,QAAQ,CAAC,sBAIX,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC7D,gBAAgB,CAAC;GACnB,CAAC;GAEH,IACE,UAAU,kBAAkB,KAC5B,oBAAoB,GAAG,KACvB,CAAC,iBAAiB,eAAe,UAAU,GAC3C;IACA,MAAM,MAAM,UAAU,OAAO,CAAC;IAC9B;GACF;GACA,MAAM;EACR;CACF;CACA,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAEA,eAAe,UACb,MACA,UACA,OACe;CAGf,IAAI;EACF,MAAM,KAAK,OAAO,SAAS,KAAK,CAAC;CACnC,QAAQ,CAER;AACF;AAEA,SAAS,gBAAgB,GAAoC;CAC3D,MAAM,MAA8B,CAAC;CACrC,EAAE,SAAS,OAAO,QAAQ;EACxB,IAAI,OAAO;CACb,CAAC;CACD,OAAO;AACT;AAEA,SAAS,gBAAwB;CAC/B,IAAI,OAAO,WAAW,QAAQ,eAAe,YAAY,OAAO,WAAW,OAAO,WAAW;CAC7F,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AAC7E;;;;;;;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;CAC9C,MAAM,SAAS,MAAM,kBAAkB,KAAK,IAAI;CAEhD,OAAO;EAAE,OADK,gBAAmB,QAAQ,KAAK,mBAAmB,SACpD;EAAG;CAAO;AACzB;;AAGA,eAAe,kBACb,KACA,OAAyB,CAAC,GACF;CACxB,IAAI;EACF,OAAO,MAAM,QAAQ;GAAE,GAAG;GAAK,UAAU,IAAI,YAAY,CAAC,IAAI;EAAW,GAAG,IAAI;CAClF,SAAS,KAAK;EACZ,IACE,KAAK,wBAAwB,iBAC7B,eAAe,gBACf,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KACtC,IAAI,YAGJ,OAAO,MAAM,QAAQ;GADiB,GAAG;GAAK,UAAU;GAAM,YAAY,KAAA;EAC3C,GAAG,IAAI;EAExC,MAAM;CACR;AACF;AAEA,SAAS,gBACP,QACA,iBACG;CACH,IAAI;EACF,IAAI,OAAO,iBAAiB,UAC1B,MAAM,IAAI,MACR,8CAA8C,OAAO,MAAM,uBAC7D;EAEF,OAAO,gBAAmB,OAAO,SAAS,OAAO,OAAO,eAAe;CACzE,SAAS,OAAO;EACd,IAAI,iBAAiB,kBAAkB,MAAM;EAC7C,MAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;EACtE,MAAM,IAAI,iBAAiB,MAAM,SAAS,QAAQ,EAAE,MAAM,CAAC;CAC7D;AACF;AAEA,SAAS,gBACP,SACA,OACA,iBACG;CACH,MAAM,UAAU,oBAAoB,UAAU,UAAU,mBAAmB,OAAO;CAClF,IAAI;EACF,OAAO,KAAK,MAAM,OAAO;CAC3B,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,wCAAwC,MAAM,KAC5C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAChD,yBAAyB,QAAQ,MAAM,GAAG,GAAG,GAChD;CACF;AACF;AAWA,IAAa,yBAAb,cAA4C,sBAAsB;CAG9C;CACA;CAHlB,YACE,SACA,QACA,SACA;EACA,MAAM,OAAO;EAHG,KAAA,SAAA;EACA,KAAA,UAAA;CAGlB;AACF;;;;;;;;;;AAmCA,SAAgB,eAAe,MAAwB,MAA4B,CAAC,GAAS;CAC3F,MAAM,kBAAkB,KAAK,YAAY,KAAA;CACzC,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CAErE,IAAI,IAAI,0BAA0B,CAAC,iBACjC,MAAM,IAAI,uBACR,gGAAgG,iBAAiB,IACjH,wBACA,OACF;CAGF,IAAI,IAAI,iBAAiB,MAAM,MAAM,SAAS,SAAS,CAAC,CAAC,GACvD,MAAM,IAAI,uBACR,2BAA2B,QAAQ,8BACnC,oBACA,OACF;CAGF,IAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,GAElD;MAAA,CADO,IAAI,gBAAgB,MAAM,MAAM,SAAS,SAAS,CAAC,CACxD,GACJ,MAAM,IAAI,uBACR,2BAA2B,QAAQ,+BAA+B,IAAI,gBAAgB,IAAI,eAAe,CAAC,CAAC,KAAK,IAAI,EAAE,KACtH,wBACA,OACF;CAAA;CAIJ,IAAI,IAAI,eAAe,CAAC,KAAK,UAAU,CAAC,KAAK,UAAU,CAAC,KAAK,YAC3D,MAAM,IAAI,uBACR,sFACA,WACA,OACF;CAGF,IAAI,IAAI,kBAAkB;EACxB,MAAM,SAAS,KAAK,YAAY,oBAAoB,OAAO;EAC3D,IAAI,WAAW,IAAI,kBACjB,MAAM,IAAI,uBACR,qCAAqC,IAAI,iBAAiB,eAAe,QAAQ,eAAe,OAAO,IACvG,kBACA,OACF;CAEJ;AACF;AAEA,SAAS,SAAS,KAAa,SAAmC;CAChE,IAAI,mBAAmB,QAAQ,OAAO,QAAQ,KAAK,GAAG;CACtD,OAAO,IAAI,YAAY,CAAC,CAAC,WAAW,QAAQ,YAAY,CAAC;AAC3D;AAEA,SAAS,gBAAgB,GAA4B;CACnD,OAAO,aAAa,SAAS,EAAE,SAAS;AAC1C;;;;;;;;;;;;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;CACnE,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EACF,MAAM,QACJ;GACE;GACA,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS;GAAO,CAAC;GAC5C,WAAW;GACX,WAAW,KAAK,aAAa;EAC/B,GACA,IACF;EACA,OAAO;GAAE,IAAI;GAAM,WAAW,KAAK,IAAI,IAAI;GAAO,OAAO;EAAK;CAChE,SAAS,KAAK;EACZ,OAAO;GACL,IAAI;GACJ,WAAW,KAAK,IAAI,IAAI;GACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;EACxD;CACF;AACF;;;;;;AAOA,IAAa,YAAb,MAAuB;CACrB;CACA;CAEA,YAAY,OAAyB,CAAC,GAAG;EACvC,KAAK,OAAO;EACZ,KAAK,kBAAkB,uBAAuB,KAAK,eAAe;CACpE;CAEA,KAAK,KAAqB,KAAgD;EACxE,MAAM,UAAU;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI;EACvC,OAAO,IAAI,aAAa,kBAAkB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO;CAChF;CAEA,SACE,KACA,KAC8C;EAC9C,OAAO,YAAe,KAAK;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI,CAAC;CACrD;AACF"}
1
+ {"version":3,"file":"llm-client-DHx8pzyJ.js","names":[],"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n * - One retry at temperature 1 when a model explicitly requires it.\n * - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n * directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n * const { value, result } = await callLlmJson<MyType>(\n * { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n * { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n * )\n *\n * `createChatClient` wraps this implementation for provider-neutral package\n * entry points. Direct callers can use `callLlm` or `callLlmJson`.\n */\n\nimport {\n type CostReceiptInput,\n type CustomTokenPricing,\n costForTokenPricing,\n type MaximumCharge,\n} from './cost-ledger'\nimport { AgentEvalError, CaptureIntegrityError } from './errors'\nimport {\n defaultProviderRedactor,\n type ProviderRedactor,\n providerFromBaseUrl,\n type RawProviderEvent,\n type RawProviderSink,\n} from './trace/raw-provider-sink'\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n role: 'system' | 'user' | 'assistant'\n /**\n * Either a plain text content string OR a multimodal content array\n * (text + image_url parts) for vision-capable models.\n */\n content:\n | string\n | Array<\n | { type: 'text'; text: string }\n | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n >\n}\n\nexport type LlmThinkingMode = 'enabled' | 'disabled'\n\nexport interface LlmCallRequest {\n model: string\n messages: LlmMessage[]\n /** Optional JSON-mode response format (response_format: json_object). */\n jsonMode?: boolean\n /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n jsonSchema?: { name: string; schema: Record<string, unknown> }\n temperature?: number\n maxTokens?: number\n /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */\n thinking?: LlmThinkingMode\n /** Per-call timeout, default 300s. */\n timeoutMs?: number\n}\n\n/** Conservative priced bound for the exact text request sent to a provider.\n * Returns undefined when output or multimodal input is not bounded, causing a\n * capped CostLedger to reject the call before execution. Pass\n * `customTokenPricing` when package pricing does not cover the model or endpoint. */\nexport function maximumChargeForLlmRequest(\n request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'maxTokens' | 'thinking'>,\n options: LlmClientOptions = {},\n): MaximumCharge | undefined {\n if (request.maxTokens === undefined) return undefined\n if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {\n throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`)\n }\n if (\n request.messages.some(\n (message) =>\n Array.isArray(message.content) && message.content.some((part) => part.type === 'image_url'),\n )\n ) {\n return undefined\n }\n\n const attempts = resolveMaximumAttempts(options.maximumAttempts)\n const forceJsonObject = options.jsonSchemaTransport === 'json-object'\n // A byte-level tokenizer cannot emit more input tokens than request bytes.\n // Pricing the complete body also covers role/schema framing omitted from content-only estimates.\n const requestBytes = new TextEncoder().encode(\n JSON.stringify(buildBody(request, forceJsonObject, options.thinking)),\n ).byteLength\n // A rejected response schema can trigger one JSON-mode batch with the same output limit.\n const batches = request.jsonSchema && !forceJsonObject ? 2 : 1\n const usage = {\n inputTokens: requestBytes * attempts * batches,\n outputTokens: request.maxTokens * attempts * batches,\n }\n return options.customTokenPricing\n ? { customTokenPricing: options.customTokenPricing, ...usage }\n : { model: request.model, ...usage }\n}\n\nexport interface LlmUsage {\n promptTokens: number\n completionTokens: number\n totalTokens: number\n /** False when the provider omitted or malformed prompt/completion usage. */\n captured?: boolean\n /** Reasoning-token subset of completionTokens, when reported. */\n reasoningTokens?: number\n /** Proxies populate this when prompt caching is on. */\n cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n /** The text content of the first choice. Empty string if none. */\n content: string\n usage: LlmUsage\n /**\n * Cost in USD. Uses the provider's reported cost when present, otherwise\n * caller-supplied token pricing. `null` when neither is available.\n */\n costUsd: number | null\n /** Model name actually used (echoed from response). */\n model: string\n /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n durationMs: number\n /**\n * `finish_reason` echoed from the first choice (`stop`, `length`,\n * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.\n * Exposed so a free-form `callLlm` caller CAN detect a truncated answer\n * (`length`) instead of treating a cut-off completion as complete. Note:\n * `callLlm` does not itself reject on it — acting on this signal is the\n * caller's responsibility (in-repo free-form drivers do not yet enforce it).\n */\n finishReason?: string | null\n /**\n * True when `content.trim()` is empty. An empty completion is a silent zero\n * for free-form `callLlm` callers; this flag is the signal a caller can\n * inspect to fail loud rather than proceed on an empty string. `callLlm`\n * surfaces it but does not throw on it.\n */\n contentEmpty?: boolean\n /** Raw response body. */\n raw: Record<string, unknown>\n}\n\nexport type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>\n\n/** Convert a provider result into the canonical paid-call receipt input. */\nexport function costReceiptFromLlm(\n result: LlmCallResult,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput {\n const cachedTokens = result.usage.cachedPromptTokens ?? 0\n const inputTokens = Math.max(0, result.usage.promptTokens - cachedTokens)\n const configuredCostUsd =\n result.costUsd === null && customTokenPricing && result.usage.captured !== false\n ? costForTokenPricing(customTokenPricing, {\n inputTokens,\n ...(cachedTokens > 0 ? { cachedTokens } : {}),\n outputTokens: result.usage.completionTokens,\n })\n : undefined\n return {\n model: result.model,\n inputTokens,\n outputTokens: result.usage.completionTokens,\n reasoningTokens: result.usage.reasoningTokens,\n cachedTokens: cachedTokens > 0 ? cachedTokens : undefined,\n actualCostUsd: result.costUsd ?? configuredCostUsd,\n usageUnknown: result.usage.captured === false,\n }\n}\n\n/** Structured-response failures retain their completed provider receipt. */\nexport function costReceiptFromLlmError(\n error: Error,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput | undefined {\n return error instanceof LlmResponseError\n ? costReceiptFromLlm(error.result, customTokenPricing)\n : undefined\n}\n\nexport class LlmCallError extends AgentEvalError {\n constructor(\n message: string,\n public readonly status: number,\n public readonly body: string,\n public readonly model: string,\n ) {\n super('judge', message)\n }\n}\n\n/** A provider response completed and incurred measurable usage, but its content\n * could not satisfy the caller's response contract. The response envelope is\n * retained so accounting can commit the receipt before the error propagates. */\nexport class LlmResponseError extends AgentEvalError {\n constructor(\n message: string,\n public readonly result: LlmCallResult,\n options?: { cause?: unknown },\n ) {\n super('judge', message, options)\n }\n}\n\nexport interface LlmClientOptions {\n /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n baseUrl?: string\n /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n apiKey?: string\n bearer?: string\n /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n authHeader?: { name: string; value: string }\n /** Stable provider idempotency key, reused across retries of this logical call. */\n idempotencyKey?: string\n /** Default timeout in ms. Per-call can override. */\n defaultTimeoutMs?: number\n /**\n * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to\n * each attempt's per-attempt timeout controller, so aborting it cancels\n * the in-flight fetch. A caller abort is FATAL: it is not retried even\n * though an AbortError otherwise matches the transient patterns.\n */\n signal?: AbortSignal\n /**\n * Cross-attempt wall-clock budget in ms, measured from the first attempt.\n * Before launching each attempt the loop checks the remaining budget and\n * stops retrying once it is exhausted, rather than waiting the full\n * per-attempt timeout on every retry. Bounds total time independent of\n * total attempts × `timeoutMs`.\n */\n deadlineMs?: number\n /** Total provider attempts. Default 3. */\n maximumAttempts?: number\n /** Token rates used when the provider omits cost or package pricing does not cover the model. */\n customTokenPricing?: CustomTokenPricing\n /**\n * Transport for requests that declare `jsonSchema`. `native` sends\n * `response_format: json_schema`; `json-object` sends the broadly supported\n * JSON mode and relies on the caller to include the schema in model-visible\n * instructions. Default: `native`.\n */\n jsonSchemaTransport?: 'native' | 'json-object'\n /**\n * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.\n * `exact` requires the complete response content to be one JSON value.\n * Default: `extract`.\n */\n jsonPayloadMode?: 'extract' | 'exact'\n /** Default provider reasoning mode. A per-call request value takes precedence. */\n thinking?: LlmThinkingMode\n /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n fetch?: typeof fetch\n /**\n * Optional raw HTTP capture sink. When provided, every request, response,\n * and error (across all retry attempts) is recorded to the sink, with auth\n * headers and credential-shaped body fields redacted by default. This is\n * the layer-1 forensics primitive: structured `LlmSpan`s record intent,\n * raw events record what actually crossed the wire.\n */\n rawSink?: RawProviderSink\n /**\n * Logical provider id attached to raw events. When omitted, derived from\n * `baseUrl` via `providerFromBaseUrl`.\n */\n provider?: string\n /** Trace context attached to raw events; populated by emitter-aware callers. */\n traceContext?: { runId?: string; spanId?: string }\n /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */\n redactor?: ProviderRedactor\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\n// Flagship / reasoning models routinely take several minutes on large prompts (a\n// reflection over many failures, a long tool transcript). A tight cap aborts a\n// legitimately-slow but healthy call — and because every retry attempt re-uses\n// the same window, such a model aborts on ALL attempts and the loop throws. The\n// default is generous enough to let those complete, bounded enough that a truly\n// hung call still fails over after retries, and tunable per deployment via\n// TANGLE_LLM_TIMEOUT_MS. Per-call `req.timeoutMs` / `opts.defaultTimeoutMs`\n// still win for callers that know their model's latency.\nconst DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 300_000\nconst DEFAULT_MAXIMUM_ATTEMPTS =\n process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS === undefined\n ? 3\n : Number(process.env.TANGLE_LLM_MAXIMUM_ATTEMPTS)\n\nfunction resolveMaximumAttempts(configured: number | undefined): number {\n const attempts = configured ?? DEFAULT_MAXIMUM_ATTEMPTS\n if (!Number.isInteger(attempts) || attempts <= 0) {\n throw new RangeError('LLM maximum attempts must be a positive integer')\n }\n return attempts\n}\n\nfunction providerTokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\n/**\n * Transient transport/network error signatures, matched against an error's\n * name, message, and `code`. Covers fetch/undici network failures, aborts\n * and timeouts, and — critically — HTTP/2 transport faults a keep-alive\n * connection raises mid-response: `terminated`, `NGHTTP2_INTERNAL_ERROR`,\n * `UND_ERR_*`, `other side closed`. Those last ones carry no clean HTTP\n * status; unrecognised, they escape the retry loop and surface as an\n * uncaught rejection.\n */\nconst TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [\n /AbortError/i,\n /TimeoutError/i,\n /this operation was aborted/i,\n /fetch failed/i,\n /ECONNRESET/i,\n /ETIMEDOUT/i,\n /EAI_AGAIN/i,\n /socket hang up/i,\n /stream.*ended.*unexpectedly/i,\n /terminated/i,\n /other side closed/i,\n /NGHTTP2/i,\n /UND_ERR/i,\n]\n\n/**\n * True when an error is a transient transport/network fault worth retrying,\n * as opposed to a deterministic failure (4xx schema reject, JSON parse) that\n * a retry cannot fix. Inspects `LlmCallError.status`, then the error's\n * name/message/code, then recurses into `error.cause` — undici nests the\n * real socket fault one or more levels under `.cause`.\n *\n * This is the retry classifier for the package: `callLlm` and\n * `withJudgeRetry` both route through it, so connection failures are treated\n * consistently across transports.\n */\nexport function isTransientLlmError(err: unknown): boolean {\n return classifyTransient(err, 0)\n}\n\nfunction classifyTransient(err: unknown, depth: number): boolean {\n if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n if (!(err instanceof Error)) return false\n // Foreign transport errors can carry a numeric HTTP status without being an\n // LlmCallError. A retryable status is decisive.\n const status = (err as { status?: unknown }).status\n if (typeof status === 'number' && RETRYABLE_STATUS.has(status)) return true\n const code = (err as { code?: unknown }).code\n const haystack = `${err.name}\\n${err.message}\\n${typeof code === 'string' ? code : ''}`\n if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true\n const cause = (err as { cause?: unknown }).cause\n if (depth < 4 && cause instanceof Error && cause !== err) {\n return classifyTransient(cause, depth + 1)\n }\n return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n const h = headers.get('retry-after')\n if (!h) return null\n const asNumber = Number(h)\n if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n const asDate = Date.parse(h)\n if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n return null\n}\n\n/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */\nexport function backoffMs(attempt: number): number {\n return Math.min(500 * 2 ** attempt, 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n Accept: 'application/json',\n }\n if (opts.authHeader) {\n headers[opts.authHeader.name] = opts.authHeader.value\n } else if (opts.bearer || opts.apiKey) {\n headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n }\n if (opts.idempotencyKey) headers['Idempotency-Key'] = opts.idempotencyKey\n return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n if (status !== 400) return false\n const lower = body.toLowerCase()\n return (\n lower.includes('response_format') ||\n lower.includes('json_schema') ||\n lower.includes('is unavailable') ||\n lower.includes('not supported')\n )\n}\n\nfunction isTemperatureOneRejection(status: number, body: string): boolean {\n if (status !== 400 || !/temperature/i.test(body)) return false\n return (\n /temperature[^.\\n]{0,120}\\b(?:only|must|should|required|requires?)\\b[^.\\n]{0,40}\\b1(?:\\.0+)?\\b/i.test(\n body,\n ) || /\\bonly\\s+1(?:\\.0+)?\\s+is\\s+allowed\\b[^.\\n]{0,120}\\btemperature\\b/i.test(body)\n )\n}\n\nfunction buildBody(\n req: LlmCallRequest,\n forceJsonObject: boolean,\n defaultThinking?: LlmThinkingMode,\n): Record<string, unknown> {\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0,\n }\n if (req.maxTokens != null) {\n if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens\n else body.max_tokens = req.maxTokens\n }\n const thinking = req.thinking ?? defaultThinking\n if (thinking !== undefined) {\n body.thinking = { type: thinking }\n }\n\n if (req.jsonSchema && !forceJsonObject) {\n body.response_format = {\n type: 'json_schema',\n json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n }\n } else if (req.jsonMode || req.jsonSchema) {\n body.response_format = { type: 'json_object' }\n }\n\n return body\n}\n\nfunction usesMaxCompletionTokens(model: string): boolean {\n return /^gpt-5(?:[.-]|$)/i.test(model)\n}\n\nasync function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n/**\n * Combine the per-attempt timeout signal with an optional caller signal into\n * one signal the fetch listens on. Prefers the native `AbortSignal.any`; falls\n * back to manual wiring on runtimes that predate it. The caller signal is also\n * propagated to the timeout controller so aborting it cancels the in-flight\n * fetch immediately.\n */\nfunction linkSignals(timeoutController: AbortController, caller?: AbortSignal): AbortSignal {\n if (!caller) return timeoutController.signal\n if (typeof (AbortSignal as { any?: unknown }).any === 'function') {\n return AbortSignal.any([timeoutController.signal, caller])\n }\n if (caller.aborted) {\n timeoutController.abort()\n } else {\n caller.addEventListener('abort', () => timeoutController.abort(), { once: true })\n }\n return timeoutController.signal\n}\n\n/** True once the cross-attempt wall-clock budget (if any) is exhausted. */\nfunction deadlineExceeded(start: number, deadlineMs: number | undefined): boolean {\n return deadlineMs != null && Date.now() - start >= deadlineMs\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n const trimmed = raw.trim()\n const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n const stripped = stripFencedJson(raw)\n try {\n JSON.parse(stripped)\n return stripped\n } catch {\n // A response that declares a JSON root must parse as that complete root.\n // Scanning onward could turn a truncated object into one of its valid nested\n // arrays or objects and silently change the response schema.\n if (stripped.startsWith('{') || stripped.startsWith('[')) return stripped\n }\n\n // Only prose-leading responses may contain a recoverable JSON payload.\n const starts = [...stripped.matchAll(/[[{]/g)]\n .map((match) => match.index)\n .filter((index) => index != null)\n for (const start of starts) {\n const candidate = extractBalancedJson(stripped, start)\n if (!candidate) continue\n try {\n JSON.parse(candidate)\n return candidate\n } catch {\n // Keep scanning; earlier braces may belong to prose.\n }\n }\n\n return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n const opener = input[start]\n const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n if (!closer) return null\n\n const stack: string[] = [closer]\n let isInString = false\n let isEscaped = false\n\n for (let i = start + 1; i < input.length; i++) {\n const char = input[i]!\n if (isEscaped) {\n isEscaped = false\n continue\n }\n if (char === '\\\\') {\n isEscaped = isInString\n continue\n }\n if (char === '\"') {\n isInString = !isInString\n continue\n }\n if (isInString) continue\n\n if (char === '{') stack.push('}')\n else if (char === '[') stack.push(']')\n else if (char === stack[stack.length - 1]) {\n stack.pop()\n if (stack.length === 0) return input.slice(start, i + 1)\n }\n }\n\n return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n const url = `${baseUrl}/chat/completions`\n const endpoint = '/chat/completions'\n const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n const maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n const fetchFn = opts.fetch ?? globalThis.fetch\n const headers = buildHeaders(opts)\n const provider = opts.provider ?? providerFromBaseUrl(baseUrl)\n const sink = opts.rawSink\n const redactor = opts.redactor ?? defaultProviderRedactor\n const traceContext = opts.traceContext\n const callerSignal = opts.signal\n const deadlineMs = opts.deadlineMs\n const deadlineStart = Date.now()\n if (opts.customTokenPricing) {\n costForTokenPricing(opts.customTokenPricing, { inputTokens: 0, outputTokens: 0 })\n }\n\n let lastErr: unknown\n let effectiveRequest = req\n for (let attempt = 0; attempt < maximumAttempts; attempt++) {\n // A caller cancel is fatal — never retried. Checking before each attempt\n // means an already-aborted signal short-circuits without firing fetch.\n if (callerSignal?.aborted) {\n throw new DOMException('callLlm aborted by caller signal', 'AbortError')\n }\n // Stop retrying once the cross-attempt budget is spent rather than burning\n // a full per-attempt timeout on each remaining retry.\n if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n }\n const controller = new AbortController()\n const attemptSignal = linkSignals(controller, callerSignal)\n const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n const started = Date.now()\n const requestBody = buildBody(\n effectiveRequest,\n opts.jsonSchemaTransport === 'json-object',\n opts.thinking,\n )\n let attemptErrorRecorded = false\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'request',\n timestamp: started,\n requestHeaders: headers,\n requestBody,\n redactedFields: [],\n })\n }\n\n try {\n const res = await fetchFn(url, {\n method: 'POST',\n headers,\n body: JSON.stringify(requestBody),\n signal: attemptSignal,\n })\n clearTimeout(timeoutHandle)\n const responseHeaders = sink ? headersToObject(res.headers) : undefined\n\n if (!res.ok) {\n const body = await res.text()\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: body,\n errorMessage: `HTTP ${res.status}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n const err = new LlmCallError(\n `LLM call ${res.status}: ${body.slice(0, 300)}`,\n res.status,\n body,\n req.model,\n )\n if (\n isTemperatureOneRejection(res.status, body) &&\n effectiveRequest.temperature !== 1 &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n effectiveRequest = { ...effectiveRequest, temperature: 1 }\n continue\n }\n if (\n RETRYABLE_STATUS.has(res.status) &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n const retryAfter = parseRetryAfter(res.headers)\n await sleep(retryAfter ?? backoffMs(attempt))\n continue\n }\n throw err\n }\n\n const text = await res.text()\n let json: Record<string, unknown>\n try {\n json = JSON.parse(text) as Record<string, unknown>\n } catch (parseErr) {\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: text,\n errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n throw parseErr\n }\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'response',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: json,\n redactedFields: [],\n })\n }\n const choice = (\n json.choices as\n | Array<{ message?: { content?: string }; finish_reason?: string | null }>\n | undefined\n )?.[0]\n const usageRaw =\n json.usage && typeof json.usage === 'object' && !Array.isArray(json.usage)\n ? (json.usage as Record<string, unknown>)\n : undefined\n const promptTokens = providerTokenCount(usageRaw?.prompt_tokens)\n const completionTokens = providerTokenCount(usageRaw?.completion_tokens)\n const totalTokens = providerTokenCount(usageRaw?.total_tokens)\n const completionDetails =\n usageRaw?.completion_tokens_details &&\n typeof usageRaw.completion_tokens_details === 'object' &&\n !Array.isArray(usageRaw.completion_tokens_details)\n ? (usageRaw.completion_tokens_details as Record<string, unknown>)\n : undefined\n const reasoningRaw = completionDetails?.reasoning_tokens\n const reasoningTokens =\n reasoningRaw === undefined ? undefined : providerTokenCount(reasoningRaw)\n const cachedRaw =\n usageRaw?.prompt_tokens_details &&\n typeof usageRaw.prompt_tokens_details === 'object' &&\n !Array.isArray(usageRaw.prompt_tokens_details)\n ? (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens\n : undefined\n const cachedPromptTokens = cachedRaw === undefined ? undefined : providerTokenCount(cachedRaw)\n const usageCaptured =\n promptTokens !== undefined &&\n completionTokens !== undefined &&\n (reasoningRaw === undefined ||\n (reasoningTokens !== undefined && reasoningTokens <= completionTokens)) &&\n (cachedRaw === undefined ||\n (cachedPromptTokens !== undefined && cachedPromptTokens <= promptTokens)) &&\n (totalTokens === undefined || totalTokens === promptTokens + completionTokens)\n const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n const content = choice?.message?.content ?? ''\n\n const configuredCost =\n typeof costFromProxy !== 'number' && usageCaptured && opts.customTokenPricing\n ? costForTokenPricing(opts.customTokenPricing, {\n inputTokens: promptTokens! - (cachedPromptTokens ?? 0),\n ...(cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {}),\n outputTokens: completionTokens!,\n })\n : undefined\n\n return {\n content,\n finishReason: choice?.finish_reason ?? null,\n contentEmpty: content.trim().length === 0,\n usage: {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),\n captured: usageCaptured,\n reasoningTokens,\n cachedPromptTokens,\n },\n costUsd: typeof costFromProxy === 'number' ? costFromProxy : (configuredCost ?? null),\n model: (json.model as string) ?? req.model,\n durationMs: Date.now() - started,\n raw: json,\n }\n } catch (err) {\n clearTimeout(timeoutHandle)\n lastErr = err\n // A caller cancel is fatal even though an AbortError matches the\n // transient patterns — a cancelled call must surface immediately, not\n // be retried against the same dead intent.\n if (callerSignal?.aborted) {\n if (sink && !attemptErrorRecorded) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n throw err\n }\n if (sink && !attemptErrorRecorded) {\n // Record only if neither the !res.ok branch nor the JSON.parse catch\n // already produced an error event for this attempt. Covers network\n // failures, timeouts, and aborts.\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n if (\n attempt < maximumAttempts - 1 &&\n isTransientLlmError(err) &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n await sleep(backoffMs(attempt))\n continue\n }\n throw err\n }\n }\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\nasync function recordRaw(\n sink: RawProviderSink,\n redactor: ProviderRedactor,\n event: RawProviderEvent,\n): Promise<void> {\n // Errors from sinks must not crash the LLM call. Forensic capture is\n // best-effort; the structured trace is the system of record.\n try {\n await sink.record(redactor(event))\n } catch {\n // Intentionally swallowed.\n }\n}\n\nfunction headersToObject(h: Headers): Record<string, string> {\n const out: Record<string, string> = {}\n h.forEach((value, key) => {\n out[key] = value\n })\n return out\n}\n\nfunction cryptoEventId(): string {\n if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n const result = await callLlmStructured(req, opts)\n const value = parseJsonResult<T>(result, opts.jsonPayloadMode ?? 'extract')\n return { value, result }\n}\n\n/** Shared schema-to-JSON-mode fallback that preserves the raw result. */\nasync function callLlmStructured(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n try {\n return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n } catch (err) {\n if (\n opts.jsonSchemaTransport !== 'json-object' &&\n err instanceof LlmCallError &&\n isSchemaRejection(err.status, err.body) &&\n req.jsonSchema\n ) {\n const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n return await callLlm(degradedReq, opts)\n }\n throw err\n }\n}\n\nfunction parseJsonResult<T>(\n result: LlmCallResult,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n try {\n if (result.finishReason === 'length') {\n throw new Error(\n `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`,\n )\n }\n return parseJsonSafely<T>(result.content, result.model, jsonPayloadMode)\n } catch (error) {\n if (error instanceof LlmResponseError) throw error\n const cause = error instanceof Error ? error : new Error(String(error))\n throw new LlmResponseError(cause.message, result, { cause })\n }\n}\n\nfunction parseJsonSafely<T>(\n content: string,\n model: string,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n const payload = jsonPayloadMode === 'exact' ? content : extractJsonPayload(content)\n try {\n return JSON.parse(payload) as T\n } catch (err) {\n throw new Error(\n `LLM returned non-JSON content (model=${model}): ${\n err instanceof Error ? err.message : String(err)\n }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n )\n }\n}\n\n// ─── Route assertion ────────────────────────────────────────────────────\n\nexport type LlmRouteAssertionReason =\n | 'no_explicit_base_url'\n | 'base_url_blocked'\n | 'base_url_not_allowed'\n | 'no_auth'\n | 'wrong_provider'\n\nexport class LlmRouteAssertionError extends CaptureIntegrityError {\n constructor(\n message: string,\n public readonly reason: LlmRouteAssertionReason,\n public readonly baseUrl: string,\n ) {\n super(message)\n }\n}\n\nexport interface LlmRouteRequirements {\n /**\n * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to\n * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using\n * the public/free-tier router is a defect — the launch reviewer needs to\n * know exactly which provider answered.\n */\n requireExplicitBaseUrl?: boolean\n /**\n * Allowlist of acceptable base URLs. Strings match by prefix\n * (case-insensitive); RegExps test against the full base URL.\n */\n allowedBaseUrls?: Array<string | RegExp>\n /** Blocklist that takes precedence over `allowedBaseUrls`. */\n blockedBaseUrls?: Array<string | RegExp>\n /** Throw if no auth header / api key is configured. */\n requireAuth?: boolean\n /**\n * Logical provider id the configured `baseUrl` is expected to match (via\n * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.\n */\n expectedProvider?: string\n}\n\n/**\n * Fail-loud assertion that the configured LLM client points at the route\n * the caller intends. Designed for the matrix-runner preflight: invoke\n * once before any LLM call to catch misconfiguration before a sweep burns\n * dollars on the wrong provider.\n *\n * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call\n * from constructors and CI gates.\n */\nexport function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {\n const baseUrlExplicit = opts.baseUrl !== undefined\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n if (req.requireExplicitBaseUrl && !baseUrlExplicit) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,\n 'no_explicit_base_url',\n baseUrl,\n )\n }\n\n if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,\n 'base_url_blocked',\n baseUrl,\n )\n }\n\n if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {\n const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))\n if (!ok) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,\n 'base_url_not_allowed',\n baseUrl,\n )\n }\n }\n\n if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,\n 'no_auth',\n baseUrl,\n )\n }\n\n if (req.expectedProvider) {\n const actual = opts.provider ?? providerFromBaseUrl(baseUrl)\n if (actual !== req.expectedProvider) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,\n 'wrong_provider',\n baseUrl,\n )\n }\n }\n}\n\nfunction matchUrl(url: string, pattern: string | RegExp): boolean {\n if (pattern instanceof RegExp) return pattern.test(url)\n return url.toLowerCase().startsWith(pattern.toLowerCase())\n}\n\nfunction describePattern(p: string | RegExp): string {\n return p instanceof RegExp ? p.source : p\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n model: string,\n opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n const start = Date.now()\n try {\n await callLlm(\n {\n model,\n messages: [{ role: 'user', content: 'ping' }],\n maxTokens: 64,\n timeoutMs: opts.timeoutMs ?? 30_000,\n },\n opts,\n )\n return { ok: true, latencyMs: Date.now() - start, error: null }\n } catch (err) {\n return {\n ok: false,\n latencyMs: Date.now() - start,\n error: err instanceof Error ? err.message : String(err),\n }\n }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n readonly maximumAttempts: number\n private readonly opts: LlmClientOptions\n\n constructor(opts: LlmClientOptions = {}) {\n this.opts = opts\n this.maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts)\n }\n\n call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n const options = { ...this.opts, ...per }\n return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)\n }\n\n callJson<T = unknown>(\n req: LlmCallRequest,\n per?: LlmClientOptions,\n ): Promise<{ value: T; result: LlmCallResult }> {\n return callLlmJson<T>(req, { ...this.opts, ...per })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,SAAgB,2BACd,SACA,UAA4B,CAAC,GACF;CAC3B,IAAI,QAAQ,cAAc,KAAA,GAAW,OAAO,KAAA;CAC5C,IAAI,CAAC,OAAO,UAAU,QAAQ,SAAS,KAAK,QAAQ,aAAa,GAC/D,MAAM,IAAI,WAAW,kEAAkE;CAEzF,IACE,QAAQ,SAAS,MACd,YACC,MAAM,QAAQ,QAAQ,OAAO,KAAK,QAAQ,QAAQ,MAAM,SAAS,KAAK,SAAS,WAAW,CAC9F,GAEA;CAGF,MAAM,WAAW,uBAAuB,QAAQ,eAAe;CAC/D,MAAM,kBAAkB,QAAQ,wBAAwB;CAGxD,MAAM,eAAe,IAAI,YAAY,CAAC,CAAC,OACrC,KAAK,UAAU,UAAU,SAAS,iBAAiB,QAAQ,QAAQ,CAAC,CACtE,CAAC,CAAC;CAEF,MAAM,UAAU,QAAQ,cAAc,CAAC,kBAAkB,IAAI;CAC7D,MAAM,QAAQ;EACZ,aAAa,eAAe,WAAW;EACvC,cAAc,QAAQ,YAAY,WAAW;CAC/C;CACA,OAAO,QAAQ,qBACX;EAAE,oBAAoB,QAAQ;EAAoB,GAAG;CAAM,IAC3D;EAAE,OAAO,QAAQ;EAAO,GAAG;CAAM;AACvC;;AAkDA,SAAgB,mBACd,QACA,oBACkB;CAClB,MAAM,eAAe,OAAO,MAAM,sBAAsB;CACxD,MAAM,cAAc,KAAK,IAAI,GAAG,OAAO,MAAM,eAAe,YAAY;CACxE,MAAM,oBACJ,OAAO,YAAY,QAAQ,sBAAsB,OAAO,MAAM,aAAa,QACvE,oBAAoB,oBAAoB;EACtC;EACA,GAAI,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;EAC3C,cAAc,OAAO,MAAM;CAC7B,CAAC,IACD,KAAA;CACN,OAAO;EACL,OAAO,OAAO;EACd;EACA,cAAc,OAAO,MAAM;EAC3B,iBAAiB,OAAO,MAAM;EAC9B,cAAc,eAAe,IAAI,eAAe,KAAA;EAChD,eAAe,OAAO,WAAW;EACjC,cAAc,OAAO,MAAM,aAAa;CAC1C;AACF;;AAGA,SAAgB,wBACd,OACA,oBAC8B;CAC9B,OAAO,iBAAiB,mBACpB,mBAAmB,MAAM,QAAQ,kBAAkB,IACnD,KAAA;AACN;AAEA,IAAa,eAAb,cAAkC,eAAe;CAG7B;CACA;CACA;CAJlB,YACE,SACA,QACA,MACA,OACA;EACA,MAAM,SAAS,OAAO;EAJN,KAAA,SAAA;EACA,KAAA,OAAA;EACA,KAAA,QAAA;CAGlB;AACF;;;;AAKA,IAAa,mBAAb,cAAsC,eAAe;CAGjC;CAFlB,YACE,SACA,QACA,SACA;EACA,MAAM,SAAS,SAAS,OAAO;EAHf,KAAA,SAAA;CAIlB;AACF;AAuEA,MAAM,mBAAmB;AASzB,MAAM,qBAAqB,OAAO,QAAQ,IAAI,qBAAqB,KAAK;AACxE,MAAM,2BACJ,QAAQ,IAAI,gCAAgC,KAAA,IACxC,IACA,OAAO,QAAQ,IAAI,2BAA2B;AAEpD,SAAS,uBAAuB,YAAwC;CACtE,MAAM,WAAW,cAAc;CAC/B,IAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,YAAY,GAC7C,MAAM,IAAI,WAAW,iDAAiD;CAExE,OAAO;AACT;AAEA,SAAS,mBAAmB,OAAoC;CAC9D,OAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAC1F;AAEA,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;;;;;;;;;;AAWrD,MAAM,2BAA8C;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;;;;;;AAaA,SAAgB,oBAAoB,KAAuB;CACzD,OAAO,kBAAkB,KAAK,CAAC;AACjC;AAEA,SAAS,kBAAkB,KAAc,OAAwB;CAC/D,IAAI,eAAe,cAAc,OAAO,iBAAiB,IAAI,IAAI,MAAM;CACvE,IAAI,EAAE,eAAe,QAAQ,OAAO;CAGpC,MAAM,SAAU,IAA6B;CAC7C,IAAI,OAAO,WAAW,YAAY,iBAAiB,IAAI,MAAM,GAAG,OAAO;CACvE,MAAM,OAAQ,IAA2B;CACzC,MAAM,WAAW,GAAG,IAAI,KAAK,IAAI,IAAI,QAAQ,IAAI,OAAO,SAAS,WAAW,OAAO;CACnF,IAAI,yBAAyB,MAAM,MAAM,EAAE,KAAK,QAAQ,CAAC,GAAG,OAAO;CACnE,MAAM,QAAS,IAA4B;CAC3C,IAAI,QAAQ,KAAK,iBAAiB,SAAS,UAAU,KACnD,OAAO,kBAAkB,OAAO,QAAQ,CAAC;CAE3C,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;CACxD,MAAM,IAAI,QAAQ,IAAI,aAAa;CACnC,IAAI,CAAC,GAAG,OAAO;CACf,MAAM,WAAW,OAAO,CAAC;CACzB,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAAG,OAAO,WAAW;CACjE,MAAM,SAAS,KAAK,MAAM,CAAC;CAC3B,IAAI,OAAO,SAAS,MAAM,GAAG,OAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;CACnE,OAAO;AACT;;AAGA,SAAgB,UAAU,SAAyB;CACjD,OAAO,KAAK,IAAI,MAAM,KAAK,SAAS,IAAM;AAC5C;AAEA,SAAS,aAAa,MAAgD;CACpE,MAAM,UAAkC;EACtC,gBAAgB;EAChB,QAAQ;CACV;CACA,IAAI,KAAK,YACP,QAAQ,KAAK,WAAW,QAAQ,KAAK,WAAW;MAC3C,IAAI,KAAK,UAAU,KAAK,QAC7B,QAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK;CAExD,IAAI,KAAK,gBAAgB,QAAQ,qBAAqB,KAAK;CAC3D,OAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;CAChE,IAAI,WAAW,KAAK,OAAO;CAC3B,MAAM,QAAQ,KAAK,YAAY;CAC/B,OACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,0BAA0B,QAAgB,MAAuB;CACxE,IAAI,WAAW,OAAO,CAAC,eAAe,KAAK,IAAI,GAAG,OAAO;CACzD,OACE,iGAAiG,KAC/F,IACF,KAAK,oEAAoE,KAAK,IAAI;AAEtF;AAEA,SAAS,UACP,KACA,iBACA,iBACyB;CACzB,MAAM,OAAgC;EACpC,OAAO,IAAI;EACX,UAAU,IAAI;EACd,aAAa,IAAI,eAAe;CAClC;CACA,IAAI,IAAI,aAAa,MACnB,IAAI,wBAAwB,IAAI,KAAK,GAAG,KAAK,wBAAwB,IAAI;MACpE,KAAK,aAAa,IAAI;CAE7B,MAAM,WAAW,IAAI,YAAY;CACjC,IAAI,aAAa,KAAA,GACf,KAAK,WAAW,EAAE,MAAM,SAAS;CAGnC,IAAI,IAAI,cAAc,CAAC,iBACrB,KAAK,kBAAkB;EACrB,MAAM;EACN,aAAa;GAAE,MAAM,IAAI,WAAW;GAAM,QAAQ,IAAI,WAAW;GAAQ,QAAQ;EAAK;CACxF;MACK,IAAI,IAAI,YAAY,IAAI,YAC7B,KAAK,kBAAkB,EAAE,MAAM,cAAc;CAG/C,OAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;CACvD,OAAO,oBAAoB,KAAK,KAAK;AACvC;AAEA,eAAe,MAAM,IAA2B;CAC9C,OAAO,IAAI,SAAS,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;;;;;;;;AASA,SAAS,YAAY,mBAAoC,QAAmC;CAC1F,IAAI,CAAC,QAAQ,OAAO,kBAAkB;CACtC,IAAI,OAAQ,YAAkC,QAAQ,YACpD,OAAO,YAAY,IAAI,CAAC,kBAAkB,QAAQ,MAAM,CAAC;CAE3D,IAAI,OAAO,SACT,kBAAkB,MAAM;MAExB,OAAO,iBAAiB,eAAe,kBAAkB,MAAM,GAAG,EAAE,MAAM,KAAK,CAAC;CAElF,OAAO,kBAAkB;AAC3B;;AAGA,SAAS,iBAAiB,OAAe,YAAyC;CAChF,OAAO,cAAc,QAAQ,KAAK,IAAI,IAAI,SAAS;AACrD;;;;;;AASA,SAAgB,gBAAgB,KAAqB;CACnD,MAAM,UAAU,IAAI,KAAK;CACzB,MAAM,IAAI,QAAQ,MAAM,yCAAyC;CACjE,OAAO,IAAI,EAAE,EAAE,CAAE,KAAK,IAAI;AAC5B;AAEA,SAAgB,mBAAmB,KAAqB;CACtD,MAAM,WAAW,gBAAgB,GAAG;CACpC,IAAI;EACF,KAAK,MAAM,QAAQ;EACnB,OAAO;CACT,QAAQ;EAIN,IAAI,SAAS,WAAW,GAAG,KAAK,SAAS,WAAW,GAAG,GAAG,OAAO;CACnE;CAGA,MAAM,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,CAAC,CAAC,CAC3C,KAAK,UAAU,MAAM,KAAK,CAAC,CAC3B,QAAQ,UAAU,SAAS,IAAI;CAClC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,YAAY,oBAAoB,UAAU,KAAK;EACrD,IAAI,CAAC,WAAW;EAChB,IAAI;GACF,KAAK,MAAM,SAAS;GACpB,OAAO;EACT,QAAQ,CAER;CACF;CAEA,OAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;CACxE,MAAM,SAAS,MAAM;CACrB,MAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;CAC7D,IAAI,CAAC,QAAQ,OAAO;CAEpB,MAAM,QAAkB,CAAC,MAAM;CAC/B,IAAI,aAAa;CACjB,IAAI,YAAY;CAEhB,KAAK,IAAI,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;EAC7C,MAAM,OAAO,MAAM;EACnB,IAAI,WAAW;GACb,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAM;GACjB,YAAY;GACZ;EACF;EACA,IAAI,SAAS,MAAK;GAChB,aAAa,CAAC;GACd;EACF;EACA,IAAI,YAAY;EAEhB,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAC3B,IAAI,SAAS,KAAK,MAAM,KAAK,GAAG;OAChC,IAAI,SAAS,MAAM,MAAM,SAAS,IAAI;GACzC,MAAM,IAAI;GACV,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;EACzD;CACF;CAEA,OAAO;AACT;;;;;;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;CACxB,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CACrE,MAAM,MAAM,GAAG,QAAQ;CACvB,MAAM,WAAW;CACjB,MAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;CAC5D,MAAM,kBAAkB,uBAAuB,KAAK,eAAe;CACnE,MAAM,UAAU,KAAK,SAAS,WAAW;CACzC,MAAM,UAAU,aAAa,IAAI;CACjC,MAAM,WAAW,KAAK,YAAY,oBAAoB,OAAO;CAC7D,MAAM,OAAO,KAAK;CAClB,MAAM,WAAW,KAAK,YAAY;CAClC,MAAM,eAAe,KAAK;CAC1B,MAAM,eAAe,KAAK;CAC1B,MAAM,aAAa,KAAK;CACxB,MAAM,gBAAgB,KAAK,IAAI;CAC/B,IAAI,KAAK,oBACP,oBAAoB,KAAK,oBAAoB;EAAE,aAAa;EAAG,cAAc;CAAE,CAAC;CAGlF,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,IAAI,UAAU,GAAG,UAAU,iBAAiB,WAAW;EAG1D,IAAI,cAAc,SAChB,MAAM,IAAI,aAAa,oCAAoC,YAAY;EAIzE,IAAI,UAAU,KAAK,iBAAiB,eAAe,UAAU,GAC3D,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;EAEtE,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,gBAAgB,YAAY,YAAY,YAAY;EAC1D,MAAM,gBAAgB,iBAAiB,WAAW,MAAM,GAAG,SAAS;EACpE,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,cAAc,UAClB,kBACA,KAAK,wBAAwB,eAC7B,KAAK,QACP;EACA,IAAI,uBAAuB;EAC3B,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;GAC9B,SAAS,cAAc;GACvB,OAAO,cAAc;GACrB,QAAQ,cAAc;GACtB;GACA,OAAO,IAAI;GACX;GACA;GACA,cAAc;GACd,WAAW;GACX,WAAW;GACX,gBAAgB;GAChB;GACA,gBAAgB,CAAC;EACnB,CAAC;EAGH,IAAI;GACF,MAAM,MAAM,MAAM,QAAQ,KAAK;IAC7B,QAAQ;IACR;IACA,MAAM,KAAK,UAAU,WAAW;IAChC,QAAQ;GACV,CAAC;GACD,aAAa,aAAa;GAC1B,MAAM,kBAAkB,OAAO,gBAAgB,IAAI,OAAO,IAAI,KAAA;GAE9D,IAAI,CAAC,IAAI,IAAI;IACX,MAAM,OAAO,MAAM,IAAI,KAAK;IAC5B,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,QAAQ,IAAI;MAC1B,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM,MAAM,IAAI,aACd,YAAY,IAAI,OAAO,IAAI,KAAK,MAAM,GAAG,GAAG,KAC5C,IAAI,QACJ,MACA,IAAI,KACN;IACA,IACE,0BAA0B,IAAI,QAAQ,IAAI,KAC1C,iBAAiB,gBAAgB,KACjC,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KACV,mBAAmB;MAAE,GAAG;MAAkB,aAAa;KAAE;KACzD;IACF;IACA,IACE,iBAAiB,IAAI,IAAI,MAAM,KAC/B,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;KACA,UAAU;KAEV,MAAM,MADa,gBAAgB,IAAI,OAClB,KAAK,UAAU,OAAO,CAAC;KAC5C;IACF;IACA,MAAM;GACR;GAEA,MAAM,OAAO,MAAM,IAAI,KAAK;GAC5B,IAAI;GACJ,IAAI;IACF,OAAO,KAAK,MAAM,IAAI;GACxB,SAAS,UAAU;IACjB,IAAI,MAAM;KACR,MAAM,UAAU,MAAM,UAAU;MAC9B,SAAS,cAAc;MACvB,OAAO,cAAc;MACrB,QAAQ,cAAc;MACtB;MACA,OAAO,IAAI;MACX;MACA;MACA,cAAc;MACd,WAAW;MACX,WAAW,KAAK,IAAI;MACpB,YAAY,KAAK,IAAI,IAAI;MACzB,YAAY,IAAI;MAChB;MACA,cAAc;MACd,cAAc,sBAAsB,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;MAClG,gBAAgB,CAAC;KACnB,CAAC;KACD,uBAAuB;IACzB;IACA,MAAM;GACR;GACA,IAAI,MACF,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,YAAY,IAAI;IAChB;IACA,cAAc;IACd,gBAAgB,CAAC;GACnB,CAAC;GAEH,MAAM,SACJ,KAAK,UAGH;GACJ,MAAM,WACJ,KAAK,SAAS,OAAO,KAAK,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,KAAK,IACpE,KAAK,QACN,KAAA;GACN,MAAM,eAAe,mBAAmB,UAAU,aAAa;GAC/D,MAAM,mBAAmB,mBAAmB,UAAU,iBAAiB;GACvE,MAAM,cAAc,mBAAmB,UAAU,YAAY;GAO7D,MAAM,gBALJ,UAAU,6BACV,OAAO,SAAS,8BAA8B,YAC9C,CAAC,MAAM,QAAQ,SAAS,yBAAyB,IAC5C,SAAS,4BACV,KAAA,EAAA,EACkC;GACxC,MAAM,kBACJ,iBAAiB,KAAA,IAAY,KAAA,IAAY,mBAAmB,YAAY;GAC1E,MAAM,YACJ,UAAU,yBACV,OAAO,SAAS,0BAA0B,YAC1C,CAAC,MAAM,QAAQ,SAAS,qBAAqB,IACxC,SAAS,sBAAkD,gBAC5D,KAAA;GACN,MAAM,qBAAqB,cAAc,KAAA,IAAY,KAAA,IAAY,mBAAmB,SAAS;GAC7F,MAAM,gBACJ,iBAAiB,KAAA,KACjB,qBAAqB,KAAA,MACpB,iBAAiB,KAAA,KACf,oBAAoB,KAAA,KAAa,mBAAmB,sBACtD,cAAc,KAAA,KACZ,uBAAuB,KAAA,KAAa,sBAAsB,kBAC5D,gBAAgB,KAAA,KAAa,gBAAgB,eAAe;GAC/D,MAAM,gBAAiB,KAAK,kBAAkB,KAAK;GACnD,MAAM,UAAU,QAAQ,SAAS,WAAW;GAE5C,MAAM,iBACJ,OAAO,kBAAkB,YAAY,iBAAiB,KAAK,qBACvD,oBAAoB,KAAK,oBAAoB;IAC3C,aAAa,gBAAiB,sBAAsB;IACpD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;IACjE,cAAc;GAChB,CAAC,IACD,KAAA;GAEN,OAAO;IACL;IACA,cAAc,QAAQ,iBAAiB;IACvC,cAAc,QAAQ,KAAK,CAAC,CAAC,WAAW;IACxC,OAAO;KACL,cAAc,gBAAgB;KAC9B,kBAAkB,oBAAoB;KACtC,aAAa,gBAAgB,gBAAgB,MAAM,oBAAoB;KACvE,UAAU;KACV;KACA;IACF;IACA,SAAS,OAAO,kBAAkB,WAAW,gBAAiB,kBAAkB;IAChF,OAAQ,KAAK,SAAoB,IAAI;IACrC,YAAY,KAAK,IAAI,IAAI;IACzB,KAAK;GACP;EACF,SAAS,KAAK;GACZ,aAAa,aAAa;GAC1B,UAAU;GAIV,IAAI,cAAc,SAAS;IACzB,IAAI,QAAQ,CAAC,sBACX,MAAM,UAAU,MAAM,UAAU;KAC9B,SAAS,cAAc;KACvB,OAAO,cAAc;KACrB,QAAQ,cAAc;KACtB;KACA,OAAO,IAAI;KACX;KACA;KACA,cAAc;KACd,WAAW;KACX,WAAW,KAAK,IAAI;KACpB,YAAY,KAAK,IAAI,IAAI;KACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;KAC7D,gBAAgB,CAAC;IACnB,CAAC;IAEH,MAAM;GACR;GACA,IAAI,QAAQ,CAAC,sBAIX,MAAM,UAAU,MAAM,UAAU;IAC9B,SAAS,cAAc;IACvB,OAAO,cAAc;IACrB,QAAQ,cAAc;IACtB;IACA,OAAO,IAAI;IACX;IACA;IACA,cAAc;IACd,WAAW;IACX,WAAW,KAAK,IAAI;IACpB,YAAY,KAAK,IAAI,IAAI;IACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC7D,gBAAgB,CAAC;GACnB,CAAC;GAEH,IACE,UAAU,kBAAkB,KAC5B,oBAAoB,GAAG,KACvB,CAAC,iBAAiB,eAAe,UAAU,GAC3C;IACA,MAAM,MAAM,UAAU,OAAO,CAAC;IAC9B;GACF;GACA,MAAM;EACR;CACF;CACA,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAEA,eAAe,UACb,MACA,UACA,OACe;CAGf,IAAI;EACF,MAAM,KAAK,OAAO,SAAS,KAAK,CAAC;CACnC,QAAQ,CAER;AACF;AAEA,SAAS,gBAAgB,GAAoC;CAC3D,MAAM,MAA8B,CAAC;CACrC,EAAE,SAAS,OAAO,QAAQ;EACxB,IAAI,OAAO;CACb,CAAC;CACD,OAAO;AACT;AAEA,SAAS,gBAAwB;CAC/B,IAAI,OAAO,WAAW,QAAQ,eAAe,YAAY,OAAO,WAAW,OAAO,WAAW;CAC7F,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AAC7E;;;;;;;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;CAC9C,MAAM,SAAS,MAAM,kBAAkB,KAAK,IAAI;CAEhD,OAAO;EAAE,OADK,gBAAmB,QAAQ,KAAK,mBAAmB,SACpD;EAAG;CAAO;AACzB;;AAGA,eAAe,kBACb,KACA,OAAyB,CAAC,GACF;CACxB,IAAI;EACF,OAAO,MAAM,QAAQ;GAAE,GAAG;GAAK,UAAU,IAAI,YAAY,CAAC,IAAI;EAAW,GAAG,IAAI;CAClF,SAAS,KAAK;EACZ,IACE,KAAK,wBAAwB,iBAC7B,eAAe,gBACf,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KACtC,IAAI,YAGJ,OAAO,MAAM,QAAQ;GADiB,GAAG;GAAK,UAAU;GAAM,YAAY,KAAA;EAC3C,GAAG,IAAI;EAExC,MAAM;CACR;AACF;AAEA,SAAS,gBACP,QACA,iBACG;CACH,IAAI;EACF,IAAI,OAAO,iBAAiB,UAC1B,MAAM,IAAI,MACR,8CAA8C,OAAO,MAAM,uBAC7D;EAEF,OAAO,gBAAmB,OAAO,SAAS,OAAO,OAAO,eAAe;CACzE,SAAS,OAAO;EACd,IAAI,iBAAiB,kBAAkB,MAAM;EAC7C,MAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;EACtE,MAAM,IAAI,iBAAiB,MAAM,SAAS,QAAQ,EAAE,MAAM,CAAC;CAC7D;AACF;AAEA,SAAS,gBACP,SACA,OACA,iBACG;CACH,MAAM,UAAU,oBAAoB,UAAU,UAAU,mBAAmB,OAAO;CAClF,IAAI;EACF,OAAO,KAAK,MAAM,OAAO;CAC3B,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,wCAAwC,MAAM,KAC5C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAChD,yBAAyB,QAAQ,MAAM,GAAG,GAAG,GAChD;CACF;AACF;AAWA,IAAa,yBAAb,cAA4C,sBAAsB;CAG9C;CACA;CAHlB,YACE,SACA,QACA,SACA;EACA,MAAM,OAAO;EAHG,KAAA,SAAA;EACA,KAAA,UAAA;CAGlB;AACF;;;;;;;;;;AAmCA,SAAgB,eAAe,MAAwB,MAA4B,CAAC,GAAS;CAC3F,MAAM,kBAAkB,KAAK,YAAY,KAAA;CACzC,MAAM,WAAW,KAAK,WAAW,iBAAA,CAAkB,QAAQ,QAAQ,EAAE;CAErE,IAAI,IAAI,0BAA0B,CAAC,iBACjC,MAAM,IAAI,uBACR,gGAAgG,iBAAiB,IACjH,wBACA,OACF;CAGF,IAAI,IAAI,iBAAiB,MAAM,MAAM,SAAS,SAAS,CAAC,CAAC,GACvD,MAAM,IAAI,uBACR,2BAA2B,QAAQ,8BACnC,oBACA,OACF;CAGF,IAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,GAElD;MAAA,CADO,IAAI,gBAAgB,MAAM,MAAM,SAAS,SAAS,CAAC,CACxD,GACJ,MAAM,IAAI,uBACR,2BAA2B,QAAQ,+BAA+B,IAAI,gBAAgB,IAAI,eAAe,CAAC,CAAC,KAAK,IAAI,EAAE,KACtH,wBACA,OACF;CAAA;CAIJ,IAAI,IAAI,eAAe,CAAC,KAAK,UAAU,CAAC,KAAK,UAAU,CAAC,KAAK,YAC3D,MAAM,IAAI,uBACR,sFACA,WACA,OACF;CAGF,IAAI,IAAI,kBAAkB;EACxB,MAAM,SAAS,KAAK,YAAY,oBAAoB,OAAO;EAC3D,IAAI,WAAW,IAAI,kBACjB,MAAM,IAAI,uBACR,qCAAqC,IAAI,iBAAiB,eAAe,QAAQ,eAAe,OAAO,IACvG,kBACA,OACF;CAEJ;AACF;AAEA,SAAS,SAAS,KAAa,SAAmC;CAChE,IAAI,mBAAmB,QAAQ,OAAO,QAAQ,KAAK,GAAG;CACtD,OAAO,IAAI,YAAY,CAAC,CAAC,WAAW,QAAQ,YAAY,CAAC;AAC3D;AAEA,SAAS,gBAAgB,GAA4B;CACnD,OAAO,aAAa,SAAS,EAAE,SAAS;AAC1C;;;;;;;;;;;;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;CACnE,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EACF,MAAM,QACJ;GACE;GACA,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS;GAAO,CAAC;GAC5C,WAAW;GACX,WAAW,KAAK,aAAa;EAC/B,GACA,IACF;EACA,OAAO;GAAE,IAAI;GAAM,WAAW,KAAK,IAAI,IAAI;GAAO,OAAO;EAAK;CAChE,SAAS,KAAK;EACZ,OAAO;GACL,IAAI;GACJ,WAAW,KAAK,IAAI,IAAI;GACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;EACxD;CACF;AACF;;;;;;AAOA,IAAa,YAAb,MAAuB;CACrB;CACA;CAEA,YAAY,OAAyB,CAAC,GAAG;EACvC,KAAK,OAAO;EACZ,KAAK,kBAAkB,uBAAuB,KAAK,eAAe;CACpE;CAEA,KAAK,KAAqB,KAAgD;EACxE,MAAM,UAAU;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI;EACvC,OAAO,IAAI,aAAa,kBAAkB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO;CAChF;CAEA,SACE,KACA,KAC8C;EAC9C,OAAO,YAAe,KAAK;GAAE,GAAG,KAAK;GAAM,GAAG;EAAI,CAAC;CACrD;AACF"}
@@ -1,3 +1,3 @@
1
1
  import { t as DefaultVerdict } from "../verdict-Dps8_okt.js";
2
- import { a as MatrixCell, i as MatrixAxis, n as AxisSummary, o as MatrixResult, r as CellResult, s as RunAgentMatrixOptions, t as runAgentMatrix } from "../index-DSC51roc2.js";
2
+ import { a as MatrixCell, i as MatrixAxis, n as AxisSummary, o as MatrixResult, r as CellResult, s as RunAgentMatrixOptions, t as runAgentMatrix } from "../index-DSC51roc.js";
3
3
  export { type AxisSummary, type CellResult, type DefaultVerdict, type MatrixAxis, type MatrixCell, type MatrixResult, type RunAgentMatrixOptions, runAgentMatrix };
@@ -1,7 +1,6 @@
1
1
  import { f as Run } from "../schema-BtVldJ3T.js";
2
2
  import { s as TraceStore } from "../store-CT9YIIve.js";
3
- import { a as ContinuousCalibrationResult, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult } from "../judge-calibration-DFtEMlde.js";
4
- import { a as CorpusAgreementReport } from "../statistics-D_4Snl-5.js";
3
+ import { Ct as GoldenItem, St as ContinuousCalibrationResult, a as CorpusAgreementReport, bt as ContinuousAgreement, vt as CalibrationResult, yt as CandidateScore } from "../statistics-CKOqre5S.js";
5
4
  import { a as OutcomeFilter, i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "../outcome-store-BYHIuO0e.js";
6
5
  import { n as SeriesConvergenceResult, t as SeriesConvergenceOptions } from "../series-convergence-ofsqPWhs.js";
7
6
  import { a as rubricPredictiveValidity, i as RubricRanking, n as RubricPredictiveValidityInput, r as RubricPredictiveValidityReport, t as RubricOutcomePair } from "../rubric-predictive-validity-C1dCLcvb.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","names":[],"sources":["../../src/meta-eval/correlation-study.ts","../../src/meta-eval/calibration.ts","../../src/meta-eval/sentinel.ts"],"mappings":";;;;;;;;UAiBiB;EACf;;EAEA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;EAEA;IAAe;IAAe;;;EAE9B;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,gBAAgB;;EAEhB;;EAEA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,aAAa,kBACb,8BACA,UAAS,0BACR,QAAQ;;;UCjDM;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,MAAM;;EAEN;;EAEA;;UAGe;EACf;;EAEA;;EAEA;IAAU;IAAY;;;UAGP;EACf;EACA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,YAAY,gBACZ,uBACA,UAAS,qBACR,QAAQ;iBAgCK,qBACd,YAAY,mBACZ,oBACA,uBACA,UAAS,qBACR;;;cCtCU;KAED,6BAA6B;UAExB;;EAEf;;EAEA;;EAEA;;UAGe;;EAEf;EACA;;EAEA;;;;;;;EAOA,SAAS;;;UAIM;EACf;EACA;EACA;;;iBAcc,yBAAyB,UAAU,kBAAkB;;;;;;iBAiCrD,wBACd,QAAQ,oBAAoB,6BAC5B,MAAM,eACL;;;;;;;;iBAmBa,sBACd,QAAQ,wBAAwB,qBAChC,MAAM,eACL;UAYc;;EAEf;;;;;;;;;iBAUc,wBACd,QAAQ,kBACR,QAAQ,cACR,MAAM,cACN,UAAS,qBACR;UA4Cc;EACf,OAAO,UAAU,mBAAmB;;EAEpC,QAAQ,mBAAmB,QAAQ;;iBAGrB,sBAAsB,UAAS,qBAA0B;;;;;;;iBAqBzD,kBAAkB,eAAe;UA0ChC;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,aAAa;;EAEb,cAAc;;UAGC;EACf;EACA,QAAQ;;EAER,OAAO;;EAEP;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;;;;;;EAOA;;iBAKc,oBACd,SAAS,oBACT,MAAM,uBACL;UA6Gc;EACf;EACA;;;;;;;;;;iBAWc,gBAAgB,QAAQ,iBAAiB"}
1
+ {"version":3,"file":"index.d.ts","names":[],"sources":["../../src/meta-eval/correlation-study.ts","../../src/meta-eval/calibration.ts","../../src/meta-eval/sentinel.ts"],"mappings":";;;;;;;UAiBiB;EACf;;EAEA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;EAEA;IAAe;IAAe;;;EAE9B;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,gBAAgB;;EAEhB;;EAEA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,aAAa,kBACb,8BACA,UAAS,0BACR,QAAQ;;;UCjDM;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,MAAM;;EAEN;;EAEA;;UAGe;EACf;;EAEA;;EAEA;IAAU;IAAY;;;UAGP;EACf;EACA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,YAAY,gBACZ,uBACA,UAAS,qBACR,QAAQ;iBAgCK,qBACd,YAAY,mBACZ,oBACA,uBACA,UAAS,qBACR;;;cCtCU;KAED,6BAA6B;UAExB;;EAEf;;EAEA;;EAEA;;UAGe;;EAEf;EACA;;EAEA;;;;;;;EAOA,SAAS;;;UAIM;EACf;EACA;EACA;;;iBAcc,yBAAyB,UAAU,kBAAkB;;;;;;iBAiCrD,wBACd,QAAQ,oBAAoB,6BAC5B,MAAM,eACL;;;;;;;;iBAmBa,sBACd,QAAQ,wBAAwB,qBAChC,MAAM,eACL;UAYc;;EAEf;;;;;;;;;iBAUc,wBACd,QAAQ,kBACR,QAAQ,cACR,MAAM,cACN,UAAS,qBACR;UA4Cc;EACf,OAAO,UAAU,mBAAmB;;EAEpC,QAAQ,mBAAmB,QAAQ;;iBAGrB,sBAAsB,UAAS,qBAA0B;;;;;;;iBAqBzD,kBAAkB,eAAe;UA0ChC;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,aAAa;;EAEb,cAAc;;UAGC;EACf;EACA,QAAQ;;EAER,OAAO;;EAEP;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;;;;;;EAOA;;iBAKc,oBACd,SAAS,oBACT,MAAM,uBACL;UA6Gc;EACf;EACA;;;;;;;;;;iBAWc,gBAAgB,QAAQ,iBAAiB"}
@@ -1,8 +1,8 @@
1
1
  import { s as ValidationError } from "../errors-8YnH8WlF.js";
2
- import { F as spearmanR, j as pearsonR } from "../statistics-RwRNu2__.js";
2
+ import { L as pearsonR, V as spearmanR } from "../statistics-CnGCLLqc.js";
3
3
  import { o as llmSpans, t as aggregateLlm } from "../query-Di7eEQ79.js";
4
4
  import { t as analyzeSeries } from "../series-convergence-CjO2QdRW.js";
5
- import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-D6Q6n9oq.js";
5
+ import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-BJf-8ejY.js";
6
6
  import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
7
7
  //#region src/meta-eval/calibration.ts
8
8
  async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
@@ -1,6 +1,6 @@
1
1
  import { p as CostProvenance } from "../cost-ledger-fGS_u_O1.js";
2
2
  import { w as JudgeScore } from "../types-DiWLru6Z.js";
3
- import { o as MatrixResult } from "../index-DSC51roc2.js";
3
+ import { o as MatrixResult } from "../index-DSC51roc.js";
4
4
  import { AgentProfile } from "@tangle-network/agent-interface";
5
5
  //#region src/multishot/types.d.ts
6
6
  interface MultishotMessage {
package/dist/openapi.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.1.0",
3
3
  "info": {
4
4
  "title": "@tangle-network/agent-eval — wire protocol",
5
- "version": "0.135.0",
5
+ "version": "0.135.2",
6
6
  "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
7
7
  "contact": {
8
8
  "name": "Tangle Network",
@@ -1,5 +1,5 @@
1
1
  import { s as ValidationError } from "./errors-8YnH8WlF.js";
2
- import { C as pairedBootstrap, E as pairedRiskDifference, R as wilcoxonSignedRank, v as mcnemar } from "./statistics-RwRNu2__.js";
2
+ import { A as pairedRiskDifference, E as pairedBootstrap, W as wilcoxonSignedRank, b as mcnemar } from "./statistics-CnGCLLqc.js";
3
3
  //#region src/paired-arms.ts
4
4
  /**
5
5
  * Matched-pair arm comparison — "did the treatment arm beat the baseline arm
@@ -257,4 +257,4 @@ function metricValue(row, name) {
257
257
  //#endregion
258
258
  export { pairArms as n, pairRunRecords as r, comparePairedArms as t };
259
259
 
260
- //# sourceMappingURL=paired-arms-CA_8pN01.js.map
260
+ //# sourceMappingURL=paired-arms-BbFKrAU-.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"paired-arms-CA_8pN01.js","names":[],"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n * 1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n * within multi-rep items), with leftovers REPORTED rather than silently\n * dropped (a silently unbalanced pairing biases every paired statistic\n * downstream). Pairing never keys on outcome content: matching reps by\n * their outcomes deflates discordant-pair counts and makes McNemar\n * anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n * 2. Composition — feeding the matched pairs to the correct paired\n * estimators that already live in `statistics`: `mcnemar` +\n * `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n * `wilcoxonSignedRank` for continuous metrics. No statistic is\n * re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n mcnemar,\n pairedBootstrap,\n pairedRiskDifference,\n type RiskDifferenceResult,\n wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n * project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n /** Matching key — rows sharing a `pairKey` across both arms form pairs\n * (typically the task/scenario/seed identity). */\n pairKey: string\n /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n * every row of a `pairKey` that has more than one rep in either arm; reps\n * then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n * content. Optional when each arm has at most one rep of the item. */\n repKey?: string\n /** Arm label this row was produced under. */\n arm: string\n /** Binary outcome; omit when the comparison has no pass/fail notion. */\n pass?: boolean\n /** Named numeric measurements (score, cost, latency, …). */\n metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n /** Arm treated as the control side of every pair. */\n baselineArm: string\n /** Arm treated as the treatment side of every pair. */\n treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n pairKey: string\n /** 0-based position of this pair within its `pairKey`, ordered by sorted\n * `repKey` (always 0 for a single-rep item). The rep identity itself is on\n * the rows (`baseline.repKey` / `treatment.repKey`). */\n repIndex: number\n baseline: PairedArmRow\n treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n pairs: MatchedPair[]\n /** Baseline rows left without a treatment counterpart — reported, never\n * silently dropped. */\n unpairedBaseline: PairedArmRow[]\n /** Treatment rows left without a baseline counterpart. */\n unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n const { baselineArm, treatmentArm } = opts\n if (baselineArm === treatmentArm) {\n throw new ValidationError(\n `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n )\n }\n\n // arm → pairKey → rows\n const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n const armsSeen = new Set<string>()\n for (const row of rows) {\n armsSeen.add(row.arm)\n if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n const group = byKey.get(row.pairKey) ?? []\n group.push(row)\n byKey.set(row.pairKey, group)\n byArm.set(row.arm, byKey)\n }\n\n for (const arm of [baselineArm, treatmentArm]) {\n if (!byArm.has(arm)) {\n const seen = [...armsSeen].sort().join(', ') || '<none>'\n throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n }\n }\n\n const baselineByKey = byArm.get(baselineArm)!\n const treatmentByKey = byArm.get(treatmentArm)!\n\n const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n const pairs: MatchedPair[] = []\n const unpairedBaseline: PairedArmRow[] = []\n const unpairedTreatment: PairedArmRow[] = []\n for (const pairKey of allKeys) {\n const b = baselineByKey.get(pairKey) ?? []\n const t = treatmentByKey.get(pairKey) ?? []\n\n if (b.length <= 1 && t.length <= 1) {\n if (b.length === 1 && t.length === 1) {\n const baseline = b[0]!\n const treatment = t[0]!\n if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n if (\n baseline.repKey === undefined ||\n treatment.repKey === undefined ||\n baseline.repKey !== treatment.repKey\n ) {\n unpairedBaseline.push(baseline)\n unpairedTreatment.push(treatment)\n continue\n }\n }\n pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n } else {\n unpairedBaseline.push(...b)\n unpairedTreatment.push(...t)\n }\n continue\n }\n\n const bByRep = indexByRepKey(b, pairKey, baselineArm)\n const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n let repIndex = 0\n for (const repKey of repKeys) {\n const baseline = bByRep.get(repKey)\n const treatment = tByRep.get(repKey)\n if (baseline !== undefined && treatment !== undefined) {\n pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n } else if (baseline !== undefined) {\n unpairedBaseline.push(baseline)\n } else if (treatment !== undefined) {\n unpairedTreatment.push(treatment)\n }\n }\n }\n\n return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n * row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n group: readonly PairedArmRow[],\n pairKey: string,\n arm: string,\n): Map<string, PairedArmRow> {\n const byRep = new Map<string, PairedArmRow>()\n for (const row of group) {\n if (row.repKey === undefined) {\n throw new ValidationError(\n `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n )\n }\n if (byRep.has(row.repKey)) {\n throw new ValidationError(\n `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n `(pairKey, repKey) must uniquely identify a rep within an arm`,\n )\n }\n byRep.set(row.repKey, row)\n }\n return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n /** Discordant pairs where the treatment passed and the baseline failed. */\n b10: number\n /** Discordant pairs where the baseline passed and the treatment failed. */\n b01: number\n /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n mcnemar: McNemarResult\n /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n name: string\n /** Pairs where BOTH sides carry a finite value for this metric. */\n n: number\n /** Pairs where at least one side does not carry the metric. */\n nMissing: number\n /** Median paired delta, or null when `n === 0`. */\n medianDelta: number | null\n /** Mean paired delta, or null when `n === 0`. */\n meanDelta: number | null\n /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n * `n === 0` — a zero-width [0, 0] interval on no data would read as a\n * measured tight null. */\n bootstrapCi: PairedBootstrapResult | null\n /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n /** Metrics to compare. Default: every metric name observed on any matched\n * pair, sorted. A name that appears on no pair is still reported (with\n * `n = 0`) so a misspelled metric is visible instead of vanishing. */\n metricNames?: string[]\n /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n nPairs: number\n nUnpairedBaseline: number\n nUnpairedTreatment: number\n /** null when no matched pair carries `pass` on both sides — a pass/fail\n * verdict over rows that never measured pass/fail would be fabricated. */\n correctness: PairedCorrectness | null\n metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n rows: readonly PairedArmRow[],\n opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n let correctness: PairedCorrectness | null = null\n const baselinePass: number[] = []\n const treatmentPass: number[] = []\n for (const pair of pairs) {\n if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n baselinePass.push(pair.baseline.pass ? 1 : 0)\n treatmentPass.push(pair.treatment.pass ? 1 : 0)\n }\n if (baselinePass.length > 0) {\n const mc = mcnemar(baselinePass, treatmentPass)\n correctness = {\n b10: mc.b,\n b01: mc.c,\n mcnemar: mc,\n riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n }\n }\n\n const metricNames =\n opts.metricNames ??\n [\n ...new Set(\n pairs.flatMap((p) => [\n ...Object.keys(p.baseline.metrics ?? {}),\n ...Object.keys(p.treatment.metrics ?? {}),\n ]),\n ),\n ].sort()\n\n const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n const before: number[] = []\n const after: number[] = []\n let nMissing = 0\n for (const pair of pairs) {\n const b = metricValue(pair.baseline, name)\n const t = metricValue(pair.treatment, name)\n if (b === undefined || t === undefined) {\n nMissing++\n continue\n }\n before.push(b)\n after.push(t)\n }\n const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n return {\n name,\n n: before.length,\n nMissing,\n medianDelta: bootstrapCi?.median ?? null,\n meanDelta: bootstrapCi?.mean ?? null,\n bootstrapCi,\n wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n }\n })\n\n return {\n nPairs: pairs.length,\n nUnpairedBaseline: unpairedBaseline.length,\n nUnpairedTreatment: unpairedTreatment.length,\n correctness,\n metricDeltas,\n }\n}\n\nexport interface MatchedRunRecordPair {\n pairKey: string\n repKey: string\n baseline: RunRecord\n treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n pairs: MatchedRunRecordPair[]\n unpairedBaseline: RunRecord[]\n unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n run: RunRecord\n repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n baselineRuns: readonly RunRecord[],\n treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n validateRunRecordArmRows(baselineRows, 'baseline')\n validateRunRecordArmRows(treatmentRows, 'treatment')\n if (baselineRows.length === 0 || treatmentRows.length === 0) {\n return {\n pairs: [],\n unpairedBaseline: baselineRows.map((row) => row.run),\n unpairedTreatment: treatmentRows.map((row) => row.run),\n }\n }\n\n const result = pairArms([...baselineRows, ...treatmentRows], {\n baselineArm: 'baseline',\n treatmentArm: 'treatment',\n })\n return {\n pairs: result.pairs.map((pair) => {\n const baseline = pair.baseline as RunRecordArmRow\n const treatment = pair.treatment as RunRecordArmRow\n return {\n pairKey: pair.pairKey,\n repKey: baseline.repKey,\n baseline: baseline.run,\n treatment: treatment.run,\n }\n }),\n unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n return runs.map((run) => {\n const scenarioId = run.scenarioId.trim()\n if (!scenarioId) {\n throw new ValidationError(\n `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n )\n }\n return {\n pairKey: JSON.stringify([run.experimentId, scenarioId]),\n repKey: String(run.seed),\n arm,\n run,\n }\n })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n const byPairKey = new Map<string, RunRecordArmRow[]>()\n for (const row of rows) {\n const group = byPairKey.get(row.pairKey) ?? []\n group.push(row)\n byPairKey.set(row.pairKey, group)\n }\n for (const [pairKey, group] of byPairKey) {\n if (group.length > 1) indexByRepKey(group, pairKey, arm)\n }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n const v = row.metrics?.[name]\n if (v === undefined) return undefined\n if (!Number.isFinite(v)) {\n throw new ValidationError(\n `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n )\n }\n return v\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuGA,SAAgB,SAAS,MAA+B,MAAuC;CAC7F,MAAM,EAAE,aAAa,iBAAiB;CACtC,IAAI,gBAAgB,cAClB,MAAM,IAAI,gBACR,oDAAoD,YAAY,wCAClE;CAIF,MAAM,wBAAQ,IAAI,IAAyC;CAC3D,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,OAAO,MAAM;EACtB,SAAS,IAAI,IAAI,GAAG;EACpB,IAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,cAAc;EACzD,MAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,qBAAK,IAAI,IAA4B;EACpE,MAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,KAAK,GAAG;EACd,MAAM,IAAI,IAAI,SAAS,KAAK;EAC5B,MAAM,IAAI,IAAI,KAAK,KAAK;CAC1B;CAEA,KAAK,MAAM,OAAO,CAAC,aAAa,YAAY,GAC1C,IAAI,CAAC,MAAM,IAAI,GAAG,GAEhB,MAAM,IAAI,gBAAgB,8BAA8B,IAAI,mBAD/C,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,KAAK,SACoC,EAAE;CAI1F,MAAM,gBAAgB,MAAM,IAAI,WAAW;CAC3C,MAAM,iBAAiB,MAAM,IAAI,YAAY;CAE7C,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACvF,MAAM,QAAuB,CAAC;CAC9B,MAAM,mBAAmC,CAAC;CAC1C,MAAM,oBAAoC,CAAC;CAC3C,KAAK,MAAM,WAAW,SAAS;EAC7B,MAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;EAE1C,IAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;GAClC,IAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;IACpC,MAAM,WAAW,EAAE;IACnB,MAAM,YAAY,EAAE;IACpB,IAAI,SAAS,WAAW,KAAA,KAAa,UAAU,WAAW,KAAA,GAEtD;SAAA,SAAS,WAAW,KAAA,KACpB,UAAU,WAAW,KAAA,KACrB,SAAS,WAAW,UAAU,QAC9B;MACA,iBAAiB,KAAK,QAAQ;MAC9B,kBAAkB,KAAK,SAAS;MAChC;KACF;;IAEF,MAAM,KAAK;KAAE;KAAS,UAAU;KAAG;KAAU;IAAU,CAAC;GAC1D,OAAO;IACL,iBAAiB,KAAK,GAAG,CAAC;IAC1B,kBAAkB,KAAK,GAAG,CAAC;GAC7B;GACA;EACF;EAEA,MAAM,SAAS,cAAc,GAAG,SAAS,WAAW;EACpD,MAAM,SAAS,cAAc,GAAG,SAAS,YAAY;EACrD,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,IAAI,WAAW;EACf,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,WAAW,OAAO,IAAI,MAAM;GAClC,MAAM,YAAY,OAAO,IAAI,MAAM;GACnC,IAAI,aAAa,KAAA,KAAa,cAAc,KAAA,GAC1C,MAAM,KAAK;IAAE;IAAS,UAAU;IAAY;IAAU;GAAU,CAAC;QAC5D,IAAI,aAAa,KAAA,GACtB,iBAAiB,KAAK,QAAQ;QACzB,IAAI,cAAc,KAAA,GACvB,kBAAkB,KAAK,SAAS;EAEpC;CACF;CAEA,OAAO;EAAE;EAAO;EAAkB;CAAkB;AACtD;;;AAIA,SAAS,cACP,OACA,SACA,KAC2B;CAC3B,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,OAAO,OAAO;EACvB,IAAI,IAAI,WAAW,KAAA,GACjB,MAAM,IAAI,gBACR,sBAAsB,QAAQ,mDAAmD,IAAI,8KAGvF;EAEF,IAAI,MAAM,IAAI,IAAI,MAAM,GACtB,MAAM,IAAI,gBACR,+BAA+B,IAAI,OAAO,iBAAiB,QAAQ,YAAY,IAAI,iEAErF;EAEF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B;CACA,OAAO;AACT;;;;;;;;;;;;;;AAiEA,SAAgB,kBACd,MACA,MACsB;CACtB,MAAM,EAAE,OAAO,kBAAkB,sBAAsB,SAAS,MAAM,IAAI;CAE1E,IAAI,cAAwC;CAC5C,MAAM,eAAyB,CAAC;CAChC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,SAAS,SAAS,KAAA,KAAa,KAAK,UAAU,SAAS,KAAA,GAAW;EAC3E,aAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;EAC5C,cAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;CAChD;CACA,IAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,KAAK,QAAQ,cAAc,aAAa;EAC9C,cAAc;GACZ,KAAK,GAAG;GACR,KAAK,GAAG;GACR,SAAS;GACT,gBAAgB,qBAAqB,cAAc,aAAa;EAClE;CACF;CAaA,MAAM,gBAVJ,KAAK,eACL,CACE,GAAG,IAAI,IACL,MAAM,SAAS,MAAM,CACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC,GACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC,CAC1C,CAAC,CACH,CACF,CAAC,CAAC,KAAK,EAAA,CAE6C,KAAK,SAAS;EAClE,MAAM,SAAmB,CAAC;EAC1B,MAAM,QAAkB,CAAC;EACzB,IAAI,WAAW;EACf,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,IAAI,YAAY,KAAK,UAAU,IAAI;GACzC,MAAM,IAAI,YAAY,KAAK,WAAW,IAAI;GAC1C,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW;IACtC;IACA;GACF;GACA,OAAO,KAAK,CAAC;GACb,MAAM,KAAK,CAAC;EACd;EACA,MAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;EAC9F,OAAO;GACL;GACA,GAAG,OAAO;GACV;GACA,aAAa,aAAa,UAAU;GACpC,WAAW,aAAa,QAAQ;GAChC;GACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;EACzE;CACF,CAAC;CAED,OAAO;EACL,QAAQ,MAAM;EACd,mBAAmB,iBAAiB;EACpC,oBAAoB,kBAAkB;EACtC;EACA;CACF;AACF;;;;;;;;AA2BA,SAAgB,eACd,cACA,eACsB;CACtB,MAAM,eAAe,iBAAiB,cAAc,UAAU;CAC9D,MAAM,gBAAgB,iBAAiB,eAAe,WAAW;CACjE,yBAAyB,cAAc,UAAU;CACjD,yBAAyB,eAAe,WAAW;CACnD,IAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GACxD,OAAO;EACL,OAAO,CAAC;EACR,kBAAkB,aAAa,KAAK,QAAQ,IAAI,GAAG;EACnD,mBAAmB,cAAc,KAAK,QAAQ,IAAI,GAAG;CACvD;CAGF,MAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;EAC3D,aAAa;EACb,cAAc;CAChB,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAM,WAAW,KAAK;GACtB,MAAM,YAAY,KAAK;GACvB,OAAO;IACL,SAAS,KAAK;IACd,QAAQ,SAAS;IACjB,UAAU,SAAS;IACnB,WAAW,UAAU;GACvB;EACF,CAAC;EACD,kBAAkB,OAAO,iBAAiB,KAAK,QAAS,IAAwB,GAAG;EACnF,mBAAmB,OAAO,kBAAkB,KAAK,QAAS,IAAwB,GAAG;CACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;CACpF,OAAO,KAAK,KAAK,QAAQ;EACvB,MAAM,aAAa,IAAI,WAAW,KAAK;EACvC,IAAI,CAAC,YACH,MAAM,IAAI,gBACR,wBAAwB,IAAI,MAAM,+EACpC;EAEF,OAAO;GACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;GACtD,QAAQ,OAAO,IAAI,IAAI;GACvB;GACA;EACF;CACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;CACrF,MAAM,4BAAY,IAAI,IAA+B;CACrD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;EAC7C,MAAM,KAAK,GAAG;EACd,UAAU,IAAI,IAAI,SAAS,KAAK;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,UAAU,WAC7B,IAAI,MAAM,SAAS,GAAG,cAAc,OAAO,SAAS,GAAG;AAE3D;AAEA,SAAS,YAAY,KAAmB,MAAkC;CACxE,MAAM,IAAI,IAAI,UAAU;CACxB,IAAI,MAAM,KAAA,GAAW,OAAO,KAAA;CAC5B,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,gBACR,mDAAmD,KAAK,gBAAgB,IAAI,QAAQ,UAAU,IAAI,IAAI,MAAM,GAC9G;CAEF,OAAO;AACT"}
1
+ {"version":3,"file":"paired-arms-BbFKrAU-.js","names":[],"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n * 1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n * within multi-rep items), with leftovers REPORTED rather than silently\n * dropped (a silently unbalanced pairing biases every paired statistic\n * downstream). Pairing never keys on outcome content: matching reps by\n * their outcomes deflates discordant-pair counts and makes McNemar\n * anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n * 2. Composition — feeding the matched pairs to the correct paired\n * estimators that already live in `statistics`: `mcnemar` +\n * `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n * `wilcoxonSignedRank` for continuous metrics. No statistic is\n * re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n mcnemar,\n pairedBootstrap,\n pairedRiskDifference,\n type RiskDifferenceResult,\n wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n * project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n /** Matching key — rows sharing a `pairKey` across both arms form pairs\n * (typically the task/scenario/seed identity). */\n pairKey: string\n /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n * every row of a `pairKey` that has more than one rep in either arm; reps\n * then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n * content. Optional when each arm has at most one rep of the item. */\n repKey?: string\n /** Arm label this row was produced under. */\n arm: string\n /** Binary outcome; omit when the comparison has no pass/fail notion. */\n pass?: boolean\n /** Named numeric measurements (score, cost, latency, …). */\n metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n /** Arm treated as the control side of every pair. */\n baselineArm: string\n /** Arm treated as the treatment side of every pair. */\n treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n pairKey: string\n /** 0-based position of this pair within its `pairKey`, ordered by sorted\n * `repKey` (always 0 for a single-rep item). The rep identity itself is on\n * the rows (`baseline.repKey` / `treatment.repKey`). */\n repIndex: number\n baseline: PairedArmRow\n treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n pairs: MatchedPair[]\n /** Baseline rows left without a treatment counterpart — reported, never\n * silently dropped. */\n unpairedBaseline: PairedArmRow[]\n /** Treatment rows left without a baseline counterpart. */\n unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n const { baselineArm, treatmentArm } = opts\n if (baselineArm === treatmentArm) {\n throw new ValidationError(\n `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n )\n }\n\n // arm → pairKey → rows\n const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n const armsSeen = new Set<string>()\n for (const row of rows) {\n armsSeen.add(row.arm)\n if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n const group = byKey.get(row.pairKey) ?? []\n group.push(row)\n byKey.set(row.pairKey, group)\n byArm.set(row.arm, byKey)\n }\n\n for (const arm of [baselineArm, treatmentArm]) {\n if (!byArm.has(arm)) {\n const seen = [...armsSeen].sort().join(', ') || '<none>'\n throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n }\n }\n\n const baselineByKey = byArm.get(baselineArm)!\n const treatmentByKey = byArm.get(treatmentArm)!\n\n const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n const pairs: MatchedPair[] = []\n const unpairedBaseline: PairedArmRow[] = []\n const unpairedTreatment: PairedArmRow[] = []\n for (const pairKey of allKeys) {\n const b = baselineByKey.get(pairKey) ?? []\n const t = treatmentByKey.get(pairKey) ?? []\n\n if (b.length <= 1 && t.length <= 1) {\n if (b.length === 1 && t.length === 1) {\n const baseline = b[0]!\n const treatment = t[0]!\n if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n if (\n baseline.repKey === undefined ||\n treatment.repKey === undefined ||\n baseline.repKey !== treatment.repKey\n ) {\n unpairedBaseline.push(baseline)\n unpairedTreatment.push(treatment)\n continue\n }\n }\n pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n } else {\n unpairedBaseline.push(...b)\n unpairedTreatment.push(...t)\n }\n continue\n }\n\n const bByRep = indexByRepKey(b, pairKey, baselineArm)\n const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n let repIndex = 0\n for (const repKey of repKeys) {\n const baseline = bByRep.get(repKey)\n const treatment = tByRep.get(repKey)\n if (baseline !== undefined && treatment !== undefined) {\n pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n } else if (baseline !== undefined) {\n unpairedBaseline.push(baseline)\n } else if (treatment !== undefined) {\n unpairedTreatment.push(treatment)\n }\n }\n }\n\n return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n * row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n group: readonly PairedArmRow[],\n pairKey: string,\n arm: string,\n): Map<string, PairedArmRow> {\n const byRep = new Map<string, PairedArmRow>()\n for (const row of group) {\n if (row.repKey === undefined) {\n throw new ValidationError(\n `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n )\n }\n if (byRep.has(row.repKey)) {\n throw new ValidationError(\n `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n `(pairKey, repKey) must uniquely identify a rep within an arm`,\n )\n }\n byRep.set(row.repKey, row)\n }\n return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n /** Discordant pairs where the treatment passed and the baseline failed. */\n b10: number\n /** Discordant pairs where the baseline passed and the treatment failed. */\n b01: number\n /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n mcnemar: McNemarResult\n /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n name: string\n /** Pairs where BOTH sides carry a finite value for this metric. */\n n: number\n /** Pairs where at least one side does not carry the metric. */\n nMissing: number\n /** Median paired delta, or null when `n === 0`. */\n medianDelta: number | null\n /** Mean paired delta, or null when `n === 0`. */\n meanDelta: number | null\n /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n * `n === 0` — a zero-width [0, 0] interval on no data would read as a\n * measured tight null. */\n bootstrapCi: PairedBootstrapResult | null\n /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n /** Metrics to compare. Default: every metric name observed on any matched\n * pair, sorted. A name that appears on no pair is still reported (with\n * `n = 0`) so a misspelled metric is visible instead of vanishing. */\n metricNames?: string[]\n /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n nPairs: number\n nUnpairedBaseline: number\n nUnpairedTreatment: number\n /** null when no matched pair carries `pass` on both sides — a pass/fail\n * verdict over rows that never measured pass/fail would be fabricated. */\n correctness: PairedCorrectness | null\n metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n rows: readonly PairedArmRow[],\n opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n let correctness: PairedCorrectness | null = null\n const baselinePass: number[] = []\n const treatmentPass: number[] = []\n for (const pair of pairs) {\n if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n baselinePass.push(pair.baseline.pass ? 1 : 0)\n treatmentPass.push(pair.treatment.pass ? 1 : 0)\n }\n if (baselinePass.length > 0) {\n const mc = mcnemar(baselinePass, treatmentPass)\n correctness = {\n b10: mc.b,\n b01: mc.c,\n mcnemar: mc,\n riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n }\n }\n\n const metricNames =\n opts.metricNames ??\n [\n ...new Set(\n pairs.flatMap((p) => [\n ...Object.keys(p.baseline.metrics ?? {}),\n ...Object.keys(p.treatment.metrics ?? {}),\n ]),\n ),\n ].sort()\n\n const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n const before: number[] = []\n const after: number[] = []\n let nMissing = 0\n for (const pair of pairs) {\n const b = metricValue(pair.baseline, name)\n const t = metricValue(pair.treatment, name)\n if (b === undefined || t === undefined) {\n nMissing++\n continue\n }\n before.push(b)\n after.push(t)\n }\n const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n return {\n name,\n n: before.length,\n nMissing,\n medianDelta: bootstrapCi?.median ?? null,\n meanDelta: bootstrapCi?.mean ?? null,\n bootstrapCi,\n wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n }\n })\n\n return {\n nPairs: pairs.length,\n nUnpairedBaseline: unpairedBaseline.length,\n nUnpairedTreatment: unpairedTreatment.length,\n correctness,\n metricDeltas,\n }\n}\n\nexport interface MatchedRunRecordPair {\n pairKey: string\n repKey: string\n baseline: RunRecord\n treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n pairs: MatchedRunRecordPair[]\n unpairedBaseline: RunRecord[]\n unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n run: RunRecord\n repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n baselineRuns: readonly RunRecord[],\n treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n validateRunRecordArmRows(baselineRows, 'baseline')\n validateRunRecordArmRows(treatmentRows, 'treatment')\n if (baselineRows.length === 0 || treatmentRows.length === 0) {\n return {\n pairs: [],\n unpairedBaseline: baselineRows.map((row) => row.run),\n unpairedTreatment: treatmentRows.map((row) => row.run),\n }\n }\n\n const result = pairArms([...baselineRows, ...treatmentRows], {\n baselineArm: 'baseline',\n treatmentArm: 'treatment',\n })\n return {\n pairs: result.pairs.map((pair) => {\n const baseline = pair.baseline as RunRecordArmRow\n const treatment = pair.treatment as RunRecordArmRow\n return {\n pairKey: pair.pairKey,\n repKey: baseline.repKey,\n baseline: baseline.run,\n treatment: treatment.run,\n }\n }),\n unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n return runs.map((run) => {\n const scenarioId = run.scenarioId.trim()\n if (!scenarioId) {\n throw new ValidationError(\n `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n )\n }\n return {\n pairKey: JSON.stringify([run.experimentId, scenarioId]),\n repKey: String(run.seed),\n arm,\n run,\n }\n })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n const byPairKey = new Map<string, RunRecordArmRow[]>()\n for (const row of rows) {\n const group = byPairKey.get(row.pairKey) ?? []\n group.push(row)\n byPairKey.set(row.pairKey, group)\n }\n for (const [pairKey, group] of byPairKey) {\n if (group.length > 1) indexByRepKey(group, pairKey, arm)\n }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n const v = row.metrics?.[name]\n if (v === undefined) return undefined\n if (!Number.isFinite(v)) {\n throw new ValidationError(\n `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n )\n }\n return v\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuGA,SAAgB,SAAS,MAA+B,MAAuC;CAC7F,MAAM,EAAE,aAAa,iBAAiB;CACtC,IAAI,gBAAgB,cAClB,MAAM,IAAI,gBACR,oDAAoD,YAAY,wCAClE;CAIF,MAAM,wBAAQ,IAAI,IAAyC;CAC3D,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,OAAO,MAAM;EACtB,SAAS,IAAI,IAAI,GAAG;EACpB,IAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,cAAc;EACzD,MAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,qBAAK,IAAI,IAA4B;EACpE,MAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,KAAK,GAAG;EACd,MAAM,IAAI,IAAI,SAAS,KAAK;EAC5B,MAAM,IAAI,IAAI,KAAK,KAAK;CAC1B;CAEA,KAAK,MAAM,OAAO,CAAC,aAAa,YAAY,GAC1C,IAAI,CAAC,MAAM,IAAI,GAAG,GAEhB,MAAM,IAAI,gBAAgB,8BAA8B,IAAI,mBAD/C,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,KAAK,SACoC,EAAE;CAI1F,MAAM,gBAAgB,MAAM,IAAI,WAAW;CAC3C,MAAM,iBAAiB,MAAM,IAAI,YAAY;CAE7C,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACvF,MAAM,QAAuB,CAAC;CAC9B,MAAM,mBAAmC,CAAC;CAC1C,MAAM,oBAAoC,CAAC;CAC3C,KAAK,MAAM,WAAW,SAAS;EAC7B,MAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;EAE1C,IAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;GAClC,IAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;IACpC,MAAM,WAAW,EAAE;IACnB,MAAM,YAAY,EAAE;IACpB,IAAI,SAAS,WAAW,KAAA,KAAa,UAAU,WAAW,KAAA,GAEtD;SAAA,SAAS,WAAW,KAAA,KACpB,UAAU,WAAW,KAAA,KACrB,SAAS,WAAW,UAAU,QAC9B;MACA,iBAAiB,KAAK,QAAQ;MAC9B,kBAAkB,KAAK,SAAS;MAChC;KACF;;IAEF,MAAM,KAAK;KAAE;KAAS,UAAU;KAAG;KAAU;IAAU,CAAC;GAC1D,OAAO;IACL,iBAAiB,KAAK,GAAG,CAAC;IAC1B,kBAAkB,KAAK,GAAG,CAAC;GAC7B;GACA;EACF;EAEA,MAAM,SAAS,cAAc,GAAG,SAAS,WAAW;EACpD,MAAM,SAAS,cAAc,GAAG,SAAS,YAAY;EACrD,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,IAAI,WAAW;EACf,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,WAAW,OAAO,IAAI,MAAM;GAClC,MAAM,YAAY,OAAO,IAAI,MAAM;GACnC,IAAI,aAAa,KAAA,KAAa,cAAc,KAAA,GAC1C,MAAM,KAAK;IAAE;IAAS,UAAU;IAAY;IAAU;GAAU,CAAC;QAC5D,IAAI,aAAa,KAAA,GACtB,iBAAiB,KAAK,QAAQ;QACzB,IAAI,cAAc,KAAA,GACvB,kBAAkB,KAAK,SAAS;EAEpC;CACF;CAEA,OAAO;EAAE;EAAO;EAAkB;CAAkB;AACtD;;;AAIA,SAAS,cACP,OACA,SACA,KAC2B;CAC3B,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,OAAO,OAAO;EACvB,IAAI,IAAI,WAAW,KAAA,GACjB,MAAM,IAAI,gBACR,sBAAsB,QAAQ,mDAAmD,IAAI,8KAGvF;EAEF,IAAI,MAAM,IAAI,IAAI,MAAM,GACtB,MAAM,IAAI,gBACR,+BAA+B,IAAI,OAAO,iBAAiB,QAAQ,YAAY,IAAI,iEAErF;EAEF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B;CACA,OAAO;AACT;;;;;;;;;;;;;;AAiEA,SAAgB,kBACd,MACA,MACsB;CACtB,MAAM,EAAE,OAAO,kBAAkB,sBAAsB,SAAS,MAAM,IAAI;CAE1E,IAAI,cAAwC;CAC5C,MAAM,eAAyB,CAAC;CAChC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,SAAS,SAAS,KAAA,KAAa,KAAK,UAAU,SAAS,KAAA,GAAW;EAC3E,aAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;EAC5C,cAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;CAChD;CACA,IAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,KAAK,QAAQ,cAAc,aAAa;EAC9C,cAAc;GACZ,KAAK,GAAG;GACR,KAAK,GAAG;GACR,SAAS;GACT,gBAAgB,qBAAqB,cAAc,aAAa;EAClE;CACF;CAaA,MAAM,gBAVJ,KAAK,eACL,CACE,GAAG,IAAI,IACL,MAAM,SAAS,MAAM,CACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC,GACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC,CAC1C,CAAC,CACH,CACF,CAAC,CAAC,KAAK,EAAA,CAE6C,KAAK,SAAS;EAClE,MAAM,SAAmB,CAAC;EAC1B,MAAM,QAAkB,CAAC;EACzB,IAAI,WAAW;EACf,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,IAAI,YAAY,KAAK,UAAU,IAAI;GACzC,MAAM,IAAI,YAAY,KAAK,WAAW,IAAI;GAC1C,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW;IACtC;IACA;GACF;GACA,OAAO,KAAK,CAAC;GACb,MAAM,KAAK,CAAC;EACd;EACA,MAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;EAC9F,OAAO;GACL;GACA,GAAG,OAAO;GACV;GACA,aAAa,aAAa,UAAU;GACpC,WAAW,aAAa,QAAQ;GAChC;GACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;EACzE;CACF,CAAC;CAED,OAAO;EACL,QAAQ,MAAM;EACd,mBAAmB,iBAAiB;EACpC,oBAAoB,kBAAkB;EACtC;EACA;CACF;AACF;;;;;;;;AA2BA,SAAgB,eACd,cACA,eACsB;CACtB,MAAM,eAAe,iBAAiB,cAAc,UAAU;CAC9D,MAAM,gBAAgB,iBAAiB,eAAe,WAAW;CACjE,yBAAyB,cAAc,UAAU;CACjD,yBAAyB,eAAe,WAAW;CACnD,IAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GACxD,OAAO;EACL,OAAO,CAAC;EACR,kBAAkB,aAAa,KAAK,QAAQ,IAAI,GAAG;EACnD,mBAAmB,cAAc,KAAK,QAAQ,IAAI,GAAG;CACvD;CAGF,MAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;EAC3D,aAAa;EACb,cAAc;CAChB,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAM,WAAW,KAAK;GACtB,MAAM,YAAY,KAAK;GACvB,OAAO;IACL,SAAS,KAAK;IACd,QAAQ,SAAS;IACjB,UAAU,SAAS;IACnB,WAAW,UAAU;GACvB;EACF,CAAC;EACD,kBAAkB,OAAO,iBAAiB,KAAK,QAAS,IAAwB,GAAG;EACnF,mBAAmB,OAAO,kBAAkB,KAAK,QAAS,IAAwB,GAAG;CACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;CACpF,OAAO,KAAK,KAAK,QAAQ;EACvB,MAAM,aAAa,IAAI,WAAW,KAAK;EACvC,IAAI,CAAC,YACH,MAAM,IAAI,gBACR,wBAAwB,IAAI,MAAM,+EACpC;EAEF,OAAO;GACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;GACtD,QAAQ,OAAO,IAAI,IAAI;GACvB;GACA;EACF;CACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;CACrF,MAAM,4BAAY,IAAI,IAA+B;CACrD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;EAC7C,MAAM,KAAK,GAAG;EACd,UAAU,IAAI,IAAI,SAAS,KAAK;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,UAAU,WAC7B,IAAI,MAAM,SAAS,GAAG,cAAc,OAAO,SAAS,GAAG;AAE3D;AAEA,SAAS,YAAY,KAAmB,MAAkC;CACxE,MAAM,IAAI,IAAI,UAAU;CACxB,IAAI,MAAM,KAAA,GAAW,OAAO,KAAA;CAC5B,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,gBACR,mDAAmD,KAAK,gBAAgB,IAAI,QAAQ,UAAU,IAAI,IAAI,MAAM,GAC9G;CAEF,OAAO;AACT"}
@@ -1,8 +1,8 @@
1
1
  import { t as executionTrackByLane } from "../execution-tracks-CpgFPpS5.js";
2
- import { h as interRaterReliability, j as pearsonR } from "../statistics-RwRNu2__.js";
2
+ import { L as pearsonR, g as interRaterReliability } from "../statistics-CnGCLLqc.js";
3
3
  import { s as isToolSpan } from "../schema-CRhEY1SO.js";
4
4
  import { t as buildTrajectory } from "../trajectory-D_7rLrvE.js";
5
- import { t as compareToBaseline } from "../baseline-BaPxoROc.js";
5
+ import { t as compareToBaseline } from "../baseline-BUeFcgrn.js";
6
6
  import { i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, s as runFailureClass, t as aggregateLlm } from "../query-Di7eEQ79.js";
7
7
  import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "../tool-use-metrics-DEGMKycK.js";
8
8
  //#region src/pipelines/budget-breach.ts
@@ -2,7 +2,7 @@ import { c as VerificationError, s as ValidationError } from "./errors-8YnH8WlF.
2
2
  import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
3
3
  import { t as FAILURE_CLASSES } from "./schema-CRhEY1SO.js";
4
4
  import { s as validateRunRecord } from "./run-record-BIwU2wdV.js";
5
- import { a as summaryTable } from "./summary-report-BxtossFi.js";
5
+ import { a as summaryTable } from "./summary-report-BEk8OFLs.js";
6
6
  //#region src/release-confidence.ts
7
7
  const DEFAULT_THRESHOLDS = {
8
8
  requireCorpus: true,
@@ -425,7 +425,8 @@ function bootstrapCi(baseline, candidate, options = {}) {
425
425
  const ciLower = deltas[Math.max(0, lowerIdx)];
426
426
  const ciUpper = deltas[Math.min(iterations - 1, upperIdx)];
427
427
  let verdict;
428
- if (ciLower > 0) verdict = "ADVANCE";
428
+ if (!Number.isFinite(ciLower) || !Number.isFinite(ciUpper) || ciLower === ciUpper) verdict = "INCONCLUSIVE";
429
+ else if (ciLower > 0) verdict = "ADVANCE";
429
430
  else if (ciUpper < 0) verdict = "REVERT";
430
431
  else if (delta >= 0) verdict = "KEEP";
431
432
  else verdict = "INCONCLUSIVE";
@@ -600,4 +601,4 @@ function duration(value) {
600
601
  //#endregion
601
602
  export { evaluateReleaseConfidence as a, assertReleaseConfidence as i, bootstrapCi as n, judgeReplayGate as r, renderReleaseReport as t };
602
603
 
603
- //# sourceMappingURL=release-report-BVZBmRZp.js.map
604
+ //# sourceMappingURL=release-report-DooPguBc.js.map