@tangle-network/agent-eval 0.126.7 → 0.128.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +6 -2
  3. package/dist/analyst/index.d.ts +52 -43
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +35 -40
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -11
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +441 -160
  11. package/dist/campaign/index.js +11 -10
  12. package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
  13. package/dist/chunk-2JX3CFMB.js.map +1 -0
  14. package/dist/chunk-2MKQIFS4.js +183 -0
  15. package/dist/chunk-2MKQIFS4.js.map +1 -0
  16. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  17. package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
  18. package/dist/chunk-7ZZMD7UK.js.map +1 -0
  19. package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
  20. package/dist/chunk-BYT7ELPS.js.map +1 -0
  21. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  22. package/dist/chunk-DPUHNQLN.js +232 -0
  23. package/dist/chunk-DPUHNQLN.js.map +1 -0
  24. package/dist/chunk-DRYIUNWY.js +622 -0
  25. package/dist/chunk-DRYIUNWY.js.map +1 -0
  26. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  27. package/dist/chunk-EJGRPCO3.js.map +1 -0
  28. package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
  29. package/dist/chunk-EOSZT7PL.js.map +1 -0
  30. package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
  31. package/dist/chunk-EZJEIH2R.js.map +1 -0
  32. package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
  33. package/dist/chunk-IHQDPH7D.js.map +1 -0
  34. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  35. package/dist/chunk-MHELPNRP.js.map +1 -0
  36. package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
  37. package/dist/chunk-NACAGYSY.js.map +1 -0
  38. package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
  39. package/dist/chunk-NKAGIDE2.js.map +1 -0
  40. package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
  41. package/dist/chunk-NYLOYM6N.js.map +1 -0
  42. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  43. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  44. package/dist/chunk-P6FYH6K4.js.map +1 -0
  45. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  46. package/dist/chunk-PBE2LOSS.js.map +1 -0
  47. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  48. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  49. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  50. package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
  51. package/dist/chunk-TBL77AUT.js.map +1 -0
  52. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  53. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  54. package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
  55. package/dist/chunk-UB2LOJ6Q.js.map +1 -0
  56. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  57. package/dist/chunk-UWZZKKU7.js.map +1 -0
  58. package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
  59. package/dist/chunk-VLOATJQ2.js.map +1 -0
  60. package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
  61. package/dist/chunk-XDWDC2MP.js.map +1 -0
  62. package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
  63. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +438 -138
  67. package/dist/contract/index.js +259 -113
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +56 -39
  70. package/dist/control.js +2 -3
  71. package/dist/hosted/index.d.ts +152 -51
  72. package/dist/hosted/index.js +24 -1
  73. package/dist/index.d.ts +399 -277
  74. package/dist/index.js +363 -244
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +33 -36
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +101 -80
  82. package/dist/reporting.js +7 -5
  83. package/dist/rl.d.ts +198 -142
  84. package/dist/rl.js +235 -127
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +54 -67
  87. package/dist/rollout/index.js +6 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +72 -50
  92. package/dist/traces.js +9 -6
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/customer-journeys.md +7 -6
  97. package/docs/hosted-ingest-spec.md +37 -40
  98. package/docs/insight-report.md +78 -7
  99. package/docs/research-report-methodology.md +20 -21
  100. package/docs/rollout.md +13 -8
  101. package/package.json +4 -3
  102. package/dist/chunk-474LBSOX.js.map +0 -1
  103. package/dist/chunk-7VYEGHWF.js.map +0 -1
  104. package/dist/chunk-ARU2PZFM.js.map +0 -1
  105. package/dist/chunk-BTACVO2W.js.map +0 -1
  106. package/dist/chunk-DPZAEKA6.js.map +0 -1
  107. package/dist/chunk-E3IADPLG.js.map +0 -1
  108. package/dist/chunk-H5UD2323.js.map +0 -1
  109. package/dist/chunk-IILEIWGW.js.map +0 -1
  110. package/dist/chunk-IPYXE555.js.map +0 -1
  111. package/dist/chunk-IR3KBHOY.js.map +0 -1
  112. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  113. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  114. package/dist/chunk-M7AH34KV.js.map +0 -1
  115. package/dist/chunk-MAX3TN3C.js.map +0 -1
  116. package/dist/chunk-MOXWMGPC.js.map +0 -1
  117. package/dist/chunk-OCFJACJU.js.map +0 -1
  118. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  119. package/dist/chunk-PJQFMIOX.js.map +0 -1
  120. package/dist/chunk-VSMTAMNK.js +0 -53
  121. package/dist/chunk-VSMTAMNK.js.map +0 -1
  122. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  123. package/dist/chunk-ZZUXHH3R.js +0 -99
  124. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  125. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  126. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  127. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  128. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  129. /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
  130. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  131. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  132. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
@@ -150,6 +150,12 @@ function isSchemaRejection(status, body) {
150
150
  const lower = body.toLowerCase();
151
151
  return lower.includes("response_format") || lower.includes("json_schema") || lower.includes("is unavailable") || lower.includes("not supported");
152
152
  }
153
+ function isTemperatureOneRejection(status, body) {
154
+ if (status !== 400 || !/temperature/i.test(body)) return false;
155
+ return /temperature[^.\n]{0,120}\b(?:only|must|should|required|requires?)\b[^.\n]{0,40}\b1(?:\.0+)?\b/i.test(
156
+ body
157
+ ) || /\bonly\s+1(?:\.0+)?\s+is\s+allowed\b[^.\n]{0,120}\btemperature\b/i.test(body);
158
+ }
153
159
  function buildBody(req, forceJsonObject, defaultThinking) {
154
160
  const body = {
155
161
  model: req.model,
@@ -270,6 +276,7 @@ async function callLlm(req, opts = {}) {
270
276
  costForTokenPricing(opts.customTokenPricing, { inputTokens: 0, outputTokens: 0 });
271
277
  }
272
278
  let lastErr;
279
+ let effectiveRequest = req;
273
280
  for (let attempt = 0; attempt < maximumAttempts; attempt++) {
274
281
  if (callerSignal?.aborted) {
275
282
  throw new DOMException("callLlm aborted by caller signal", "AbortError");
@@ -281,7 +288,11 @@ async function callLlm(req, opts = {}) {
281
288
  const attemptSignal = linkSignals(controller, callerSignal);
282
289
  const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs);
283
290
  const started = Date.now();
284
- const requestBody = buildBody(req, opts.jsonSchemaTransport === "json-object", opts.thinking);
291
+ const requestBody = buildBody(
292
+ effectiveRequest,
293
+ opts.jsonSchemaTransport === "json-object",
294
+ opts.thinking
295
+ );
285
296
  let attemptErrorRecorded = false;
286
297
  if (sink) {
287
298
  await recordRaw(sink, redactor, {
@@ -338,6 +349,11 @@ async function callLlm(req, opts = {}) {
338
349
  body,
339
350
  req.model
340
351
  );
352
+ if (isTemperatureOneRejection(res.status, body) && effectiveRequest.temperature !== 1 && attempt < maximumAttempts - 1 && !deadlineExceeded(deadlineStart, deadlineMs)) {
353
+ lastErr = err;
354
+ effectiveRequest = { ...effectiveRequest, temperature: 1 };
355
+ continue;
356
+ }
341
357
  if (RETRYABLE_STATUS.has(res.status) && attempt < maximumAttempts - 1 && !deadlineExceeded(deadlineStart, deadlineMs)) {
342
358
  lastErr = err;
343
359
  const retryAfter = parseRetryAfter(res.headers);
@@ -650,4 +666,4 @@ export {
650
666
  probeLlm,
651
667
  LlmClient
652
668
  };
653
- //# sourceMappingURL=chunk-J5SQWP6Y.js.map
669
+ //# sourceMappingURL=chunk-PBE2LOSS.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n * - One retry at temperature 1 when a model explicitly requires it.\n * - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n * directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n * const { value, result } = await callLlmJson<MyType>(\n * { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n * { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n * )\n *\n * This is THE llm-calling seam for agent-eval primitives that need structured\n * output (semantic concept judge, reviewer directives, critic scores). Primitives\n * that need free-form text use `callLlm` and parse output themselves.\n */\n\nimport {\n type CostReceiptInput,\n type CustomTokenPricing,\n costForTokenPricing,\n type MaximumCharge,\n} from './cost-ledger'\nimport { AgentEvalError, CaptureIntegrityError } from './errors'\nimport {\n defaultProviderRedactor,\n type ProviderRedactor,\n providerFromBaseUrl,\n type RawProviderEvent,\n type RawProviderSink,\n} from './trace/raw-provider-sink'\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n role: 'system' | 'user' | 'assistant'\n /**\n * Either a plain text content string OR a multimodal content array\n * (text + image_url parts) for vision-capable models.\n */\n content:\n | string\n | Array<\n | { type: 'text'; text: string }\n | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n >\n}\n\nexport type LlmThinkingMode = 'enabled' | 'disabled'\n\nexport interface LlmCallRequest {\n model: string\n messages: LlmMessage[]\n /** Optional JSON-mode response format (response_format: json_object). */\n jsonMode?: boolean\n /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n jsonSchema?: { name: string; schema: Record<string, unknown> }\n temperature?: number\n maxTokens?: number\n /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */\n thinking?: LlmThinkingMode\n /** Per-call timeout, default 300s. */\n timeoutMs?: number\n}\n\n/** Conservative priced bound for the exact text request sent to a provider.\n * Returns undefined when output or multimodal input is not bounded, causing a\n * capped CostLedger to reject the call before execution. Pass\n * `customTokenPricing` when package pricing does not cover the model or endpoint. */\nexport function maximumChargeForLlmRequest(\n request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'maxTokens' | 'thinking'>,\n options: LlmClientOptions = {},\n): MaximumCharge | undefined {\n if (request.maxTokens === undefined) return undefined\n if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {\n throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`)\n }\n if (\n request.messages.some(\n (message) =>\n Array.isArray(message.content) && message.content.some((part) => part.type === 'image_url'),\n )\n ) {\n return undefined\n }\n\n const attempts = resolveMaximumAttempts(options.maxRetries)\n const forceJsonObject = options.jsonSchemaTransport === 'json-object'\n // A byte-level tokenizer cannot emit more input tokens than request bytes.\n // Pricing the complete body also covers role/schema framing omitted from content-only estimates.\n const requestBytes = new TextEncoder().encode(\n JSON.stringify(buildBody(request, forceJsonObject, options.thinking)),\n ).byteLength\n // A rejected response schema can trigger one JSON-mode batch with the same output limit.\n const batches = request.jsonSchema && !forceJsonObject ? 2 : 1\n const usage = {\n inputTokens: requestBytes * attempts * batches,\n outputTokens: request.maxTokens * attempts * batches,\n }\n return options.customTokenPricing\n ? { customTokenPricing: options.customTokenPricing, ...usage }\n : { model: request.model, ...usage }\n}\n\nexport interface LlmUsage {\n promptTokens: number\n completionTokens: number\n totalTokens: number\n /** False when the provider omitted or malformed prompt/completion usage. */\n captured?: boolean\n /** Reasoning-token subset of completionTokens, when reported. */\n reasoningTokens?: number\n /** Proxies populate this when prompt caching is on. */\n cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n /** The text content of the first choice. Empty string if none. */\n content: string\n usage: LlmUsage\n /**\n * Cost in USD. Uses the provider's reported cost when present, otherwise\n * caller-supplied token pricing. `null` when neither is available.\n */\n costUsd: number | null\n /** Model name actually used (echoed from response). */\n model: string\n /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n durationMs: number\n /**\n * `finish_reason` echoed from the first choice (`stop`, `length`,\n * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.\n * Exposed so a free-form `callLlm` caller CAN detect a truncated answer\n * (`length`) instead of treating a cut-off completion as complete. Note:\n * `callLlm` does not itself reject on it — acting on this signal is the\n * caller's responsibility (in-repo free-form drivers do not yet enforce it).\n */\n finishReason?: string | null\n /**\n * True when `content.trim()` is empty. An empty completion is a silent zero\n * for free-form `callLlm` callers; this flag is the signal a caller can\n * inspect to fail loud rather than proceed on an empty string. `callLlm`\n * surfaces it but does not throw on it.\n */\n contentEmpty?: boolean\n /** Raw response body. */\n raw: Record<string, unknown>\n}\n\nexport type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>\n\n/** Convert a provider result into the canonical paid-call receipt input. */\nexport function costReceiptFromLlm(\n result: LlmCallResult,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput {\n const cachedTokens = result.usage.cachedPromptTokens ?? 0\n const inputTokens = Math.max(0, result.usage.promptTokens - cachedTokens)\n const configuredCostUsd =\n result.costUsd === null && customTokenPricing && result.usage.captured !== false\n ? costForTokenPricing(customTokenPricing, {\n inputTokens,\n ...(cachedTokens > 0 ? { cachedTokens } : {}),\n outputTokens: result.usage.completionTokens,\n })\n : undefined\n return {\n model: result.model,\n inputTokens,\n outputTokens: result.usage.completionTokens,\n reasoningTokens: result.usage.reasoningTokens,\n cachedTokens: cachedTokens > 0 ? cachedTokens : undefined,\n actualCostUsd: result.costUsd ?? configuredCostUsd,\n usageUnknown: result.usage.captured === false,\n }\n}\n\n/** Structured-response failures retain their completed provider receipt. */\nexport function costReceiptFromLlmError(\n error: Error,\n customTokenPricing?: CustomTokenPricing,\n): CostReceiptInput | undefined {\n return error instanceof LlmResponseError\n ? costReceiptFromLlm(error.result, customTokenPricing)\n : undefined\n}\n\nexport class LlmCallError extends AgentEvalError {\n constructor(\n message: string,\n public readonly status: number,\n public readonly body: string,\n public readonly model: string,\n ) {\n super('judge', message)\n }\n}\n\n/** A provider response completed and incurred measurable usage, but its content\n * could not satisfy the caller's response contract. The response envelope is\n * retained so accounting can commit the receipt before the error propagates. */\nexport class LlmResponseError extends AgentEvalError {\n constructor(\n message: string,\n public readonly result: LlmCallResult,\n options?: { cause?: unknown },\n ) {\n super('judge', message, options)\n }\n}\n\nexport interface LlmClientOptions {\n /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n baseUrl?: string\n /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n apiKey?: string\n bearer?: string\n /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n authHeader?: { name: string; value: string }\n /** Stable provider idempotency key, reused across retries of this logical call. */\n idempotencyKey?: string\n /** Default timeout in ms. Per-call can override. */\n defaultTimeoutMs?: number\n /**\n * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to\n * each attempt's per-attempt timeout controller, so aborting it cancels\n * the in-flight fetch. A caller abort is FATAL: it is not retried even\n * though an AbortError otherwise matches the transient patterns.\n */\n signal?: AbortSignal\n /**\n * Cross-attempt wall-clock budget in ms, measured from the first attempt.\n * Before launching each attempt the loop checks the remaining budget and\n * stops retrying once it is exhausted, rather than waiting the full\n * per-attempt timeout on every retry. Bounds total time independent of\n * total attempts × `timeoutMs`.\n */\n deadlineMs?: number\n /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */\n maxRetries?: number\n /** Token rates used when the provider omits cost or package pricing does not cover the model. */\n customTokenPricing?: CustomTokenPricing\n /**\n * Transport for requests that declare `jsonSchema`. `native` sends\n * `response_format: json_schema`; `json-object` sends the broadly supported\n * JSON mode and relies on the caller to include the schema in model-visible\n * instructions. Default: `native`.\n */\n jsonSchemaTransport?: 'native' | 'json-object'\n /**\n * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.\n * `exact` requires the complete response content to be one JSON value.\n * Default: `extract`.\n */\n jsonPayloadMode?: 'extract' | 'exact'\n /** Default provider reasoning mode. A per-call request value takes precedence. */\n thinking?: LlmThinkingMode\n /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n fetch?: typeof fetch\n /**\n * Optional raw HTTP capture sink. When provided, every request, response,\n * and error (across all retry attempts) is recorded to the sink, with auth\n * headers and credential-shaped body fields redacted by default. This is\n * the layer-1 forensics primitive: structured `LlmSpan`s record intent,\n * raw events record what actually crossed the wire.\n */\n rawSink?: RawProviderSink\n /**\n * Logical provider id attached to raw events. When omitted, derived from\n * `baseUrl` via `providerFromBaseUrl`.\n */\n provider?: string\n /** Trace context attached to raw events; populated by emitter-aware callers. */\n traceContext?: { runId?: string; spanId?: string }\n /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */\n redactor?: ProviderRedactor\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\n// Flagship / reasoning models routinely take several minutes on large prompts (a\n// reflection over many failures, a long tool transcript). A tight cap aborts a\n// legitimately-slow but healthy call — and because every retry attempt re-uses\n// the same window, such a model aborts on ALL attempts and the loop throws. The\n// default is generous enough to let those complete, bounded enough that a truly\n// hung call still fails over after retries, and tunable per deployment via\n// TANGLE_LLM_TIMEOUT_MS. Per-call `req.timeoutMs` / `opts.defaultTimeoutMs`\n// still win for callers that know their model's latency.\nconst DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 300_000\nconst DEFAULT_MAX_RETRIES = Number(process.env.TANGLE_LLM_MAX_RETRIES) || 3\n\nfunction resolveMaximumAttempts(configured: number | undefined): number {\n const attempts = configured ?? DEFAULT_MAX_RETRIES\n if (!Number.isInteger(attempts) || attempts <= 0) {\n throw new RangeError('LLM maximum attempts must be a positive integer')\n }\n return attempts\n}\n\nfunction providerTokenCount(value: unknown): number | undefined {\n return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\n/**\n * Transient transport/network error signatures, matched against an error's\n * name, message, and `code`. Covers fetch/undici network failures, aborts\n * and timeouts, and — critically — HTTP/2 transport faults a keep-alive\n * connection raises mid-response: `terminated`, `NGHTTP2_INTERNAL_ERROR`,\n * `UND_ERR_*`, `other side closed`. Those last ones carry no clean HTTP\n * status; unrecognised, they escape the retry loop and surface as an\n * uncaught rejection.\n */\nconst TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [\n /AbortError/i,\n /TimeoutError/i,\n /this operation was aborted/i,\n /fetch failed/i,\n /ECONNRESET/i,\n /ETIMEDOUT/i,\n /EAI_AGAIN/i,\n /socket hang up/i,\n /stream.*ended.*unexpectedly/i,\n /terminated/i,\n /other side closed/i,\n /NGHTTP2/i,\n /UND_ERR/i,\n]\n\n/**\n * True when an error is a transient transport/network fault worth retrying,\n * as opposed to a deterministic failure (4xx schema reject, JSON parse) that\n * a retry cannot fix. Inspects `LlmCallError.status`, then the error's\n * name/message/code, then recurses into `error.cause` — undici nests the\n * real socket fault one or more levels under `.cause`.\n *\n * This is THE retry classifier for the package: `callLlm` and\n * `withJudgeRetry` both route through it, so a connection-class error is\n * treated identically whether it surfaces in the HTTP client or a\n * TCloud-backed judge.\n */\nexport function isTransientLlmError(err: unknown): boolean {\n return classifyTransient(err, 0)\n}\n\nfunction classifyTransient(err: unknown, depth: number): boolean {\n if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n if (!(err instanceof Error)) return false\n // Foreign errors (e.g. a TCloud judge SDK error) can carry a numeric HTTP\n // status without being an LlmCallError — a retryable status is decisive.\n const status = (err as { status?: unknown }).status\n if (typeof status === 'number' && RETRYABLE_STATUS.has(status)) return true\n const code = (err as { code?: unknown }).code\n const haystack = `${err.name}\\n${err.message}\\n${typeof code === 'string' ? code : ''}`\n if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true\n const cause = (err as { cause?: unknown }).cause\n if (depth < 4 && cause instanceof Error && cause !== err) {\n return classifyTransient(cause, depth + 1)\n }\n return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n const h = headers.get('retry-after')\n if (!h) return null\n const asNumber = Number(h)\n if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n const asDate = Date.parse(h)\n if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n return null\n}\n\n/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */\nexport function backoffMs(attempt: number): number {\n return Math.min(500 * 2 ** attempt, 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n Accept: 'application/json',\n }\n if (opts.authHeader) {\n headers[opts.authHeader.name] = opts.authHeader.value\n } else if (opts.bearer || opts.apiKey) {\n headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n }\n if (opts.idempotencyKey) headers['Idempotency-Key'] = opts.idempotencyKey\n return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n if (status !== 400) return false\n const lower = body.toLowerCase()\n return (\n lower.includes('response_format') ||\n lower.includes('json_schema') ||\n lower.includes('is unavailable') ||\n lower.includes('not supported')\n )\n}\n\nfunction isTemperatureOneRejection(status: number, body: string): boolean {\n if (status !== 400 || !/temperature/i.test(body)) return false\n return (\n /temperature[^.\\n]{0,120}\\b(?:only|must|should|required|requires?)\\b[^.\\n]{0,40}\\b1(?:\\.0+)?\\b/i.test(\n body,\n ) || /\\bonly\\s+1(?:\\.0+)?\\s+is\\s+allowed\\b[^.\\n]{0,120}\\btemperature\\b/i.test(body)\n )\n}\n\nfunction buildBody(\n req: LlmCallRequest,\n forceJsonObject: boolean,\n defaultThinking?: LlmThinkingMode,\n): Record<string, unknown> {\n const body: Record<string, unknown> = {\n model: req.model,\n messages: req.messages,\n temperature: req.temperature ?? 0,\n }\n if (req.maxTokens != null) {\n if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens\n else body.max_tokens = req.maxTokens\n }\n const thinking = req.thinking ?? defaultThinking\n if (thinking !== undefined) {\n body.thinking = { type: thinking }\n }\n\n if (req.jsonSchema && !forceJsonObject) {\n body.response_format = {\n type: 'json_schema',\n json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n }\n } else if (req.jsonMode || req.jsonSchema) {\n body.response_format = { type: 'json_object' }\n }\n\n return body\n}\n\nfunction usesMaxCompletionTokens(model: string): boolean {\n return /^gpt-5(?:[.-]|$)/i.test(model)\n}\n\nasync function sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n/**\n * Combine the per-attempt timeout signal with an optional caller signal into\n * one signal the fetch listens on. Prefers the native `AbortSignal.any`; falls\n * back to manual wiring on runtimes that predate it. The caller signal is also\n * propagated to the timeout controller so aborting it cancels the in-flight\n * fetch immediately.\n */\nfunction linkSignals(timeoutController: AbortController, caller?: AbortSignal): AbortSignal {\n if (!caller) return timeoutController.signal\n if (typeof (AbortSignal as { any?: unknown }).any === 'function') {\n return AbortSignal.any([timeoutController.signal, caller])\n }\n if (caller.aborted) {\n timeoutController.abort()\n } else {\n caller.addEventListener('abort', () => timeoutController.abort(), { once: true })\n }\n return timeoutController.signal\n}\n\n/** True once the cross-attempt wall-clock budget (if any) is exhausted. */\nfunction deadlineExceeded(start: number, deadlineMs: number | undefined): boolean {\n return deadlineMs != null && Date.now() - start >= deadlineMs\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n const trimmed = raw.trim()\n const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n const stripped = stripFencedJson(raw)\n try {\n JSON.parse(stripped)\n return stripped\n } catch {\n // A response that declares a JSON root must parse as that complete root.\n // Scanning onward could turn a truncated object into one of its valid nested\n // arrays or objects and silently change the response schema.\n if (stripped.startsWith('{') || stripped.startsWith('[')) return stripped\n }\n\n // Only prose-leading responses may contain a recoverable JSON payload.\n const starts = [...stripped.matchAll(/[[{]/g)]\n .map((match) => match.index)\n .filter((index) => index != null)\n for (const start of starts) {\n const candidate = extractBalancedJson(stripped, start)\n if (!candidate) continue\n try {\n JSON.parse(candidate)\n return candidate\n } catch {\n // Keep scanning; earlier braces may belong to prose.\n }\n }\n\n return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n const opener = input[start]\n const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n if (!closer) return null\n\n const stack: string[] = [closer]\n let isInString = false\n let isEscaped = false\n\n for (let i = start + 1; i < input.length; i++) {\n const char = input[i]!\n if (isEscaped) {\n isEscaped = false\n continue\n }\n if (char === '\\\\') {\n isEscaped = isInString\n continue\n }\n if (char === '\"') {\n isInString = !isInString\n continue\n }\n if (isInString) continue\n\n if (char === '{') stack.push('}')\n else if (char === '[') stack.push(']')\n else if (char === stack[stack.length - 1]) {\n stack.pop()\n if (stack.length === 0) return input.slice(start, i + 1)\n }\n }\n\n return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n const url = `${baseUrl}/chat/completions`\n const endpoint = '/chat/completions'\n const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n const maximumAttempts = resolveMaximumAttempts(opts.maxRetries)\n const fetchFn = opts.fetch ?? globalThis.fetch\n const headers = buildHeaders(opts)\n const provider = opts.provider ?? providerFromBaseUrl(baseUrl)\n const sink = opts.rawSink\n const redactor = opts.redactor ?? defaultProviderRedactor\n const traceContext = opts.traceContext\n const callerSignal = opts.signal\n const deadlineMs = opts.deadlineMs\n const deadlineStart = Date.now()\n if (opts.customTokenPricing) {\n costForTokenPricing(opts.customTokenPricing, { inputTokens: 0, outputTokens: 0 })\n }\n\n let lastErr: unknown\n let effectiveRequest = req\n for (let attempt = 0; attempt < maximumAttempts; attempt++) {\n // A caller cancel is fatal — never retried. Checking before each attempt\n // means an already-aborted signal short-circuits without firing fetch.\n if (callerSignal?.aborted) {\n throw new DOMException('callLlm aborted by caller signal', 'AbortError')\n }\n // Stop retrying once the cross-attempt budget is spent rather than burning\n // a full per-attempt timeout on each remaining retry.\n if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n }\n const controller = new AbortController()\n const attemptSignal = linkSignals(controller, callerSignal)\n const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n const started = Date.now()\n const requestBody = buildBody(\n effectiveRequest,\n opts.jsonSchemaTransport === 'json-object',\n opts.thinking,\n )\n let attemptErrorRecorded = false\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'request',\n timestamp: started,\n requestHeaders: headers,\n requestBody,\n redactedFields: [],\n })\n }\n\n try {\n const res = await fetchFn(url, {\n method: 'POST',\n headers,\n body: JSON.stringify(requestBody),\n signal: attemptSignal,\n })\n clearTimeout(timeoutHandle)\n const responseHeaders = sink ? headersToObject(res.headers) : undefined\n\n if (!res.ok) {\n const body = await res.text()\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: body,\n errorMessage: `HTTP ${res.status}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n const err = new LlmCallError(\n `LLM call ${res.status}: ${body.slice(0, 300)}`,\n res.status,\n body,\n req.model,\n )\n if (\n isTemperatureOneRejection(res.status, body) &&\n effectiveRequest.temperature !== 1 &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n effectiveRequest = { ...effectiveRequest, temperature: 1 }\n continue\n }\n if (\n RETRYABLE_STATUS.has(res.status) &&\n attempt < maximumAttempts - 1 &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n lastErr = err\n const retryAfter = parseRetryAfter(res.headers)\n await sleep(retryAfter ?? backoffMs(attempt))\n continue\n }\n throw err\n }\n\n const text = await res.text()\n let json: Record<string, unknown>\n try {\n json = JSON.parse(text) as Record<string, unknown>\n } catch (parseErr) {\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: text,\n errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,\n redactedFields: [],\n })\n attemptErrorRecorded = true\n }\n throw parseErr\n }\n if (sink) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'response',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n statusCode: res.status,\n responseHeaders,\n responseBody: json,\n redactedFields: [],\n })\n }\n const choice = (\n json.choices as\n | Array<{ message?: { content?: string }; finish_reason?: string | null }>\n | undefined\n )?.[0]\n const usageRaw =\n json.usage && typeof json.usage === 'object' && !Array.isArray(json.usage)\n ? (json.usage as Record<string, unknown>)\n : undefined\n const promptTokens = providerTokenCount(usageRaw?.prompt_tokens)\n const completionTokens = providerTokenCount(usageRaw?.completion_tokens)\n const totalTokens = providerTokenCount(usageRaw?.total_tokens)\n const completionDetails =\n usageRaw?.completion_tokens_details &&\n typeof usageRaw.completion_tokens_details === 'object' &&\n !Array.isArray(usageRaw.completion_tokens_details)\n ? (usageRaw.completion_tokens_details as Record<string, unknown>)\n : undefined\n const reasoningRaw = completionDetails?.reasoning_tokens\n const reasoningTokens =\n reasoningRaw === undefined ? undefined : providerTokenCount(reasoningRaw)\n const cachedRaw =\n usageRaw?.prompt_tokens_details &&\n typeof usageRaw.prompt_tokens_details === 'object' &&\n !Array.isArray(usageRaw.prompt_tokens_details)\n ? (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens\n : undefined\n const cachedPromptTokens = cachedRaw === undefined ? undefined : providerTokenCount(cachedRaw)\n const usageCaptured =\n promptTokens !== undefined &&\n completionTokens !== undefined &&\n (reasoningRaw === undefined ||\n (reasoningTokens !== undefined && reasoningTokens <= completionTokens)) &&\n (cachedRaw === undefined ||\n (cachedPromptTokens !== undefined && cachedPromptTokens <= promptTokens)) &&\n (totalTokens === undefined || totalTokens === promptTokens + completionTokens)\n const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n const content = choice?.message?.content ?? ''\n\n const configuredCost =\n typeof costFromProxy !== 'number' && usageCaptured && opts.customTokenPricing\n ? costForTokenPricing(opts.customTokenPricing, {\n inputTokens: promptTokens! - (cachedPromptTokens ?? 0),\n ...(cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {}),\n outputTokens: completionTokens!,\n })\n : undefined\n\n return {\n content,\n finishReason: choice?.finish_reason ?? null,\n contentEmpty: content.trim().length === 0,\n usage: {\n promptTokens: promptTokens ?? 0,\n completionTokens: completionTokens ?? 0,\n totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),\n captured: usageCaptured,\n reasoningTokens,\n cachedPromptTokens,\n },\n costUsd: typeof costFromProxy === 'number' ? costFromProxy : (configuredCost ?? null),\n model: (json.model as string) ?? req.model,\n durationMs: Date.now() - started,\n raw: json,\n }\n } catch (err) {\n clearTimeout(timeoutHandle)\n lastErr = err\n // A caller cancel is fatal even though an AbortError matches the\n // transient patterns — a cancelled call must surface immediately, not\n // be retried against the same dead intent.\n if (callerSignal?.aborted) {\n if (sink && !attemptErrorRecorded) {\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n throw err\n }\n if (sink && !attemptErrorRecorded) {\n // Record only if neither the !res.ok branch nor the JSON.parse catch\n // already produced an error event for this attempt. Covers network\n // failures, timeouts, and aborts.\n await recordRaw(sink, redactor, {\n eventId: cryptoEventId(),\n runId: traceContext?.runId,\n spanId: traceContext?.spanId,\n provider,\n model: req.model,\n endpoint,\n baseUrl,\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n }\n if (\n attempt < maximumAttempts - 1 &&\n isTransientLlmError(err) &&\n !deadlineExceeded(deadlineStart, deadlineMs)\n ) {\n await sleep(backoffMs(attempt))\n continue\n }\n throw err\n }\n }\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\nasync function recordRaw(\n sink: RawProviderSink,\n redactor: ProviderRedactor,\n event: RawProviderEvent,\n): Promise<void> {\n // Errors from sinks must not crash the LLM call. Forensic capture is\n // best-effort; the structured trace is the system of record.\n try {\n await sink.record(redactor(event))\n } catch {\n // Intentionally swallowed.\n }\n}\n\nfunction headersToObject(h: Headers): Record<string, string> {\n const out: Record<string, string> = {}\n h.forEach((value, key) => {\n out[key] = value\n })\n return out\n}\n\nfunction cryptoEventId(): string {\n if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n const result = await callLlmStructured(req, opts)\n const value = parseJsonResult<T>(result, opts.jsonPayloadMode ?? 'extract')\n return { value, result }\n}\n\n/** Shared schema-to-JSON-mode fallback that preserves the raw result. */\nasync function callLlmStructured(\n req: LlmCallRequest,\n opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n try {\n return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n } catch (err) {\n if (\n opts.jsonSchemaTransport !== 'json-object' &&\n err instanceof LlmCallError &&\n isSchemaRejection(err.status, err.body) &&\n req.jsonSchema\n ) {\n const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n return await callLlm(degradedReq, opts)\n }\n throw err\n }\n}\n\nfunction parseJsonResult<T>(\n result: LlmCallResult,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n try {\n if (result.finishReason === 'length') {\n throw new Error(\n `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`,\n )\n }\n return parseJsonSafely<T>(result.content, result.model, jsonPayloadMode)\n } catch (error) {\n if (error instanceof LlmResponseError) throw error\n const cause = error instanceof Error ? error : new Error(String(error))\n throw new LlmResponseError(cause.message, result, { cause })\n }\n}\n\nfunction parseJsonSafely<T>(\n content: string,\n model: string,\n jsonPayloadMode: NonNullable<LlmClientOptions['jsonPayloadMode']>,\n): T {\n const payload = jsonPayloadMode === 'exact' ? content : extractJsonPayload(content)\n try {\n return JSON.parse(payload) as T\n } catch (err) {\n throw new Error(\n `LLM returned non-JSON content (model=${model}): ${\n err instanceof Error ? err.message : String(err)\n }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n )\n }\n}\n\n// ─── Route assertion ────────────────────────────────────────────────────\n\nexport type LlmRouteAssertionReason =\n | 'no_explicit_base_url'\n | 'base_url_blocked'\n | 'base_url_not_allowed'\n | 'no_auth'\n | 'wrong_provider'\n\nexport class LlmRouteAssertionError extends CaptureIntegrityError {\n constructor(\n message: string,\n public readonly reason: LlmRouteAssertionReason,\n public readonly baseUrl: string,\n ) {\n super(message)\n }\n}\n\nexport interface LlmRouteRequirements {\n /**\n * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to\n * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using\n * the public/free-tier router is a defect — the launch reviewer needs to\n * know exactly which provider answered.\n */\n requireExplicitBaseUrl?: boolean\n /**\n * Allowlist of acceptable base URLs. Strings match by prefix\n * (case-insensitive); RegExps test against the full base URL.\n */\n allowedBaseUrls?: Array<string | RegExp>\n /** Blocklist that takes precedence over `allowedBaseUrls`. */\n blockedBaseUrls?: Array<string | RegExp>\n /** Throw if no auth header / api key is configured. */\n requireAuth?: boolean\n /**\n * Logical provider id the configured `baseUrl` is expected to match (via\n * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.\n */\n expectedProvider?: string\n}\n\n/**\n * Fail-loud assertion that the configured LLM client points at the route\n * the caller intends. Designed for the matrix-runner preflight: invoke\n * once before any LLM call to catch misconfiguration before a sweep burns\n * dollars on the wrong provider.\n *\n * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call\n * from constructors and CI gates.\n */\nexport function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {\n const baseUrlExplicit = opts.baseUrl !== undefined\n const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n if (req.requireExplicitBaseUrl && !baseUrlExplicit) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,\n 'no_explicit_base_url',\n baseUrl,\n )\n }\n\n if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,\n 'base_url_blocked',\n baseUrl,\n )\n }\n\n if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {\n const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))\n if (!ok) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,\n 'base_url_not_allowed',\n baseUrl,\n )\n }\n }\n\n if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,\n 'no_auth',\n baseUrl,\n )\n }\n\n if (req.expectedProvider) {\n const actual = opts.provider ?? providerFromBaseUrl(baseUrl)\n if (actual !== req.expectedProvider) {\n throw new LlmRouteAssertionError(\n `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,\n 'wrong_provider',\n baseUrl,\n )\n }\n }\n}\n\nfunction matchUrl(url: string, pattern: string | RegExp): boolean {\n if (pattern instanceof RegExp) return pattern.test(url)\n return url.toLowerCase().startsWith(pattern.toLowerCase())\n}\n\nfunction describePattern(p: string | RegExp): string {\n return p instanceof RegExp ? p.source : p\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n model: string,\n opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n const start = Date.now()\n try {\n await callLlm(\n {\n model,\n messages: [{ role: 'user', content: 'ping' }],\n maxTokens: 64,\n timeoutMs: opts.timeoutMs ?? 30_000,\n },\n opts,\n )\n return { ok: true, latencyMs: Date.now() - start, error: null }\n } catch (err) {\n return {\n ok: false,\n latencyMs: Date.now() - start,\n error: err instanceof Error ? err.message : String(err),\n }\n }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n readonly maximumAttempts: number\n private readonly opts: LlmClientOptions\n\n constructor(opts: LlmClientOptions = {}) {\n this.opts = opts\n this.maximumAttempts = resolveMaximumAttempts(opts.maxRetries)\n }\n\n call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n const options = { ...this.opts, ...per }\n return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)\n }\n\n callJson<T = unknown>(\n req: LlmCallRequest,\n per?: LlmClientOptions,\n ): Promise<{ value: T; result: LlmCallResult }> {\n return callLlmJson<T>(req, { ...this.opts, ...per })\n }\n}\n"],"mappings":";;;;;;;;;;;;;AA2EO,SAAS,2BACd,SACA,UAA4B,CAAC,GACF;AAC3B,MAAI,QAAQ,cAAc,OAAW,QAAO;AAC5C,MAAI,CAAC,OAAO,UAAU,QAAQ,SAAS,KAAK,QAAQ,aAAa,GAAG;AAClE,UAAM,IAAI,WAAW,kEAAkE;AAAA,EACzF;AACA,MACE,QAAQ,SAAS;AAAA,IACf,CAAC,YACC,MAAM,QAAQ,QAAQ,OAAO,KAAK,QAAQ,QAAQ,KAAK,CAAC,SAAS,KAAK,SAAS,WAAW;AAAA,EAC9F,GACA;AACA,WAAO;AAAA,EACT;AAEA,QAAM,WAAW,uBAAuB,QAAQ,UAAU;AAC1D,QAAM,kBAAkB,QAAQ,wBAAwB;AAGxD,QAAM,eAAe,IAAI,YAAY,EAAE;AAAA,IACrC,KAAK,UAAU,UAAU,SAAS,iBAAiB,QAAQ,QAAQ,CAAC;AAAA,EACtE,EAAE;AAEF,QAAM,UAAU,QAAQ,cAAc,CAAC,kBAAkB,IAAI;AAC7D,QAAM,QAAQ;AAAA,IACZ,aAAa,eAAe,WAAW;AAAA,IACvC,cAAc,QAAQ,YAAY,WAAW;AAAA,EAC/C;AACA,SAAO,QAAQ,qBACX,EAAE,oBAAoB,QAAQ,oBAAoB,GAAG,MAAM,IAC3D,EAAE,OAAO,QAAQ,OAAO,GAAG,MAAM;AACvC;AAkDO,SAAS,mBACd,QACA,oBACkB;AAClB,QAAM,eAAe,OAAO,MAAM,sBAAsB;AACxD,QAAM,cAAc,KAAK,IAAI,GAAG,OAAO,MAAM,eAAe,YAAY;AACxE,QAAM,oBACJ,OAAO,YAAY,QAAQ,sBAAsB,OAAO,MAAM,aAAa,QACvE,oBAAoB,oBAAoB;AAAA,IACtC;AAAA,IACA,GAAI,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;AAAA,IAC3C,cAAc,OAAO,MAAM;AAAA,EAC7B,CAAC,IACD;AACN,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd;AAAA,IACA,cAAc,OAAO,MAAM;AAAA,IAC3B,iBAAiB,OAAO,MAAM;AAAA,IAC9B,cAAc,eAAe,IAAI,eAAe;AAAA,IAChD,eAAe,OAAO,WAAW;AAAA,IACjC,cAAc,OAAO,MAAM,aAAa;AAAA,EAC1C;AACF;AAGO,SAAS,wBACd,OACA,oBAC8B;AAC9B,SAAO,iBAAiB,mBACpB,mBAAmB,MAAM,QAAQ,kBAAkB,IACnD;AACN;AAEO,IAAM,eAAN,cAA2B,eAAe;AAAA,EAC/C,YACE,SACgB,QACA,MACA,OAChB;AACA,UAAM,SAAS,OAAO;AAJN;AACA;AACA;AAAA,EAGlB;AAAA,EALkB;AAAA,EACA;AAAA,EACA;AAIpB;AAKO,IAAM,mBAAN,cAA+B,eAAe;AAAA,EACnD,YACE,SACgB,QAChB,SACA;AACA,UAAM,SAAS,SAAS,OAAO;AAHf;AAAA,EAIlB;AAAA,EAJkB;AAKpB;AAuEA,IAAM,mBAAmB;AASzB,IAAM,qBAAqB,OAAO,QAAQ,IAAI,qBAAqB,KAAK;AACxE,IAAM,sBAAsB,OAAO,QAAQ,IAAI,sBAAsB,KAAK;AAE1E,SAAS,uBAAuB,YAAwC;AACtE,QAAM,WAAW,cAAc;AAC/B,MAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,YAAY,GAAG;AAChD,UAAM,IAAI,WAAW,iDAAiD;AAAA,EACxE;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAoC;AAC9D,SAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ;AAC1F;AAEA,IAAM,mBAAmB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAWrD,IAAM,2BAA8C;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcO,SAAS,oBAAoB,KAAuB;AACzD,SAAO,kBAAkB,KAAK,CAAC;AACjC;AAEA,SAAS,kBAAkB,KAAc,OAAwB;AAC/D,MAAI,eAAe,aAAc,QAAO,iBAAiB,IAAI,IAAI,MAAM;AACvE,MAAI,EAAE,eAAe,OAAQ,QAAO;AAGpC,QAAM,SAAU,IAA6B;AAC7C,MAAI,OAAO,WAAW,YAAY,iBAAiB,IAAI,MAAM,EAAG,QAAO;AACvE,QAAM,OAAQ,IAA2B;AACzC,QAAM,WAAW,GAAG,IAAI,IAAI;AAAA,EAAK,IAAI,OAAO;AAAA,EAAK,OAAO,SAAS,WAAW,OAAO,EAAE;AACrF,MAAI,yBAAyB,KAAK,CAAC,MAAM,EAAE,KAAK,QAAQ,CAAC,EAAG,QAAO;AACnE,QAAM,QAAS,IAA4B;AAC3C,MAAI,QAAQ,KAAK,iBAAiB,SAAS,UAAU,KAAK;AACxD,WAAO,kBAAkB,OAAO,QAAQ,CAAC;AAAA,EAC3C;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;AACxD,QAAM,IAAI,QAAQ,IAAI,aAAa;AACnC,MAAI,CAAC,EAAG,QAAO;AACf,QAAM,WAAW,OAAO,CAAC;AACzB,MAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,EAAG,QAAO,WAAW;AACjE,QAAM,SAAS,KAAK,MAAM,CAAC;AAC3B,MAAI,OAAO,SAAS,MAAM,EAAG,QAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;AACnE,SAAO;AACT;AAGO,SAAS,UAAU,SAAyB;AACjD,SAAO,KAAK,IAAI,MAAM,KAAK,SAAS,IAAM;AAC5C;AAEA,SAAS,aAAa,MAAgD;AACpE,QAAM,UAAkC;AAAA,IACtC,gBAAgB;AAAA,IAChB,QAAQ;AAAA,EACV;AACA,MAAI,KAAK,YAAY;AACnB,YAAQ,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW;AAAA,EAClD,WAAW,KAAK,UAAU,KAAK,QAAQ;AACrC,YAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK,MAAM;AAAA,EAC9D;AACA,MAAI,KAAK,eAAgB,SAAQ,iBAAiB,IAAI,KAAK;AAC3D,SAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;AAChE,MAAI,WAAW,IAAK,QAAO;AAC3B,QAAM,QAAQ,KAAK,YAAY;AAC/B,SACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,0BAA0B,QAAgB,MAAuB;AACxE,MAAI,WAAW,OAAO,CAAC,eAAe,KAAK,IAAI,EAAG,QAAO;AACzD,SACE,iGAAiG;AAAA,IAC/F;AAAA,EACF,KAAK,oEAAoE,KAAK,IAAI;AAEtF;AAEA,SAAS,UACP,KACA,iBACA,iBACyB;AACzB,QAAM,OAAgC;AAAA,IACpC,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,aAAa,IAAI,eAAe;AAAA,EAClC;AACA,MAAI,IAAI,aAAa,MAAM;AACzB,QAAI,wBAAwB,IAAI,KAAK,EAAG,MAAK,wBAAwB,IAAI;AAAA,QACpE,MAAK,aAAa,IAAI;AAAA,EAC7B;AACA,QAAM,WAAW,IAAI,YAAY;AACjC,MAAI,aAAa,QAAW;AAC1B,SAAK,WAAW,EAAE,MAAM,SAAS;AAAA,EACnC;AAEA,MAAI,IAAI,cAAc,CAAC,iBAAiB;AACtC,SAAK,kBAAkB;AAAA,MACrB,MAAM;AAAA,MACN,aAAa,EAAE,MAAM,IAAI,WAAW,MAAM,QAAQ,IAAI,WAAW,QAAQ,QAAQ,KAAK;AAAA,IACxF;AAAA,EACF,WAAW,IAAI,YAAY,IAAI,YAAY;AACzC,SAAK,kBAAkB,EAAE,MAAM,cAAc;AAAA,EAC/C;AAEA,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;AACvD,SAAO,oBAAoB,KAAK,KAAK;AACvC;AAEA,eAAe,MAAM,IAA2B;AAC9C,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AASA,SAAS,YAAY,mBAAoC,QAAmC;AAC1F,MAAI,CAAC,OAAQ,QAAO,kBAAkB;AACtC,MAAI,OAAQ,YAAkC,QAAQ,YAAY;AAChE,WAAO,YAAY,IAAI,CAAC,kBAAkB,QAAQ,MAAM,CAAC;AAAA,EAC3D;AACA,MAAI,OAAO,SAAS;AAClB,sBAAkB,MAAM;AAAA,EAC1B,OAAO;AACL,WAAO,iBAAiB,SAAS,MAAM,kBAAkB,MAAM,GAAG,EAAE,MAAM,KAAK,CAAC;AAAA,EAClF;AACA,SAAO,kBAAkB;AAC3B;AAGA,SAAS,iBAAiB,OAAe,YAAyC;AAChF,SAAO,cAAc,QAAQ,KAAK,IAAI,IAAI,SAAS;AACrD;AASO,SAAS,gBAAgB,KAAqB;AACnD,QAAM,UAAU,IAAI,KAAK;AACzB,QAAM,IAAI,QAAQ,MAAM,yCAAyC;AACjE,SAAO,IAAI,EAAE,CAAC,EAAG,KAAK,IAAI;AAC5B;AAEO,SAAS,mBAAmB,KAAqB;AACtD,QAAM,WAAW,gBAAgB,GAAG;AACpC,MAAI;AACF,SAAK,MAAM,QAAQ;AACnB,WAAO;AAAA,EACT,QAAQ;AAIN,QAAI,SAAS,WAAW,GAAG,KAAK,SAAS,WAAW,GAAG,EAAG,QAAO;AAAA,EACnE;AAGA,QAAM,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,CAAC,EAC1C,IAAI,CAAC,UAAU,MAAM,KAAK,EAC1B,OAAO,CAAC,UAAU,SAAS,IAAI;AAClC,aAAW,SAAS,QAAQ;AAC1B,UAAM,YAAY,oBAAoB,UAAU,KAAK;AACrD,QAAI,CAAC,UAAW;AAChB,QAAI;AACF,WAAK,MAAM,SAAS;AACpB,aAAO;AAAA,IACT,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;AACxE,QAAM,SAAS,MAAM,KAAK;AAC1B,QAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;AAC7D,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAkB,CAAC,MAAM;AAC/B,MAAI,aAAa;AACjB,MAAI,YAAY;AAEhB,WAAS,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;AAC7C,UAAM,OAAO,MAAM,CAAC;AACpB,QAAI,WAAW;AACb,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,MAAM;AACjB,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,KAAK;AAChB,mBAAa,CAAC;AACd;AAAA,IACF;AACA,QAAI,WAAY;AAEhB,QAAI,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aACvB,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aAC5B,SAAS,MAAM,MAAM,SAAS,CAAC,GAAG;AACzC,YAAM,IAAI;AACV,UAAI,MAAM,WAAW,EAAG,QAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO;AACT;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;AACxB,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AACrE,QAAM,MAAM,GAAG,OAAO;AACtB,QAAM,WAAW;AACjB,QAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;AAC5D,QAAM,kBAAkB,uBAAuB,KAAK,UAAU;AAC9D,QAAM,UAAU,KAAK,SAAS,WAAW;AACzC,QAAM,UAAU,aAAa,IAAI;AACjC,QAAM,WAAW,KAAK,YAAY,oBAAoB,OAAO;AAC7D,QAAM,OAAO,KAAK;AAClB,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,eAAe,KAAK;AAC1B,QAAM,eAAe,KAAK;AAC1B,QAAM,aAAa,KAAK;AACxB,QAAM,gBAAgB,KAAK,IAAI;AAC/B,MAAI,KAAK,oBAAoB;AAC3B,wBAAoB,KAAK,oBAAoB,EAAE,aAAa,GAAG,cAAc,EAAE,CAAC;AAAA,EAClF;AAEA,MAAI;AACJ,MAAI,mBAAmB;AACvB,WAAS,UAAU,GAAG,UAAU,iBAAiB,WAAW;AAG1D,QAAI,cAAc,SAAS;AACzB,YAAM,IAAI,aAAa,oCAAoC,YAAY;AAAA,IACzE;AAGA,QAAI,UAAU,KAAK,iBAAiB,eAAe,UAAU,GAAG;AAC9D,YAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AAAA,IACtE;AACA,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,gBAAgB,YAAY,YAAY,YAAY;AAC1D,UAAM,gBAAgB,WAAW,MAAM,WAAW,MAAM,GAAG,SAAS;AACpE,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,cAAc;AAAA,MAClB;AAAA,MACA,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IACP;AACA,QAAI,uBAAuB;AAC3B,QAAI,MAAM;AACR,YAAM,UAAU,MAAM,UAAU;AAAA,QAC9B,SAAS,cAAc;AAAA,QACvB,OAAO,cAAc;AAAA,QACrB,QAAQ,cAAc;AAAA,QACtB;AAAA,QACA,OAAO,IAAI;AAAA,QACX;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,WAAW;AAAA,QACX,WAAW;AAAA,QACX,gBAAgB;AAAA,QAChB;AAAA,QACA,gBAAgB,CAAC;AAAA,MACnB,CAAC;AAAA,IACH;AAEA,QAAI;AACF,YAAM,MAAM,MAAM,QAAQ,KAAK;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU,WAAW;AAAA,QAChC,QAAQ;AAAA,MACV,CAAC;AACD,mBAAa,aAAa;AAC1B,YAAM,kBAAkB,OAAO,gBAAgB,IAAI,OAAO,IAAI;AAE9D,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,YAAI,MAAM;AACR,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,YAAY,IAAI;AAAA,YAChB;AAAA,YACA,cAAc;AAAA,YACd,cAAc,QAAQ,IAAI,MAAM;AAAA,YAChC,gBAAgB,CAAC;AAAA,UACnB,CAAC;AACD,iCAAuB;AAAA,QACzB;AACA,cAAM,MAAM,IAAI;AAAA,UACd,YAAY,IAAI,MAAM,KAAK,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA,UAC7C,IAAI;AAAA,UACJ;AAAA,UACA,IAAI;AAAA,QACN;AACA,YACE,0BAA0B,IAAI,QAAQ,IAAI,KAC1C,iBAAiB,gBAAgB,KACjC,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;AACA,oBAAU;AACV,6BAAmB,EAAE,GAAG,kBAAkB,aAAa,EAAE;AACzD;AAAA,QACF;AACA,YACE,iBAAiB,IAAI,IAAI,MAAM,KAC/B,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;AACA,oBAAU;AACV,gBAAM,aAAa,gBAAgB,IAAI,OAAO;AAC9C,gBAAM,MAAM,cAAc,UAAU,OAAO,CAAC;AAC5C;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAEA,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI;AACJ,UAAI;AACF,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB,SAAS,UAAU;AACjB,YAAI,MAAM;AACR,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,YAAY,IAAI;AAAA,YAChB;AAAA,YACA,cAAc;AAAA,YACd,cAAc,sBAAsB,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ,CAAC;AAAA,YACnG,gBAAgB,CAAC;AAAA,UACnB,CAAC;AACD,iCAAuB;AAAA,QACzB;AACA,cAAM;AAAA,MACR;AACA,UAAI,MAAM;AACR,cAAM,UAAU,MAAM,UAAU;AAAA,UAC9B,SAAS,cAAc;AAAA,UACvB,OAAO,cAAc;AAAA,UACrB,QAAQ,cAAc;AAAA,UACtB;AAAA,UACA,OAAO,IAAI;AAAA,UACX;AAAA,UACA;AAAA,UACA,cAAc;AAAA,UACd,WAAW;AAAA,UACX,WAAW,KAAK,IAAI;AAAA,UACpB,YAAY,KAAK,IAAI,IAAI;AAAA,UACzB,YAAY,IAAI;AAAA,UAChB;AAAA,UACA,cAAc;AAAA,UACd,gBAAgB,CAAC;AAAA,QACnB,CAAC;AAAA,MACH;AACA,YAAM,SACJ,KAAK,UAGH,CAAC;AACL,YAAM,WACJ,KAAK,SAAS,OAAO,KAAK,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,KAAK,IACpE,KAAK,QACN;AACN,YAAM,eAAe,mBAAmB,UAAU,aAAa;AAC/D,YAAM,mBAAmB,mBAAmB,UAAU,iBAAiB;AACvE,YAAM,cAAc,mBAAmB,UAAU,YAAY;AAC7D,YAAM,oBACJ,UAAU,6BACV,OAAO,SAAS,8BAA8B,YAC9C,CAAC,MAAM,QAAQ,SAAS,yBAAyB,IAC5C,SAAS,4BACV;AACN,YAAM,eAAe,mBAAmB;AACxC,YAAM,kBACJ,iBAAiB,SAAY,SAAY,mBAAmB,YAAY;AAC1E,YAAM,YACJ,UAAU,yBACV,OAAO,SAAS,0BAA0B,YAC1C,CAAC,MAAM,QAAQ,SAAS,qBAAqB,IACxC,SAAS,sBAAkD,gBAC5D;AACN,YAAM,qBAAqB,cAAc,SAAY,SAAY,mBAAmB,SAAS;AAC7F,YAAM,gBACJ,iBAAiB,UACjB,qBAAqB,WACpB,iBAAiB,UACf,oBAAoB,UAAa,mBAAmB,sBACtD,cAAc,UACZ,uBAAuB,UAAa,sBAAsB,kBAC5D,gBAAgB,UAAa,gBAAgB,eAAe;AAC/D,YAAM,gBAAiB,KAAK,kBAAkB,KAAK;AACnD,YAAM,UAAU,QAAQ,SAAS,WAAW;AAE5C,YAAM,iBACJ,OAAO,kBAAkB,YAAY,iBAAiB,KAAK,qBACvD,oBAAoB,KAAK,oBAAoB;AAAA,QAC3C,aAAa,gBAAiB,sBAAsB;AAAA,QACpD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;AAAA,QACjE,cAAc;AAAA,MAChB,CAAC,IACD;AAEN,aAAO;AAAA,QACL;AAAA,QACA,cAAc,QAAQ,iBAAiB;AAAA,QACvC,cAAc,QAAQ,KAAK,EAAE,WAAW;AAAA,QACxC,OAAO;AAAA,UACL,cAAc,gBAAgB;AAAA,UAC9B,kBAAkB,oBAAoB;AAAA,UACtC,aAAa,gBAAgB,gBAAgB,MAAM,oBAAoB;AAAA,UACvE,UAAU;AAAA,UACV;AAAA,UACA;AAAA,QACF;AAAA,QACA,SAAS,OAAO,kBAAkB,WAAW,gBAAiB,kBAAkB;AAAA,QAChF,OAAQ,KAAK,SAAoB,IAAI;AAAA,QACrC,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB,KAAK;AAAA,MACP;AAAA,IACF,SAAS,KAAK;AACZ,mBAAa,aAAa;AAC1B,gBAAU;AAIV,UAAI,cAAc,SAAS;AACzB,YAAI,QAAQ,CAAC,sBAAsB;AACjC,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,YAC7D,gBAAgB,CAAC;AAAA,UACnB,CAAC;AAAA,QACH;AACA,cAAM;AAAA,MACR;AACA,UAAI,QAAQ,CAAC,sBAAsB;AAIjC,cAAM,UAAU,MAAM,UAAU;AAAA,UAC9B,SAAS,cAAc;AAAA,UACvB,OAAO,cAAc;AAAA,UACrB,QAAQ,cAAc;AAAA,UACtB;AAAA,UACA,OAAO,IAAI;AAAA,UACX;AAAA,UACA;AAAA,UACA,cAAc;AAAA,UACd,WAAW;AAAA,UACX,WAAW,KAAK,IAAI;AAAA,UACpB,YAAY,KAAK,IAAI,IAAI;AAAA,UACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UAC7D,gBAAgB,CAAC;AAAA,QACnB,CAAC;AAAA,MACH;AACA,UACE,UAAU,kBAAkB,KAC5B,oBAAoB,GAAG,KACvB,CAAC,iBAAiB,eAAe,UAAU,GAC3C;AACA,cAAM,MAAM,UAAU,OAAO,CAAC;AAC9B;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AACA,QAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAEA,eAAe,UACb,MACA,UACA,OACe;AAGf,MAAI;AACF,UAAM,KAAK,OAAO,SAAS,KAAK,CAAC;AAAA,EACnC,QAAQ;AAAA,EAER;AACF;AAEA,SAAS,gBAAgB,GAAoC;AAC3D,QAAM,MAA8B,CAAC;AACrC,IAAE,QAAQ,CAAC,OAAO,QAAQ;AACxB,QAAI,GAAG,IAAI;AAAA,EACb,CAAC;AACD,SAAO;AACT;AAEA,SAAS,gBAAwB;AAC/B,MAAI,OAAO,WAAW,QAAQ,eAAe,WAAY,QAAO,WAAW,OAAO,WAAW;AAC7F,SAAO,GAAG,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9E;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;AAC9C,QAAM,SAAS,MAAM,kBAAkB,KAAK,IAAI;AAChD,QAAM,QAAQ,gBAAmB,QAAQ,KAAK,mBAAmB,SAAS;AAC1E,SAAO,EAAE,OAAO,OAAO;AACzB;AAGA,eAAe,kBACb,KACA,OAAyB,CAAC,GACF;AACxB,MAAI;AACF,WAAO,MAAM,QAAQ,EAAE,GAAG,KAAK,UAAU,IAAI,YAAY,CAAC,IAAI,WAAW,GAAG,IAAI;AAAA,EAClF,SAAS,KAAK;AACZ,QACE,KAAK,wBAAwB,iBAC7B,eAAe,gBACf,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KACtC,IAAI,YACJ;AACA,YAAM,cAA8B,EAAE,GAAG,KAAK,UAAU,MAAM,YAAY,OAAU;AACpF,aAAO,MAAM,QAAQ,aAAa,IAAI;AAAA,IACxC;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,gBACP,QACA,iBACG;AACH,MAAI;AACF,QAAI,OAAO,iBAAiB,UAAU;AACpC,YAAM,IAAI;AAAA,QACR,8CAA8C,OAAO,KAAK;AAAA,MAC5D;AAAA,IACF;AACA,WAAO,gBAAmB,OAAO,SAAS,OAAO,OAAO,eAAe;AAAA,EACzE,SAAS,OAAO;AACd,QAAI,iBAAiB,iBAAkB,OAAM;AAC7C,UAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACtE,UAAM,IAAI,iBAAiB,MAAM,SAAS,QAAQ,EAAE,MAAM,CAAC;AAAA,EAC7D;AACF;AAEA,SAAS,gBACP,SACA,OACA,iBACG;AACH,QAAM,UAAU,oBAAoB,UAAU,UAAU,mBAAmB,OAAO;AAClF,MAAI;AACF,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,wCAAwC,KAAK,MAC3C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CACjD;AAAA;AAAA,EAA0B,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA,IACjD;AAAA,EACF;AACF;AAWO,IAAM,yBAAN,cAAqC,sBAAsB;AAAA,EAChE,YACE,SACgB,QACA,SAChB;AACA,UAAM,OAAO;AAHG;AACA;AAAA,EAGlB;AAAA,EAJkB;AAAA,EACA;AAIpB;AAmCO,SAAS,eAAe,MAAwB,MAA4B,CAAC,GAAS;AAC3F,QAAM,kBAAkB,KAAK,YAAY;AACzC,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AAErE,MAAI,IAAI,0BAA0B,CAAC,iBAAiB;AAClD,UAAM,IAAI;AAAA,MACR,gGAAgG,gBAAgB;AAAA,MAChH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,iBAAiB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,CAAC,GAAG;AAC1D,UAAM,IAAI;AAAA,MACR,2BAA2B,OAAO;AAAA,MAClC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,GAAG;AACzD,UAAM,KAAK,IAAI,gBAAgB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,CAAC;AAC/D,QAAI,CAAC,IAAI;AACP,YAAM,IAAI;AAAA,QACR,2BAA2B,OAAO,gCAAgC,IAAI,gBAAgB,IAAI,eAAe,EAAE,KAAK,IAAI,CAAC;AAAA,QACrH;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,eAAe,CAAC,KAAK,UAAU,CAAC,KAAK,UAAU,CAAC,KAAK,YAAY;AACvE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,kBAAkB;AACxB,UAAM,SAAS,KAAK,YAAY,oBAAoB,OAAO;AAC3D,QAAI,WAAW,IAAI,kBAAkB;AACnC,YAAM,IAAI;AAAA,QACR,qCAAqC,IAAI,gBAAgB,gBAAgB,OAAO,gBAAgB,MAAM;AAAA,QACtG;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,SAAS,KAAa,SAAmC;AAChE,MAAI,mBAAmB,OAAQ,QAAO,QAAQ,KAAK,GAAG;AACtD,SAAO,IAAI,YAAY,EAAE,WAAW,QAAQ,YAAY,CAAC;AAC3D;AAEA,SAAS,gBAAgB,GAA4B;AACnD,SAAO,aAAa,SAAS,EAAE,SAAS;AAC1C;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;AACnE,QAAM,QAAQ,KAAK,IAAI;AACvB,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,QACE;AAAA,QACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,QAC5C,WAAW;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,WAAW,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK;AAAA,EAChE,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,WAAW,KAAK,IAAI,IAAI;AAAA,MACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;AAOO,IAAM,YAAN,MAAgB;AAAA,EACZ;AAAA,EACQ;AAAA,EAEjB,YAAY,OAAyB,CAAC,GAAG;AACvC,SAAK,OAAO;AACZ,SAAK,kBAAkB,uBAAuB,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,KAAK,KAAqB,KAAgD;AACxE,UAAM,UAAU,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI;AACvC,WAAO,IAAI,aAAa,kBAAkB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO;AAAA,EAChF;AAAA,EAEA,SACE,KACA,KAC8C;AAC9C,WAAO,YAAe,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EACrD;AACF;","names":[]}
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  pearsonR,
3
3
  spearmanR
4
- } from "./chunk-PJQFMIOX.js";
4
+ } from "./chunk-MHELPNRP.js";
5
5
 
6
6
  // src/meta-eval/rubric-predictive-validity.ts
7
7
  async function rubricPredictiveValidity(input) {
@@ -133,4 +133,4 @@ function makeRng(seed) {
133
133
  export {
134
134
  rubricPredictiveValidity
135
135
  };
136
- //# sourceMappingURL=chunk-X4UCIOTZ.js.map
136
+ //# sourceMappingURL=chunk-S5YLIBFX.js.map
@@ -660,7 +660,6 @@ function fromCodeAgentSession(source, options) {
660
660
  sourcePath: options.sourcePath,
661
661
  entries: 0,
662
662
  malformedLines: options.malformedLines ?? 0,
663
- inferredScore: true,
664
663
  hasExplicitTerminalSignal: false,
665
664
  hasFinalOutput: false,
666
665
  hasQualityLabel: false,
@@ -692,7 +691,8 @@ function fromCodeAgentSession(source, options) {
692
691
  ...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
693
692
  };
694
693
  const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
695
- const score = clamp01(options.score ?? metrics.processScore);
694
+ const qualityScore = finiteScore(options.score);
695
+ const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
696
696
  const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
697
697
  const configHash = options.configHash ?? hashJson({
698
698
  source,
@@ -706,9 +706,13 @@ function fromCodeAgentSession(source, options) {
706
706
  model,
707
707
  explicitTerminal,
708
708
  malformedLines: options.malformedLines ?? 0,
709
- scoreOverridden: options.score !== void 0,
709
+ hasQualityLabel: qualityScore !== void 0,
710
+ invalidQualityScore,
710
711
  costKind: costProvenance.kind
711
712
  });
713
+ const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
714
+ const processExitCode = options.execution?.exitCode;
715
+ const splitTag = options.splitTag ?? "holdout";
712
716
  const run = {
713
717
  runId: `${source}:${sessionId}`,
714
718
  experimentId: options.experimentId ?? `${source}-local-sessions`,
@@ -722,8 +726,12 @@ function fromCodeAgentSession(source, options) {
722
726
  costUsd,
723
727
  costProvenance,
724
728
  tokenUsage,
729
+ terminalOutcome,
730
+ ...terminalOutcome === "failed" ? {
731
+ terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure"
732
+ } : {},
725
733
  outcome: {
726
- holdoutScore: score,
734
+ ...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
727
735
  raw: {
728
736
  entries: metrics.entries,
729
737
  user_messages: metrics.userMessages,
@@ -732,6 +740,10 @@ function fromCodeAgentSession(source, options) {
732
740
  tool_calls: metrics.toolCalls,
733
741
  tool_outputs: metrics.toolOutputs,
734
742
  tool_errors: metrics.toolErrors,
743
+ execution_error_count: metrics.toolErrors,
744
+ process_error_count: terminalOutcome === "failed" ? 1 : 0,
745
+ unclassified_error_count: metrics.unclassifiedErrors,
746
+ ...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
735
747
  patch_attempts: metrics.patchAttempts,
736
748
  patch_successes: metrics.patchSuccesses,
737
749
  patch_failures: metrics.patchFailures,
@@ -761,20 +773,18 @@ function fromCodeAgentSession(source, options) {
761
773
  cache_write_tokens: metrics.cacheWriteTokens,
762
774
  observed_cost_usd: metrics.observedCostUsd,
763
775
  observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
764
- estimated_cost_usd: costProvenance.kind === "estimated" ? costUsd : 0,
765
- process_score: score,
766
- inferred_score: options.score === void 0 ? 1 : 0,
776
+ estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
777
+ process_score: metrics.processScore,
767
778
  explicit_terminal_signal: explicitTerminal ? 1 : 0,
768
- quality_label_present: options.score !== void 0 ? 1 : 0,
779
+ quality_label_present: qualityScore !== void 0 ? 1 : 0,
769
780
  cost_observed: costProvenance.kind === "observed" ? 1 : 0,
770
781
  cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
771
782
  cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
772
783
  cost_unknown: costProvenance.kind === "uncaptured" ? 1 : 0
773
784
  }
774
785
  },
775
- splitTag: options.splitTag ?? "holdout",
776
- ...options.scenarioId ? { scenarioId: options.scenarioId } : {},
777
- ...metrics.toolErrors > 0 || metrics.turnsAborted > 0 ? { failureMode: metrics.turnsAborted > 0 ? "turn_aborted" : "tool_error" } : {}
786
+ splitTag,
787
+ scenarioId: options.scenarioId ?? sessionId
778
788
  };
779
789
  return {
780
790
  runs: [run],
@@ -785,10 +795,9 @@ function fromCodeAgentSession(source, options) {
785
795
  sourcePath: options.sourcePath,
786
796
  entries: entries.length,
787
797
  malformedLines: options.malformedLines ?? 0,
788
- inferredScore: options.score === void 0,
789
798
  hasExplicitTerminalSignal: explicitTerminal,
790
799
  hasFinalOutput: observation.finalText !== null,
791
- hasQualityLabel: options.score !== void 0,
800
+ hasQualityLabel: qualityScore !== void 0,
792
801
  hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
793
802
  hasCost: costProvenance.kind !== "uncaptured",
794
803
  costKind: costProvenance.kind,
@@ -799,6 +808,12 @@ function fromCodeAgentSession(source, options) {
799
808
  observations: [{ ...observation, sessionId }]
800
809
  };
801
810
  }
811
+ function terminalOutcomeFromSession(terminal) {
812
+ if (!terminal.explicit) return "unknown";
813
+ if (terminal.status === "completed") return "succeeded";
814
+ if (terminal.status === "failed") return "failed";
815
+ return "unknown";
816
+ }
802
817
  function metricsFor(source, entries) {
803
818
  switch (source) {
804
819
  case "codex":
@@ -833,7 +848,7 @@ function codexMetrics(entries) {
833
848
  addCodexExecUsage(metrics, record2(entry.usage));
834
849
  }
835
850
  if (entryType === "turn.failed") metrics.turnsAborted += 1;
836
- if (entryType === "error") metrics.toolErrors += 1;
851
+ if (entryType === "error") metrics.unclassifiedErrors += 1;
837
852
  const item = record2(entry.item);
838
853
  if (item && (entryType === "item.started" || entryType === "item.completed")) {
839
854
  addCodexExecItem(metrics, startedToolIds, entryType, item);
@@ -885,7 +900,7 @@ function codexMetrics(entries) {
885
900
  if (payloadType === "token_count")
886
901
  setCodexCumulativeUsage(metrics, record2(record2(payload.info)?.total_token_usage));
887
902
  const result = record2(payload.result);
888
- if (result && "Err" in result) metrics.toolErrors += 1;
903
+ if (result && "Err" in result) metrics.unclassifiedErrors += 1;
889
904
  }
890
905
  }
891
906
  if (metrics.wallMs === 0 && startedAt !== void 0 && completedAt !== void 0) {
@@ -918,7 +933,8 @@ function addCodexExecItem(metrics, startedToolIds, eventType, item) {
918
933
  metrics.toolErrors += 1;
919
934
  }
920
935
  }
921
- if (itemType === "error" || codexExecToolFailed(itemType, item)) metrics.toolErrors += 1;
936
+ if (itemType === "error") metrics.unclassifiedErrors += 1;
937
+ else if (codexExecToolFailed(itemType, item)) metrics.toolErrors += 1;
922
938
  }
923
939
  function codexExecToolFailed(itemType, item) {
924
940
  const status = stringField2(item, "status");
@@ -1148,6 +1164,7 @@ function emptyMetrics(entries) {
1148
1164
  toolCalls: 0,
1149
1165
  toolOutputs: 0,
1150
1166
  toolErrors: 0,
1167
+ unclassifiedErrors: 0,
1151
1168
  patchAttempts: 0,
1152
1169
  patchSuccesses: 0,
1153
1170
  patchFailures: 0,
@@ -1236,7 +1253,7 @@ function resolveSessionCost(explicit, metrics, model) {
1236
1253
  if (explicit) {
1237
1254
  if (explicit.kind === "uncaptured") {
1238
1255
  if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
1239
- return { costUsd: 0, costProvenance: explicit };
1256
+ return { costUsd: null, costProvenance: explicit };
1240
1257
  }
1241
1258
  if (!Number.isFinite(explicit.usd) || explicit.usd < 0) {
1242
1259
  throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
@@ -1252,11 +1269,12 @@ function resolveSessionCost(explicit, metrics, model) {
1252
1269
  const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
1253
1270
  return { costUsd, costProvenance: { kind: "estimated", usd: costUsd } };
1254
1271
  }
1255
- return { costUsd: 0, costProvenance: { kind: "uncaptured", usd: null } };
1272
+ return { costUsd: null, costProvenance: { kind: "uncaptured", usd: null } };
1256
1273
  }
1257
1274
  function diagnosticsFor(metrics, options) {
1258
1275
  const warnings = [];
1259
- if (!options.scoreOverridden) warnings.push("outcome score is inferred from process telemetry");
1276
+ if (options.invalidQualityScore) warnings.push("non-finite task quality score omitted");
1277
+ else if (!options.hasQualityLabel) warnings.push("task quality score not supplied");
1260
1278
  if (!options.explicitTerminal) warnings.push("no explicit terminal success/failure signal");
1261
1279
  if (metrics.inputTokens === 0 && metrics.outputTokens === 0 && metrics.reasoningTokens === 0 && metrics.cachedTokens === 0 && metrics.cacheWriteTokens === 0) {
1262
1280
  warnings.push("missing token usage");
@@ -1387,6 +1405,9 @@ function clamp01(value) {
1387
1405
  if (!Number.isFinite(value)) return 0;
1388
1406
  return Math.max(0, Math.min(1, value));
1389
1407
  }
1408
+ function finiteScore(value) {
1409
+ return value !== void 0 && Number.isFinite(value) ? clamp01(value) : void 0;
1410
+ }
1390
1411
  function isRecord2(value) {
1391
1412
  return value !== null && typeof value === "object" && !Array.isArray(value);
1392
1413
  }
@@ -1422,4 +1443,4 @@ export {
1422
1443
  fromPiSession,
1423
1444
  fromPigraphSession
1424
1445
  };
1425
- //# sourceMappingURL=chunk-7VYEGHWF.js.map
1446
+ //# sourceMappingURL=chunk-SZLVEKMJ.js.map