@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/chunk-NJC7U437.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n *     directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n *   const { value, result } = await callLlmJson<MyType>(\n *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n *   )\n *\n * This is THE llm-calling seam for agent-eval primitives that need structured\n * output (semantic concept judge, reviewer directives, critic scores). Primitives\n * that need free-form text use `callLlm` and parse output themselves.\n */\n\nimport type { CostReceiptInput, MaximumCharge } from './cost-ledger'\nimport { AgentEvalError, CaptureIntegrityError } from './errors'\nimport {\n  defaultProviderRedactor,\n  type ProviderRedactor,\n  providerFromBaseUrl,\n  type RawProviderEvent,\n  type RawProviderSink,\n} from './trace/raw-provider-sink'\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n  role: 'system' | 'user' | 'assistant'\n  /**\n   * Either a plain text content string OR a multimodal content array\n   * (text + image_url parts) for vision-capable models.\n   */\n  content:\n    | string\n    | Array<\n        | { type: 'text'; text: string }\n        | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n      >\n}\n\nexport interface LlmCallRequest {\n  model: string\n  messages: LlmMessage[]\n  /** Optional JSON-mode response format (response_format: json_object). */\n  jsonMode?: boolean\n  /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n  jsonSchema?: { name: string; schema: Record<string, unknown> }\n  temperature?: number\n  maxTokens?: number\n  /** Per-call timeout, default 300s. */\n  timeoutMs?: number\n}\n\n/** Conservative priced bound for the exact text request sent to a provider.\n * Returns undefined when output or multimodal input is not bounded, causing a\n * capped CostLedger to reject the call before execution. */\nexport function maximumChargeForLlmRequest(\n  request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'maxTokens'>,\n  options: LlmClientOptions = {},\n): MaximumCharge | undefined {\n  if (request.maxTokens === undefined) return undefined\n  if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {\n    throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`)\n  }\n  if (\n    request.messages.some(\n      (message) =>\n        Array.isArray(message.content) && message.content.some((part) => part.type === 'image_url'),\n    )\n  ) {\n    return undefined\n  }\n\n  const attempts = resolveMaximumAttempts(options.maxRetries)\n  // A byte-level tokenizer cannot emit more input tokens than request bytes.\n  // Pricing the complete body also covers role/schema framing omitted from content-only estimates.\n  const requestBytes = new TextEncoder().encode(\n    JSON.stringify(buildBody(request, false)),\n  ).byteLength\n  // A rejected response schema can trigger one JSON-mode batch with the same output limit.\n  const batches = request.jsonSchema ? 2 : 1\n  return {\n    model: request.model,\n    inputTokens: requestBytes * attempts * batches,\n    outputTokens: request.maxTokens * attempts * batches,\n  }\n}\n\nexport interface LlmUsage {\n  promptTokens: number\n  completionTokens: number\n  totalTokens: number\n  /** False when the provider omitted or malformed prompt/completion usage. */\n  captured?: boolean\n  /** Proxies populate this when prompt caching is on. */\n  cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n  /** The text content of the first choice. Empty string if none. */\n  content: string\n  usage: LlmUsage\n  /**\n   * Cost in USD. Pulled from proxy's `_response_cost` field when present;\n   * `null` when neither the proxy nor the caller can derive it.\n   */\n  costUsd: number | null\n  /** Model name actually used (echoed from response). */\n  model: string\n  /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n  durationMs: number\n  /**\n   * `finish_reason` echoed from the first choice (`stop`, `length`,\n   * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.\n   * Exposed so a free-form `callLlm` caller CAN detect a truncated answer\n   * (`length`) instead of treating a cut-off completion as complete. Note:\n   * `callLlm` does not itself reject on it — acting on this signal is the\n   * caller's responsibility (in-repo free-form drivers do not yet enforce it).\n   */\n  finishReason?: string | null\n  /**\n   * True when `content.trim()` is empty. An empty completion is a silent zero\n   * for free-form `callLlm` callers; this flag is the signal a caller can\n   * inspect to fail loud rather than proceed on an empty string. `callLlm`\n   * surfaces it but does not throw on it.\n   */\n  contentEmpty?: boolean\n  /** Raw response body. */\n  raw: Record<string, unknown>\n}\n\nexport type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>\n\n/** Convert a provider result into the canonical paid-call receipt input. */\nexport function costReceiptFromLlm(result: LlmCallResult): CostReceiptInput {\n  const cachedTokens = result.usage.cachedPromptTokens ?? 0\n  return {\n    model: result.model,\n    inputTokens: Math.max(0, result.usage.promptTokens - cachedTokens),\n    outputTokens: result.usage.completionTokens,\n    cachedTokens: cachedTokens > 0 ? cachedTokens : undefined,\n    actualCostUsd: result.costUsd ?? undefined,\n    usageUnknown: result.usage.captured === false,\n  }\n}\n\n/** Structured-response failures retain their completed provider receipt. */\nexport function costReceiptFromLlmError(error: Error): CostReceiptInput | undefined {\n  return error instanceof LlmResponseError ? costReceiptFromLlm(error.result) : undefined\n}\n\nexport class LlmCallError extends AgentEvalError {\n  constructor(\n    message: string,\n    public readonly status: number,\n    public readonly body: string,\n    public readonly model: string,\n  ) {\n    super('judge', message)\n  }\n}\n\n/** A provider response completed and incurred measurable usage, but its content\n *  could not satisfy the caller's response contract. The response envelope is\n *  retained so accounting can commit the receipt before the error propagates. */\nexport class LlmResponseError extends AgentEvalError {\n  constructor(\n    message: string,\n    public readonly result: LlmCallResult,\n    options?: { cause?: unknown },\n  ) {\n    super('judge', message, options)\n  }\n}\n\nexport interface LlmClientOptions {\n  /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n  baseUrl?: string\n  /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n  apiKey?: string\n  bearer?: string\n  /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n  authHeader?: { name: string; value: string }\n  /** Stable provider idempotency key, reused across retries of this logical call. */\n  idempotencyKey?: string\n  /** Default timeout in ms. Per-call can override. */\n  defaultTimeoutMs?: number\n  /**\n   * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to\n   * each attempt's per-attempt timeout controller, so aborting it cancels\n   * the in-flight fetch. A caller abort is FATAL: it is not retried even\n   * though an AbortError otherwise matches the transient patterns.\n   */\n  signal?: AbortSignal\n  /**\n   * Cross-attempt wall-clock budget in ms, measured from the first attempt.\n   * Before launching each attempt the loop checks the remaining budget and\n   * stops retrying once it is exhausted, rather than waiting the full\n   * per-attempt timeout on every retry. Bounds total time independent of\n   * total attempts × `timeoutMs`.\n   */\n  deadlineMs?: number\n  /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */\n  maxRetries?: number\n  /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n  fetch?: typeof fetch\n  /**\n   * Optional raw HTTP capture sink. When provided, every request, response,\n   * and error (across all retry attempts) is recorded to the sink, with auth\n   * headers and credential-shaped body fields redacted by default. This is\n   * the layer-1 forensics primitive: structured `LlmSpan`s record intent,\n   * raw events record what actually crossed the wire.\n   */\n  rawSink?: RawProviderSink\n  /**\n   * Logical provider id attached to raw events. When omitted, derived from\n   * `baseUrl` via `providerFromBaseUrl`.\n   */\n  provider?: string\n  /** Trace context attached to raw events; populated by emitter-aware callers. */\n  traceContext?: { runId?: string; spanId?: string }\n  /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */\n  redactor?: ProviderRedactor\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\n// Flagship / reasoning models routinely take several minutes on large prompts (a\n// reflection over many failures, a long tool transcript). A tight cap aborts a\n// legitimately-slow but healthy call — and because every retry attempt re-uses\n// the same window, such a model aborts on ALL attempts and the loop throws. The\n// default is generous enough to let those complete, bounded enough that a truly\n// hung call still fails over after retries, and tunable per deployment via\n// TANGLE_LLM_TIMEOUT_MS. Per-call `req.timeoutMs` / `opts.defaultTimeoutMs`\n// still win for callers that know their model's latency.\nconst DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 300_000\nconst DEFAULT_MAX_RETRIES = Number(process.env.TANGLE_LLM_MAX_RETRIES) || 3\n\nfunction resolveMaximumAttempts(configured: number | undefined): number {\n  const attempts = configured ?? DEFAULT_MAX_RETRIES\n  if (!Number.isInteger(attempts) || attempts <= 0) {\n    throw new RangeError('LLM maximum attempts must be a positive integer')\n  }\n  return attempts\n}\n\nfunction providerTokenCount(value: unknown): number | undefined {\n  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined\n}\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\n/**\n * Transient transport/network error signatures, matched against an error's\n * name, message, and `code`. Covers fetch/undici network failures, aborts\n * and timeouts, and — critically — HTTP/2 transport faults a keep-alive\n * connection raises mid-response: `terminated`, `NGHTTP2_INTERNAL_ERROR`,\n * `UND_ERR_*`, `other side closed`. Those last ones carry no clean HTTP\n * status; unrecognised, they escape the retry loop and surface as an\n * uncaught rejection.\n */\nconst TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [\n  /AbortError/i,\n  /TimeoutError/i,\n  /this operation was aborted/i,\n  /fetch failed/i,\n  /ECONNRESET/i,\n  /ETIMEDOUT/i,\n  /EAI_AGAIN/i,\n  /socket hang up/i,\n  /stream.*ended.*unexpectedly/i,\n  /terminated/i,\n  /other side closed/i,\n  /NGHTTP2/i,\n  /UND_ERR/i,\n]\n\n/**\n * True when an error is a transient transport/network fault worth retrying,\n * as opposed to a deterministic failure (4xx schema reject, JSON parse) that\n * a retry cannot fix. Inspects `LlmCallError.status`, then the error's\n * name/message/code, then recurses into `error.cause` — undici nests the\n * real socket fault one or more levels under `.cause`.\n *\n * This is THE retry classifier for the package: `callLlm` and\n * `withJudgeRetry` both route through it, so a connection-class error is\n * treated identically whether it surfaces in the HTTP client or a\n * TCloud-backed judge.\n */\nexport function isTransientLlmError(err: unknown): boolean {\n  return classifyTransient(err, 0)\n}\n\nfunction classifyTransient(err: unknown, depth: number): boolean {\n  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n  if (!(err instanceof Error)) return false\n  // Foreign errors (e.g. a TCloud judge SDK error) can carry a numeric HTTP\n  // status without being an LlmCallError — a retryable status is decisive.\n  const status = (err as { status?: unknown }).status\n  if (typeof status === 'number' && RETRYABLE_STATUS.has(status)) return true\n  const code = (err as { code?: unknown }).code\n  const haystack = `${err.name}\\n${err.message}\\n${typeof code === 'string' ? code : ''}`\n  if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true\n  const cause = (err as { cause?: unknown }).cause\n  if (depth < 4 && cause instanceof Error && cause !== err) {\n    return classifyTransient(cause, depth + 1)\n  }\n  return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n  const h = headers.get('retry-after')\n  if (!h) return null\n  const asNumber = Number(h)\n  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n  const asDate = Date.parse(h)\n  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n  return null\n}\n\n/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */\nexport function backoffMs(attempt: number): number {\n  return Math.min(500 * 2 ** attempt, 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n    Accept: 'application/json',\n  }\n  if (opts.authHeader) {\n    headers[opts.authHeader.name] = opts.authHeader.value\n  } else if (opts.bearer || opts.apiKey) {\n    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n  }\n  if (opts.idempotencyKey) headers['Idempotency-Key'] = opts.idempotencyKey\n  return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n  if (status !== 400) return false\n  const lower = body.toLowerCase()\n  return (\n    lower.includes('response_format') ||\n    lower.includes('json_schema') ||\n    lower.includes('is unavailable') ||\n    lower.includes('not supported')\n  )\n}\n\nfunction buildBody(req: LlmCallRequest, forceJsonObject: boolean): Record<string, unknown> {\n  const body: Record<string, unknown> = {\n    model: req.model,\n    messages: req.messages,\n    temperature: req.temperature ?? 0,\n  }\n  if (req.maxTokens != null) {\n    if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens\n    else body.max_tokens = req.maxTokens\n  }\n\n  if (req.jsonSchema && !forceJsonObject) {\n    body.response_format = {\n      type: 'json_schema',\n      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n    }\n  } else if (req.jsonMode || req.jsonSchema) {\n    body.response_format = { type: 'json_object' }\n  }\n\n  return body\n}\n\nfunction usesMaxCompletionTokens(model: string): boolean {\n  return /^gpt-5(?:[.-]|$)/i.test(model)\n}\n\nasync function sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n/**\n * Combine the per-attempt timeout signal with an optional caller signal into\n * one signal the fetch listens on. Prefers the native `AbortSignal.any`; falls\n * back to manual wiring on runtimes that predate it. The caller signal is also\n * propagated to the timeout controller so aborting it cancels the in-flight\n * fetch immediately.\n */\nfunction linkSignals(timeoutController: AbortController, caller?: AbortSignal): AbortSignal {\n  if (!caller) return timeoutController.signal\n  if (typeof (AbortSignal as { any?: unknown }).any === 'function') {\n    return AbortSignal.any([timeoutController.signal, caller])\n  }\n  if (caller.aborted) {\n    timeoutController.abort()\n  } else {\n    caller.addEventListener('abort', () => timeoutController.abort(), { once: true })\n  }\n  return timeoutController.signal\n}\n\n/** True once the cross-attempt wall-clock budget (if any) is exhausted. */\nfunction deadlineExceeded(start: number, deadlineMs: number | undefined): boolean {\n  return deadlineMs != null && Date.now() - start >= deadlineMs\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n  const trimmed = raw.trim()\n  const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n  return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n  const stripped = stripFencedJson(raw)\n  try {\n    JSON.parse(stripped)\n    return stripped\n  } catch {\n    // A response that declares a JSON root must parse as that complete root.\n    // Scanning onward could turn a truncated object into one of its valid nested\n    // arrays or objects and silently change the response schema.\n    if (stripped.startsWith('{') || stripped.startsWith('[')) return stripped\n  }\n\n  // Only prose-leading responses may contain a recoverable JSON payload.\n  const starts = [...stripped.matchAll(/[[{]/g)]\n    .map((match) => match.index)\n    .filter((index) => index != null)\n  for (const start of starts) {\n    const candidate = extractBalancedJson(stripped, start)\n    if (!candidate) continue\n    try {\n      JSON.parse(candidate)\n      return candidate\n    } catch {\n      // Keep scanning; earlier braces may belong to prose.\n    }\n  }\n\n  return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n  const opener = input[start]\n  const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n  if (!closer) return null\n\n  const stack: string[] = [closer]\n  let isInString = false\n  let isEscaped = false\n\n  for (let i = start + 1; i < input.length; i++) {\n    const char = input[i]!\n    if (isEscaped) {\n      isEscaped = false\n      continue\n    }\n    if (char === '\\\\') {\n      isEscaped = isInString\n      continue\n    }\n    if (char === '\"') {\n      isInString = !isInString\n      continue\n    }\n    if (isInString) continue\n\n    if (char === '{') stack.push('}')\n    else if (char === '[') stack.push(']')\n    else if (char === stack[stack.length - 1]) {\n      stack.pop()\n      if (stack.length === 0) return input.slice(start, i + 1)\n    }\n  }\n\n  return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n  const url = `${baseUrl}/chat/completions`\n  const endpoint = '/chat/completions'\n  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n  const maximumAttempts = resolveMaximumAttempts(opts.maxRetries)\n  const fetchFn = opts.fetch ?? globalThis.fetch\n  const headers = buildHeaders(opts)\n  const provider = opts.provider ?? providerFromBaseUrl(baseUrl)\n  const sink = opts.rawSink\n  const redactor = opts.redactor ?? defaultProviderRedactor\n  const traceContext = opts.traceContext\n  const callerSignal = opts.signal\n  const deadlineMs = opts.deadlineMs\n  const deadlineStart = Date.now()\n\n  let lastErr: unknown\n  for (let attempt = 0; attempt < maximumAttempts; attempt++) {\n    // A caller cancel is fatal — never retried. Checking before each attempt\n    // means an already-aborted signal short-circuits without firing fetch.\n    if (callerSignal?.aborted) {\n      throw new DOMException('callLlm aborted by caller signal', 'AbortError')\n    }\n    // Stop retrying once the cross-attempt budget is spent rather than burning\n    // a full per-attempt timeout on each remaining retry.\n    if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {\n      throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n    }\n    const controller = new AbortController()\n    const attemptSignal = linkSignals(controller, callerSignal)\n    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n    const started = Date.now()\n    const requestBody = buildBody(req, false)\n    let attemptErrorRecorded = false\n    if (sink) {\n      await recordRaw(sink, redactor, {\n        eventId: cryptoEventId(),\n        runId: traceContext?.runId,\n        spanId: traceContext?.spanId,\n        provider,\n        model: req.model,\n        endpoint,\n        baseUrl,\n        attemptIndex: attempt,\n        direction: 'request',\n        timestamp: started,\n        requestHeaders: headers,\n        requestBody,\n        redactedFields: [],\n      })\n    }\n\n    try {\n      const res = await fetchFn(url, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify(requestBody),\n        signal: attemptSignal,\n      })\n      clearTimeout(timeoutHandle)\n      const responseHeaders = sink ? headersToObject(res.headers) : undefined\n\n      if (!res.ok) {\n        const body = await res.text()\n        if (sink) {\n          await recordRaw(sink, redactor, {\n            eventId: cryptoEventId(),\n            runId: traceContext?.runId,\n            spanId: traceContext?.spanId,\n            provider,\n            model: req.model,\n            endpoint,\n            baseUrl,\n            attemptIndex: attempt,\n            direction: 'error',\n            timestamp: Date.now(),\n            durationMs: Date.now() - started,\n            statusCode: res.status,\n            responseHeaders,\n            responseBody: body,\n            errorMessage: `HTTP ${res.status}`,\n            redactedFields: [],\n          })\n          attemptErrorRecorded = true\n        }\n        const err = new LlmCallError(\n          `LLM call ${res.status}: ${body.slice(0, 300)}`,\n          res.status,\n          body,\n          req.model,\n        )\n        if (\n          RETRYABLE_STATUS.has(res.status) &&\n          attempt < maximumAttempts - 1 &&\n          !deadlineExceeded(deadlineStart, deadlineMs)\n        ) {\n          lastErr = err\n          const retryAfter = parseRetryAfter(res.headers)\n          await sleep(retryAfter ?? backoffMs(attempt))\n          continue\n        }\n        throw err\n      }\n\n      const text = await res.text()\n      let json: Record<string, unknown>\n      try {\n        json = JSON.parse(text) as Record<string, unknown>\n      } catch (parseErr) {\n        if (sink) {\n          await recordRaw(sink, redactor, {\n            eventId: cryptoEventId(),\n            runId: traceContext?.runId,\n            spanId: traceContext?.spanId,\n            provider,\n            model: req.model,\n            endpoint,\n            baseUrl,\n            attemptIndex: attempt,\n            direction: 'error',\n            timestamp: Date.now(),\n            durationMs: Date.now() - started,\n            statusCode: res.status,\n            responseHeaders,\n            responseBody: text,\n            errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,\n            redactedFields: [],\n          })\n          attemptErrorRecorded = true\n        }\n        throw parseErr\n      }\n      if (sink) {\n        await recordRaw(sink, redactor, {\n          eventId: cryptoEventId(),\n          runId: traceContext?.runId,\n          spanId: traceContext?.spanId,\n          provider,\n          model: req.model,\n          endpoint,\n          baseUrl,\n          attemptIndex: attempt,\n          direction: 'response',\n          timestamp: Date.now(),\n          durationMs: Date.now() - started,\n          statusCode: res.status,\n          responseHeaders,\n          responseBody: json,\n          redactedFields: [],\n        })\n      }\n      const choice = (\n        json.choices as\n          | Array<{ message?: { content?: string }; finish_reason?: string | null }>\n          | undefined\n      )?.[0]\n      const usageRaw =\n        json.usage && typeof json.usage === 'object' && !Array.isArray(json.usage)\n          ? (json.usage as Record<string, unknown>)\n          : undefined\n      const promptTokens = providerTokenCount(usageRaw?.prompt_tokens)\n      const completionTokens = providerTokenCount(usageRaw?.completion_tokens)\n      const totalTokens = providerTokenCount(usageRaw?.total_tokens)\n      const cachedRaw =\n        usageRaw?.prompt_tokens_details &&\n        typeof usageRaw.prompt_tokens_details === 'object' &&\n        !Array.isArray(usageRaw.prompt_tokens_details)\n          ? (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens\n          : undefined\n      const cachedPromptTokens = cachedRaw === undefined ? undefined : providerTokenCount(cachedRaw)\n      const usageCaptured =\n        promptTokens !== undefined &&\n        completionTokens !== undefined &&\n        (cachedRaw === undefined ||\n          (cachedPromptTokens !== undefined && cachedPromptTokens <= promptTokens)) &&\n        (totalTokens === undefined || totalTokens === promptTokens + completionTokens)\n      const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n      const content = choice?.message?.content ?? ''\n\n      return {\n        content,\n        finishReason: choice?.finish_reason ?? null,\n        contentEmpty: content.trim().length === 0,\n        usage: {\n          promptTokens: promptTokens ?? 0,\n          completionTokens: completionTokens ?? 0,\n          totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),\n          captured: usageCaptured,\n          cachedPromptTokens,\n        },\n        costUsd: typeof costFromProxy === 'number' ? costFromProxy : null,\n        model: (json.model as string) ?? req.model,\n        durationMs: Date.now() - started,\n        raw: json,\n      }\n    } catch (err) {\n      clearTimeout(timeoutHandle)\n      lastErr = err\n      // A caller cancel is fatal even though an AbortError matches the\n      // transient patterns — a cancelled call must surface immediately, not\n      // be retried against the same dead intent.\n      if (callerSignal?.aborted) {\n        if (sink && !attemptErrorRecorded) {\n          await recordRaw(sink, redactor, {\n            eventId: cryptoEventId(),\n            runId: traceContext?.runId,\n            spanId: traceContext?.spanId,\n            provider,\n            model: req.model,\n            endpoint,\n            baseUrl,\n            attemptIndex: attempt,\n            direction: 'error',\n            timestamp: Date.now(),\n            durationMs: Date.now() - started,\n            errorMessage: err instanceof Error ? err.message : String(err),\n            redactedFields: [],\n          })\n        }\n        throw err\n      }\n      if (sink && !attemptErrorRecorded) {\n        // Record only if neither the !res.ok branch nor the JSON.parse catch\n        // already produced an error event for this attempt. Covers network\n        // failures, timeouts, and aborts.\n        await recordRaw(sink, redactor, {\n          eventId: cryptoEventId(),\n          runId: traceContext?.runId,\n          spanId: traceContext?.spanId,\n          provider,\n          model: req.model,\n          endpoint,\n          baseUrl,\n          attemptIndex: attempt,\n          direction: 'error',\n          timestamp: Date.now(),\n          durationMs: Date.now() - started,\n          errorMessage: err instanceof Error ? err.message : String(err),\n          redactedFields: [],\n        })\n      }\n      if (\n        attempt < maximumAttempts - 1 &&\n        isTransientLlmError(err) &&\n        !deadlineExceeded(deadlineStart, deadlineMs)\n      ) {\n        await sleep(backoffMs(attempt))\n        continue\n      }\n      throw err\n    }\n  }\n  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\nasync function recordRaw(\n  sink: RawProviderSink,\n  redactor: ProviderRedactor,\n  event: RawProviderEvent,\n): Promise<void> {\n  // Errors from sinks must not crash the LLM call. Forensic capture is\n  // best-effort; the structured trace is the system of record.\n  try {\n    await sink.record(redactor(event))\n  } catch {\n    // Intentionally swallowed.\n  }\n}\n\nfunction headersToObject(h: Headers): Record<string, string> {\n  const out: Record<string, string> = {}\n  h.forEach((value, key) => {\n    out[key] = value\n  })\n  return out\n}\n\nfunction cryptoEventId(): string {\n  if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n  const result = await callLlmStructured(req, opts)\n  const value = parseJsonResult<T>(result)\n  return { value, result }\n}\n\n/** Shared schema-to-JSON-mode fallback that preserves the raw result. */\nasync function callLlmStructured(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n  try {\n    return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n  } catch (err) {\n    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {\n      const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n      return await callLlm(degradedReq, opts)\n    }\n    throw err\n  }\n}\n\nfunction parseJsonResult<T>(result: LlmCallResult): T {\n  try {\n    if (result.finishReason === 'length') {\n      throw new Error(\n        `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`,\n      )\n    }\n    return parseJsonSafely<T>(result.content, result.model)\n  } catch (error) {\n    if (error instanceof LlmResponseError) throw error\n    const cause = error instanceof Error ? error : new Error(String(error))\n    throw new LlmResponseError(cause.message, result, { cause })\n  }\n}\n\nfunction parseJsonSafely<T>(content: string, model: string): T {\n  const stripped = extractJsonPayload(content)\n  try {\n    return JSON.parse(stripped) as T\n  } catch (err) {\n    throw new Error(\n      `LLM returned non-JSON content (model=${model}): ${\n        err instanceof Error ? err.message : String(err)\n      }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n    )\n  }\n}\n\n// ─── Route assertion ────────────────────────────────────────────────────\n\nexport type LlmRouteAssertionReason =\n  | 'no_explicit_base_url'\n  | 'base_url_blocked'\n  | 'base_url_not_allowed'\n  | 'no_auth'\n  | 'wrong_provider'\n\nexport class LlmRouteAssertionError extends CaptureIntegrityError {\n  constructor(\n    message: string,\n    public readonly reason: LlmRouteAssertionReason,\n    public readonly baseUrl: string,\n  ) {\n    super(message)\n  }\n}\n\nexport interface LlmRouteRequirements {\n  /**\n   * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to\n   * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using\n   * the public/free-tier router is a defect — the launch reviewer needs to\n   * know exactly which provider answered.\n   */\n  requireExplicitBaseUrl?: boolean\n  /**\n   * Allowlist of acceptable base URLs. Strings match by prefix\n   * (case-insensitive); RegExps test against the full base URL.\n   */\n  allowedBaseUrls?: Array<string | RegExp>\n  /** Blocklist that takes precedence over `allowedBaseUrls`. */\n  blockedBaseUrls?: Array<string | RegExp>\n  /** Throw if no auth header / api key is configured. */\n  requireAuth?: boolean\n  /**\n   * Logical provider id the configured `baseUrl` is expected to match (via\n   * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.\n   */\n  expectedProvider?: string\n}\n\n/**\n * Fail-loud assertion that the configured LLM client points at the route\n * the caller intends. Designed for the matrix-runner preflight: invoke\n * once before any LLM call to catch misconfiguration before a sweep burns\n * dollars on the wrong provider.\n *\n * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call\n * from constructors and CI gates.\n */\nexport function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {\n  const baseUrlExplicit = opts.baseUrl !== undefined\n  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n  if (req.requireExplicitBaseUrl && !baseUrlExplicit) {\n    throw new LlmRouteAssertionError(\n      `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,\n      'no_explicit_base_url',\n      baseUrl,\n    )\n  }\n\n  if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {\n    throw new LlmRouteAssertionError(\n      `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,\n      'base_url_blocked',\n      baseUrl,\n    )\n  }\n\n  if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {\n    const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))\n    if (!ok) {\n      throw new LlmRouteAssertionError(\n        `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,\n        'base_url_not_allowed',\n        baseUrl,\n      )\n    }\n  }\n\n  if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {\n    throw new LlmRouteAssertionError(\n      `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,\n      'no_auth',\n      baseUrl,\n    )\n  }\n\n  if (req.expectedProvider) {\n    const actual = opts.provider ?? providerFromBaseUrl(baseUrl)\n    if (actual !== req.expectedProvider) {\n      throw new LlmRouteAssertionError(\n        `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,\n        'wrong_provider',\n        baseUrl,\n      )\n    }\n  }\n}\n\nfunction matchUrl(url: string, pattern: string | RegExp): boolean {\n  if (pattern instanceof RegExp) return pattern.test(url)\n  return url.toLowerCase().startsWith(pattern.toLowerCase())\n}\n\nfunction describePattern(p: string | RegExp): string {\n  return p instanceof RegExp ? p.source : p\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n  model: string,\n  opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n  const start = Date.now()\n  try {\n    await callLlm(\n      {\n        model,\n        messages: [{ role: 'user', content: 'ping' }],\n        maxTokens: 64,\n        timeoutMs: opts.timeoutMs ?? 30_000,\n      },\n      opts,\n    )\n    return { ok: true, latencyMs: Date.now() - start, error: null }\n  } catch (err) {\n    return {\n      ok: false,\n      latencyMs: Date.now() - start,\n      error: err instanceof Error ? err.message : String(err),\n    }\n  }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n  readonly maximumAttempts: number\n  private readonly opts: LlmClientOptions\n\n  constructor(opts: LlmClientOptions = {}) {\n    this.opts = opts\n    this.maximumAttempts = resolveMaximumAttempts(opts.maxRetries)\n  }\n\n  call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n    const options = { ...this.opts, ...per }\n    return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)\n  }\n\n  callJson<T = unknown>(\n    req: LlmCallRequest,\n    per?: LlmClientOptions,\n  ): Promise<{ value: T; result: LlmCallResult }> {\n    return callLlmJson<T>(req, { ...this.opts, ...per })\n  }\n}\n"],"mappings":";;;;;;;;;;AAgEO,SAAS,2BACd,SACA,UAA4B,CAAC,GACF;AAC3B,MAAI,QAAQ,cAAc,OAAW,QAAO;AAC5C,MAAI,CAAC,OAAO,UAAU,QAAQ,SAAS,KAAK,QAAQ,aAAa,GAAG;AAClE,UAAM,IAAI,WAAW,kEAAkE;AAAA,EACzF;AACA,MACE,QAAQ,SAAS;AAAA,IACf,CAAC,YACC,MAAM,QAAQ,QAAQ,OAAO,KAAK,QAAQ,QAAQ,KAAK,CAAC,SAAS,KAAK,SAAS,WAAW;AAAA,EAC9F,GACA;AACA,WAAO;AAAA,EACT;AAEA,QAAM,WAAW,uBAAuB,QAAQ,UAAU;AAG1D,QAAM,eAAe,IAAI,YAAY,EAAE;AAAA,IACrC,KAAK,UAAU,UAAU,SAAS,KAAK,CAAC;AAAA,EAC1C,EAAE;AAEF,QAAM,UAAU,QAAQ,aAAa,IAAI;AACzC,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf,aAAa,eAAe,WAAW;AAAA,IACvC,cAAc,QAAQ,YAAY,WAAW;AAAA,EAC/C;AACF;AAgDO,SAAS,mBAAmB,QAAyC;AAC1E,QAAM,eAAe,OAAO,MAAM,sBAAsB;AACxD,SAAO;AAAA,IACL,OAAO,OAAO;AAAA,IACd,aAAa,KAAK,IAAI,GAAG,OAAO,MAAM,eAAe,YAAY;AAAA,IACjE,cAAc,OAAO,MAAM;AAAA,IAC3B,cAAc,eAAe,IAAI,eAAe;AAAA,IAChD,eAAe,OAAO,WAAW;AAAA,IACjC,cAAc,OAAO,MAAM,aAAa;AAAA,EAC1C;AACF;AAGO,SAAS,wBAAwB,OAA4C;AAClF,SAAO,iBAAiB,mBAAmB,mBAAmB,MAAM,MAAM,IAAI;AAChF;AAEO,IAAM,eAAN,cAA2B,eAAe;AAAA,EAC/C,YACE,SACgB,QACA,MACA,OAChB;AACA,UAAM,SAAS,OAAO;AAJN;AACA;AACA;AAAA,EAGlB;AAAA,EALkB;AAAA,EACA;AAAA,EACA;AAIpB;AAKO,IAAM,mBAAN,cAA+B,eAAe;AAAA,EACnD,YACE,SACgB,QAChB,SACA;AACA,UAAM,SAAS,SAAS,OAAO;AAHf;AAAA,EAIlB;AAAA,EAJkB;AAKpB;AAsDA,IAAM,mBAAmB;AASzB,IAAM,qBAAqB,OAAO,QAAQ,IAAI,qBAAqB,KAAK;AACxE,IAAM,sBAAsB,OAAO,QAAQ,IAAI,sBAAsB,KAAK;AAE1E,SAAS,uBAAuB,YAAwC;AACtE,QAAM,WAAW,cAAc;AAC/B,MAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,YAAY,GAAG;AAChD,UAAM,IAAI,WAAW,iDAAiD;AAAA,EACxE;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAoC;AAC9D,SAAO,OAAO,UAAU,YAAY,OAAO,cAAc,KAAK,KAAK,SAAS,IAAI,QAAQ;AAC1F;AAEA,IAAM,mBAAmB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAWrD,IAAM,2BAA8C;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcO,SAAS,oBAAoB,KAAuB;AACzD,SAAO,kBAAkB,KAAK,CAAC;AACjC;AAEA,SAAS,kBAAkB,KAAc,OAAwB;AAC/D,MAAI,eAAe,aAAc,QAAO,iBAAiB,IAAI,IAAI,MAAM;AACvE,MAAI,EAAE,eAAe,OAAQ,QAAO;AAGpC,QAAM,SAAU,IAA6B;AAC7C,MAAI,OAAO,WAAW,YAAY,iBAAiB,IAAI,MAAM,EAAG,QAAO;AACvE,QAAM,OAAQ,IAA2B;AACzC,QAAM,WAAW,GAAG,IAAI,IAAI;AAAA,EAAK,IAAI,OAAO;AAAA,EAAK,OAAO,SAAS,WAAW,OAAO,EAAE;AACrF,MAAI,yBAAyB,KAAK,CAAC,MAAM,EAAE,KAAK,QAAQ,CAAC,EAAG,QAAO;AACnE,QAAM,QAAS,IAA4B;AAC3C,MAAI,QAAQ,KAAK,iBAAiB,SAAS,UAAU,KAAK;AACxD,WAAO,kBAAkB,OAAO,QAAQ,CAAC;AAAA,EAC3C;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;AACxD,QAAM,IAAI,QAAQ,IAAI,aAAa;AACnC,MAAI,CAAC,EAAG,QAAO;AACf,QAAM,WAAW,OAAO,CAAC;AACzB,MAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,EAAG,QAAO,WAAW;AACjE,QAAM,SAAS,KAAK,MAAM,CAAC;AAC3B,MAAI,OAAO,SAAS,MAAM,EAAG,QAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;AACnE,SAAO;AACT;AAGO,SAAS,UAAU,SAAyB;AACjD,SAAO,KAAK,IAAI,MAAM,KAAK,SAAS,IAAM;AAC5C;AAEA,SAAS,aAAa,MAAgD;AACpE,QAAM,UAAkC;AAAA,IACtC,gBAAgB;AAAA,IAChB,QAAQ;AAAA,EACV;AACA,MAAI,KAAK,YAAY;AACnB,YAAQ,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW;AAAA,EAClD,WAAW,KAAK,UAAU,KAAK,QAAQ;AACrC,YAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK,MAAM;AAAA,EAC9D;AACA,MAAI,KAAK,eAAgB,SAAQ,iBAAiB,IAAI,KAAK;AAC3D,SAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;AAChE,MAAI,WAAW,IAAK,QAAO;AAC3B,QAAM,QAAQ,KAAK,YAAY;AAC/B,SACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,UAAU,KAAqB,iBAAmD;AACzF,QAAM,OAAgC;AAAA,IACpC,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,aAAa,IAAI,eAAe;AAAA,EAClC;AACA,MAAI,IAAI,aAAa,MAAM;AACzB,QAAI,wBAAwB,IAAI,KAAK,EAAG,MAAK,wBAAwB,IAAI;AAAA,QACpE,MAAK,aAAa,IAAI;AAAA,EAC7B;AAEA,MAAI,IAAI,cAAc,CAAC,iBAAiB;AACtC,SAAK,kBAAkB;AAAA,MACrB,MAAM;AAAA,MACN,aAAa,EAAE,MAAM,IAAI,WAAW,MAAM,QAAQ,IAAI,WAAW,QAAQ,QAAQ,KAAK;AAAA,IACxF;AAAA,EACF,WAAW,IAAI,YAAY,IAAI,YAAY;AACzC,SAAK,kBAAkB,EAAE,MAAM,cAAc;AAAA,EAC/C;AAEA,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;AACvD,SAAO,oBAAoB,KAAK,KAAK;AACvC;AAEA,eAAe,MAAM,IAA2B;AAC9C,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AASA,SAAS,YAAY,mBAAoC,QAAmC;AAC1F,MAAI,CAAC,OAAQ,QAAO,kBAAkB;AACtC,MAAI,OAAQ,YAAkC,QAAQ,YAAY;AAChE,WAAO,YAAY,IAAI,CAAC,kBAAkB,QAAQ,MAAM,CAAC;AAAA,EAC3D;AACA,MAAI,OAAO,SAAS;AAClB,sBAAkB,MAAM;AAAA,EAC1B,OAAO;AACL,WAAO,iBAAiB,SAAS,MAAM,kBAAkB,MAAM,GAAG,EAAE,MAAM,KAAK,CAAC;AAAA,EAClF;AACA,SAAO,kBAAkB;AAC3B;AAGA,SAAS,iBAAiB,OAAe,YAAyC;AAChF,SAAO,cAAc,QAAQ,KAAK,IAAI,IAAI,SAAS;AACrD;AASO,SAAS,gBAAgB,KAAqB;AACnD,QAAM,UAAU,IAAI,KAAK;AACzB,QAAM,IAAI,QAAQ,MAAM,yCAAyC;AACjE,SAAO,IAAI,EAAE,CAAC,EAAG,KAAK,IAAI;AAC5B;AAEO,SAAS,mBAAmB,KAAqB;AACtD,QAAM,WAAW,gBAAgB,GAAG;AACpC,MAAI;AACF,SAAK,MAAM,QAAQ;AACnB,WAAO;AAAA,EACT,QAAQ;AAIN,QAAI,SAAS,WAAW,GAAG,KAAK,SAAS,WAAW,GAAG,EAAG,QAAO;AAAA,EACnE;AAGA,QAAM,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,CAAC,EAC1C,IAAI,CAAC,UAAU,MAAM,KAAK,EAC1B,OAAO,CAAC,UAAU,SAAS,IAAI;AAClC,aAAW,SAAS,QAAQ;AAC1B,UAAM,YAAY,oBAAoB,UAAU,KAAK;AACrD,QAAI,CAAC,UAAW;AAChB,QAAI;AACF,WAAK,MAAM,SAAS;AACpB,aAAO;AAAA,IACT,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;AACxE,QAAM,SAAS,MAAM,KAAK;AAC1B,QAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;AAC7D,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAkB,CAAC,MAAM;AAC/B,MAAI,aAAa;AACjB,MAAI,YAAY;AAEhB,WAAS,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;AAC7C,UAAM,OAAO,MAAM,CAAC;AACpB,QAAI,WAAW;AACb,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,MAAM;AACjB,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,KAAK;AAChB,mBAAa,CAAC;AACd;AAAA,IACF;AACA,QAAI,WAAY;AAEhB,QAAI,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aACvB,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aAC5B,SAAS,MAAM,MAAM,SAAS,CAAC,GAAG;AACzC,YAAM,IAAI;AACV,UAAI,MAAM,WAAW,EAAG,QAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO;AACT;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;AACxB,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AACrE,QAAM,MAAM,GAAG,OAAO;AACtB,QAAM,WAAW;AACjB,QAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;AAC5D,QAAM,kBAAkB,uBAAuB,KAAK,UAAU;AAC9D,QAAM,UAAU,KAAK,SAAS,WAAW;AACzC,QAAM,UAAU,aAAa,IAAI;AACjC,QAAM,WAAW,KAAK,YAAY,oBAAoB,OAAO;AAC7D,QAAM,OAAO,KAAK;AAClB,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,eAAe,KAAK;AAC1B,QAAM,eAAe,KAAK;AAC1B,QAAM,aAAa,KAAK;AACxB,QAAM,gBAAgB,KAAK,IAAI;AAE/B,MAAI;AACJ,WAAS,UAAU,GAAG,UAAU,iBAAiB,WAAW;AAG1D,QAAI,cAAc,SAAS;AACzB,YAAM,IAAI,aAAa,oCAAoC,YAAY;AAAA,IACzE;AAGA,QAAI,UAAU,KAAK,iBAAiB,eAAe,UAAU,GAAG;AAC9D,YAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AAAA,IACtE;AACA,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,gBAAgB,YAAY,YAAY,YAAY;AAC1D,UAAM,gBAAgB,WAAW,MAAM,WAAW,MAAM,GAAG,SAAS;AACpE,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,cAAc,UAAU,KAAK,KAAK;AACxC,QAAI,uBAAuB;AAC3B,QAAI,MAAM;AACR,YAAM,UAAU,MAAM,UAAU;AAAA,QAC9B,SAAS,cAAc;AAAA,QACvB,OAAO,cAAc;AAAA,QACrB,QAAQ,cAAc;AAAA,QACtB;AAAA,QACA,OAAO,IAAI;AAAA,QACX;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,WAAW;AAAA,QACX,WAAW;AAAA,QACX,gBAAgB;AAAA,QAChB;AAAA,QACA,gBAAgB,CAAC;AAAA,MACnB,CAAC;AAAA,IACH;AAEA,QAAI;AACF,YAAM,MAAM,MAAM,QAAQ,KAAK;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU,WAAW;AAAA,QAChC,QAAQ;AAAA,MACV,CAAC;AACD,mBAAa,aAAa;AAC1B,YAAM,kBAAkB,OAAO,gBAAgB,IAAI,OAAO,IAAI;AAE9D,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,YAAI,MAAM;AACR,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,YAAY,IAAI;AAAA,YAChB;AAAA,YACA,cAAc;AAAA,YACd,cAAc,QAAQ,IAAI,MAAM;AAAA,YAChC,gBAAgB,CAAC;AAAA,UACnB,CAAC;AACD,iCAAuB;AAAA,QACzB;AACA,cAAM,MAAM,IAAI;AAAA,UACd,YAAY,IAAI,MAAM,KAAK,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA,UAC7C,IAAI;AAAA,UACJ;AAAA,UACA,IAAI;AAAA,QACN;AACA,YACE,iBAAiB,IAAI,IAAI,MAAM,KAC/B,UAAU,kBAAkB,KAC5B,CAAC,iBAAiB,eAAe,UAAU,GAC3C;AACA,oBAAU;AACV,gBAAM,aAAa,gBAAgB,IAAI,OAAO;AAC9C,gBAAM,MAAM,cAAc,UAAU,OAAO,CAAC;AAC5C;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAEA,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI;AACJ,UAAI;AACF,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB,SAAS,UAAU;AACjB,YAAI,MAAM;AACR,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,YAAY,IAAI;AAAA,YAChB;AAAA,YACA,cAAc;AAAA,YACd,cAAc,sBAAsB,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ,CAAC;AAAA,YACnG,gBAAgB,CAAC;AAAA,UACnB,CAAC;AACD,iCAAuB;AAAA,QACzB;AACA,cAAM;AAAA,MACR;AACA,UAAI,MAAM;AACR,cAAM,UAAU,MAAM,UAAU;AAAA,UAC9B,SAAS,cAAc;AAAA,UACvB,OAAO,cAAc;AAAA,UACrB,QAAQ,cAAc;AAAA,UACtB;AAAA,UACA,OAAO,IAAI;AAAA,UACX;AAAA,UACA;AAAA,UACA,cAAc;AAAA,UACd,WAAW;AAAA,UACX,WAAW,KAAK,IAAI;AAAA,UACpB,YAAY,KAAK,IAAI,IAAI;AAAA,UACzB,YAAY,IAAI;AAAA,UAChB;AAAA,UACA,cAAc;AAAA,UACd,gBAAgB,CAAC;AAAA,QACnB,CAAC;AAAA,MACH;AACA,YAAM,SACJ,KAAK,UAGH,CAAC;AACL,YAAM,WACJ,KAAK,SAAS,OAAO,KAAK,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,KAAK,IACpE,KAAK,QACN;AACN,YAAM,eAAe,mBAAmB,UAAU,aAAa;AAC/D,YAAM,mBAAmB,mBAAmB,UAAU,iBAAiB;AACvE,YAAM,cAAc,mBAAmB,UAAU,YAAY;AAC7D,YAAM,YACJ,UAAU,yBACV,OAAO,SAAS,0BAA0B,YAC1C,CAAC,MAAM,QAAQ,SAAS,qBAAqB,IACxC,SAAS,sBAAkD,gBAC5D;AACN,YAAM,qBAAqB,cAAc,SAAY,SAAY,mBAAmB,SAAS;AAC7F,YAAM,gBACJ,iBAAiB,UACjB,qBAAqB,WACpB,cAAc,UACZ,uBAAuB,UAAa,sBAAsB,kBAC5D,gBAAgB,UAAa,gBAAgB,eAAe;AAC/D,YAAM,gBAAiB,KAAK,kBAAkB,KAAK;AACnD,YAAM,UAAU,QAAQ,SAAS,WAAW;AAE5C,aAAO;AAAA,QACL;AAAA,QACA,cAAc,QAAQ,iBAAiB;AAAA,QACvC,cAAc,QAAQ,KAAK,EAAE,WAAW;AAAA,QACxC,OAAO;AAAA,UACL,cAAc,gBAAgB;AAAA,UAC9B,kBAAkB,oBAAoB;AAAA,UACtC,aAAa,gBAAgB,gBAAgB,MAAM,oBAAoB;AAAA,UACvE,UAAU;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS,OAAO,kBAAkB,WAAW,gBAAgB;AAAA,QAC7D,OAAQ,KAAK,SAAoB,IAAI;AAAA,QACrC,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB,KAAK;AAAA,MACP;AAAA,IACF,SAAS,KAAK;AACZ,mBAAa,aAAa;AAC1B,gBAAU;AAIV,UAAI,cAAc,SAAS;AACzB,YAAI,QAAQ,CAAC,sBAAsB;AACjC,gBAAM,UAAU,MAAM,UAAU;AAAA,YAC9B,SAAS,cAAc;AAAA,YACvB,OAAO,cAAc;AAAA,YACrB,QAAQ,cAAc;AAAA,YACtB;AAAA,YACA,OAAO,IAAI;AAAA,YACX;AAAA,YACA;AAAA,YACA,cAAc;AAAA,YACd,WAAW;AAAA,YACX,WAAW,KAAK,IAAI;AAAA,YACpB,YAAY,KAAK,IAAI,IAAI;AAAA,YACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,YAC7D,gBAAgB,CAAC;AAAA,UACnB,CAAC;AAAA,QACH;AACA,cAAM;AAAA,MACR;AACA,UAAI,QAAQ,CAAC,sBAAsB;AAIjC,cAAM,UAAU,MAAM,UAAU;AAAA,UAC9B,SAAS,cAAc;AAAA,UACvB,OAAO,cAAc;AAAA,UACrB,QAAQ,cAAc;AAAA,UACtB;AAAA,UACA,OAAO,IAAI;AAAA,UACX;AAAA,UACA;AAAA,UACA,cAAc;AAAA,UACd,WAAW;AAAA,UACX,WAAW,KAAK,IAAI;AAAA,UACpB,YAAY,KAAK,IAAI,IAAI;AAAA,UACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UAC7D,gBAAgB,CAAC;AAAA,QACnB,CAAC;AAAA,MACH;AACA,UACE,UAAU,kBAAkB,KAC5B,oBAAoB,GAAG,KACvB,CAAC,iBAAiB,eAAe,UAAU,GAC3C;AACA,cAAM,MAAM,UAAU,OAAO,CAAC;AAC9B;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AACA,QAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAEA,eAAe,UACb,MACA,UACA,OACe;AAGf,MAAI;AACF,UAAM,KAAK,OAAO,SAAS,KAAK,CAAC;AAAA,EACnC,QAAQ;AAAA,EAER;AACF;AAEA,SAAS,gBAAgB,GAAoC;AAC3D,QAAM,MAA8B,CAAC;AACrC,IAAE,QAAQ,CAAC,OAAO,QAAQ;AACxB,QAAI,GAAG,IAAI;AAAA,EACb,CAAC;AACD,SAAO;AACT;AAEA,SAAS,gBAAwB;AAC/B,MAAI,OAAO,WAAW,QAAQ,eAAe,WAAY,QAAO,WAAW,OAAO,WAAW;AAC7F,SAAO,GAAG,KAAK,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC;AAC9E;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;AAC9C,QAAM,SAAS,MAAM,kBAAkB,KAAK,IAAI;AAChD,QAAM,QAAQ,gBAAmB,MAAM;AACvC,SAAO,EAAE,OAAO,OAAO;AACzB;AAGA,eAAe,kBACb,KACA,OAAyB,CAAC,GACF;AACxB,MAAI;AACF,WAAO,MAAM,QAAQ,EAAE,GAAG,KAAK,UAAU,IAAI,YAAY,CAAC,IAAI,WAAW,GAAG,IAAI;AAAA,EAClF,SAAS,KAAK;AACZ,QAAI,eAAe,gBAAgB,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KAAK,IAAI,YAAY;AAC5F,YAAM,cAA8B,EAAE,GAAG,KAAK,UAAU,MAAM,YAAY,OAAU;AACpF,aAAO,MAAM,QAAQ,aAAa,IAAI;AAAA,IACxC;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,gBAAmB,QAA0B;AACpD,MAAI;AACF,QAAI,OAAO,iBAAiB,UAAU;AACpC,YAAM,IAAI;AAAA,QACR,8CAA8C,OAAO,KAAK;AAAA,MAC5D;AAAA,IACF;AACA,WAAO,gBAAmB,OAAO,SAAS,OAAO,KAAK;AAAA,EACxD,SAAS,OAAO;AACd,QAAI,iBAAiB,iBAAkB,OAAM;AAC7C,UAAM,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACtE,UAAM,IAAI,iBAAiB,MAAM,SAAS,QAAQ,EAAE,MAAM,CAAC;AAAA,EAC7D;AACF;AAEA,SAAS,gBAAmB,SAAiB,OAAkB;AAC7D,QAAM,WAAW,mBAAmB,OAAO;AAC3C,MAAI;AACF,WAAO,KAAK,MAAM,QAAQ;AAAA,EAC5B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,wCAAwC,KAAK,MAC3C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CACjD;AAAA;AAAA,EAA0B,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA,IACjD;AAAA,EACF;AACF;AAWO,IAAM,yBAAN,cAAqC,sBAAsB;AAAA,EAChE,YACE,SACgB,QACA,SAChB;AACA,UAAM,OAAO;AAHG;AACA;AAAA,EAGlB;AAAA,EAJkB;AAAA,EACA;AAIpB;AAmCO,SAAS,eAAe,MAAwB,MAA4B,CAAC,GAAS;AAC3F,QAAM,kBAAkB,KAAK,YAAY;AACzC,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AAErE,MAAI,IAAI,0BAA0B,CAAC,iBAAiB;AAClD,UAAM,IAAI;AAAA,MACR,gGAAgG,gBAAgB;AAAA,MAChH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,iBAAiB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,CAAC,GAAG;AAC1D,UAAM,IAAI;AAAA,MACR,2BAA2B,OAAO;AAAA,MAClC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,GAAG;AACzD,UAAM,KAAK,IAAI,gBAAgB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,CAAC;AAC/D,QAAI,CAAC,IAAI;AACP,YAAM,IAAI;AAAA,QACR,2BAA2B,OAAO,gCAAgC,IAAI,gBAAgB,IAAI,eAAe,EAAE,KAAK,IAAI,CAAC;AAAA,QACrH;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,eAAe,CAAC,KAAK,UAAU,CAAC,KAAK,UAAU,CAAC,KAAK,YAAY;AACvE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,kBAAkB;AACxB,UAAM,SAAS,KAAK,YAAY,oBAAoB,OAAO;AAC3D,QAAI,WAAW,IAAI,kBAAkB;AACnC,YAAM,IAAI;AAAA,QACR,qCAAqC,IAAI,gBAAgB,gBAAgB,OAAO,gBAAgB,MAAM;AAAA,QACtG;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,SAAS,KAAa,SAAmC;AAChE,MAAI,mBAAmB,OAAQ,QAAO,QAAQ,KAAK,GAAG;AACtD,SAAO,IAAI,YAAY,EAAE,WAAW,QAAQ,YAAY,CAAC;AAC3D;AAEA,SAAS,gBAAgB,GAA4B;AACnD,SAAO,aAAa,SAAS,EAAE,SAAS;AAC1C;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;AACnE,QAAM,QAAQ,KAAK,IAAI;AACvB,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,QACE;AAAA,QACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,QAC5C,WAAW;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,WAAW,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK;AAAA,EAChE,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,WAAW,KAAK,IAAI,IAAI;AAAA,MACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;AAOO,IAAM,YAAN,MAAgB;AAAA,EACZ;AAAA,EACQ;AAAA,EAEjB,YAAY,OAAyB,CAAC,GAAG;AACvC,SAAK,OAAO;AACZ,SAAK,kBAAkB,uBAAuB,KAAK,UAAU;AAAA,EAC/D;AAAA,EAEA,KAAK,KAAqB,KAAgD;AACxE,UAAM,UAAU,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI;AACvC,WAAO,IAAI,aAAa,kBAAkB,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO;AAAA,EAChF;AAAA,EAEA,SACE,KACA,KAC8C;AAC9C,WAAO,YAAe,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EACrD;AACF;","names":[]}

package/dist/chunk-NPCTHQIO.js

@@ -1,91 +0,0 @@
-// src/meta-eval/calibration.ts
-async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
-  const runs = await traceStore.listRuns();
-  const outcomes = await outcomeStore.list();
-  const byRun = /* @__PURE__ */ new Map();
-  for (const o of outcomes) {
-    const arr = byRun.get(o.runId) ?? [];
-    arr.push(o);
-    byRun.set(o.runId, arr);
-  }
-  const extract = evalMetric.extract ?? defaultExtract(evalMetric.id);
-  const pairs = [];
-  for (const run of runs) {
-    const os = byRun.get(run.runId);
-    if (!os?.length) continue;
-    const x = await extract(run, traceStore);
-    if (x === null || !Number.isFinite(x)) continue;
-    const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0];
-    const y = latest.metrics[outcomeMetric];
-    if (typeof y !== "number" || !Number.isFinite(y)) continue;
-    pairs.push({ x, y });
-  }
-  if (pairs.length < 2) return null;
-  return calibrationFromPairs(
-    pairs.map((p) => ({ evalScore: p.x, outcome: p.y })),
-    evalMetric.id,
-    outcomeMetric,
-    options
-  );
-}
-function calibrationFromPairs(inputPairs, evalMetric, outcomeMetric, options = {}) {
-  const pairs = inputPairs.filter(
-    (pair) => Number.isFinite(pair.evalScore) && Number.isFinite(pair.outcome)
-  );
-  if (pairs.length < 2) return null;
-  const numBins = options.bins ?? 10;
-  const binning = options.binning ?? "equal-width";
-  const xs = pairs.map((p) => p.evalScore);
-  const lo = options.range?.lo ?? Math.min(...xs);
-  const hi = options.range?.hi ?? Math.max(...xs);
-  const bins = [];
-  if (binning === "equal-frequency") {
-    const sorted = [...pairs].sort((a, b) => a.evalScore - b.evalScore);
-    const perBin = Math.max(1, Math.floor(sorted.length / numBins));
-    for (let i = 0; i < sorted.length; i += perBin) {
-      const chunk = sorted.slice(i, i + perBin);
-      if (chunk.length === 0) continue;
-      bins.push(toBin(chunk));
-    }
-  } else {
-    const width = (hi - lo) / numBins;
-    if (width === 0) return null;
-    for (let i = 0; i < numBins; i++) {
-      const binLo = lo + i * width;
-      const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width;
-      const chunk = pairs.filter((p) => p.evalScore >= binLo && p.evalScore < binHi);
-      if (chunk.length === 0) continue;
-      bins.push(toBin(chunk, binLo, binHi));
-    }
-  }
-  const total = bins.reduce((a, b) => a + b.n, 0);
-  const ece = bins.reduce((a, b) => a + b.n / total * b.gap, 0);
-  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0);
-  return { evalMetric, outcomeMetric, n: pairs.length, bins, ece, maxGap };
-}
-function toBin(chunk, lower, upper) {
-  const xs = chunk.map((c) => c.evalScore);
-  const ys = chunk.map((c) => c.outcome);
-  const evalMean = mean(xs);
-  const outcomeMean = mean(ys);
-  return {
-    lower: lower ?? Math.min(...xs),
-    upper: upper ?? Math.max(...xs),
-    n: chunk.length,
-    evalMean,
-    outcomeMean,
-    gap: Math.abs(outcomeMean - evalMean)
-  };
-}
-function mean(xs) {
-  return xs.reduce((a, b) => a + b, 0) / xs.length;
-}
-function defaultExtract(metric) {
-  return async (run) => run.outcome?.score ?? (metric === "pass" ? run.outcome?.pass === true ? 1 : 0 : null);
-}
-
-export {
-  calibrationCurve,
-  calibrationFromPairs
-};
-//# sourceMappingURL=chunk-NPCTHQIO.js.map

package/dist/chunk-NPCTHQIO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/meta-eval/calibration.ts"],"sourcesContent":["/**\n * Calibration curve — binned \"if eval says X, what does reality show?\"\n *\n * Companion to correlationStudy. Raw correlation is a single number;\n * the calibration curve shows *where* the eval is well-calibrated vs\n * overconfident / underconfident. Buckets the eval metric, computes\n * mean outcome per bucket, reports expected-calibration-error (ECE).\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport type { EvalMetricSpec } from './correlation-study'\nimport type { DeploymentOutcome, OutcomeStore } from './outcome-store'\n\nexport interface CalibrationBin {\n  lower: number\n  upper: number\n  n: number\n  evalMean: number\n  outcomeMean: number\n  /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */\n  gap: number\n}\n\nexport interface CalibrationReport {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  bins: CalibrationBin[]\n  /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */\n  ece: number\n  /** Max bin gap — upper bound on miscalibration. */\n  maxGap: number\n}\n\nexport interface CalibrationOptions {\n  bins?: number\n  /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */\n  binning?: 'equal-width' | 'equal-frequency'\n  /** Clip eval values to [lo, hi] before binning. */\n  range?: { lo: number; hi: number }\n}\n\nexport interface CalibrationPair {\n  evalScore: number\n  outcome: number\n}\n\nexport async function calibrationCurve(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetric: EvalMetricSpec,\n  outcomeMetric: string,\n  options: CalibrationOptions = {},\n): Promise<CalibrationReport | null> {\n  const runs = await traceStore.listRuns()\n  const outcomes = await outcomeStore.list()\n  const byRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = byRun.get(o.runId) ?? []\n    arr.push(o)\n    byRun.set(o.runId, arr)\n  }\n\n  const extract = evalMetric.extract ?? defaultExtract(evalMetric.id)\n  const pairs: Array<{ x: number; y: number }> = []\n  for (const run of runs) {\n    const os = byRun.get(run.runId)\n    if (!os?.length) continue\n    const x = await extract(run, traceStore)\n    if (x === null || !Number.isFinite(x)) continue\n    const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0]!\n    const y = latest.metrics[outcomeMetric]\n    if (typeof y !== 'number' || !Number.isFinite(y)) continue\n    pairs.push({ x, y })\n  }\n  if (pairs.length < 2) return null\n\n  return calibrationFromPairs(\n    pairs.map((p) => ({ evalScore: p.x, outcome: p.y })),\n    evalMetric.id,\n    outcomeMetric,\n    options,\n  )\n}\n\nexport function calibrationFromPairs(\n  inputPairs: CalibrationPair[],\n  evalMetric: string,\n  outcomeMetric: string,\n  options: CalibrationOptions = {},\n): CalibrationReport | null {\n  const pairs = inputPairs.filter(\n    (pair) => Number.isFinite(pair.evalScore) && Number.isFinite(pair.outcome),\n  )\n  if (pairs.length < 2) return null\n\n  const numBins = options.bins ?? 10\n  const binning = options.binning ?? 'equal-width'\n  const xs = pairs.map((p) => p.evalScore)\n  const lo = options.range?.lo ?? Math.min(...xs)\n  const hi = options.range?.hi ?? Math.max(...xs)\n\n  const bins: CalibrationBin[] = []\n  if (binning === 'equal-frequency') {\n    const sorted = [...pairs].sort((a, b) => a.evalScore - b.evalScore)\n    const perBin = Math.max(1, Math.floor(sorted.length / numBins))\n    for (let i = 0; i < sorted.length; i += perBin) {\n      const chunk = sorted.slice(i, i + perBin)\n      if (chunk.length === 0) continue\n      bins.push(toBin(chunk))\n    }\n  } else {\n    const width = (hi - lo) / numBins\n    if (width === 0) return null\n    for (let i = 0; i < numBins; i++) {\n      const binLo = lo + i * width\n      const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width\n      const chunk = pairs.filter((p) => p.evalScore >= binLo && p.evalScore < binHi)\n      if (chunk.length === 0) continue\n      bins.push(toBin(chunk, binLo, binHi))\n    }\n  }\n\n  const total = bins.reduce((a, b) => a + b.n, 0)\n  const ece = bins.reduce((a, b) => a + (b.n / total) * b.gap, 0)\n  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0)\n\n  return { evalMetric, outcomeMetric, n: pairs.length, bins, ece, maxGap }\n}\n\nfunction toBin(chunk: CalibrationPair[], lower?: number, upper?: number): CalibrationBin {\n  const xs = chunk.map((c) => c.evalScore)\n  const ys = chunk.map((c) => c.outcome)\n  const evalMean = mean(xs)\n  const outcomeMean = mean(ys)\n  return {\n    lower: lower ?? Math.min(...xs),\n    upper: upper ?? Math.max(...xs),\n    n: chunk.length,\n    evalMean,\n    outcomeMean,\n    gap: Math.abs(outcomeMean - evalMean),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n  return async (run) =>\n    run.outcome?.score ?? (metric === 'pass' ? (run.outcome?.pass === true ? 1 : 0) : null)\n}\n"],"mappings":";AAgDA,eAAsB,iBACpB,YACA,cACA,YACA,eACA,UAA8B,CAAC,GACI;AACnC,QAAM,OAAO,MAAM,WAAW,SAAS;AACvC,QAAM,WAAW,MAAM,aAAa,KAAK;AACzC,QAAM,QAAQ,oBAAI,IAAiC;AACnD,aAAW,KAAK,UAAU;AACxB,UAAM,MAAM,MAAM,IAAI,EAAE,KAAK,KAAK,CAAC;AACnC,QAAI,KAAK,CAAC;AACV,UAAM,IAAI,EAAE,OAAO,GAAG;AAAA,EACxB;AAEA,QAAM,UAAU,WAAW,WAAW,eAAe,WAAW,EAAE;AAClE,QAAM,QAAyC,CAAC;AAChD,aAAW,OAAO,MAAM;AACtB,UAAM,KAAK,MAAM,IAAI,IAAI,KAAK;AAC9B,QAAI,CAAC,IAAI,OAAQ;AACjB,UAAM,IAAI,MAAM,QAAQ,KAAK,UAAU;AACvC,QAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,EAAG;AACvC,UAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;AACpE,UAAM,IAAI,OAAO,QAAQ,aAAa;AACtC,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG;AAClD,UAAM,KAAK,EAAE,GAAG,EAAE,CAAC;AAAA,EACrB;AACA,MAAI,MAAM,SAAS,EAAG,QAAO;AAE7B,SAAO;AAAA,IACL,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE;AAAA,IACnD,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,qBACd,YACA,YACA,eACA,UAA8B,CAAC,GACL;AAC1B,QAAM,QAAQ,WAAW;AAAA,IACvB,CAAC,SAAS,OAAO,SAAS,KAAK,SAAS,KAAK,OAAO,SAAS,KAAK,OAAO;AAAA,EAC3E;AACA,MAAI,MAAM,SAAS,EAAG,QAAO;AAE7B,QAAM,UAAU,QAAQ,QAAQ;AAChC,QAAM,UAAU,QAAQ,WAAW;AACnC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS;AACvC,QAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;AAC9C,QAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;AAE9C,QAAM,OAAyB,CAAC;AAChC,MAAI,YAAY,mBAAmB;AACjC,UAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAClE,UAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,SAAS,OAAO,CAAC;AAC9D,aAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,QAAQ;AAC9C,YAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,MAAM;AACxC,UAAI,MAAM,WAAW,EAAG;AACxB,WAAK,KAAK,MAAM,KAAK,CAAC;AAAA,IACxB;AAAA,EACF,OAAO;AACL,UAAM,SAAS,KAAK,MAAM;AAC1B,QAAI,UAAU,EAAG,QAAO;AACxB,aAAS,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,YAAM,QAAQ,KAAK,IAAI;AACvB,YAAM,QAAQ,MAAM,UAAU,IAAI,KAAK,OAAO,MAAM,IAAI,KAAK;AAC7D,YAAM,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,aAAa,SAAS,EAAE,YAAY,KAAK;AAC7E,UAAI,MAAM,WAAW,EAAG;AACxB,WAAK,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;AAAA,IACtC;AAAA,EACF;AAEA,QAAM,QAAQ,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,GAAG,CAAC;AAC9C,QAAM,MAAM,KAAK,OAAO,CAAC,GAAG,MAAM,IAAK,EAAE,IAAI,QAAS,EAAE,KAAK,CAAC;AAC9D,QAAM,SAAS,KAAK,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;AAE1D,SAAO,EAAE,YAAY,eAAe,GAAG,MAAM,QAAQ,MAAM,KAAK,OAAO;AACzE;AAEA,SAAS,MAAM,OAA0B,OAAgB,OAAgC;AACvF,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS;AACvC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO;AACrC,QAAM,WAAW,KAAK,EAAE;AACxB,QAAM,cAAc,KAAK,EAAE;AAC3B,SAAO;AAAA,IACL,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;AAAA,IAC9B,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;AAAA,IAC9B,GAAG,MAAM;AAAA,IACT;AAAA,IACA;AAAA,IACA,KAAK,KAAK,IAAI,cAAc,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,KAAK,IAAsB;AAClC,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAAS,eAAe,QAAyE;AAC/F,SAAO,OAAO,QACZ,IAAI,SAAS,UAAU,WAAW,SAAU,IAAI,SAAS,SAAS,OAAO,IAAI,IAAK;AACtF;","names":[]}

package/dist/chunk-ONWEPEDO.js

@@ -1,57 +0,0 @@
-// src/errors.ts
-var AgentEvalError = class extends Error {
-  /** Stable string code. Survives minification; safe to switch on. */
-  code;
-  constructor(code, message, options) {
-    super(message, options);
-    this.name = this.constructor.name;
-    this.code = code;
-  }
-};
-var ValidationError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("validation", message, options);
-  }
-};
-var NotFoundError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("not_found", message, options);
-  }
-};
-var ConfigError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("config", message, options);
-  }
-};
-var CaptureIntegrityError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("capture_integrity", message, options);
-  }
-};
-var JudgeError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("judge", message, options);
-  }
-};
-var VerificationError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("verification", message, options);
-  }
-};
-var ReplayError = class extends AgentEvalError {
-  constructor(message, options) {
-    super("replay", message, options);
-  }
-};
-
-export {
-  AgentEvalError,
-  ValidationError,
-  NotFoundError,
-  ConfigError,
-  CaptureIntegrityError,
-  JudgeError,
-  VerificationError,
-  ReplayError
-};
-//# sourceMappingURL=chunk-ONWEPEDO.js.map

package/dist/chunk-ONWEPEDO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/errors.ts"],"sourcesContent":["/**\n * Error taxonomy for `@tangle-network/agent-eval`.\n *\n * Every error this package throws as part of its *public contract* extends\n * `AgentEvalError`. Consumers can pattern-match by `instanceof <Subclass>` or\n * by the stable string `code` carried on the base class.\n *\n * The codes are stable across minor versions; new codes can be added, but\n * existing codes never change meaning. New subclasses are non-breaking.\n *\n * Internal invariant guards (`throw new Error('this should never happen')`)\n * remain plain `Error`s on purpose — they're programmer-mistake assertions,\n * not consumer-catchable contract failures.\n */\n\nexport type AgentEvalErrorCode =\n  | 'validation'\n  | 'not_found'\n  | 'config'\n  | 'capture_integrity'\n  | 'judge'\n  | 'verification'\n  | 'replay'\n  | 'backend_integrity'\n  | 'profile_matrix'\n\n/**\n * Base class for every contract error this package throws — carries the stable\n * string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.\n */\nexport class AgentEvalError extends Error {\n  /** Stable string code. Survives minification; safe to switch on. */\n  readonly code: AgentEvalErrorCode\n\n  constructor(code: AgentEvalErrorCode, message: string, options?: { cause?: unknown }) {\n    super(message, options)\n    this.name = this.constructor.name\n    this.code = code\n  }\n}\n\n/** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */\nexport class ValidationError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('validation', message, options)\n  }\n}\n\n/** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */\nexport class NotFoundError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('not_found', message, options)\n  }\n}\n\n/** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */\nexport class ConfigError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('config', message, options)\n  }\n}\n\n/**\n * A run is missing the artifacts a launch-grade check requires:\n * raw HTTP capture absent, no LLM spans, route assertion failed, run-end\n * assertion tripped. Block ship on this; do not catch and move on.\n */\nexport class CaptureIntegrityError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('capture_integrity', message, options)\n  }\n}\n\n/** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */\nexport class JudgeError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('judge', message, options)\n  }\n}\n\n/** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */\nexport class VerificationError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('verification', message, options)\n  }\n}\n\n/** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */\nexport class ReplayError extends AgentEvalError {\n  constructor(message: string, options?: { cause?: unknown }) {\n    super('replay', message, options)\n  }\n}\n"],"mappings":";AA8BO,IAAM,iBAAN,cAA6B,MAAM;AAAA;AAAA,EAE/B;AAAA,EAET,YAAY,MAA0B,SAAiB,SAA+B;AACpF,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO;AAAA,EACd;AACF;AAGO,IAAM,kBAAN,cAA8B,eAAe;AAAA,EAClD,YAAY,SAAiB,SAA+B;AAC1D,UAAM,cAAc,SAAS,OAAO;AAAA,EACtC;AACF;AAGO,IAAM,gBAAN,cAA4B,eAAe;AAAA,EAChD,YAAY,SAAiB,SAA+B;AAC1D,UAAM,aAAa,SAAS,OAAO;AAAA,EACrC;AACF;AAGO,IAAM,cAAN,cAA0B,eAAe;AAAA,EAC9C,YAAY,SAAiB,SAA+B;AAC1D,UAAM,UAAU,SAAS,OAAO;AAAA,EAClC;AACF;AAOO,IAAM,wBAAN,cAAoC,eAAe;AAAA,EACxD,YAAY,SAAiB,SAA+B;AAC1D,UAAM,qBAAqB,SAAS,OAAO;AAAA,EAC7C;AACF;AAGO,IAAM,aAAN,cAAyB,eAAe;AAAA,EAC7C,YAAY,SAAiB,SAA+B;AAC1D,UAAM,SAAS,SAAS,OAAO;AAAA,EACjC;AACF;AAGO,IAAM,oBAAN,cAAgC,eAAe;AAAA,EACpD,YAAY,SAAiB,SAA+B;AAC1D,UAAM,gBAAgB,SAAS,OAAO;AAAA,EACxC;AACF;AAGO,IAAM,cAAN,cAA0B,eAAe;AAAA,EAC9C,YAAY,SAAiB,SAA+B;AAC1D,UAAM,UAAU,SAAS,OAAO;AAAA,EAClC;AACF;","names":[]}