@tangle-network/agent-eval 0.20.11 → 0.20.12

package/dist/traces.js

@@ -0,0 +1,100 @@
+import {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemTraceStore,
+  InMemoryTraceStore,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpans,
+  planTraceInsightQuestions,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup
+} from "./chunk-KWUAAIHR.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-PKCVBYTQ.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemTraceStore,
+  InMemoryTraceStore,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceEmitter,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpanFromProvider,
+  llmSpans,
+  planTraceInsightQuestions,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup
+};
+//# sourceMappingURL=traces.js.map

package/dist/traces.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/wire/index.js

@@ -24,8 +24,8 @@ import {
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "../chunk-LSR4IAYN.js";
-import "../chunk-JAOLXRIA.js";
+} from "../chunk-HNJLMAJ2.js";
+import "../chunk-75MCTH7P.js";
 import "../chunk-PZ5AY32C.js";
 export {
   BUILTIN_RUBRICS,

package/docs/concepts.md

@@ -1,14 +1,15 @@
 # Concepts
 
-Read this once and the rest of agent-eval makes sense.
+`agent-eval` is for deciding whether an agent run should pass, keep working, be
+replayed, be optimized, or be promoted.
 
-## What is agent-eval?
+It exists because agent output is not evidence. A model can say a task is done
+while the build fails, the browser flow is broken, the integration was never
+connected, or the answer lacks required sources. The package gives products a
+shared way to record runs, check outcomes, classify failures, compare variants,
+and make release decisions.
 
-A library for **deciding whether a code generator or content generator did its job.** You give it a thing the generator produced (a scaffold, a patch, a tweet, a JSON config), and you get back a structured verdict: pass/fail, dimension scores, a reason in plain English.
-
-It exists because LLMs lie about whether they succeeded. A model will say "Done!" and ship code that doesn't compile. agent-eval is the layer between the model's output and your decision to ship.
-
-## The three things you'll touch most
+## Main Objects
 
 | Thing | What it is | One-line example |
 |---|---|---|
@@ -17,7 +18,8 @@ It exists because LLMs lie about whether they succeeded. A model will say "Done!
 | **Verifier** | A pipeline of judges run in order, with dependencies. | "install → typecheck → build → semantic" |
 | **Feedback trajectory** | A multi-shot record of attempts, approvals, rejections, edits, metrics, and policy outcomes. | "draft → user rejects → revised draft → approved → measured" |
 
-That's the whole framework. Everything else (sessions, traces, layers) is plumbing around those three.
+Everything else exists to make those objects useful in real product loops:
+traces, datasets, control runtime, optimizers, statistics, and reports.
 
 When the thing being evaluated is an agent that should keep working, use
 [`runAgentControlLoop`](./control-runtime.md). It turns validators into a
@@ -62,7 +64,7 @@ shape stays the same.
 Those trajectories can be converted into preference memory, `DatasetScenario`
 rows, optimizer rows, and held-out examples for overfit checks.
 
-## The three-layer eval (for code generators)
+## Code Generator Eval
 
 When the artifact is generated code, agent-eval scores it at three independent layers. Each layer fails differently, and you want to know which one broke:
 
@@ -125,7 +127,7 @@ Two rules that will save you bugs:
 
 2. **Pair LLM judges with build outcomes.** An LLM judge will rate non-compiling code as "looks right" (0.8). Always short-circuit on `buildOutcome.passed === false` before any LLM judging.
 
-## The trace model (skip on first read)
+## Trace Model
 
 Every operation emits structured spans into a `TraceStore`. A run is a tree:
 
@@ -142,7 +144,10 @@ builder-session                 [span]
 
 Spans are append-only and have stable ids — replay is reading the same store back. OTLP export ships them out for distributed tracing.
 
-You don't need to build the trace tree by hand. `BuilderSession` does it for you. Look at the trace store when you're debugging a flaky run; ignore it otherwise.
+You usually should not build this tree by hand. Product runtimes,
+`runAgentControlLoop`, harnesses, and verifiers should emit it while they run.
+Use traces when debugging a flaky run, building replay data, or explaining a
+release decision.
 
 ## Where to go next
 

package/docs/feature-guide.md

@@ -3,7 +3,7 @@
 This page explains the main `agent-eval` primitives in plain English first,
 then shows when to use each one.
 
-## ELI5
+## Overview
 
 LLM agents can write code, drafts, research, plans, and actions. The hard part
 is knowing whether they actually did a good job, whether they should keep
@@ -41,7 +41,7 @@ trying, and whether a change made them better or worse.
 
 ## Integration Patterns
 
-### Recommended Agent Product Shape
+### Recommended Product Shape
 
 Use this shape when the product needs to keep pushing work forward instead of
 only answering once:
@@ -175,21 +175,6 @@ Store as `FeedbackTrajectory`, then derive:
   logs, screenshots, or browser state. Use separate sandboxes for parallel
   variants or destructive checks.
 
-## Same-Sandbox Example
-
-`examples/same-sandbox-harness/` shows the common coding/browser pattern:
-
-```text
-one sandbox/workdir -> install/build/test -> inspect evidence -> emit judge span
-```
-
-Use this when a judge needs evidence produced by earlier harness phases. Use
-isolated sandboxes when variants run in parallel or a phase can corrupt the
-workspace.
-- Treat telemetry as evidence, not control flow. A trace sink outage should be
-  visible in `runtimeErrors`, but it should not stop the worker from completing
-  the user task.
-
 ## Highest-ROI Adoption Order
 
 1. Wrap one real product workflow in `runAgentControlLoop`.
@@ -211,3 +196,11 @@ reusable:
 
 Core should provide shapes, stores, runners, scoring, traces, and converters.
 Downstream integrations provide domain state, policy, tools, and storage.
+
+## Examples
+
+- `examples/same-sandbox-harness`: one workdir for install/build/test plus
+  evidence inspection.
+- `examples/multi-shot-optimization`: full-trajectory optimization with a
+  holdout gate.
+- `examples/benchmarks`: benchmark adapter contracts and reference wrappers.

package/docs/integration-launch-gates.md

@@ -0,0 +1,77 @@
+# Integration Launch Gates
+
+Use these gates when a product lets generated apps or agents use user-owned
+connections through an integration hub.
+
+The eval should wrap the real product path:
+
+```txt
+user prompt
+  -> product emits IntegrationManifest
+  -> platform resolves connections and grants
+  -> sandbox receives capability bundle
+  -> generated app invokes integration action
+  -> platform enforces policy, approval, idempotency, audit
+```
+
+## Deterministic Gates
+
+- The generated app declares an integration manifest before sandbox launch.
+- Manifest validation passes.
+- Required connections and scopes are present before execution.
+- Sandbox environment contains a capability bundle, not raw provider tokens.
+- Reads invoke through the platform bridge.
+- Writes return `approval_required` unless product policy explicitly allows
+  them.
+- Approved writes are bound to the same action, input hash, connection, and
+  subject.
+- Revoked grants or expired capabilities stop invocation.
+- Resumed or long-running sandboxes receive a refreshed bundle before expiry.
+- Audit includes grant creation, capability issue, invoke success/failure,
+  approval resolution, and revoke events.
+
+## Failure Classes
+
+`agent-eval` classifies integration failures separately from prompt/tool
+failures:
+
+- `bad_integration_manifest`
+- `missing_integration_connection`
+- `missing_integration_scope`
+- `integration_approval_required`
+- `integration_auth_expired`
+- `integration_provider_failure`
+- `unsafe_integration_write_denied`
+
+Use the helper payload builders and eval builders so products emit the same
+trace evidence:
+
+```ts
+const gate = {
+  connectorId: 'google-calendar',
+  actionId: 'events.create',
+  valid: true,
+  missingConnections: [],
+  missingScopes: ['calendar.events.write'],
+}
+
+const evals = integrationGateEvals(gate)
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationManifestResolvedPayload(gate),
+})
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationInvokeFailedPayload({
+    connectorId: 'google-calendar',
+    actionId: 'events.create',
+    code: 'scope_denied',
+    message: 'calendar.events.write was not granted',
+  }),
+})
+```
+
+The classifier then reports the real missing surface instead of burying the
+failure under `tool_recovery_failure` or `unknown`.

package/docs/product-eval-adoption.md

@@ -98,6 +98,23 @@ replayable eval data.
 production run -> feedback trajectory -> dataset scenario -> optimizer row
 ```
 
+For promotion-grade runs, also project the completed control result into a
+strict `RunRecord`:
+
+```ts
+const record = controlRunToRunRecord(controlResult, {
+  experimentId,
+  candidateId,
+  seed,
+  model: 'gpt-4o-2024-11-20',
+  promptHash,
+  configHash,
+  commitSha,
+  splitTag: 'holdout',
+  tokenUsage,
+})
+```
+
 ## Datasets And Holdouts
 
 Use four splits:
@@ -107,6 +124,10 @@ Use four splits:
 - `test`: normal reporting.
 - `holdout`: promotion-only gate.
 
+The low-level `RunRecord` schema uses `search | dev | holdout`; map `train`
+and normal non-holdout test/report rows to `search` when producing promotion
+tables.
+
 Do not inspect or tune against holdout failures during optimization. If a
 holdout failure reveals a real product bug, fix the bug and rotate the holdout
 set with a signed note.
@@ -149,6 +170,7 @@ A launch or promotion should require:
 - cost and latency within budget
 - no unresolved canary or contamination failures
 - trace evidence for representative successes and failures
+- TraceAnalyst findings for failure-heavy or regression-heavy corpora
 - human-readable report with failure clusters and next actions
 
 `evaluateReleaseConfidence()` and the paired statistics helpers provide the
@@ -177,6 +199,11 @@ Use `@tangle-network/agent-integrations` manifests as readiness inputs. Gate
 missing connections, missing scopes, approval-required writes, and stale tokens
 before blaming the agent prompt.
 
+For generated apps and sandbox agents, also run the
+[Integration Launch Gates](./integration-launch-gates.md). The eval should prove
+that app code invokes through the integration bridge, not provider SDKs with raw
+OAuth tokens.
+
 ### Voice Agent
 
 Record transcript, timing, interruptions, tool calls, and task outcome. Judge

package/docs/trace-analysis.md

@@ -0,0 +1,75 @@
+# Trace Analysis
+
+Trace analysis is the bridge between raw product telemetry and useful eval work.
+
+```txt
+live product run
+  -> TraceEmitter / TraceStore
+  -> TraceAnalyst investigates trace corpora
+  -> findings become ASI, failures, replay cases, and release actions
+```
+
+## When To Use TraceAnalyst
+
+Use `TraceAnalyst` when you have more than a few traces and need to answer:
+
+- which failure modes are recurring?
+- which spans explain a regression?
+- did retrieval, integrations, sandbox, or policy block the run?
+- are failed runs missing evidence that the optimizer needs?
+- which product surfaces deserve the next fix?
+
+Use summary tables and release confidence for promotion decisions. Use
+TraceAnalyst to explain the evidence behind those decisions.
+
+## Minimal Flow
+
+```ts
+import {
+  OtlpFileTraceStore,
+  analyzeTraces,
+} from '@tangle-network/agent-eval'
+
+const result = await analyzeTraces({
+  question: 'Why did app-runtime holdout runs fail this week?',
+}, {
+  source: new OtlpFileTraceStore({ path: 'traces/otlp.jsonl' }),
+  ai,
+  model: 'gpt-4o-2024-11-20',
+})
+
+console.log(result.findings)
+```
+
+Products can pass any `TraceAnalysisStore`; they do not need to use the file
+store in production.
+
+## Required Trace Shape
+
+Every serious product run should include:
+
+- `runId`, `projectId`, `scenarioId`, `variantId`, and `layer`
+- commit, prompt hash, config hash, model fingerprint, and dataset version
+- LLM spans with model, inputs, outputs, token counts, and cost
+- tool/integration spans with arguments, result summaries, and error codes
+- retrieval spans with query, source ids, hit scores, and freshness metadata
+- sandbox/build/test/deploy spans with exit codes and log artifacts
+- custom events for knowledge readiness and integration gates
+- final run outcome with pass/score/failure class
+
+Do not put secrets, raw OAuth tokens, or unredacted PII in traces.
+
+## Product Loop
+
+The product loop should not treat traces as a separate debug dump. The intended
+path is:
+
+1. Wrap the real workflow in `runAgentControlLoop` or the product runtime.
+2. Emit canonical spans/events while the user task runs.
+3. Convert the completed run to `FeedbackTrajectory` for replay.
+4. Convert promotion-grade runs to `RunRecord` with `controlRunToRunRecord`.
+5. Run TraceAnalyst over failure-heavy trace sets.
+6. Feed findings into `ActionableSideInfo`, failure clusters, and release
+   reports.
+
+That makes normal product usage become eval data instead of isolated logs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.20.11",
+  "version": "0.20.12",
   "description": "Trace-first evaluation infrastructure for agent systems: traces, harnesses, verifier pipelines, judges, datasets, gates, optimization, and reporting.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -19,6 +19,26 @@
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     },
+    "./control": {
+      "types": "./dist/control.d.ts",
+      "import": "./dist/control.js",
+      "default": "./dist/control.js"
+    },
+    "./optimization": {
+      "types": "./dist/optimization.d.ts",
+      "import": "./dist/optimization.js",
+      "default": "./dist/optimization.js"
+    },
+    "./reporting": {
+      "types": "./dist/reporting.d.ts",
+      "import": "./dist/reporting.js",
+      "default": "./dist/reporting.js"
+    },
+    "./traces": {
+      "types": "./dist/traces.d.ts",
+      "import": "./dist/traces.js",
+      "default": "./dist/traces.js"
+    },
     "./telemetry": {
       "types": "./dist/telemetry/index.d.ts",
       "import": "./dist/telemetry/index.js",

package/dist/chunk-JAOLXRIA.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n *     directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n *   const { value, result } = await callLlmJson<MyType>(\n *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n *   )\n *\n * This is THE llm-calling seam for agent-eval primitives that need structured\n * output (semantic concept judge, reviewer directives, critic scores). Primitives\n * that need free-form text use `callLlm` and parse output themselves.\n */\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n  role: 'system' | 'user' | 'assistant'\n  /**\n   * Either a plain text content string OR a multimodal content array\n   * (text + image_url parts) for vision-capable models.\n   */\n  content:\n    | string\n    | Array<\n        | { type: 'text'; text: string }\n        | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n      >\n}\n\nexport interface LlmCallRequest {\n  model: string\n  messages: LlmMessage[]\n  /** Optional JSON-mode response format (response_format: json_object). */\n  jsonMode?: boolean\n  /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n  jsonSchema?: { name: string; schema: Record<string, unknown> }\n  temperature?: number\n  maxTokens?: number\n  /** Per-call timeout, default 60s. */\n  timeoutMs?: number\n}\n\nexport interface LlmUsage {\n  promptTokens: number\n  completionTokens: number\n  totalTokens: number\n  /** Proxies populate this when prompt caching is on. */\n  cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n  /** The text content of the first choice. Empty string if none. */\n  content: string\n  usage: LlmUsage\n  /**\n   * Cost in USD. Pulled from proxy's `_response_cost` field when present;\n   * `null` when neither the proxy nor the caller can derive it.\n   */\n  costUsd: number | null\n  /** Model name actually used (echoed from response). */\n  model: string\n  /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n  durationMs: number\n  /** Raw response body. */\n  raw: Record<string, unknown>\n}\n\nexport class LlmCallError extends Error {\n  constructor(\n    message: string,\n    public readonly status: number,\n    public readonly body: string,\n    public readonly model: string,\n  ) {\n    super(message)\n    this.name = 'LlmCallError'\n  }\n}\n\nexport interface LlmClientOptions {\n  /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n  baseUrl?: string\n  /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n  apiKey?: string\n  bearer?: string\n  /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n  authHeader?: { name: string; value: string }\n  /** Default timeout in ms. Per-call can override. */\n  defaultTimeoutMs?: number\n  /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */\n  maxRetries?: number\n  /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n  fetch?: typeof fetch\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\nconst DEFAULT_TIMEOUT_MS = 60_000\nconst DEFAULT_MAX_RETRIES = 3\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\nfunction isRetryableError(err: unknown): boolean {\n  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n  if (err instanceof Error) {\n    return (\n      err.name === 'AbortError' ||\n      err.name === 'TimeoutError' ||\n      /fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN/i.test(err.message)\n    )\n  }\n  return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n  const h = headers.get('retry-after')\n  if (!h) return null\n  const asNumber = Number(h)\n  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n  const asDate = Date.parse(h)\n  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n  return null\n}\n\nfunction backoffMs(attempt: number): number {\n  // 500ms, 1s, 2s, 4s, ...\n  return Math.min(500 * Math.pow(2, attempt), 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n    Accept: 'application/json',\n  }\n  if (opts.authHeader) {\n    headers[opts.authHeader.name] = opts.authHeader.value\n  } else if (opts.bearer || opts.apiKey) {\n    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n  }\n  return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n  if (status !== 400) return false\n  const lower = body.toLowerCase()\n  return (\n    lower.includes('response_format') ||\n    lower.includes('json_schema') ||\n    lower.includes('is unavailable') ||\n    lower.includes('not supported')\n  )\n}\n\nfunction buildBody(req: LlmCallRequest, forceJsonObject: boolean): Record<string, unknown> {\n  const body: Record<string, unknown> = {\n    model: req.model,\n    messages: req.messages,\n    temperature: req.temperature ?? 0,\n  }\n  if (req.maxTokens != null) body.max_tokens = req.maxTokens\n\n  if (req.jsonSchema && !forceJsonObject) {\n    body.response_format = {\n      type: 'json_schema',\n      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n    }\n  } else if (req.jsonMode || req.jsonSchema) {\n    body.response_format = { type: 'json_object' }\n  }\n\n  return body\n}\n\nasync function sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n  const trimmed = raw.trim()\n  const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n  return m ? m[1]!.trim() : trimmed\n}\n\nexport function extractJsonPayload(raw: string): string {\n  const stripped = stripFencedJson(raw)\n  try {\n    JSON.parse(stripped)\n    return stripped\n  } catch {\n    // Continue with balanced extraction below.\n  }\n\n  const starts = [...stripped.matchAll(/[\\[{]/g)].map((match) => match.index).filter((index) => index != null)\n  for (const start of starts) {\n    const candidate = extractBalancedJson(stripped, start)\n    if (!candidate) continue\n    try {\n      JSON.parse(candidate)\n      return candidate\n    } catch {\n      // Keep scanning; earlier braces may belong to prose.\n    }\n  }\n\n  return stripped\n}\n\nfunction extractBalancedJson(input: string, start: number): string | null {\n  const opener = input[start]\n  const closer = opener === '{' ? '}' : opener === '[' ? ']' : null\n  if (!closer) return null\n\n  const stack: string[] = [closer]\n  let isInString = false\n  let isEscaped = false\n\n  for (let i = start + 1; i < input.length; i++) {\n    const char = input[i]!\n    if (isEscaped) {\n      isEscaped = false\n      continue\n    }\n    if (char === '\\\\') {\n      isEscaped = isInString\n      continue\n    }\n    if (char === '\"') {\n      isInString = !isInString\n      continue\n    }\n    if (isInString) continue\n\n    if (char === '{') stack.push('}')\n    else if (char === '[') stack.push(']')\n    else if (char === stack[stack.length - 1]) {\n      stack.pop()\n      if (stack.length === 0) return input.slice(start, i + 1)\n    }\n  }\n\n  return null\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n  const url = `${baseUrl}/chat/completions`\n  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n  const maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES\n  const fetchFn = opts.fetch ?? globalThis.fetch\n  const headers = buildHeaders(opts)\n\n  let lastErr: unknown\n  for (let attempt = 0; attempt < maxRetries; attempt++) {\n    const controller = new AbortController()\n    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n    const started = Date.now()\n\n    try {\n      const res = await fetchFn(url, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify(buildBody(req, false)),\n        signal: controller.signal,\n      })\n      clearTimeout(timeoutHandle)\n\n      if (!res.ok) {\n        const body = await res.text()\n        const err = new LlmCallError(\n          `LLM call ${res.status}: ${body.slice(0, 300)}`,\n          res.status,\n          body,\n          req.model,\n        )\n        if (RETRYABLE_STATUS.has(res.status) && attempt < maxRetries - 1) {\n          lastErr = err\n          const retryAfter = parseRetryAfter(res.headers)\n          await sleep(retryAfter ?? backoffMs(attempt))\n          continue\n        }\n        throw err\n      }\n\n      const json = (await res.json()) as Record<string, unknown>\n      const choice = (json.choices as Array<{ message?: { content?: string } }> | undefined)?.[0]\n      const usageRaw = (json.usage as Record<string, unknown> | undefined) ?? {}\n      const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n\n      return {\n        content: choice?.message?.content ?? '',\n        usage: {\n          promptTokens: Number(usageRaw.prompt_tokens ?? 0),\n          completionTokens: Number(usageRaw.completion_tokens ?? 0),\n          totalTokens: Number(usageRaw.total_tokens ?? 0),\n          cachedPromptTokens:\n            usageRaw.prompt_tokens_details &&\n            typeof usageRaw.prompt_tokens_details === 'object'\n              ? Number(\n                  (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens ?? 0,\n                )\n              : undefined,\n        },\n        costUsd: typeof costFromProxy === 'number' ? costFromProxy : null,\n        model: (json.model as string) ?? req.model,\n        durationMs: Date.now() - started,\n        raw: json,\n      }\n    } catch (err) {\n      clearTimeout(timeoutHandle)\n      lastErr = err\n      if (attempt < maxRetries - 1 && isRetryableError(err)) {\n        await sleep(backoffMs(attempt))\n        continue\n      }\n      throw err\n    }\n  }\n  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n  try {\n    const result = await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n    const value = parseJsonSafely<T>(result.content, result.model)\n    return { value, result }\n  } catch (err) {\n    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {\n      // Degrade to json_object + retry.\n      const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n      const result = await callLlm(degradedReq, opts)\n      const value = parseJsonSafely<T>(result.content, result.model)\n      return { value, result }\n    }\n    throw err\n  }\n}\n\nfunction parseJsonSafely<T>(content: string, model: string): T {\n  const stripped = extractJsonPayload(content)\n  try {\n    return JSON.parse(stripped) as T\n  } catch (err) {\n    throw new Error(\n      `LLM returned non-JSON content (model=${model}): ${\n        err instanceof Error ? err.message : String(err)\n      }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n    )\n  }\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n  model: string,\n  opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n  const start = Date.now()\n  try {\n    await callLlm(\n      {\n        model,\n        messages: [{ role: 'user', content: 'ping' }],\n        maxTokens: 64,\n        timeoutMs: opts.timeoutMs ?? 30_000,\n      },\n      opts,\n    )\n    return { ok: true, latencyMs: Date.now() - start, error: null }\n  } catch (err) {\n    return {\n      ok: false,\n      latencyMs: Date.now() - start,\n      error: err instanceof Error ? err.message : String(err),\n    }\n  }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n  constructor(private readonly opts: LlmClientOptions = {}) {}\n\n  call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n    return callLlm(req, { ...this.opts, ...per })\n  }\n\n  callJson<T = unknown>(\n    req: LlmCallRequest,\n    per?: LlmClientOptions,\n  ): Promise<{ value: T; result: LlmCallResult }> {\n    return callLlmJson<T>(req, { ...this.opts, ...per })\n  }\n}\n"],"mappings":";AA4EO,IAAM,eAAN,cAA2B,MAAM;AAAA,EACtC,YACE,SACgB,QACA,MACA,OAChB;AACA,UAAM,OAAO;AAJG;AACA;AACA;AAGhB,SAAK,OAAO;AAAA,EACd;AAAA,EANkB;AAAA,EACA;AAAA,EACA;AAKpB;AAoBA,IAAM,mBAAmB;AACzB,IAAM,qBAAqB;AAC3B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAErD,SAAS,iBAAiB,KAAuB;AAC/C,MAAI,eAAe,aAAc,QAAO,iBAAiB,IAAI,IAAI,MAAM;AACvE,MAAI,eAAe,OAAO;AACxB,WACE,IAAI,SAAS,gBACb,IAAI,SAAS,kBACb,+CAA+C,KAAK,IAAI,OAAO;AAAA,EAEnE;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;AACxD,QAAM,IAAI,QAAQ,IAAI,aAAa;AACnC,MAAI,CAAC,EAAG,QAAO;AACf,QAAM,WAAW,OAAO,CAAC;AACzB,MAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,EAAG,QAAO,WAAW;AACjE,QAAM,SAAS,KAAK,MAAM,CAAC;AAC3B,MAAI,OAAO,SAAS,MAAM,EAAG,QAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;AACnE,SAAO;AACT;AAEA,SAAS,UAAU,SAAyB;AAE1C,SAAO,KAAK,IAAI,MAAM,KAAK,IAAI,GAAG,OAAO,GAAG,IAAM;AACpD;AAEA,SAAS,aAAa,MAAgD;AACpE,QAAM,UAAkC;AAAA,IACtC,gBAAgB;AAAA,IAChB,QAAQ;AAAA,EACV;AACA,MAAI,KAAK,YAAY;AACnB,YAAQ,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW;AAAA,EAClD,WAAW,KAAK,UAAU,KAAK,QAAQ;AACrC,YAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK,MAAM;AAAA,EAC9D;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;AAChE,MAAI,WAAW,IAAK,QAAO;AAC3B,QAAM,QAAQ,KAAK,YAAY;AAC/B,SACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,UAAU,KAAqB,iBAAmD;AACzF,QAAM,OAAgC;AAAA,IACpC,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,aAAa,IAAI,eAAe;AAAA,EAClC;AACA,MAAI,IAAI,aAAa,KAAM,MAAK,aAAa,IAAI;AAEjD,MAAI,IAAI,cAAc,CAAC,iBAAiB;AACtC,SAAK,kBAAkB;AAAA,MACrB,MAAM;AAAA,MACN,aAAa,EAAE,MAAM,IAAI,WAAW,MAAM,QAAQ,IAAI,WAAW,QAAQ,QAAQ,KAAK;AAAA,IACxF;AAAA,EACF,WAAW,IAAI,YAAY,IAAI,YAAY;AACzC,SAAK,kBAAkB,EAAE,MAAM,cAAc;AAAA,EAC/C;AAEA,SAAO;AACT;AAEA,eAAe,MAAM,IAA2B;AAC9C,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AASO,SAAS,gBAAgB,KAAqB;AACnD,QAAM,UAAU,IAAI,KAAK;AACzB,QAAM,IAAI,QAAQ,MAAM,yCAAyC;AACjE,SAAO,IAAI,EAAE,CAAC,EAAG,KAAK,IAAI;AAC5B;AAEO,SAAS,mBAAmB,KAAqB;AACtD,QAAM,WAAW,gBAAgB,GAAG;AACpC,MAAI;AACF,SAAK,MAAM,QAAQ;AACnB,WAAO;AAAA,EACT,QAAQ;AAAA,EAER;AAEA,QAAM,SAAS,CAAC,GAAG,SAAS,SAAS,QAAQ,CAAC,EAAE,IAAI,CAAC,UAAU,MAAM,KAAK,EAAE,OAAO,CAAC,UAAU,SAAS,IAAI;AAC3G,aAAW,SAAS,QAAQ;AAC1B,UAAM,YAAY,oBAAoB,UAAU,KAAK;AACrD,QAAI,CAAC,UAAW;AAChB,QAAI;AACF,WAAK,MAAM,SAAS;AACpB,aAAO;AAAA,IACT,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,OAA8B;AACxE,QAAM,SAAS,MAAM,KAAK;AAC1B,QAAM,SAAS,WAAW,MAAM,MAAM,WAAW,MAAM,MAAM;AAC7D,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAkB,CAAC,MAAM;AAC/B,MAAI,aAAa;AACjB,MAAI,YAAY;AAEhB,WAAS,IAAI,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAK;AAC7C,UAAM,OAAO,MAAM,CAAC;AACpB,QAAI,WAAW;AACb,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,MAAM;AACjB,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,SAAS,KAAK;AAChB,mBAAa,CAAC;AACd;AAAA,IACF;AACA,QAAI,WAAY;AAEhB,QAAI,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aACvB,SAAS,IAAK,OAAM,KAAK,GAAG;AAAA,aAC5B,SAAS,MAAM,MAAM,SAAS,CAAC,GAAG;AACzC,YAAM,IAAI;AACV,UAAI,MAAM,WAAW,EAAG,QAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO;AACT;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;AACxB,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AACrE,QAAM,MAAM,GAAG,OAAO;AACtB,QAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;AAC5D,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,UAAU,KAAK,SAAS,WAAW;AACzC,QAAM,UAAU,aAAa,IAAI;AAEjC,MAAI;AACJ,WAAS,UAAU,GAAG,UAAU,YAAY,WAAW;AACrD,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,gBAAgB,WAAW,MAAM,WAAW,MAAM,GAAG,SAAS;AACpE,UAAM,UAAU,KAAK,IAAI;AAEzB,QAAI;AACF,YAAM,MAAM,MAAM,QAAQ,KAAK;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU,UAAU,KAAK,KAAK,CAAC;AAAA,QAC1C,QAAQ,WAAW;AAAA,MACrB,CAAC;AACD,mBAAa,aAAa;AAE1B,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,cAAM,MAAM,IAAI;AAAA,UACd,YAAY,IAAI,MAAM,KAAK,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA,UAC7C,IAAI;AAAA,UACJ;AAAA,UACA,IAAI;AAAA,QACN;AACA,YAAI,iBAAiB,IAAI,IAAI,MAAM,KAAK,UAAU,aAAa,GAAG;AAChE,oBAAU;AACV,gBAAM,aAAa,gBAAgB,IAAI,OAAO;AAC9C,gBAAM,MAAM,cAAc,UAAU,OAAO,CAAC;AAC5C;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAEA,YAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,YAAM,SAAU,KAAK,UAAoE,CAAC;AAC1F,YAAM,WAAY,KAAK,SAAiD,CAAC;AACzE,YAAM,gBAAiB,KAAK,kBAAkB,KAAK;AAEnD,aAAO;AAAA,QACL,SAAS,QAAQ,SAAS,WAAW;AAAA,QACrC,OAAO;AAAA,UACL,cAAc,OAAO,SAAS,iBAAiB,CAAC;AAAA,UAChD,kBAAkB,OAAO,SAAS,qBAAqB,CAAC;AAAA,UACxD,aAAa,OAAO,SAAS,gBAAgB,CAAC;AAAA,UAC9C,oBACE,SAAS,yBACT,OAAO,SAAS,0BAA0B,WACtC;AAAA,YACG,SAAS,sBAAkD,iBAAiB;AAAA,UAC/E,IACA;AAAA,QACR;AAAA,QACA,SAAS,OAAO,kBAAkB,WAAW,gBAAgB;AAAA,QAC7D,OAAQ,KAAK,SAAoB,IAAI;AAAA,QACrC,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB,KAAK;AAAA,MACP;AAAA,IACF,SAAS,KAAK;AACZ,mBAAa,aAAa;AAC1B,gBAAU;AACV,UAAI,UAAU,aAAa,KAAK,iBAAiB,GAAG,GAAG;AACrD,cAAM,MAAM,UAAU,OAAO,CAAC;AAC9B;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AACA,QAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;AAC9C,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,EAAE,GAAG,KAAK,UAAU,IAAI,YAAY,CAAC,IAAI,WAAW,GAAG,IAAI;AACxF,UAAM,QAAQ,gBAAmB,OAAO,SAAS,OAAO,KAAK;AAC7D,WAAO,EAAE,OAAO,OAAO;AAAA,EACzB,SAAS,KAAK;AACZ,QAAI,eAAe,gBAAgB,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KAAK,IAAI,YAAY;AAE5F,YAAM,cAA8B,EAAE,GAAG,KAAK,UAAU,MAAM,YAAY,OAAU;AACpF,YAAM,SAAS,MAAM,QAAQ,aAAa,IAAI;AAC9C,YAAM,QAAQ,gBAAmB,OAAO,SAAS,OAAO,KAAK;AAC7D,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,gBAAmB,SAAiB,OAAkB;AAC7D,QAAM,WAAW,mBAAmB,OAAO;AAC3C,MAAI;AACF,WAAO,KAAK,MAAM,QAAQ;AAAA,EAC5B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,wCAAwC,KAAK,MAC3C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CACjD;AAAA;AAAA,EAA0B,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA,IACjD;AAAA,EACF;AACF;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;AACnE,QAAM,QAAQ,KAAK,IAAI;AACvB,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,QACE;AAAA,QACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,QAC5C,WAAW;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,WAAW,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK;AAAA,EAChE,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,WAAW,KAAK,IAAI,IAAI;AAAA,MACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;AAOO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAA6B,OAAyB,CAAC,GAAG;AAA7B;AAAA,EAA8B;AAAA,EAA9B;AAAA,EAE7B,KAAK,KAAqB,KAAgD;AACxE,WAAO,QAAQ,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EAC9C;AAAA,EAEA,SACE,KACA,KAC8C;AAC9C,WAAO,YAAe,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EACrD;AACF;","names":[]}