@tangle-network/agent-eval 0.20.0 → 0.20.3

package/dist/wire/index.js

@@ -24,8 +24,8 @@ import {
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "../chunk-OZPRSK4A.js";
-import "../chunk-ITN4YOZY.js";
+} from "../chunk-CJJSB6ZQ.js";
+import "../chunk-JAOLXRIA.js";
 import "../chunk-PZ5AY32C.js";
 export {
   BUILTIN_RUBRICS,

package/examples/benchmarks/README.md

@@ -0,0 +1,44 @@
+# Example benchmark wrappers
+
+Reference implementations of `BenchmarkAdapter` for two public benchmarks. They are NOT bundled — they're intentionally shipped as source you read, copy, and adapt.
+
+| Wrapper | What it does | Why it's an example, not core |
+|---|---|---|
+| [`gsm8k/`](./gsm8k) | Exact-match grading on the final numeric answer of GSM8K (Cobbe et al.) | The dataset isn't ours and isn't bundled. The wrapper points to a local JSONL via `AGENT_EVAL_GSM8K_PATH`. |
+| [`swebench-lite/`](./swebench-lite) | Pass/fail grading via an external SWE-Bench grader command | The grader is a separate binary; the wrapper stubs the integration via `AGENT_EVAL_SWEBENCH_GRADER_CMD`. |
+
+The novel benchmark we ship and own — the synthetic routing task — lives in `src/benchmarks/routing/` and IS in the bundle.
+
+## Using these wrappers
+
+Two paths.
+
+**Option A — read and inline.** Copy the wrapper file into your project. Replace the import paths from `../../../src/benchmarks/types` and `../../../src/run-record` with `@tangle-network/agent-eval`. Done.
+
+**Option B — import from agent-eval source.** If your project sits in this monorepo (or you've cloned the repo), import directly:
+
+```ts
+import * as gsm8k from '@tangle-network/agent-eval/examples/benchmarks/gsm8k'
+```
+
+This requires adding `examples/**/*.ts` to your TypeScript paths. Easier to just copy.
+
+## What every BenchmarkAdapter exports
+
+```ts
+loadDataset(split: 'search' | 'dev' | 'holdout'): Promise<DatasetItem[]>
+evaluate(item, response): Promise<{ score: number, raw: Record<string, unknown> }>
+assignSplit(itemId: string): 'search' | 'dev' | 'holdout'
+```
+
+`assignSplit` uses `deterministicSplit(itemId, BENCHMARK_SPLIT_SEED)` — same item gets the same split everywhere. Don't change the seed; it's load-bearing for reproducibility.
+
+## Adding a new benchmark
+
+1. Create `examples/benchmarks/<your-benchmark>/index.ts`.
+2. Export `loadDataset`, `evaluate`, `assignSplit`. Optionally a typed `Adapter` class.
+3. Use `deterministicSplit` from `@tangle-network/agent-eval` for split assignment.
+4. Fail loud on missing config (env vars, paths). Never default to silent-pass.
+5. Document config requirements in a per-benchmark README.
+
+If your benchmark is novel and broadly useful, propose moving it into `src/benchmarks/` as core surface (PR welcome). The bar is: novel rubric, reusable across projects, low maintenance burden.

package/examples/benchmarks/gsm8k/index.ts

@@ -0,0 +1,126 @@
+/**
+ * GSM8K wrapper — exact-match grading on the final numeric answer.
+ *
+ * The dataset itself is NOT bundled. `loadDataset` will:
+ *   1. read from `process.env.AGENT_EVAL_GSM8K_PATH` if set (a JSONL
+ *      file with `{ id, question, answer }` records — the standard
+ *      HF mirror layout converted to JSONL);
+ *   2. otherwise throw a clearly-marked error pointing to the loader.
+ *
+ * `evaluate` parses the final number out of the response (last
+ * occurrence of a signed-decimal-or-integer literal, optionally after
+ * `####`, the GSM8K answer convention) and compares to the ground-
+ * truth integer. Floating-point comparisons use a 1e-6 tolerance.
+ */
+
+import { existsSync, readFileSync } from 'node:fs'
+
+import type {
+  BenchmarkAdapter,
+  BenchmarkDatasetItem,
+  BenchmarkEvaluation,
+} from '../../../src/benchmarks/types'
+import { deterministicSplit } from '../../../src/benchmarks/types'
+import type { RunSplitTag } from '../../../src/run-record'
+
+export interface Gsm8kPayload {
+  question: string
+  /** Reference answer, post-#### normalization. May be a number or
+   *  a numeric string ("72", "1.5"). */
+  answer: string
+}
+
+export type Gsm8kItem = BenchmarkDatasetItem<Gsm8kPayload>
+
+class Gsm8kAdapter implements BenchmarkAdapter<Gsm8kItem, Gsm8kPayload> {
+  async loadDataset(split: RunSplitTag): Promise<Gsm8kItem[]> {
+    const path = process.env.AGENT_EVAL_GSM8K_PATH
+    if (!path) {
+      throw new Error(
+        'GSM8K dataset not provided. Set AGENT_EVAL_GSM8K_PATH to a JSONL file ' +
+          'with {id, question, answer} records (the HF GSM8K mirror converted to JSONL).',
+      )
+    }
+    if (!existsSync(path)) {
+      throw new Error(`AGENT_EVAL_GSM8K_PATH=${path} does not exist`)
+    }
+    const items = parseJsonl(path).filter((it) => assignSplitImpl(it.id) === split)
+    return items
+  }
+
+  async evaluate(item: Gsm8kItem, response: string): Promise<BenchmarkEvaluation> {
+    const expected = parseGsm8kAnswer(item.payload.answer)
+    const observed = parseGsm8kAnswer(response)
+    if (expected === null) {
+      // Defensive: the dataset should never ship a non-numeric ref.
+      return { score: 0, raw: { reason: 'reference_not_numeric', expected: item.payload.answer } }
+    }
+    if (observed === null) {
+      return { score: 0, raw: { reason: 'no_numeric_in_response', expected, observed: null } }
+    }
+    const ok = Math.abs(expected - observed) < 1e-6
+    return { score: ok ? 1 : 0, raw: { expected, observed, exactMatch: ok } }
+  }
+
+  assignSplit(itemId: string): RunSplitTag {
+    return assignSplitImpl(itemId)
+  }
+}
+
+function assignSplitImpl(itemId: string): RunSplitTag {
+  return deterministicSplit(`gsm8k::${itemId}`)
+}
+
+function parseJsonl(path: string): Gsm8kItem[] {
+  const raw = readFileSync(path, 'utf8')
+  const out: Gsm8kItem[] = []
+  let lineNo = 0
+  for (const line of raw.split('\n')) {
+    lineNo++
+    const trimmed = line.trim()
+    if (!trimmed) continue
+    let row: Record<string, unknown>
+    try {
+      row = JSON.parse(trimmed) as Record<string, unknown>
+    } catch (e) {
+      throw new Error(`GSM8K JSONL parse error at line ${lineNo}: ${(e as Error).message}`)
+    }
+    const id = String(row.id ?? `gsm8k_${lineNo}`)
+    const question = String(row.question ?? '')
+    const answer = String(row.answer ?? '')
+    if (!question || !answer) {
+      throw new Error(`GSM8K JSONL line ${lineNo} missing question/answer`)
+    }
+    out.push({ id, payload: { question, answer } })
+  }
+  return out
+}
+
+/**
+ * Parse a GSM8K-style answer. Honors the dataset's `#### N`
+ * convention (the canonical answer comes after `####`); otherwise
+ * returns the LAST signed numeric literal in the string.
+ */
+export function parseGsm8kAnswer(text: string): number | null {
+  if (!text) return null
+  const afterMarker = text.match(/####\s*(-?\d[\d,]*\.?\d*)/)
+  if (afterMarker) {
+    const cleaned = afterMarker[1]!.replace(/,/g, '')
+    const v = Number(cleaned)
+    if (Number.isFinite(v)) return v
+  }
+  // Last numeric literal anywhere in the string.
+  const matches = text.match(/-?\d[\d,]*\.?\d*/g)
+  if (!matches || matches.length === 0) return null
+  const last = matches[matches.length - 1]!
+  const cleaned = last.replace(/,/g, '')
+  const v = Number(cleaned)
+  return Number.isFinite(v) ? v : null
+}
+
+const adapter = new Gsm8kAdapter()
+
+export const loadDataset = adapter.loadDataset.bind(adapter)
+export const evaluate = adapter.evaluate.bind(adapter)
+export const assignSplit = adapter.assignSplit.bind(adapter)
+export { Gsm8kAdapter }

package/examples/benchmarks/swebench-lite/index.ts

@@ -0,0 +1,178 @@
+/**
+ * SWE-Bench Lite wrapper — 30-instance subset.
+ *
+ * Status: STUB. The actual SWE-Bench harness needs a Docker host and
+ * is too heavy to ship inside this package. We expose the contract
+ * (loadDataset, evaluate, assignSplit) so consumers can plug in their
+ * own grader without touching call sites.
+ *
+ * Wire-up paths in priority order:
+ *
+ *   1. `process.env.AGENT_EVAL_SWEBENCH_PATH` → JSONL with the 30
+ *      lite instances + per-instance metadata (instance_id,
+ *      problem_statement, base_commit, repo, FAIL_TO_PASS,
+ *      PASS_TO_PASS).
+ *   2. `process.env.AGENT_EVAL_SWEBENCH_GRADER_CMD` → executable
+ *      that reads `{instance_id, patch}` JSON on stdin and writes
+ *      `{passed, fail_to_pass_passed, pass_to_pass_passed, log}`
+ *      JSON on stdout. Implementations can shell out to the
+ *      official `swebench` runner here.
+ *
+ * If neither is set, every public method throws a clearly-marked
+ * "not implemented" error. The stub fails LOUD; it never silently
+ * scores zero.
+ */
+
+import { existsSync, readFileSync } from 'node:fs'
+import { spawn } from 'node:child_process'
+
+import type {
+  BenchmarkAdapter,
+  BenchmarkDatasetItem,
+  BenchmarkEvaluation,
+} from '../../../src/benchmarks/types'
+import { deterministicSplit } from '../../../src/benchmarks/types'
+import type { RunSplitTag } from '../../../src/run-record'
+
+export interface SweBenchLitePayload {
+  instanceId: string
+  problemStatement: string
+  baseCommit: string
+  repo: string
+  failToPass: string[]
+  passToPass: string[]
+}
+
+export type SweBenchLiteItem = BenchmarkDatasetItem<SweBenchLitePayload>
+
+class SweBenchLiteAdapter
+  implements BenchmarkAdapter<SweBenchLiteItem, SweBenchLitePayload>
+{
+  async loadDataset(split: RunSplitTag): Promise<SweBenchLiteItem[]> {
+    const path = process.env.AGENT_EVAL_SWEBENCH_PATH
+    if (!path) {
+      throw new Error(
+        'SWE-Bench Lite dataset not provided. Set AGENT_EVAL_SWEBENCH_PATH to a JSONL file ' +
+          'with the 30 lite instances. STUB: this wrapper does not bundle the dataset; ' +
+          'see https://www.swebench.com/lite.html for the canonical source.',
+      )
+    }
+    if (!existsSync(path)) {
+      throw new Error(`AGENT_EVAL_SWEBENCH_PATH=${path} does not exist`)
+    }
+    const all = parseJsonl(path)
+    return all.filter((it) => assignSplitImpl(it.id) === split)
+  }
+
+  async evaluate(item: SweBenchLiteItem, response: string): Promise<BenchmarkEvaluation> {
+    const cmd = process.env.AGENT_EVAL_SWEBENCH_GRADER_CMD
+    if (!cmd) {
+      throw new Error(
+        'SWE-Bench Lite grader not configured. Set AGENT_EVAL_SWEBENCH_GRADER_CMD to an ' +
+          'executable that reads {instance_id, patch} JSON on stdin and writes ' +
+          '{passed, fail_to_pass_passed, pass_to_pass_passed, log} JSON on stdout. ' +
+          'TODO(swebench-lite): bundle a default Docker-based runner once the SDK ' +
+          'stabilises (https://github.com/swe-bench/SWE-bench).',
+      )
+    }
+    const stdinPayload = JSON.stringify({ instance_id: item.payload.instanceId, patch: response })
+    const result = await runGrader(cmd, stdinPayload)
+    let parsed: Record<string, unknown>
+    try {
+      parsed = JSON.parse(result.stdout) as Record<string, unknown>
+    } catch (e) {
+      throw new Error(
+        `SWE-Bench grader emitted non-JSON stdout: ${(e as Error).message}\n` +
+          `stdout=${result.stdout.slice(0, 400)}\nstderr=${result.stderr.slice(0, 400)}`,
+      )
+    }
+    const passed = Boolean(parsed.passed)
+    return {
+      score: passed ? 1 : 0,
+      raw: {
+        passed,
+        failToPassPassed: Boolean(parsed.fail_to_pass_passed),
+        passToPassPassed: Boolean(parsed.pass_to_pass_passed),
+        graderLog: typeof parsed.log === 'string' ? parsed.log.slice(0, 4000) : '',
+      },
+    }
+  }
+
+  assignSplit(itemId: string): RunSplitTag {
+    return assignSplitImpl(itemId)
+  }
+}
+
+function assignSplitImpl(itemId: string): RunSplitTag {
+  return deterministicSplit(`swebench-lite::${itemId}`)
+}
+
+function parseJsonl(path: string): SweBenchLiteItem[] {
+  const raw = readFileSync(path, 'utf8')
+  const out: SweBenchLiteItem[] = []
+  let lineNo = 0
+  for (const line of raw.split('\n')) {
+    lineNo++
+    const trimmed = line.trim()
+    if (!trimmed) continue
+    const row = JSON.parse(trimmed) as Record<string, unknown>
+    const instanceId = String(row.instance_id ?? row.instanceId ?? '')
+    if (!instanceId) {
+      throw new Error(`swebench-lite line ${lineNo} missing instance_id`)
+    }
+    out.push({
+      id: instanceId,
+      payload: {
+        instanceId,
+        problemStatement: String(row.problem_statement ?? row.problemStatement ?? ''),
+        baseCommit: String(row.base_commit ?? row.baseCommit ?? ''),
+        repo: String(row.repo ?? ''),
+        failToPass: asStringArray(row.FAIL_TO_PASS ?? row.failToPass),
+        passToPass: asStringArray(row.PASS_TO_PASS ?? row.passToPass),
+      },
+    })
+  }
+  return out
+}
+
+function asStringArray(v: unknown): string[] {
+  if (Array.isArray(v)) return v.filter((x): x is string => typeof x === 'string')
+  if (typeof v === 'string') {
+    try {
+      const parsed = JSON.parse(v)
+      if (Array.isArray(parsed)) return parsed.filter((x): x is string => typeof x === 'string')
+    } catch {
+      // Plain string; treat as a single-element list.
+      return [v]
+    }
+  }
+  return []
+}
+
+function runGrader(cmd: string, stdin: string): Promise<{ stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    const parts = cmd.split(/\s+/)
+    const child = spawn(parts[0]!, parts.slice(1), { stdio: ['pipe', 'pipe', 'pipe'] })
+    let stdout = ''
+    let stderr = ''
+    child.stdout.on('data', (b: Buffer) => (stdout += b.toString('utf8')))
+    child.stderr.on('data', (b: Buffer) => (stderr += b.toString('utf8')))
+    child.on('error', reject)
+    child.on('close', (code) => {
+      if (code !== 0) {
+        reject(new Error(`grader exited with code ${code}: ${stderr.slice(0, 400)}`))
+        return
+      }
+      resolve({ stdout, stderr })
+    })
+    child.stdin.write(stdin)
+    child.stdin.end()
+  })
+}
+
+const adapter = new SweBenchLiteAdapter()
+
+export const loadDataset = adapter.loadDataset.bind(adapter)
+export const evaluate = adapter.evaluate.bind(adapter)
+export const assignSplit = adapter.assignSplit.bind(adapter)
+export { SweBenchLiteAdapter }

package/examples/multi-shot-optimization/index.ts

@@ -0,0 +1,114 @@
+import {
+  runMultiShotOptimization,
+  trialTraceFromMultiShotTrial,
+  type MultiShotVariant,
+  type RunRecord,
+} from '@tangle-network/agent-eval'
+
+type Payload = {
+  instruction: string
+  quality: number
+}
+
+const baseline: MultiShotVariant<Payload> = {
+  id: 'baseline',
+  label: 'baseline',
+  generation: 0,
+  payload: {
+    instruction: 'Complete the user task.',
+    quality: 0.45,
+  },
+}
+
+const result = await runMultiShotOptimization<Payload>({
+  runId: 'demo-multi-shot',
+  target: 'demo-agent-system-prompt',
+  seedVariants: [baseline],
+  searchScenarioIds: ['search-brief', 'search-code-review', 'search-research'],
+  reps: 1,
+  generations: 2,
+  populationSize: 2,
+  scoreConcurrency: 2,
+  runner: {
+    async run({ variant, scenarioId }) {
+      return {
+        trace: {
+          scenarioId,
+          turns: [
+            { role: 'user', content: `Run ${scenarioId}` },
+            { role: 'assistant', content: `${variant.payload.instruction} quality=${variant.payload.quality}` },
+          ],
+          output: `quality=${variant.payload.quality}`,
+        },
+        costUsd: 0.01,
+        durationMs: 50,
+      }
+    },
+  },
+  scorer: {
+    async score({ variant }) {
+      return {
+        score: variant.payload.quality,
+        ok: true,
+        asi: variant.payload.quality >= 0.8
+          ? []
+          : [{
+              expectationId: 'complete-task',
+              message: 'The agent did not fully complete the task.',
+              severity: 'error',
+              responsibleSurface: 'system-prompt',
+              suggestion: 'Make completion criteria explicit before final response.',
+            }],
+      }
+    },
+  },
+  mutateAdapter: {
+    async mutate({ parent, bottomTrials, childCount, generation }) {
+      const traces = bottomTrials.map((trial) => trialTraceFromMultiShotTrial(trial))
+      const rationale = traces.flatMap((trace) => (trace.expectations ?? []).map((e) => e.phrase)).join('\n')
+      return Array.from({ length: childCount }, (_, i) => ({
+        id: `${parent.id}.g${generation}.${i}`,
+        label: 'completion-focused',
+        generation,
+        payload: {
+          instruction: `${parent.payload.instruction} Verify every requested step before final answer.`,
+          quality: 0.9,
+        },
+        rationale,
+      }))
+    },
+  },
+  gate: {
+    holdoutScenarioIds: ['holdout-brief', 'holdout-code-review', 'holdout-research'],
+    gate: {
+      baselineKey: 'baseline',
+      minProductiveRuns: 3,
+      pairedDeltaThreshold: 0,
+      seed: 7,
+    },
+    toRunRecord: ({ variant, scenarioId, rep, split, seed, trial }): RunRecord => ({
+      runId: `demo-${variant.id}-${scenarioId}-${rep}-${split}`,
+      experimentId: scenarioId,
+      candidateId: variant.id,
+      seed,
+      model: 'demo-model@2026-01-01',
+      promptHash: 'p'.repeat(64),
+      configHash: 'c'.repeat(64),
+      commitSha: 'deadbeef',
+      wallMs: trial.durationMs ?? 0,
+      costUsd: trial.cost ?? 0,
+      tokenUsage: { input: 1, output: 1 },
+      outcome: {
+        [split === 'holdout' ? 'holdoutScore' : 'searchScore']: trial.score,
+        raw: { score: trial.score },
+      },
+      splitTag: split,
+    }),
+  },
+})
+
+console.log({
+  searchBest: result.searchBestVariant.id,
+  promoted: result.promotedVariant.id,
+  gate: result.gate?.decision ?? null,
+})

package/examples/same-sandbox-harness/index.ts

@@ -0,0 +1,63 @@
+import {
+  InMemoryTraceStore,
+  SandboxHarness,
+  SubprocessSandboxDriver,
+  TraceEmitter,
+} from '@tangle-network/agent-eval'
+
+/**
+ * Same-sandbox pattern:
+ * - one driver owns one workdir
+ * - the harness runs setup/build/test there
+ * - later checks can inspect files/logs/screenshots produced by those phases
+ *
+ * Replace `workdir` with a generated app, browser automation checkout, or
+ * remote computer-use workspace.
+ */
+export async function runSameSandboxExample(workdir: string) {
+  const store = new InMemoryTraceStore()
+  const driver = new SubprocessSandboxDriver({ cwd: workdir })
+  const harness = new SandboxHarness(driver)
+  const emitter = new TraceEmitter(store)
+  await emitter.startRun({
+    scenarioId: 'same-sandbox-example',
+    layer: 'app-build',
+  })
+
+  const result = await harness.run({
+    setupCommand: 'pnpm install --frozen-lockfile',
+    runCommand: 'pnpm build',
+    testCommand: 'pnpm test',
+    timeoutMs: 180_000,
+  }, emitter)
+
+  const summary = [
+    `passed=${result.passed}`,
+    `score=${result.score}`,
+    `build=${result.run?.exitCode ?? 'not-run'}`,
+    `test=${result.test?.exitCode ?? 'not-run'}`,
+    result.test?.stdout?.slice(-2000) ?? '',
+  ].join('\n')
+
+  const judged = {
+    score: result.passed && summary.includes('test=0') ? 1 : 0,
+    rationale: result.passed
+      ? 'Shared sandbox produced passing build/test evidence.'
+      : 'Shared sandbox did not produce passing build/test evidence.',
+  }
+  await emitter.recordJudge({
+    judgeId: 'same-sandbox-evidence',
+    name: 'same-sandbox-evidence',
+    dimension: 'evidence',
+    score: judged.score,
+    rationale: judged.rationale,
+    evidence: summary,
+  })
+  await emitter.endRun({
+    pass: result.passed,
+    score: result.score,
+    notes: judged.rationale,
+  })
+
+  return { result, judged, traces: await store.listRuns() }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.20.0",
-  "description": "Trace-first evaluation framework for Tangle agents. Core (spans, pipelines, sandbox harness, OTLP export), trust (dataset, red-team, calibration, behavior DSL), builder-of-builders (three-layer eval, resumable sessions, meta-runtime correlation), and frontier (meta-eval correlation study, Process Reward Modeling, bisector).",
+  "version": "0.20.3",
+  "description": "Trace-first evaluation infrastructure for agent systems: traces, harnesses, verifier pipelines, judges, datasets, gates, optimization, and reporting.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
     "type": "git",
@@ -40,7 +40,8 @@
   },
   "files": [
     "dist",
-    "docs"
+    "docs",
+    "examples"
   ],
   "publishConfig": {
     "access": "public"

package/dist/chunk-ITN4YOZY.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/llm-client.ts"],"sourcesContent":["/**\n * LLM client with graceful degrade.\n *\n * OpenAI-compatible `/v1/chat/completions` client with:\n *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).\n *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).\n *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.\n *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.\n *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI\n *     directly, cli-bridge subscriptions, and any router that speaks the spec.\n *\n * Usage:\n *   const { value, result } = await callLlmJson<MyType>(\n *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },\n *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },\n *   )\n *\n * This is THE llm-calling seam for agent-eval primitives that need structured\n * output (semantic concept judge, reviewer directives, critic scores). Primitives\n * that need free-form text use `callLlm` and parse output themselves.\n */\n\n// ─── Types ──────────────────────────────────────────────────────────────\n\nexport interface LlmMessage {\n  role: 'system' | 'user' | 'assistant'\n  /**\n   * Either a plain text content string OR a multimodal content array\n   * (text + image_url parts) for vision-capable models.\n   */\n  content:\n    | string\n    | Array<\n        | { type: 'text'; text: string }\n        | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }\n      >\n}\n\nexport interface LlmCallRequest {\n  model: string\n  messages: LlmMessage[]\n  /** Optional JSON-mode response format (response_format: json_object). */\n  jsonMode?: boolean\n  /** Optional structured output via JSON Schema. Falls back to json_object on 400. */\n  jsonSchema?: { name: string; schema: Record<string, unknown> }\n  temperature?: number\n  maxTokens?: number\n  /** Per-call timeout, default 60s. */\n  timeoutMs?: number\n}\n\nexport interface LlmUsage {\n  promptTokens: number\n  completionTokens: number\n  totalTokens: number\n  /** Proxies populate this when prompt caching is on. */\n  cachedPromptTokens?: number\n}\n\nexport interface LlmCallResult {\n  /** The text content of the first choice. Empty string if none. */\n  content: string\n  usage: LlmUsage\n  /**\n   * Cost in USD. Pulled from proxy's `_response_cost` field when present;\n   * `null` when neither the proxy nor the caller can derive it.\n   */\n  costUsd: number | null\n  /** Model name actually used (echoed from response). */\n  model: string\n  /** Wall-clock duration of the HTTP call (last attempt, if retried). */\n  durationMs: number\n  /** Raw response body. */\n  raw: Record<string, unknown>\n}\n\nexport class LlmCallError extends Error {\n  constructor(\n    message: string,\n    public readonly status: number,\n    public readonly body: string,\n    public readonly model: string,\n  ) {\n    super(message)\n    this.name = 'LlmCallError'\n  }\n}\n\nexport interface LlmClientOptions {\n  /** Base URL (without trailing slash). Must end at the `/v1` prefix. */\n  baseUrl?: string\n  /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */\n  apiKey?: string\n  bearer?: string\n  /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */\n  authHeader?: { name: string; value: string }\n  /** Default timeout in ms. Per-call can override. */\n  defaultTimeoutMs?: number\n  /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */\n  maxRetries?: number\n  /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */\n  fetch?: typeof fetch\n}\n\n// ─── Internals ──────────────────────────────────────────────────────────\n\nconst DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'\nconst DEFAULT_TIMEOUT_MS = 60_000\nconst DEFAULT_MAX_RETRIES = 3\n\nconst RETRYABLE_STATUS = new Set([429, 502, 503, 504])\n\nfunction isRetryableError(err: unknown): boolean {\n  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status)\n  if (err instanceof Error) {\n    return (\n      err.name === 'AbortError' ||\n      err.name === 'TimeoutError' ||\n      /fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN/i.test(err.message)\n    )\n  }\n  return false\n}\n\nfunction parseRetryAfter(headers: Headers): number | null {\n  const h = headers.get('retry-after')\n  if (!h) return null\n  const asNumber = Number(h)\n  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1000\n  const asDate = Date.parse(h)\n  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now())\n  return null\n}\n\nfunction backoffMs(attempt: number): number {\n  // 500ms, 1s, 2s, 4s, ...\n  return Math.min(500 * Math.pow(2, attempt), 16_000)\n}\n\nfunction buildHeaders(opts: LlmClientOptions): Record<string, string> {\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n    Accept: 'application/json',\n  }\n  if (opts.authHeader) {\n    headers[opts.authHeader.name] = opts.authHeader.value\n  } else if (opts.bearer || opts.apiKey) {\n    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`\n  }\n  return headers\n}\n\nfunction isSchemaRejection(status: number, body: string): boolean {\n  if (status !== 400) return false\n  const lower = body.toLowerCase()\n  return (\n    lower.includes('response_format') ||\n    lower.includes('json_schema') ||\n    lower.includes('is unavailable') ||\n    lower.includes('not supported')\n  )\n}\n\nfunction buildBody(req: LlmCallRequest, forceJsonObject: boolean): Record<string, unknown> {\n  const body: Record<string, unknown> = {\n    model: req.model,\n    messages: req.messages,\n    temperature: req.temperature ?? 0,\n  }\n  if (req.maxTokens != null) body.max_tokens = req.maxTokens\n\n  if (req.jsonSchema && !forceJsonObject) {\n    body.response_format = {\n      type: 'json_schema',\n      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true },\n    }\n  } else if (req.jsonMode || req.jsonSchema) {\n    body.response_format = { type: 'json_object' }\n  }\n\n  return body\n}\n\nasync function sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\n// ─── Public API ─────────────────────────────────────────────────────────\n\n/**\n * Strip a ```json / ``` code fence if the model emitted one.\n * Idempotent for naked JSON. Some models (claude-code via router, certain\n * deepseek models) wrap output even under json_object.\n */\nexport function stripFencedJson(raw: string): string {\n  const trimmed = raw.trim()\n  const m = trimmed.match(/^```(?:json)?\\s*\\n?([\\s\\S]*?)\\n?```\\s*$/)\n  return m ? m[1]!.trim() : trimmed\n}\n\n/**\n * Low-level call. Returns raw content + usage + cost. Retries on transient\n * failures; does NOT degrade schema here — callers that want graceful\n * degrade use `callLlmJson`.\n */\nexport async function callLlm(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<LlmCallResult> {\n  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n  const url = `${baseUrl}/chat/completions`\n  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS\n  const maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES\n  const fetchFn = opts.fetch ?? globalThis.fetch\n  const headers = buildHeaders(opts)\n\n  let lastErr: unknown\n  for (let attempt = 0; attempt < maxRetries; attempt++) {\n    const controller = new AbortController()\n    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs)\n    const started = Date.now()\n\n    try {\n      const res = await fetchFn(url, {\n        method: 'POST',\n        headers,\n        body: JSON.stringify(buildBody(req, false)),\n        signal: controller.signal,\n      })\n      clearTimeout(timeoutHandle)\n\n      if (!res.ok) {\n        const body = await res.text()\n        const err = new LlmCallError(\n          `LLM call ${res.status}: ${body.slice(0, 300)}`,\n          res.status,\n          body,\n          req.model,\n        )\n        if (RETRYABLE_STATUS.has(res.status) && attempt < maxRetries - 1) {\n          lastErr = err\n          const retryAfter = parseRetryAfter(res.headers)\n          await sleep(retryAfter ?? backoffMs(attempt))\n          continue\n        }\n        throw err\n      }\n\n      const json = (await res.json()) as Record<string, unknown>\n      const choice = (json.choices as Array<{ message?: { content?: string } }> | undefined)?.[0]\n      const usageRaw = (json.usage as Record<string, unknown> | undefined) ?? {}\n      const costFromProxy = (json._response_cost ?? json.cost_usd) as number | undefined\n\n      return {\n        content: choice?.message?.content ?? '',\n        usage: {\n          promptTokens: Number(usageRaw.prompt_tokens ?? 0),\n          completionTokens: Number(usageRaw.completion_tokens ?? 0),\n          totalTokens: Number(usageRaw.total_tokens ?? 0),\n          cachedPromptTokens:\n            usageRaw.prompt_tokens_details &&\n            typeof usageRaw.prompt_tokens_details === 'object'\n              ? Number(\n                  (usageRaw.prompt_tokens_details as Record<string, unknown>).cached_tokens ?? 0,\n                )\n              : undefined,\n        },\n        costUsd: typeof costFromProxy === 'number' ? costFromProxy : null,\n        model: (json.model as string) ?? req.model,\n        durationMs: Date.now() - started,\n        raw: json,\n      }\n    } catch (err) {\n      clearTimeout(timeoutHandle)\n      lastErr = err\n      if (attempt < maxRetries - 1 && isRetryableError(err)) {\n        await sleep(backoffMs(attempt))\n        continue\n      }\n      throw err\n    }\n  }\n  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n}\n\n/**\n * Structured-output call. Returns parsed JSON plus the raw result envelope.\n * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —\n * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept\n * the `response_format.json_schema` shape but DO accept `json_object`.\n */\nexport async function callLlmJson<T = unknown>(\n  req: LlmCallRequest,\n  opts: LlmClientOptions = {},\n): Promise<{ value: T; result: LlmCallResult }> {\n  try {\n    const result = await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)\n    const value = parseJsonSafely<T>(result.content, result.model)\n    return { value, result }\n  } catch (err) {\n    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {\n      // Degrade to json_object + retry.\n      const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }\n      const result = await callLlm(degradedReq, opts)\n      const value = parseJsonSafely<T>(result.content, result.model)\n      return { value, result }\n    }\n    throw err\n  }\n}\n\nfunction parseJsonSafely<T>(content: string, model: string): T {\n  const stripped = stripFencedJson(content)\n  try {\n    return JSON.parse(stripped) as T\n  } catch (err) {\n    throw new Error(\n      `LLM returned non-JSON content (model=${model}): ${\n        err instanceof Error ? err.message : String(err)\n      }\\n--- raw content ---\\n${content.slice(0, 800)}`,\n    )\n  }\n}\n\n/**\n * Probe whether a model is reachable. Returns latency + null error on\n * success; `ok=false` + error message on any failure (HTTP, timeout,\n * network, parse). Designed for sweep preflights — fail loud at the\n * boundary before burning a 30-leaf run on a misconfigured router.\n *\n * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models\n * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning\n * for short prompts, so don't tighten this further. We don't validate\n * content; HTTP 200 means reachable.\n */\nexport async function probeLlm(\n  model: string,\n  opts: LlmClientOptions & { timeoutMs?: number } = {},\n): Promise<{ ok: boolean; latencyMs: number; error: string | null }> {\n  const start = Date.now()\n  try {\n    await callLlm(\n      {\n        model,\n        messages: [{ role: 'user', content: 'ping' }],\n        maxTokens: 64,\n        timeoutMs: opts.timeoutMs ?? 30_000,\n      },\n      opts,\n    )\n    return { ok: true, latencyMs: Date.now() - start, error: null }\n  } catch (err) {\n    return {\n      ok: false,\n      latencyMs: Date.now() - start,\n      error: err instanceof Error ? err.message : String(err),\n    }\n  }\n}\n\n/**\n * Stateful client — construct once with defaults, call many times.\n * Thin wrapper around the free functions; exists for callers that want\n * to inject a single configured instance into multiple primitives.\n */\nexport class LlmClient {\n  constructor(private readonly opts: LlmClientOptions = {}) {}\n\n  call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {\n    return callLlm(req, { ...this.opts, ...per })\n  }\n\n  callJson<T = unknown>(\n    req: LlmCallRequest,\n    per?: LlmClientOptions,\n  ): Promise<{ value: T; result: LlmCallResult }> {\n    return callLlmJson<T>(req, { ...this.opts, ...per })\n  }\n}\n"],"mappings":";AA4EO,IAAM,eAAN,cAA2B,MAAM;AAAA,EACtC,YACE,SACgB,QACA,MACA,OAChB;AACA,UAAM,OAAO;AAJG;AACA;AACA;AAGhB,SAAK,OAAO;AAAA,EACd;AAAA,EANkB;AAAA,EACA;AAAA,EACA;AAKpB;AAoBA,IAAM,mBAAmB;AACzB,IAAM,qBAAqB;AAC3B,IAAM,sBAAsB;AAE5B,IAAM,mBAAmB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAErD,SAAS,iBAAiB,KAAuB;AAC/C,MAAI,eAAe,aAAc,QAAO,iBAAiB,IAAI,IAAI,MAAM;AACvE,MAAI,eAAe,OAAO;AACxB,WACE,IAAI,SAAS,gBACb,IAAI,SAAS,kBACb,+CAA+C,KAAK,IAAI,OAAO;AAAA,EAEnE;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,SAAiC;AACxD,QAAM,IAAI,QAAQ,IAAI,aAAa;AACnC,MAAI,CAAC,EAAG,QAAO;AACf,QAAM,WAAW,OAAO,CAAC;AACzB,MAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,EAAG,QAAO,WAAW;AACjE,QAAM,SAAS,KAAK,MAAM,CAAC;AAC3B,MAAI,OAAO,SAAS,MAAM,EAAG,QAAO,KAAK,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC;AACnE,SAAO;AACT;AAEA,SAAS,UAAU,SAAyB;AAE1C,SAAO,KAAK,IAAI,MAAM,KAAK,IAAI,GAAG,OAAO,GAAG,IAAM;AACpD;AAEA,SAAS,aAAa,MAAgD;AACpE,QAAM,UAAkC;AAAA,IACtC,gBAAgB;AAAA,IAChB,QAAQ;AAAA,EACV;AACA,MAAI,KAAK,YAAY;AACnB,YAAQ,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW;AAAA,EAClD,WAAW,KAAK,UAAU,KAAK,QAAQ;AACrC,YAAQ,gBAAgB,UAAU,KAAK,UAAU,KAAK,MAAM;AAAA,EAC9D;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,QAAgB,MAAuB;AAChE,MAAI,WAAW,IAAK,QAAO;AAC3B,QAAM,QAAQ,KAAK,YAAY;AAC/B,SACE,MAAM,SAAS,iBAAiB,KAChC,MAAM,SAAS,aAAa,KAC5B,MAAM,SAAS,gBAAgB,KAC/B,MAAM,SAAS,eAAe;AAElC;AAEA,SAAS,UAAU,KAAqB,iBAAmD;AACzF,QAAM,OAAgC;AAAA,IACpC,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,aAAa,IAAI,eAAe;AAAA,EAClC;AACA,MAAI,IAAI,aAAa,KAAM,MAAK,aAAa,IAAI;AAEjD,MAAI,IAAI,cAAc,CAAC,iBAAiB;AACtC,SAAK,kBAAkB;AAAA,MACrB,MAAM;AAAA,MACN,aAAa,EAAE,MAAM,IAAI,WAAW,MAAM,QAAQ,IAAI,WAAW,QAAQ,QAAQ,KAAK;AAAA,IACxF;AAAA,EACF,WAAW,IAAI,YAAY,IAAI,YAAY;AACzC,SAAK,kBAAkB,EAAE,MAAM,cAAc;AAAA,EAC/C;AAEA,SAAO;AACT;AAEA,eAAe,MAAM,IAA2B;AAC9C,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AASO,SAAS,gBAAgB,KAAqB;AACnD,QAAM,UAAU,IAAI,KAAK;AACzB,QAAM,IAAI,QAAQ,MAAM,yCAAyC;AACjE,SAAO,IAAI,EAAE,CAAC,EAAG,KAAK,IAAI;AAC5B;AAOA,eAAsB,QACpB,KACA,OAAyB,CAAC,GACF;AACxB,QAAM,WAAW,KAAK,WAAW,kBAAkB,QAAQ,QAAQ,EAAE;AACrE,QAAM,MAAM,GAAG,OAAO;AACtB,QAAM,YAAY,IAAI,aAAa,KAAK,oBAAoB;AAC5D,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,UAAU,KAAK,SAAS,WAAW;AACzC,QAAM,UAAU,aAAa,IAAI;AAEjC,MAAI;AACJ,WAAS,UAAU,GAAG,UAAU,YAAY,WAAW;AACrD,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,gBAAgB,WAAW,MAAM,WAAW,MAAM,GAAG,SAAS;AACpE,UAAM,UAAU,KAAK,IAAI;AAEzB,QAAI;AACF,YAAM,MAAM,MAAM,QAAQ,KAAK;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU,UAAU,KAAK,KAAK,CAAC;AAAA,QAC1C,QAAQ,WAAW;AAAA,MACrB,CAAC;AACD,mBAAa,aAAa;AAE1B,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,cAAM,MAAM,IAAI;AAAA,UACd,YAAY,IAAI,MAAM,KAAK,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA,UAC7C,IAAI;AAAA,UACJ;AAAA,UACA,IAAI;AAAA,QACN;AACA,YAAI,iBAAiB,IAAI,IAAI,MAAM,KAAK,UAAU,aAAa,GAAG;AAChE,oBAAU;AACV,gBAAM,aAAa,gBAAgB,IAAI,OAAO;AAC9C,gBAAM,MAAM,cAAc,UAAU,OAAO,CAAC;AAC5C;AAAA,QACF;AACA,cAAM;AAAA,MACR;AAEA,YAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,YAAM,SAAU,KAAK,UAAoE,CAAC;AAC1F,YAAM,WAAY,KAAK,SAAiD,CAAC;AACzE,YAAM,gBAAiB,KAAK,kBAAkB,KAAK;AAEnD,aAAO;AAAA,QACL,SAAS,QAAQ,SAAS,WAAW;AAAA,QACrC,OAAO;AAAA,UACL,cAAc,OAAO,SAAS,iBAAiB,CAAC;AAAA,UAChD,kBAAkB,OAAO,SAAS,qBAAqB,CAAC;AAAA,UACxD,aAAa,OAAO,SAAS,gBAAgB,CAAC;AAAA,UAC9C,oBACE,SAAS,yBACT,OAAO,SAAS,0BAA0B,WACtC;AAAA,YACG,SAAS,sBAAkD,iBAAiB;AAAA,UAC/E,IACA;AAAA,QACR;AAAA,QACA,SAAS,OAAO,kBAAkB,WAAW,gBAAgB;AAAA,QAC7D,OAAQ,KAAK,SAAoB,IAAI;AAAA,QACrC,YAAY,KAAK,IAAI,IAAI;AAAA,QACzB,KAAK;AAAA,MACP;AAAA,IACF,SAAS,KAAK;AACZ,mBAAa,aAAa;AAC1B,gBAAU;AACV,UAAI,UAAU,aAAa,KAAK,iBAAiB,GAAG,GAAG;AACrD,cAAM,MAAM,UAAU,OAAO,CAAC;AAC9B;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AACA,QAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;AACtE;AAQA,eAAsB,YACpB,KACA,OAAyB,CAAC,GACoB;AAC9C,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,EAAE,GAAG,KAAK,UAAU,IAAI,YAAY,CAAC,IAAI,WAAW,GAAG,IAAI;AACxF,UAAM,QAAQ,gBAAmB,OAAO,SAAS,OAAO,KAAK;AAC7D,WAAO,EAAE,OAAO,OAAO;AAAA,EACzB,SAAS,KAAK;AACZ,QAAI,eAAe,gBAAgB,kBAAkB,IAAI,QAAQ,IAAI,IAAI,KAAK,IAAI,YAAY;AAE5F,YAAM,cAA8B,EAAE,GAAG,KAAK,UAAU,MAAM,YAAY,OAAU;AACpF,YAAM,SAAS,MAAM,QAAQ,aAAa,IAAI;AAC9C,YAAM,QAAQ,gBAAmB,OAAO,SAAS,OAAO,KAAK;AAC7D,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,gBAAmB,SAAiB,OAAkB;AAC7D,QAAM,WAAW,gBAAgB,OAAO;AACxC,MAAI;AACF,WAAO,KAAK,MAAM,QAAQ;AAAA,EAC5B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,wCAAwC,KAAK,MAC3C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CACjD;AAAA;AAAA,EAA0B,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA,IACjD;AAAA,EACF;AACF;AAaA,eAAsB,SACpB,OACA,OAAkD,CAAC,GACgB;AACnE,QAAM,QAAQ,KAAK,IAAI;AACvB,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,QACE;AAAA,QACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,QAC5C,WAAW;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,WAAW,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK;AAAA,EAChE,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,WAAW,KAAK,IAAI,IAAI;AAAA,MACxB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;AAOO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAA6B,OAAyB,CAAC,GAAG;AAA7B;AAAA,EAA8B;AAAA,EAA9B;AAAA,EAE7B,KAAK,KAAqB,KAAgD;AACxE,WAAO,QAAQ,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EAC9C;AAAA,EAEA,SACE,KACA,KAC8C;AAC9C,WAAO,YAAe,KAAK,EAAE,GAAG,KAAK,MAAM,GAAG,IAAI,CAAC;AAAA,EACrD;AACF;","names":[]}