npm - @tangle-network/agent-eval - Versions diffs - 0.216.1 → 0.217.1 - Mend

@tangle-network/agent-eval 0.216.1 → 0.217.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (521) hide show

package/dist/cli.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"cli.js","names":[],"sources":["../src/cli-config.ts","../src/search-command.ts","../src/cli.ts"],"sourcesContent":["/*\n Provider configuration for the `agent-eval` binary.\n \n The binary reads endpoint and credential values from its environment.\n * TypeScript library callers bind a transport explicitly. The maintained\n * HTTP transport accepts their credential and sends provider requests.\n /\n\nimport { type ChatClient, createChatClient } from './analyst/chat-client'\n\nexport interface CliLlmConfig {\n /* Judge transport for the wire handlers. Absent when no provider is configured. /\n chat?: ChatClient\n model?: string\n}\n\n/* The endpoint the binary resolved from its environment. /\nexport interface CliProviderRoute {\n baseUrl: string\n apiKey: string\n model?: string\n}\n\n/\n Environment precedence for the binary's provider route, with no client built.\n \n Both halves are required: a base URL without a key, or a key without an\n * endpoint, is a half-configured server. `/v1/judge` then refuses with\n * `llm_not_configured` instead of calling an unintended endpoint.\n /\nexport function resolveCliProviderRoute(\n env: NodeJS.ProcessEnv = process.env,\n): CliProviderRoute \| undefined {\n const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n const baseUrl =\n explicitBaseUrl ??\n nonEmpty(env.OPENAI_BASE_URL) ??\n nonEmpty(env.TANGLE_ROUTER_URL) ??\n (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n const model =\n nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n if (!baseUrl \|\| !apiKey) return undefined\n return { baseUrl, apiKey, ...(model ? { model } : {}) }\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n const route = resolveCliProviderRoute(env)\n if (!route) return {}\n return {\n chat: cliChatClient({ baseUrl: route.baseUrl, apiKey: route.apiKey }, route.model),\n ...(route.model ? { model: route.model } : {}),\n }\n}\n\n/\n The binary's transport IS the published one — `createChatClient({ transport:\n * 'openai-compatible' })` — so the server and a library consumer cannot drift\n * onto two different clients for the same endpoint.\n \n The one thing kept local is the error text. A generic \"no model on the\n * request\" tells a server operator nothing; this names the variable to set.\n /\nfunction cliChatClient(\n route: { baseUrl: string; apiKey: string },\n defaultModel: string \| undefined,\n): ChatClient {\n const client = createChatClient({\n transport: 'openai-compatible',\n baseUrl: route.baseUrl,\n apiKey: route.apiKey,\n ...(defaultModel ? { defaultModel } : {}),\n })\n return {\n ...client,\n chat: (req, callOpts) => {\n if (!req.model && !defaultModel) {\n throw new Error(\n 'agent-eval: no model on the request and no AGENT_EVAL_LLM_MODEL configured',\n )\n }\n return client.chat(req, callOpts)\n },\n }\n}\n\nfunction nonEmpty(value: string \| undefined): string \| undefined {\n const trimmed = value?.trim()\n return trimmed ? trimmed : undefined\n}\n","/\n `agent-eval search <subcommand>`: work with a search ledger from a terminal.\n \n agent-eval search ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n * agent-eval search show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n * [--edit-credit] [--landscape] [--skill-manifold] [--json]\n * agent-eval search show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n \n `ship` sends the ledger to the hosted store named by `TANGLE_INGEST_URL`,\n * `TANGLE_INGEST_API_KEY` and `TANGLE_TENANT_ID`, starting from the store's\n * head, so it finishes or resumes a ship that a loop could not complete.\n \n `show` verifies the ledger and prints `renderSearchSummary`: the same\n * compact text a proposer reads, on the search's own ranking split (the\n * selection split when the search declares one, else train). There is no\n * local HTML renderer; the hosted store's page is the visual view (§5 of the\n * search-tree design). A lens flag adds that lens's text form after the\n * summary, so an agent reads what a person sees; `--json` prints the lens\n * results as JSON instead, the shape a view renders.\n \n `show --meta` reads every ledger it is given and prints the `metaSearch`\n * lens over them instead of one search's summary: each search as one node\n * scored by its held-out lift per known dollar, the configurations best first,\n * and the searches arranged by derivation and containment.\n /\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { open, readFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { replaySearchLedgerText } from './campaign/search-ledger'\nimport type { SearchArtifactRef } from './campaign/search-ledger-types'\nimport type { SearchStateView } from './campaign/search-state'\nimport { renderSearchSummary } from './campaign/search-summary'\nimport { hostedTenantFromEnv } from './hosted/client'\nimport { SEARCH_LEDGER_BATCH_MAX_BYTES, SearchRunKindSchema } from './hosted/search-ledger-wire'\nimport { SearchShipConflictError, shipSearchLedger } from './hosted/search-shipper'\nimport { editCredit, editCreditText } from './search/lenses/edit-credit'\nimport { type FrontData, front } from './search/lenses/front'\nimport { formatLandscape, landscape, surfaceTextEdits } from './search/lenses/landscape'\nimport { metaSearch } from './search/lenses/meta-search'\nimport { renderMetaSearchText } from './search/lenses/meta-search-text'\nimport { type OperatorYieldData, operatorYield } from './search/lenses/operator-yield'\nimport { INSUFFICIENT_FROM } from './search/lenses/shared'\nimport { formatSkillManifold, skillManifold } from './search/lenses/skill-manifold'\nimport { type TaskMatrixData, taskMatrix } from './search/lenses/task-matrix'\nimport { type TreeData, type TreeNode, tree } from './search/lenses/tree'\n\nconst USAGE = `usage: agent-eval search <subcommand> ...\n\n ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n Ships the ledger's entries, and the blobs they name, to the hosted store in\n TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL) as tenant TANGLE_TENANT_ID with\n TANGLE_INGEST_API_KEY (or TANGLE_API_KEY). It starts from the store's head, so\n running it again sends only what the store lacks. Prints the result as JSON.\n Exits 1 when the store holds a different chain for the search.\n\n show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n [--edit-credit] [--landscape] [--skill-manifold] [--json]\n Verifies the ledger and prints its search summary: the leading nodes\n against the root, the most recently discarded nodes and why, and a log\n of recent proposals. The same text a proposer reads as context, on the\n search's own ranking split. Each lens flag adds that lens's text form\n (search-tree-design §12) below the summary — the same JSON\n Intelligence, discovery lab, VerticalBench and agent-runtime read.\n --tree the tidy tree of nodes and edges.\n --operator-yield outcome counts and yield per known dollar, by edge operator.\n --front the Pareto front over score and known cost per cell.\n --task-matrix nodes × units, clustered, with specialist gain per cluster.\n --edit-credit every edit as a gene followed down the lineage, its\n credit, interacting pairs, and skill candidates. It\n reads node artifacts from the blobs the ledger names,\n verified by digest.\n --landscape nodes placed by line edits between their profile\n surfaces (read from the blobs, verified by digest), the\n kriged score surface, its basins, and the plateau score\n a policy drafts on.\n --skill-manifold the node × unit score matrix factored into skill axes,\n its cross-validated intrinsic dimension, and the unit\n that best separates the leaders.\n --json prints the requested lenses as JSON instead of text.\n\n show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n Verifies every ledger and prints the meta-search lens over them: each\n search as one node, scored by its claim's held-out lift per known\n dollar (unscored searches say why), configurations best first with\n their interval, method and n, and the searches arranged by derivation\n and containment. --objective names the objective the best\n configuration is chosen within when the searches span several.`\n\nexport async function runSearchCommand(argv: string[]): Promise<number> {\n const [subcommand, ...rest] = argv\n if (subcommand === undefined \|\| subcommand === '--help' \|\| subcommand === '-h') {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (subcommand === 'show') return await runShowCommand(rest)\n if (subcommand !== 'ship') {\n process.stderr.write(`unknown search subcommand: ${subcommand}\\n${USAGE}\\n`)\n return 1\n }\n if (rest.some((arg) => arg === undefined \|\| arg === '--help' \|\| arg === '-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n const { path, flags } = parseShipArgs(rest)\n const runKind = SearchRunKindSchema.safeParse(flags['run-kind'])\n if (!runKind.success) throw new Error(`--run-kind must be optimization or eval\\n${USAGE}`)\n const content = flags.content ?? 'full'\n if (content !== 'full' && content !== 'digests') {\n throw new Error(`--content must be full or digests\\n${USAGE}`)\n }\n const tenant = hostedTenantFromEnv()\n if (!tenant) {\n throw new Error(\n 'set TANGLE_INGEST_URL, TANGLE_INGEST_API_KEY and TANGLE_TENANT_ID to name the store',\n )\n }\n const searchId = await firstLineSearchId(path)\n try {\n const shipped = await shipSearchLedger({\n tenant,\n ledger: { path, searchId },\n runKind: runKind.data,\n content,\n })\n process.stdout.write(`${JSON.stringify(shipped, null, 2)}\\n`)\n return shipped.head.nextSequence === shipped.localLines ? 0 : 1\n } catch (error) {\n if (!(error instanceof SearchShipConflictError)) throw error\n process.stderr.write(`${error.message}\\n`)\n return 1\n }\n}\n\nasync function runShowCommand(argv: string[]): Promise<number> {\n if (argv.includes('--help') \|\| argv.includes('-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (argv.includes('--meta')) return await runShowMetaCommand(argv)\n const positional = argv.filter((arg) => !arg.startsWith('--'))\n const flags = new Set(argv.filter((arg) => arg.startsWith('--')))\n const unknown = [...flags].filter((flag) => !SHOW_FLAGS.has(flag))\n if (positional.length !== 1 \|\| unknown.length > 0) {\n if (unknown.length > 0) process.stderr.write(`unknown flag ${unknown.join(', ')}\\n`)\n process.stdout.write(`${USAGE}\\n`)\n return 1\n }\n const path = positional[0]!\n const searchId = await firstLineSearchId(path)\n const text = await readFile(path, 'utf8')\n const state = replaySearchLedgerText(text, searchId, path)\n const split =\n state.header && state.header.splits.selection.tasks.length > 0 ? 'selection' : 'train'\n const lenses = showLenses(state, path, flags)\n if (flags.has('--json')) {\n if (lenses.length === 0) {\n process.stderr.write('--json needs a lens flag\\n')\n return 1\n }\n const json = Object.fromEntries(lenses.map((lens) => [lens.name, lens.result]))\n process.stdout.write(`${JSON.stringify(json, null, 2)}\\n`)\n return 0\n }\n const sections = [renderSearchSummary(state, { split }), ...lenses.map((lens) => lens.text)]\n process.stdout.write(`${sections.join('\\n\\n')}\\n`)\n return 0\n}\n\nasync function runShowMetaCommand(argv: string[]): Promise<number> {\n const paths: string[] = []\n let objective: string \| undefined\n let json = false\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (token === '--meta') continue\n if (token === '--json') {\n json = true\n continue\n }\n if (token === '--objective') {\n objective = argv[++index]\n if (objective === undefined) throw new Error(`--objective needs a value\\n${USAGE}`)\n continue\n }\n if (token.startsWith('--')) throw new Error(`unknown flag ${token}\\n${USAGE}`)\n paths.push(token)\n }\n if (paths.length === 0) throw new Error(`show --meta needs at least one ledger\\n${USAGE}`)\n const states = []\n for (const path of paths) {\n const searchId = await firstLineSearchId(path)\n states.push(replaySearchLedgerText(await readFile(path, 'utf8'), searchId, path))\n }\n const lens = metaSearch(states, objective === undefined ? {} : { objective })\n process.stdout.write(\n json ? `${JSON.stringify({ metaSearch: lens }, null, 2)}\\n` : `${renderMetaSearchText(lens)}\\n`,\n )\n return 0\n}\n\nconst SHOW_FLAGS = new Set([\n '--edit-credit',\n '--json',\n '--tree',\n '--operator-yield',\n '--front',\n '--task-matrix',\n '--landscape',\n '--skill-manifold',\n])\n\nfunction showLenses(\n state: SearchStateView,\n ledgerPath: string,\n flags: ReadonlySet<string>,\n): Array<{ name: string; result: unknown; text: string }> {\n const lenses: Array<{ name: string; result: unknown; text: string }> = []\n if (flags.has('--tree')) {\n const result = tree(state)\n lenses.push({ name: 'tree', result, text: renderTreeText(result.data) })\n }\n if (flags.has('--operator-yield')) {\n const result = operatorYield(state)\n lenses.push({ name: 'operatorYield', result, text: renderOperatorYieldText(result.data) })\n }\n if (flags.has('--front')) {\n const result = front(state)\n lenses.push({ name: 'front', result, text: renderFrontText(result.data) })\n }\n if (flags.has('--task-matrix')) {\n const result = taskMatrix(state)\n lenses.push({ name: 'taskMatrix', result, text: renderTaskMatrixText(result.data) })\n }\n if (flags.has('--edit-credit')) {\n const result = editCredit(state, { readArtifact: ledgerBlobReader(ledgerPath) })\n lenses.push({ name: 'editCredit', result, text: editCreditText(result) })\n }\n if (flags.has('--landscape')) {\n const read = ledgerBlobReader(ledgerPath)\n const result = landscape(\n state,\n surfaceTextEdits((surface) => read(surface.artifact) ?? null),\n )\n lenses.push({ name: 'landscape', result, text: formatLandscape(result) })\n }\n if (flags.has('--skill-manifold')) {\n const result = skillManifold(state)\n lenses.push({ name: 'skillManifold', result, text: formatSkillManifold(result) })\n }\n return lenses\n}\n\n/\n Reads a blob a ledger names: at its `file:` URI, else under `blobs/` beside\n * the ledger, where `SearchRecorder` writes them. Bytes that do not hash to the\n * reference's digest and length read as unavailable, never as content.\n /\nfunction ledgerBlobReader(ledgerPath: string): (ref: SearchArtifactRef) => unknown \| undefined {\n const cache = new Map<string, unknown>()\n return (ref) => {\n if (cache.has(ref.sha256)) return cache.get(ref.sha256)\n const hex = ref.sha256.slice('sha256:'.length)\n const candidates = [join(dirname(ledgerPath), 'blobs', `${hex}.json`)]\n if (ref.uri.startsWith('file:')) candidates.unshift(fileURLToPath(ref.uri))\n let value: unknown\n for (const candidate of candidates) {\n if (!existsSync(candidate)) continue\n const bytes = readFileSync(candidate)\n const digest = createHash('sha256').update(bytes).digest('hex')\n if (digest !== hex \|\| bytes.byteLength !== ref.byteLength) continue\n try {\n value = JSON.parse(bytes.toString('utf8'))\n } catch {\n // Verified bytes that are not JSON are not content a lens can read.\n }\n break\n }\n cache.set(ref.sha256, value)\n return value\n }\n}\n\nfunction renderTreeText(data: TreeData): string {\n const lines = [\n `tree: ${data.nodeCount} node${data.nodeCount === 1 ? '' : 's'}, ${data.edgeCount} edge${data.edgeCount === 1 ? '' : 's'}`,\n ]\n const walk = (node: TreeNode, prefix: string, isLast: boolean): void => {\n const branch = prefix === '' ? '' : isLast ? '└─ ' : '├─ '\n const op = node.operator ? `${node.operator} → ` : ''\n const rung = node.rung === null ? '' : ` rung ${node.rung}`\n const depth = node.depth === null ? ' depth unknown' : ''\n const cost =\n node.unknownCostCells === 0\n ? `$${node.knownCostUsd.toFixed(2)}`\n : `$${node.knownCostUsd.toFixed(2)} known + at least $${node.floorUsd.toFixed(2)} over ${node.unknownCostCells} unknown-cost cell${node.unknownCostCells === 1 ? '' : 's'}`\n lines.push(\n `${prefix}${branch}${op}${node.nodeId} (${node.status ?? 'undecided'}${rung}${depth}) ${cost}`,\n )\n const childPrefix = prefix + (prefix === '' ? '' : isLast ? ' ' : '│ ')\n node.children.forEach((child, index) => {\n walk(child, childPrefix, index === node.children.length - 1)\n })\n }\n data.roots.forEach((root, index) => {\n walk(root, '', index === data.roots.length - 1)\n })\n return lines.join('\\n')\n}\n\nfunction renderOperatorYieldText(data: OperatorYieldData): string {\n const lines = [`operator yield (${data.split} split, gain per known $ of proposal + screen):`]\n if (data.rows.length === 0) lines.push(' no edges yet')\n for (const row of data.rows) {\n const y = row.yield\n const outcomes = Object.entries(row.outcomes)\n .filter(([, count]) => count > 0)\n .map(([status, count]) => `${status}=${count}`)\n .join(' ')\n const yieldText =\n y.method === 'none'\n ? y.n === 0\n ? 'yield: no measured children'\n : `yield unknown (${y.n} of ${INSUFFICIENT_FROM}, none)`\n : y.method === 'insufficient'\n ? `yield≈${y.mean!.toFixed(4)}/$ (${y.n} of ${INSUFFICIENT_FROM}, insufficient)`\n : `yield≈${y.mean!.toFixed(4)}/$ [${y.interval![0].toFixed(4)}, ${y.interval![1].toFixed(4)}] (${y.method}, ${y.intervalMethod}, n=${y.n})`\n const nodes = Object.values(row.outcomes).reduce((sum, count) => sum + count, 0)\n const reproposals = row.reproposals > 0 ? `, ${row.reproposals} re-proposals` : ''\n lines.push(\n ` ${row.operator}: ${row.proposals} proposals${reproposals} → ${nodes} node${nodes === 1 ? '' : 's'} (${outcomes \|\| 'no decisions yet'}); ${yieldText}; ${row.excluded} excluded`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderFrontText(data: FrontData): string {\n const onFront = data.rows.filter((row) => row.onFront)\n const reasons = Object.entries(data.exclusions)\n .filter(([, count]) => count > 0)\n .map(([reason, count]) => `${reason} ${count}`)\n .join(', ')\n const lines = [\n `front (${data.split} split, axes: ${data.axes.join(', ')}): ${data.frontierSize} of ${data.rows.length} node(s) on the frontier, ${data.excludedCount} excluded${reasons ? ` (${reasons})` : ''}`,\n ]\n for (const row of onFront) {\n lines.push(\n ` ${row.nodeId}: score=${row.score!.toFixed(4)} on ${row.units} units (${row.method}) costPerCell=$${row.costPerCellUsd!.toFixed(4)}`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderTaskMatrixText(data: TaskMatrixData): string {\n const lines = [\n `task matrix (${data.split} split, ${data.direction}): ${data.nodeIds.length} node(s) × ${data.unitIds.length} unit(s), ${data.nodeClusters.length} node cluster(s), ${data.unitClusters.length} unit cluster(s)`,\n ]\n if (data.sampling) {\n lines.push(\n ` clustered ${data.sampling.nodesClustered} of ${data.sampling.nodesTotal} nodes (cap ${data.sampling.cap}), evenly sampled in registration order; unit clusters and specialist gains use the same nodes`,\n )\n }\n for (const row of data.specialistGain) {\n const gain =\n row.gain === null\n ? `insufficient (${row.insufficient})`\n : `${row.gain.toFixed(4)} (best ${row.bestNodeId}, ${row.nodesContributing} nodes on ${row.unitIds.length} units)`\n lines.push(` cluster ${row.clusterId} [${row.unitIds.join(', ')}]: specialist gain ${gain}`)\n }\n return lines.join('\\n')\n}\n\nfunction parseShipArgs(argv: string[]): { path: string; flags: Record<string, string> } {\n const flags: Record<string, string> = {}\n const positional: string[] = []\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (!token.startsWith('--')) {\n positional.push(token)\n continue\n }\n const [name, inline] = token.slice(2).split('=', 2) as [string, string \| undefined]\n if (name !== 'run-kind' && name !== 'content') throw new Error(`unknown flag --${name}`)\n const value = inline ?? argv[++index]\n if (value === undefined) throw new Error(`--${name} needs a value`)\n flags[name] = value\n }\n if (positional.length !== 1) throw new Error(USAGE)\n return { path: positional[0]!, flags }\n}\n\n/* The ledger names its search on every line; read the first. /\nasync function firstLineSearchId(path: string): Promise<string> {\n const handle = await open(path, 'r')\n try {\n // A line longer than one request's limit cannot ship anyway.\n const bytes = Buffer.alloc(SEARCH_LEDGER_BATCH_MAX_BYTES + 1)\n const { bytesRead } = await handle.read(bytes, 0, bytes.byteLength, 0)\n const newline = bytes.subarray(0, bytesRead).indexOf(0x0a)\n if (newline < 0) throw new Error(`${path} holds no complete ledger line`)\n const first = JSON.parse(bytes.toString('utf8', 0, newline)) as { searchId?: unknown }\n if (typeof first.searchId !== 'string') throw new Error(`${path} line 1 names no searchId`)\n return first.searchId\n } finally {\n await handle.close()\n }\n}\n","#!/usr/bin/env node\n/\n agent-eval CLI.\n \n agent-eval serve [--port 5005] [--host 127.0.0.1]\n * agent-eval rpc <method> # one request from stdin → one response on stdout\n * agent-eval rpc-batch <method> # JSONL stdin → JSONL stdout\n * agent-eval analyst-benchmark ... # benchmark a real-model trace analyst\n * agent-eval supervisor-run report <runDir> [--format headline\|markdown\|json]\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization\|eval\n * agent-eval search show <search-ledger.jsonl>\n * agent-eval openapi [--out path] # write OpenAPI spec\n * agent-eval version\n \n <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n /\nimport { writeFileSync } from 'node:fs'\nimport { runAnalystBenchmarkCommand } from './analyst/benchmark-command'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { runRolloutReleaseCli } from './rollout/release/hf-dataset'\nimport { runSearchCommand } from './search-command'\nimport { runSupervisorRunCommand } from './supervisor-run/report-command'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n command: string\n positional: string[]\n flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n const [command, ...rest] = argv\n const positional: string[] = []\n const flags: Record<string, string> = {}\n for (let i = 0; i < rest.length; i++) {\n const tok = rest[i]!\n if (tok.startsWith('--')) {\n const raw = tok.slice(2)\n const equalsAt = raw.indexOf('=')\n if (equalsAt >= 0) {\n flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n continue\n }\n const key = raw\n if (key === 'help') {\n flags[key] = 'true'\n continue\n }\n const next = rest[i + 1]\n if (next != null && !next.startsWith('--')) {\n flags[key] = next\n i++\n } else {\n flags[key] = 'true'\n }\n } else if (tok === '-h') {\n flags.help = 'true'\n } else {\n positional.push(tok)\n }\n }\n return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n serve [--port 5005] [--host 127.0.0.1]\n Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n rpc <method>\n Read one JSON object from stdin (the params for <method>), write one\n JSON object to stdout. Methods: judge, listRubrics, version.\n rpc-batch <method>\n Like 'rpc' but JSONL in / JSONL out.\n openapi [--out openapi.json]\n Write the OpenAPI 3.1 spec.\n rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]\n Build a HuggingFace-ready dataset dir from tangle.rollout.v1 ledgers:\n validate, fail-closed split filter, scrub, export formats + card.\n analyst-benchmark --dataset <agentrx\|codetracebench> --labels <path> --trace-dir <path> [--artifact-dir <path>] --out <dir> ...\n Compare an empty baseline with a real-model trace analyst on public labels.\n Run with --help for all required inputs and controls.\n supervisor-run report <runDir> [--format headline\|markdown\|json]\n Report one Runtime or loops supervisor run directory. The status comes\n from Runtime's own result.json / failure.json; every missing measurement\n stays named. Exits 1 when the directory cannot be read.\n search ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n Ship a search ledger and its blobs to the hosted store in TANGLE_INGEST_URL,\n starting from the store's head. Run with --help for the environment it reads.\n search show <search-ledger.jsonl>\n Verify a search ledger and print its summary: leading nodes, recently\n discarded nodes and recent proposals, the same text a proposer reads.\n version\n Print server + wire-protocol version JSON.\n\nJudge provider:\n Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n OPENAI_ and TANGLE_* equivalents are also accepted. This binary is the only\n place that reads a provider credential; the library never does. Without both\n a base URL and a key, /v1/judge refuses with llm_not_configured.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n const { command, positional, flags } = parseArgs(process.argv.slice(2))\n assertKnownFlags(command, flags)\n\n if (command === 'analyst-benchmark' && flags.help === 'true') {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n if (command === 'search') return await runSearchCommand(process.argv.slice(3))\n if (flags.help === 'true') {\n process.stdout.write(`${HELP}\\n`)\n return 0\n }\n\n switch (command) {\n case 'serve': {\n const port = parsePort(flags.port ?? '5005')\n const host = flags.host ?? '127.0.0.1'\n const llm = resolveCliLlmConfig()\n const { server } = await startServerAsync({\n port,\n host,\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n // Keep process alive on SIGINT/SIGTERM\n const shutdown = (sig: string) => {\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] received ${sig}, shutting down`)\n server.close(() => process.exit(0))\n // Force exit after 5s if close hangs\n setTimeout(() => process.exit(1), 5000).unref()\n }\n process.on('SIGINT', () => shutdown('SIGINT'))\n process.on('SIGTERM', () => shutdown('SIGTERM'))\n // Block forever\n await new Promise(() => {})\n return 0\n }\n case 'rpc': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcOnce(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'rpc-batch': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcBatch(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'openapi': {\n const out = flags.out ?? 'openapi.json'\n const spec = buildOpenApi(handleVersion().version)\n writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n return 0\n }\n case 'rollout-release': {\n // The subcommand owns its own flag grammar (multi-value --formats,\n // boolean --include-proposers) — pass raw argv through untouched.\n return await runRolloutReleaseCli(process.argv.slice(3))\n }\n case 'analyst-benchmark': {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n case 'supervisor-run': {\n // The subcommand owns its own flag grammar; pass raw argv through.\n return await runSupervisorRunCommand(process.argv.slice(3))\n }\n case 'version': {\n process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n return 0\n }\n case '--version': {\n process.stdout.write(`${handleVersion().version}\\n`)\n return 0\n }\n case 'help':\n case '--help':\n case '-h':\n case '':\n process.stdout.write(`${HELP}\\n`)\n return 0\n default:\n process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n return 1\n }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n serve: new Set(['help', 'host', 'port']),\n rpc: new Set(['help']),\n 'rpc-batch': new Set(['help']),\n openapi: new Set(['help', 'out']),\n version: new Set(['help']),\n help: new Set(),\n '--help': new Set(),\n '-h': new Set(),\n '--version': new Set(),\n '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n // These subcommands parse their own argv.\n if (\n command === 'rollout-release' \|\|\n command === 'analyst-benchmark' \|\|\n command === 'supervisor-run' \|\|\n command === 'search'\n ) {\n return\n }\n const allowed = FLAGS_BY_COMMAND[command]\n if (!allowed) return\n const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n if (unknown.length > 0) {\n throw new Error(`unknown flag for ${command \|\| 'help'}: --${unknown[0]}`)\n }\n}\n\nfunction parsePort(raw: string): number {\n const port = Number(raw)\n if (!Number.isInteger(port) \|\| port < 0 \|\| port > 65_535) {\n throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n }\n return port\n}\n\nmain()\n .then((code) => {\n process.exitCode = code\n })\n .catch((err) => {\n // eslint-disable-next-line no-console\n console.error('[agent-eval] cli error:', err)\n process.exitCode = 1\n })\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAgB,wBACd,MAAyB,QAAQ,KACH;CAC9B,MAAM,kBAAkB,SAAS,IAAI,uBAAuB;CAC5D,MAAM,iBAAiB,SAAS,IAAI,sBAAsB;CAC1D,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,KAAA,OAC7C,eAAe,mCAAmC,KAAA;CACrD,MAAM,SAAS,kBAAkB,gBAAgB;CACjD,MAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;CAE/F,IAAI,CAAC,WAAW,CAAC,QAAQ,OAAO,KAAA;CAChC,OAAO;EAAE;EAAS;EAAQ,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;CAAG;AACxD;AAEA,SAAgB,oBAAoB,MAAyB,QAAQ,KAAmB;CACtF,MAAM,QAAQ,wBAAwB,GAAG;CACzC,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,OAAO;EACL,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,QAAQ,MAAM;EAAO,GAAG,MAAM,KAAK;EACjF,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;CAC9C;AACF;;;;;;;;;AAUA,SAAS,cACP,OACA,cACY;CACZ,MAAM,SAAS,iBAAiB;EAC9B,WAAW;EACX,SAAS,MAAM;EACf,QAAQ,MAAM;EACd,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EACL,GAAG;EACH,OAAO,KAAK,aAAa;GACvB,IAAI,CAAC,IAAI,SAAS,CAAC,cACjB,MAAM,IAAI,MACR,4EACF;GAEF,OAAO,OAAO,KAAK,KAAK,QAAQ;EAClC;CACF;AACF;AAEA,SAAS,SAAS,OAA+C;CAC/D,MAAM,UAAU,OAAO,KAAK;CAC5B,OAAO,UAAU,UAAU,KAAA;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC7CA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0Cd,eAAsB,iBAAiB,MAAiC;CACtE,MAAM,CAAC,YAAY,GAAG,QAAQ;CAC9B,IAAI,eAAe,KAAA,KAAa,eAAe,YAAY,eAAe,MAAM;EAC9E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,eAAe,QAAQ,OAAO,MAAM,eAAe,IAAI;CAC3D,IAAI,eAAe,QAAQ;EACzB,QAAQ,OAAO,MAAM,8BAA8B,WAAW,IAAI,MAAM,GAAG;EAC3E,OAAO;CACT;CACA,IAAI,KAAK,MAAM,QAAQ,QAAQ,KAAA,KAAa,QAAQ,YAAY,QAAQ,IAAI,GAAG;EAC7E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,EAAE,MAAM,UAAU,cAAc,IAAI;CAC1C,MAAM,UAAU,oBAAoB,UAAU,MAAM,WAAW;CAC/D,IAAI,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,4CAA4C,OAAO;CACzF,MAAM,UAAU,MAAM,WAAW;CACjC,IAAI,YAAY,UAAU,YAAY,WACpC,MAAM,IAAI,MAAM,sCAAsC,OAAO;CAE/D,MAAM,SAAS,oBAAoB;CACnC,IAAI,CAAC,QACH,MAAM,IAAI,MACR,qFACF;CAEF,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,IAAI;EACF,MAAM,UAAU,MAAM,iBAAiB;GACrC;GACA,QAAQ;IAAE;IAAM;GAAS;GACzB,SAAS,QAAQ;GACjB;EACF,CAAC;EACD,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,EAAE,GAAG;EAC5D,OAAO,QAAQ,KAAK,iBAAiB,QAAQ,aAAa,IAAI;CAChE,SAAS,OAAO;EACd,IAAI,EAAE,iBAAiB,0BAA0B,MAAM;EACvD,QAAQ,OAAO,MAAM,GAAG,MAAM,QAAQ,GAAG;EACzC,OAAO;CACT;AACF;AAEA,eAAe,eAAe,MAAiC;CAC7D,IAAI,KAAK,SAAS,QAAQ,KAAK,KAAK,SAAS,IAAI,GAAG;EAClD,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,KAAK,SAAS,QAAQ,GAAG,OAAO,MAAM,mBAAmB,IAAI;CACjE,MAAM,aAAa,KAAK,QAAQ,QAAQ,CAAC,IAAI,WAAW,IAAI,CAAC;CAC7D,MAAM,QAAQ,IAAI,IAAI,KAAK,QAAQ,QAAQ,IAAI,WAAW,IAAI,CAAC,CAAC;CAChE,MAAM,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC;CACjE,IAAI,WAAW,WAAW,KAAK,QAAQ,SAAS,GAAG;EACjD,IAAI,QAAQ,SAAS,GAAG,QAAQ,OAAO,MAAM,gBAAgB,QAAQ,KAAK,IAAI,EAAE,GAAG;EACnF,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,OAAO,WAAW;CACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM;CACxC,MAAM,QAAQ,uBAAuB,MAAM,UAAU,IAAI;CACzD,MAAM,QACJ,MAAM,UAAU,MAAM,OAAO,OAAO,UAAU,MAAM,SAAS,IAAI,cAAc;CACjF,MAAM,SAAS,WAAW,OAAO,MAAM,KAAK;CAC5C,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,IAAI,OAAO,WAAW,GAAG;GACvB,QAAQ,OAAO,MAAM,4BAA4B;GACjD,OAAO;EACT;EACA,MAAM,OAAO,OAAO,YAAY,OAAO,KAAK,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;EAC9E,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,GAAG;EACzD,OAAO;CACT;CACA,MAAM,WAAW,CAAC,oBAAoB,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,OAAO,KAAK,SAAS,KAAK,IAAI,CAAC;CAC3F,QAAQ,OAAO,MAAM,GAAG,SAAS,KAAK,MAAM,EAAE,GAAG;CACjD,OAAO;AACT;AAEA,eAAe,mBAAmB,MAAiC;CACjE,MAAM,QAAkB,CAAC;CACzB,IAAI;CACJ,IAAI,OAAO;CACX,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,UAAU;EACxB,IAAI,UAAU,UAAU;GACtB,OAAO;GACP;EACF;EACA,IAAI,UAAU,eAAe;GAC3B,YAAY,KAAK,EAAE;GACnB,IAAI,cAAc,KAAA,GAAW,MAAM,IAAI,MAAM,8BAA8B,OAAO;GAClF;EACF;EACA,IAAI,MAAM,WAAW,IAAI,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,IAAI,OAAO;EAC7E,MAAM,KAAK,KAAK;CAClB;CACA,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,0CAA0C,OAAO;CACzF,MAAM,SAAS,CAAC;CAChB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;EAC7C,OAAO,KAAK,uBAAuB,MAAM,SAAS,MAAM,MAAM,GAAG,UAAU,IAAI,CAAC;CAClF;CACA,MAAM,OAAO,WAAW,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,CAAC;CAC5E,QAAQ,OAAO,MACb,OAAO,GAAG,KAAK,UAAU,EAAE,YAAY,KAAK,GAAG,MAAM,CAAC,EAAE,MAAM,GAAG,qBAAqB,IAAI,EAAE,GAC9F;CACA,OAAO;AACT;AAEA,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,SAAS,WACP,OACA,YACA,OACwD;CACxD,MAAM,SAAiE,CAAC;CACxE,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,MAAM,SAAS,KAAK,KAAK;EACzB,OAAO,KAAK;GAAE,MAAM;GAAQ;GAAQ,MAAM,eAAe,OAAO,IAAI;EAAE,CAAC;CACzE;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,wBAAwB,OAAO,IAAI;EAAE,CAAC;CAC3F;CACA,IAAI,MAAM,IAAI,SAAS,GAAG;EACxB,MAAM,SAAS,MAAM,KAAK;EAC1B,OAAO,KAAK;GAAE,MAAM;GAAS;GAAQ,MAAM,gBAAgB,OAAO,IAAI;EAAE,CAAC;CAC3E;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,KAAK;EAC/B,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,qBAAqB,OAAO,IAAI;EAAE,CAAC;CACrF;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,OAAO,EAAE,cAAc,iBAAiB,UAAU,EAAE,CAAC;EAC/E,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,eAAe,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,aAAa,GAAG;EAC5B,MAAM,OAAO,iBAAiB,UAAU;EACxC,MAAM,SAAS,UACb,OACA,kBAAkB,YAAY,KAAK,QAAQ,QAAQ,KAAK,IAAI,CAC9D;EACA,OAAO,KAAK;GAAE,MAAM;GAAa;GAAQ,MAAM,gBAAgB,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,oBAAoB,MAAM;EAAE,CAAC;CAClF;CACA,OAAO;AACT;;;;;;AAOA,SAAS,iBAAiB,YAAqE;CAC7F,MAAM,wBAAQ,IAAI,IAAqB;CACvC,QAAQ,QAAQ;EACd,IAAI,MAAM,IAAI,IAAI,MAAM,GAAG,OAAO,MAAM,IAAI,IAAI,MAAM;EACtD,MAAM,MAAM,IAAI,OAAO,MAAM,CAAgB;EAC7C,MAAM,aAAa,CAAC,KAAK,QAAQ,UAAU,GAAG,SAAS,GAAG,IAAI,MAAM,CAAC;EACrE,IAAI,IAAI,IAAI,WAAW,OAAO,GAAG,WAAW,QAAQ,cAAc,IAAI,GAAG,CAAC;EAC1E,IAAI;EACJ,KAAK,MAAM,aAAa,YAAY;GAClC,IAAI,CAAC,WAAW,SAAS,GAAG;GAC5B,MAAM,QAAQ,aAAa,SAAS;GAEpC,IADe,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAChD,MAAM,OAAO,MAAM,eAAe,IAAI,YAAY;GAC3D,IAAI;IACF,QAAQ,KAAK,MAAM,MAAM,SAAS,MAAM,CAAC;GAC3C,QAAQ,CAER;GACA;EACF;EACA,MAAM,IAAI,IAAI,QAAQ,KAAK;EAC3B,OAAO;CACT;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,MAAM,QAAQ,CACZ,SAAS,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,IAAI,IAAI,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,KACvH;CACA,MAAM,QAAQ,MAAgB,QAAgB,WAA0B;EACtE,MAAM,SAAS,WAAW,KAAK,KAAK,SAAS,QAAQ;EACrD,MAAM,KAAK,KAAK,WAAW,GAAG,KAAK,SAAS,OAAO;EACnD,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,SAAS,KAAK;EACrD,MAAM,QAAQ,KAAK,UAAU,OAAO,mBAAmB;EACvD,MAAM,OACJ,KAAK,qBAAqB,IACtB,IAAI,KAAK,aAAa,QAAQ,CAAC,MAC/B,IAAI,KAAK,aAAa,QAAQ,CAAC,EAAE,qBAAqB,KAAK,SAAS,QAAQ,CAAC,EAAE,QAAQ,KAAK,iBAAiB,oBAAoB,KAAK,qBAAqB,IAAI,KAAK;EAC1K,MAAM,KACJ,GAAG,SAAS,SAAS,KAAK,KAAK,OAAO,IAAI,KAAK,UAAU,cAAc,OAAO,MAAM,IAAI,MAC1F;EACA,MAAM,cAAc,UAAU,WAAW,KAAK,KAAK,SAAS,QAAQ;EACpE,KAAK,SAAS,SAAS,OAAO,UAAU;GACtC,KAAK,OAAO,aAAa,UAAU,KAAK,SAAS,SAAS,CAAC;EAC7D,CAAC;CACH;CACA,KAAK,MAAM,SAAS,MAAM,UAAU;EAClC,KAAK,MAAM,IAAI,UAAU,KAAK,MAAM,SAAS,CAAC;CAChD,CAAC;CACD,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,wBAAwB,MAAiC;CAChE,MAAM,QAAQ,CAAC,mBAAmB,KAAK,MAAM,gDAAgD;CAC7F,IAAI,KAAK,KAAK,WAAW,GAAG,MAAM,KAAK,gBAAgB;CACvD,KAAK,MAAM,OAAO,KAAK,MAAM;EAC3B,MAAM,IAAI,IAAI;EACd,MAAM,WAAW,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAC1C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,GAAG;EACX,MAAM,YACJ,EAAE,WAAW,SACT,EAAE,MAAM,IACN,gCACA,kBAAkB,EAAE,EAAE,MAAM,kBAAkB,WAChD,EAAE,WAAW,iBACX,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,kBAAkB,mBAC9D,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,eAAe,MAAM,EAAE,EAAE;EAC/I,MAAM,QAAQ,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;EAC/E,MAAM,cAAc,IAAI,cAAc,IAAI,KAAK,IAAI,YAAY,iBAAiB;EAChF,MAAM,KACJ,KAAK,IAAI,SAAS,IAAI,IAAI,UAAU,YAAY,YAAY,KAAK,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,YAAY,mBAAmB,KAAK,UAAU,IAAI,IAAI,SAAS,UAC1K;CACF;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,gBAAgB,MAAyB;CAChD,MAAM,UAAU,KAAK,KAAK,QAAQ,QAAQ,IAAI,OAAO;CACrD,MAAM,UAAU,OAAO,QAAQ,KAAK,UAAU,CAAC,CAC5C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,IAAI;CACZ,MAAM,QAAQ,CACZ,UAAU,KAAK,MAAM,gBAAgB,KAAK,KAAK,KAAK,IAAI,EAAE,KAAK,KAAK,aAAa,MAAM,KAAK,KAAK,OAAO,4BAA4B,KAAK,cAAc,WAAW,UAAU,KAAK,QAAQ,KAAK,IAChM;CACA,KAAK,MAAM,OAAO,SAChB,MAAM,KACJ,KAAK,IAAI,OAAO,UAAU,IAAI,MAAO,QAAQ,CAAC,EAAE,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,iBAAiB,IAAI,eAAgB,QAAQ,CAAC,GACrI;CAEF,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,MAA8B;CAC1D,MAAM,QAAQ,CACZ,gBAAgB,KAAK,MAAM,UAAU,KAAK,UAAU,KAAK,KAAK,QAAQ,OAAO,aAAa,KAAK,QAAQ,OAAO,YAAY,KAAK,aAAa,OAAO,oBAAoB,KAAK,aAAa,OAAO,iBAClM;CACA,IAAI,KAAK,UACP,MAAM,KACJ,eAAe,KAAK,SAAS,eAAe,MAAM,KAAK,SAAS,WAAW,cAAc,KAAK,SAAS,IAAI,+FAC7G;CAEF,KAAK,MAAM,OAAO,KAAK,gBAAgB;EACrC,MAAM,OACJ,IAAI,SAAS,OACT,iBAAiB,IAAI,aAAa,KAClC,GAAG,IAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,IAAI,WAAW,IAAI,IAAI,kBAAkB,YAAY,IAAI,QAAQ,OAAO;EAC9G,MAAM,KAAK,aAAa,IAAI,UAAU,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,qBAAqB,MAAM;CAC9F;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,cAAc,MAAiE;CACtF,MAAM,QAAgC,CAAC;CACvC,MAAM,aAAuB,CAAC;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,CAAC,MAAM,WAAW,IAAI,GAAG;GAC3B,WAAW,KAAK,KAAK;GACrB;EACF;EACA,MAAM,CAAC,MAAM,UAAU,MAAM,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;EAClD,IAAI,SAAS,cAAc,SAAS,WAAW,MAAM,IAAI,MAAM,kBAAkB,MAAM;EACvF,MAAM,QAAQ,UAAU,KAAK,EAAE;EAC/B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,KAAK,KAAK,eAAe;EAClE,MAAM,QAAQ;CAChB;CACA,IAAI,WAAW,WAAW,GAAG,MAAM,IAAI,MAAM,KAAK;CAClD,OAAO;EAAE,MAAM,WAAW;EAAK;CAAM;AACvC;;AAGA,eAAe,kBAAkB,MAA+B;CAC9D,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;CACnC,IAAI;EAEF,MAAM,QAAQ,OAAO,MAAM,gCAAgC,CAAC;EAC5D,MAAM,EAAE,cAAc,MAAM,OAAO,KAAK,OAAO,GAAG,MAAM,YAAY,CAAC;EACrE,MAAM,UAAU,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ,EAAI;EACzD,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,GAAG,KAAK,+BAA+B;EACxE,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,QAAQ,GAAG,OAAO,CAAC;EAC3D,IAAI,OAAO,MAAM,aAAa,UAAU,MAAM,IAAI,MAAM,GAAG,KAAK,0BAA0B;EAC1F,OAAO,MAAM;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;AACF;;;;;;;;;;;;;;;;;;;ACtXA,SAAS,UAAU,MAAsB;CACvC,MAAM,CAAC,SAAS,GAAG,QAAQ;CAC3B,MAAM,aAAuB,CAAC;CAC9B,MAAM,QAAgC,CAAC;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,IAAI,WAAW,IAAI,GAAG;GACxB,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,WAAW,IAAI,QAAQ,GAAG;GAChC,IAAI,YAAY,GAAG;IACjB,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,IAAI,MAAM,WAAW,CAAC;IACtD;GACF;GACA,MAAM,MAAM;GACZ,IAAI,QAAQ,QAAQ;IAClB,MAAM,OAAO;IACb;GACF;GACA,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;IAC1C,MAAM,OAAO;IACb;GACF,OACE,MAAM,OAAO;EAEjB,OAAO,IAAI,QAAQ,MACjB,MAAM,OAAO;OAEb,WAAW,KAAK,GAAG;CAEvB;CACA,OAAO;EAAE,SAAS,WAAW;EAAQ;EAAY;CAAM;AACzD;AAEA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCb,eAAe,OAAwB;CACrC,MAAM,EAAE,SAAS,YAAY,UAAU,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;CACtE,iBAAiB,SAAS,KAAK;CAE/B,IAAI,YAAY,uBAAuB,MAAM,SAAS,QACpD,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;CAE/D,IAAI,YAAY,UAAU,OAAO,MAAM,iBAAiB,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7E,IAAI,MAAM,SAAS,QAAQ;EACzB,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;EAChC,OAAO;CACT;CAEA,QAAQ,SAAR;EACE,KAAK,SAAS;GACZ,MAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;GAC3C,MAAM,OAAO,MAAM,QAAQ;GAC3B,MAAM,MAAM,oBAAoB;GAChC,MAAM,EAAE,WAAW,MAAM,iBAAiB;IACxC;IACA;IACA,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;GAED,MAAM,YAAY,QAAgB;IAEhC,QAAQ,IAAI,yBAAyB,IAAI,gBAAgB;IACzD,OAAO,YAAY,QAAQ,KAAK,CAAC,CAAC;IAElC,iBAAiB,QAAQ,KAAK,CAAC,GAAG,GAAI,CAAC,CAAC,MAAM;GAChD;GACA,QAAQ,GAAG,gBAAgB,SAAS,QAAQ,CAAC;GAC7C,QAAQ,GAAG,iBAAiB,SAAS,SAAS,CAAC;GAE/C,MAAM,IAAI,cAAc,CAAC,CAAC;GAC1B,OAAO;EACT;EACA,KAAK,OAAO;GACV,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,WAAW,QAAQ;IAC9B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,aAAa;GAChB,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,YAAY,QAAQ;IAC/B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,WAAW;GACd,MAAM,MAAM,MAAM,OAAO;GACzB,MAAM,OAAO,aAAa,cAAc,CAAC,CAAC,OAAO;GACjD,cAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,KAAK,OAAO;GAEhE,QAAQ,IAAI,0CAA0C,KAAK;GAC3D,OAAO;EACT;EACA,KAAK,mBAGH,OAAO,MAAM,qBAAqB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAEzD,KAAK,qBACH,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE/D,KAAK,kBAEH,OAAO,MAAM,wBAAwB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE5D,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,EAAE,GAAG;GACpE,OAAO;EAET,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,cAAc,CAAC,CAAC,QAAQ,GAAG;GACnD,OAAO;EAET,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;GAChC,OAAO;EACT;GACE,QAAQ,OAAO,MAAM,oBAAoB,QAAQ,IAAI,KAAK,GAAG;GAC7D,OAAO;CACX;AACF;AAEA,MAAM,mBAAwD;CAC5D,uBAAO,IAAI,IAAI;EAAC;EAAQ;EAAQ;CAAM,CAAC;CACvC,qBAAK,IAAI,IAAI,CAAC,MAAM,CAAC;CACrB,6BAAa,IAAI,IAAI,CAAC,MAAM,CAAC;CAC7B,yBAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;CAChC,yBAAS,IAAI,IAAI,CAAC,MAAM,CAAC;CACzB,sBAAM,IAAI,IAAI;CACd,0BAAU,IAAI,IAAI;CAClB,sBAAM,IAAI,IAAI;CACd,6BAAa,IAAI,IAAI;CACrB,oBAAI,IAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;CAE9E,IACE,YAAY,qBACZ,YAAY,uBACZ,YAAY,oBACZ,YAAY,UAEZ;CAEF,MAAM,UAAU,iBAAiB;CACjC,IAAI,CAAC,SAAS;CACd,MAAM,UAAU,OAAO,KAAK,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;CACtE,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MAAM,oBAAoB,WAAW,OAAO,MAAM,QAAQ,IAAI;AAE5E;AAEA,SAAS,UAAU,KAAqB;CACtC,MAAM,OAAO,OAAO,GAAG;CACvB,IAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAChD,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,GAAG;CAE9F,OAAO;AACT;AAEA,KAAK,CAAC,CACH,MAAM,SAAS;CACd,QAAQ,WAAW;AACrB,CAAC,CAAC,CACD,OAAO,QAAQ;CAEd,QAAQ,MAAM,2BAA2B,GAAG;CAC5C,QAAQ,WAAW;AACrB,CAAC"}
1	+ {"version":3,"file":"cli.js","names":[],"sources":["../src/cli-config.ts","../src/search-command.ts","../src/supervisor-run/report-command.ts","../src/cli.ts"],"sourcesContent":["/*\n Provider configuration for the `agent-eval` binary.\n \n The binary reads endpoint and credential values from its environment.\n * TypeScript library callers bind a transport explicitly. The maintained\n * HTTP transport accepts their credential and sends provider requests.\n /\n\nimport { type ChatClient, createChatClient } from './analyst/chat-client'\n\ninterface CliLlmConfig {\n /* Judge transport for the wire handlers. Absent when no provider is configured. /\n chat?: ChatClient\n model?: string\n}\n\n/* The endpoint the binary resolved from its environment. /\ninterface CliProviderRoute {\n baseUrl: string\n apiKey: string\n model?: string\n}\n\n/\n Environment precedence for the binary's provider route, with no client built.\n \n Both halves are required: a base URL without a key, or a key without an\n * endpoint, is a half-configured server. `/v1/judge` then refuses with\n * `llm_not_configured` instead of calling an unintended endpoint.\n /\nfunction resolveCliProviderRoute(\n env: NodeJS.ProcessEnv = process.env,\n): CliProviderRoute \| undefined {\n const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n const baseUrl =\n explicitBaseUrl ??\n nonEmpty(env.OPENAI_BASE_URL) ??\n nonEmpty(env.TANGLE_ROUTER_URL) ??\n (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n const model =\n nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n if (!baseUrl \|\| !apiKey) return undefined\n return { baseUrl, apiKey, ...(model ? { model } : {}) }\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n const route = resolveCliProviderRoute(env)\n if (!route) return {}\n return {\n chat: cliChatClient({ baseUrl: route.baseUrl, apiKey: route.apiKey }, route.model),\n ...(route.model ? { model: route.model } : {}),\n }\n}\n\n/\n The binary's transport IS the published one — `createChatClient({ transport:\n * 'openai-compatible' })` — so the server and a library consumer cannot drift\n * onto two different clients for the same endpoint.\n \n The one thing kept local is the error text. A generic \"no model on the\n * request\" tells a server operator nothing; this names the variable to set.\n /\nfunction cliChatClient(\n route: { baseUrl: string; apiKey: string },\n defaultModel: string \| undefined,\n): ChatClient {\n const client = createChatClient({\n transport: 'openai-compatible',\n baseUrl: route.baseUrl,\n apiKey: route.apiKey,\n ...(defaultModel ? { defaultModel } : {}),\n })\n return {\n ...client,\n chat: (req, callOpts) => {\n if (!req.model && !defaultModel) {\n throw new Error(\n 'agent-eval: no model on the request and no AGENT_EVAL_LLM_MODEL configured',\n )\n }\n return client.chat(req, callOpts)\n },\n }\n}\n\nfunction nonEmpty(value: string \| undefined): string \| undefined {\n const trimmed = value?.trim()\n return trimmed ? trimmed : undefined\n}\n","/\n `agent-eval search <subcommand>`: work with a search ledger from a terminal.\n \n agent-eval search ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n * agent-eval search show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n * [--edit-credit] [--landscape] [--skill-manifold] [--json]\n * agent-eval search show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n \n `ship` sends the ledger to the hosted store named by `TANGLE_INGEST_URL`,\n * `TANGLE_INGEST_API_KEY` and `TANGLE_TENANT_ID`, starting from the store's\n * head, so it finishes or resumes a ship that a loop could not complete.\n \n `show` verifies the ledger and prints `renderSearchSummary`: the same\n * compact text a proposer reads, on the search's own ranking split (the\n * selection split when the search declares one, else train). There is no\n * local HTML renderer; the hosted store's page is the visual view (§5 of the\n * search-tree design). A lens flag adds that lens's text form after the\n * summary, so an agent reads what a person sees; `--json` prints the lens\n * results as JSON instead, the shape a view renders.\n \n `show --meta` reads every ledger it is given and prints the `metaSearch`\n * lens over them instead of one search's summary: each search as one node\n * scored by its held-out lift per known dollar, the configurations best first,\n * and the searches arranged by derivation and containment.\n /\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { open, readFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { replaySearchLedgerText } from './campaign/search-ledger'\nimport type { SearchArtifactRef } from './campaign/search-ledger-types'\nimport type { SearchStateView } from './campaign/search-state'\nimport { renderSearchSummary } from './campaign/search-summary'\nimport { hostedTenantFromEnv } from './hosted/client'\nimport { SEARCH_LEDGER_BATCH_MAX_BYTES, SearchRunKindSchema } from './hosted/search-ledger-wire'\nimport { SearchShipConflictError, shipSearchLedger } from './hosted/search-shipper'\nimport { editCredit, editCreditText } from './search/lenses/edit-credit'\nimport { type FrontData, front } from './search/lenses/front'\nimport { formatLandscape, landscape, surfaceTextEdits } from './search/lenses/landscape'\nimport { metaSearch } from './search/lenses/meta-search'\nimport { renderMetaSearchText } from './search/lenses/meta-search-text'\nimport { type OperatorYieldData, operatorYield } from './search/lenses/operator-yield'\nimport { INSUFFICIENT_FROM } from './search/lenses/shared'\nimport { formatSkillManifold, skillManifold } from './search/lenses/skill-manifold'\nimport { type TaskMatrixData, taskMatrix } from './search/lenses/task-matrix'\nimport { type TreeData, type TreeNode, tree } from './search/lenses/tree'\n\nconst USAGE = `usage: agent-eval search <subcommand> ...\n\n ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n Ships the ledger's entries, and the blobs they name, to the hosted store in\n TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL) as tenant TANGLE_TENANT_ID with\n TANGLE_INGEST_API_KEY (or TANGLE_API_KEY). It starts from the store's head, so\n running it again sends only what the store lacks. Prints the result as JSON.\n Exits 1 when the store holds a different chain for the search.\n\n show <search-ledger.jsonl> [--tree] [--operator-yield] [--front] [--task-matrix]\n [--edit-credit] [--landscape] [--skill-manifold] [--json]\n Verifies the ledger and prints its search summary: the leading nodes\n against the root, the most recently discarded nodes and why, and a log\n of recent proposals. The same text a proposer reads as context, on the\n search's own ranking split. Each lens flag adds that lens's text form\n (search-tree-design §12) below the summary — the same JSON\n Intelligence, discovery lab, VerticalBench and agent-runtime read.\n --tree the tidy tree of nodes and edges.\n --operator-yield outcome counts and yield per known dollar, by edge operator.\n --front the Pareto front over score and known cost per cell.\n --task-matrix nodes × units, clustered, with specialist gain per cluster.\n --edit-credit every edit as a gene followed down the lineage, its\n credit, interacting pairs, and skill candidates. It\n reads node artifacts from the blobs the ledger names,\n verified by digest.\n --landscape nodes placed by line edits between their profile\n surfaces (read from the blobs, verified by digest), the\n kriged score surface, its basins, and the plateau score\n a policy drafts on.\n --skill-manifold the node × unit score matrix factored into skill axes,\n its cross-validated intrinsic dimension, and the unit\n that best separates the leaders.\n --json prints the requested lenses as JSON instead of text.\n\n show <search-ledger.jsonl> [<search-ledger.jsonl> ...] --meta [--objective <key>] [--json]\n Verifies every ledger and prints the meta-search lens over them: each\n search as one node, scored by its claim's held-out lift per known\n dollar (unscored searches say why), configurations best first with\n their interval, method and n, and the searches arranged by derivation\n and containment. --objective names the objective the best\n configuration is chosen within when the searches span several.`\n\nexport async function runSearchCommand(argv: string[]): Promise<number> {\n const [subcommand, ...rest] = argv\n if (subcommand === undefined \|\| subcommand === '--help' \|\| subcommand === '-h') {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (subcommand === 'show') return await runShowCommand(rest)\n if (subcommand !== 'ship') {\n process.stderr.write(`unknown search subcommand: ${subcommand}\\n${USAGE}\\n`)\n return 1\n }\n if (rest.some((arg) => arg === undefined \|\| arg === '--help' \|\| arg === '-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n const { path, flags } = parseShipArgs(rest)\n const runKind = SearchRunKindSchema.safeParse(flags['run-kind'])\n if (!runKind.success) throw new Error(`--run-kind must be optimization or eval\\n${USAGE}`)\n const content = flags.content ?? 'full'\n if (content !== 'full' && content !== 'digests') {\n throw new Error(`--content must be full or digests\\n${USAGE}`)\n }\n const tenant = hostedTenantFromEnv()\n if (!tenant) {\n throw new Error(\n 'set TANGLE_INGEST_URL, TANGLE_INGEST_API_KEY and TANGLE_TENANT_ID to name the store',\n )\n }\n const searchId = await firstLineSearchId(path)\n try {\n const shipped = await shipSearchLedger({\n tenant,\n ledger: { path, searchId },\n runKind: runKind.data,\n content,\n })\n process.stdout.write(`${JSON.stringify(shipped, null, 2)}\\n`)\n return shipped.head.nextSequence === shipped.localLines ? 0 : 1\n } catch (error) {\n if (!(error instanceof SearchShipConflictError)) throw error\n process.stderr.write(`${error.message}\\n`)\n return 1\n }\n}\n\nasync function runShowCommand(argv: string[]): Promise<number> {\n if (argv.includes('--help') \|\| argv.includes('-h')) {\n process.stdout.write(`${USAGE}\\n`)\n return 0\n }\n if (argv.includes('--meta')) return await runShowMetaCommand(argv)\n const positional = argv.filter((arg) => !arg.startsWith('--'))\n const flags = new Set(argv.filter((arg) => arg.startsWith('--')))\n const unknown = [...flags].filter((flag) => !SHOW_FLAGS.has(flag))\n if (positional.length !== 1 \|\| unknown.length > 0) {\n if (unknown.length > 0) process.stderr.write(`unknown flag ${unknown.join(', ')}\\n`)\n process.stdout.write(`${USAGE}\\n`)\n return 1\n }\n const path = positional[0]!\n const searchId = await firstLineSearchId(path)\n const text = await readFile(path, 'utf8')\n const state = replaySearchLedgerText(text, searchId, path)\n const split =\n state.header && state.header.splits.selection.tasks.length > 0 ? 'selection' : 'train'\n const lenses = showLenses(state, path, flags)\n if (flags.has('--json')) {\n if (lenses.length === 0) {\n process.stderr.write('--json needs a lens flag\\n')\n return 1\n }\n const json = Object.fromEntries(lenses.map((lens) => [lens.name, lens.result]))\n process.stdout.write(`${JSON.stringify(json, null, 2)}\\n`)\n return 0\n }\n const sections = [renderSearchSummary(state, { split }), ...lenses.map((lens) => lens.text)]\n process.stdout.write(`${sections.join('\\n\\n')}\\n`)\n return 0\n}\n\nasync function runShowMetaCommand(argv: string[]): Promise<number> {\n const paths: string[] = []\n let objective: string \| undefined\n let json = false\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (token === '--meta') continue\n if (token === '--json') {\n json = true\n continue\n }\n if (token === '--objective') {\n objective = argv[++index]\n if (objective === undefined) throw new Error(`--objective needs a value\\n${USAGE}`)\n continue\n }\n if (token.startsWith('--')) throw new Error(`unknown flag ${token}\\n${USAGE}`)\n paths.push(token)\n }\n if (paths.length === 0) throw new Error(`show --meta needs at least one ledger\\n${USAGE}`)\n const states = []\n for (const path of paths) {\n const searchId = await firstLineSearchId(path)\n states.push(replaySearchLedgerText(await readFile(path, 'utf8'), searchId, path))\n }\n const lens = metaSearch(states, objective === undefined ? {} : { objective })\n process.stdout.write(\n json ? `${JSON.stringify({ metaSearch: lens }, null, 2)}\\n` : `${renderMetaSearchText(lens)}\\n`,\n )\n return 0\n}\n\nconst SHOW_FLAGS = new Set([\n '--edit-credit',\n '--json',\n '--tree',\n '--operator-yield',\n '--front',\n '--task-matrix',\n '--landscape',\n '--skill-manifold',\n])\n\nfunction showLenses(\n state: SearchStateView,\n ledgerPath: string,\n flags: ReadonlySet<string>,\n): Array<{ name: string; result: unknown; text: string }> {\n const lenses: Array<{ name: string; result: unknown; text: string }> = []\n if (flags.has('--tree')) {\n const result = tree(state)\n lenses.push({ name: 'tree', result, text: renderTreeText(result.data) })\n }\n if (flags.has('--operator-yield')) {\n const result = operatorYield(state)\n lenses.push({ name: 'operatorYield', result, text: renderOperatorYieldText(result.data) })\n }\n if (flags.has('--front')) {\n const result = front(state)\n lenses.push({ name: 'front', result, text: renderFrontText(result.data) })\n }\n if (flags.has('--task-matrix')) {\n const result = taskMatrix(state)\n lenses.push({ name: 'taskMatrix', result, text: renderTaskMatrixText(result.data) })\n }\n if (flags.has('--edit-credit')) {\n const result = editCredit(state, { readArtifact: ledgerBlobReader(ledgerPath) })\n lenses.push({ name: 'editCredit', result, text: editCreditText(result) })\n }\n if (flags.has('--landscape')) {\n const read = ledgerBlobReader(ledgerPath)\n const result = landscape(\n state,\n surfaceTextEdits((surface) => read(surface.artifact) ?? null),\n )\n lenses.push({ name: 'landscape', result, text: formatLandscape(result) })\n }\n if (flags.has('--skill-manifold')) {\n const result = skillManifold(state)\n lenses.push({ name: 'skillManifold', result, text: formatSkillManifold(result) })\n }\n return lenses\n}\n\n/\n Reads a blob a ledger names: at its `file:` URI, else under `blobs/` beside\n * the ledger, where `SearchRecorder` writes them. Bytes that do not hash to the\n * reference's digest and length read as unavailable, never as content.\n /\nfunction ledgerBlobReader(ledgerPath: string): (ref: SearchArtifactRef) => unknown \| undefined {\n const cache = new Map<string, unknown>()\n return (ref) => {\n if (cache.has(ref.sha256)) return cache.get(ref.sha256)\n const hex = ref.sha256.slice('sha256:'.length)\n const candidates = [join(dirname(ledgerPath), 'blobs', `${hex}.json`)]\n if (ref.uri.startsWith('file:')) candidates.unshift(fileURLToPath(ref.uri))\n let value: unknown\n for (const candidate of candidates) {\n if (!existsSync(candidate)) continue\n const bytes = readFileSync(candidate)\n const digest = createHash('sha256').update(bytes).digest('hex')\n if (digest !== hex \|\| bytes.byteLength !== ref.byteLength) continue\n try {\n value = JSON.parse(bytes.toString('utf8'))\n } catch {\n // Verified bytes that are not JSON are not content a lens can read.\n }\n break\n }\n cache.set(ref.sha256, value)\n return value\n }\n}\n\nfunction renderTreeText(data: TreeData): string {\n const lines = [\n `tree: ${data.nodeCount} node${data.nodeCount === 1 ? '' : 's'}, ${data.edgeCount} edge${data.edgeCount === 1 ? '' : 's'}`,\n ]\n const walk = (node: TreeNode, prefix: string, isLast: boolean): void => {\n const branch = prefix === '' ? '' : isLast ? '└─ ' : '├─ '\n const op = node.operator ? `${node.operator} → ` : ''\n const rung = node.rung === null ? '' : ` rung ${node.rung}`\n const depth = node.depth === null ? ' depth unknown' : ''\n const cost =\n node.unknownCostCells === 0\n ? `$${node.knownCostUsd.toFixed(2)}`\n : `$${node.knownCostUsd.toFixed(2)} known + at least $${node.floorUsd.toFixed(2)} over ${node.unknownCostCells} unknown-cost cell${node.unknownCostCells === 1 ? '' : 's'}`\n lines.push(\n `${prefix}${branch}${op}${node.nodeId} (${node.status ?? 'undecided'}${rung}${depth}) ${cost}`,\n )\n const childPrefix = prefix + (prefix === '' ? '' : isLast ? ' ' : '│ ')\n node.children.forEach((child, index) => {\n walk(child, childPrefix, index === node.children.length - 1)\n })\n }\n data.roots.forEach((root, index) => {\n walk(root, '', index === data.roots.length - 1)\n })\n return lines.join('\\n')\n}\n\nfunction renderOperatorYieldText(data: OperatorYieldData): string {\n const lines = [`operator yield (${data.split} split, gain per known $ of proposal + screen):`]\n if (data.rows.length === 0) lines.push(' no edges yet')\n for (const row of data.rows) {\n const y = row.yield\n const outcomes = Object.entries(row.outcomes)\n .filter(([, count]) => count > 0)\n .map(([status, count]) => `${status}=${count}`)\n .join(' ')\n const yieldText =\n y.method === 'none'\n ? y.n === 0\n ? 'yield: no measured children'\n : `yield unknown (${y.n} of ${INSUFFICIENT_FROM}, none)`\n : y.method === 'insufficient'\n ? `yield≈${y.mean!.toFixed(4)}/$ (${y.n} of ${INSUFFICIENT_FROM}, insufficient)`\n : `yield≈${y.mean!.toFixed(4)}/$ [${y.interval![0].toFixed(4)}, ${y.interval![1].toFixed(4)}] (${y.method}, ${y.intervalMethod}, n=${y.n})`\n const nodes = Object.values(row.outcomes).reduce((sum, count) => sum + count, 0)\n const reproposals = row.reproposals > 0 ? `, ${row.reproposals} re-proposals` : ''\n lines.push(\n ` ${row.operator}: ${row.proposals} proposals${reproposals} → ${nodes} node${nodes === 1 ? '' : 's'} (${outcomes \|\| 'no decisions yet'}); ${yieldText}; ${row.excluded} excluded`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderFrontText(data: FrontData): string {\n const onFront = data.rows.filter((row) => row.onFront)\n const reasons = Object.entries(data.exclusions)\n .filter(([, count]) => count > 0)\n .map(([reason, count]) => `${reason} ${count}`)\n .join(', ')\n const lines = [\n `front (${data.split} split, axes: ${data.axes.join(', ')}): ${data.frontierSize} of ${data.rows.length} node(s) on the frontier, ${data.excludedCount} excluded${reasons ? ` (${reasons})` : ''}`,\n ]\n for (const row of onFront) {\n lines.push(\n ` ${row.nodeId}: score=${row.score!.toFixed(4)} on ${row.units} units (${row.method}) costPerCell=$${row.costPerCellUsd!.toFixed(4)}`,\n )\n }\n return lines.join('\\n')\n}\n\nfunction renderTaskMatrixText(data: TaskMatrixData): string {\n const lines = [\n `task matrix (${data.split} split, ${data.direction}): ${data.nodeIds.length} node(s) × ${data.unitIds.length} unit(s), ${data.nodeClusters.length} node cluster(s), ${data.unitClusters.length} unit cluster(s)`,\n ]\n if (data.sampling) {\n lines.push(\n ` clustered ${data.sampling.nodesClustered} of ${data.sampling.nodesTotal} nodes (cap ${data.sampling.cap}), evenly sampled in registration order; unit clusters and specialist gains use the same nodes`,\n )\n }\n for (const row of data.specialistGain) {\n const gain =\n row.gain === null\n ? `insufficient (${row.insufficient})`\n : `${row.gain.toFixed(4)} (best ${row.bestNodeId}, ${row.nodesContributing} nodes on ${row.unitIds.length} units)`\n lines.push(` cluster ${row.clusterId} [${row.unitIds.join(', ')}]: specialist gain ${gain}`)\n }\n return lines.join('\\n')\n}\n\nfunction parseShipArgs(argv: string[]): { path: string; flags: Record<string, string> } {\n const flags: Record<string, string> = {}\n const positional: string[] = []\n for (let index = 0; index < argv.length; index++) {\n const token = argv[index]!\n if (!token.startsWith('--')) {\n positional.push(token)\n continue\n }\n const [name, inline] = token.slice(2).split('=', 2) as [string, string \| undefined]\n if (name !== 'run-kind' && name !== 'content') throw new Error(`unknown flag --${name}`)\n const value = inline ?? argv[++index]\n if (value === undefined) throw new Error(`--${name} needs a value`)\n flags[name] = value\n }\n if (positional.length !== 1) throw new Error(USAGE)\n return { path: positional[0]!, flags }\n}\n\n/* The ledger names its search on every line; read the first. /\nasync function firstLineSearchId(path: string): Promise<string> {\n const handle = await open(path, 'r')\n try {\n // A line longer than one request's limit cannot ship anyway.\n const bytes = Buffer.alloc(SEARCH_LEDGER_BATCH_MAX_BYTES + 1)\n const { bytesRead } = await handle.read(bytes, 0, bytes.byteLength, 0)\n const newline = bytes.subarray(0, bytesRead).indexOf(0x0a)\n if (newline < 0) throw new Error(`${path} holds no complete ledger line`)\n const first = JSON.parse(bytes.toString('utf8', 0, newline)) as { searchId?: unknown }\n if (typeof first.searchId !== 'string') throw new Error(`${path} line 1 names no searchId`)\n return first.searchId\n } finally {\n await handle.close()\n }\n}\n","/\n `agent-eval supervisor-run report <runDir>` takes one run directory and\n * prints the report the module already renders. The command reads through\n * `analyzeSupervisorRun`, so a Runtime run directory takes the\n * same path and the status comes from the record `terminal-record.ts` names.\n \n Exit codes follow the other self-parsing subcommands: 2 for a usage error,\n * 1 when the directory cannot be read (missing, not a directory, or a corrupt\n * journal or terminal record), 0 after the report was written to stdout.\n /\n\nimport { stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { errorMessage } from '../errors'\nimport { analyzeSupervisorRun } from './reader'\nimport { renderSupervisorRunHeadline, renderSupervisorRunMarkdown } from './render'\n\nexport type SupervisorRunReportFormat = 'headline' \| 'markdown' \| 'json'\n\nconst FORMATS: readonly SupervisorRunReportFormat[] = ['headline', 'markdown', 'json']\n\nconst SUPERVISOR_RUN_USAGE =\n 'usage: agent-eval supervisor-run report <runDir> [--format headline\|markdown\|json]'\n\n/* Output sinks, injectable so tests read the exact bytes a terminal would. /\nexport interface SupervisorRunCommandIo {\n readonly stdout: (text: string) => void\n readonly stderr: (text: string) => void\n}\n\nconst PROCESS_IO: SupervisorRunCommandIo = {\n stdout: (text) => {\n process.stdout.write(text)\n },\n stderr: (text) => {\n process.stderr.write(text)\n },\n}\n\ninterface ReportArgs {\n readonly runDir: string\n readonly format: SupervisorRunReportFormat\n}\n\nfunction isFormat(value: string): value is SupervisorRunReportFormat {\n return (FORMATS as readonly string[]).includes(value)\n}\n\nfunction parseReportArgs(argv: readonly string[]): ReportArgs {\n let runDir: string \| null = null\n let format: SupervisorRunReportFormat = 'headline'\n for (let i = 0; i < argv.length; i++) {\n const arg = argv[i] as string\n if (arg === '--format') {\n const raw = argv[++i]\n if (raw === undefined \|\| !isFormat(raw)) {\n throw new Error(\n `--format expects one of ${FORMATS.join('\|')}, got ${JSON.stringify(raw ?? '')}`,\n )\n }\n format = raw\n } else if (arg.startsWith('--format=')) {\n const raw = arg.slice('--format='.length)\n if (!isFormat(raw)) {\n throw new Error(`--format expects one of ${FORMATS.join('\|')}, got ${JSON.stringify(raw)}`)\n }\n format = raw\n } else if (arg.startsWith('--')) {\n throw new Error(`unknown flag \"${arg}\"`)\n } else if (runDir === null) {\n runDir = arg\n } else {\n throw new Error(`unexpected argument \"${arg}\"`)\n }\n }\n if (runDir === null) throw new Error(SUPERVISOR_RUN_USAGE)\n return { runDir, format }\n}\n\n/\n CLI driver for `agent-eval supervisor-run`. `argv` is everything after the\n * command name, so `['report', runDir, ...flags]`. Returns the process exit code.\n /\nexport async function runSupervisorRunCommand(\n argv: readonly string[],\n io: SupervisorRunCommandIo = PROCESS_IO,\n): Promise<number> {\n const [verb, ...rest] = argv\n if (verb === undefined \|\| verb === '--help' \|\| verb === '-h' \|\| verb === 'help') {\n io.stdout(`${SUPERVISOR_RUN_USAGE}\\n`)\n return verb === undefined ? 2 : 0\n }\n if (verb !== 'report') {\n io.stderr(`unknown supervisor-run subcommand \"${verb}\"\\n${SUPERVISOR_RUN_USAGE}\\n`)\n return 2\n }\n if (rest.includes('--help') \|\| rest.includes('-h')) {\n io.stdout(`${SUPERVISOR_RUN_USAGE}\\n`)\n return 0\n }\n\n let args: ReportArgs\n try {\n args = parseReportArgs(rest)\n } catch (error) {\n io.stderr(`${errorMessage(error)}\\n`)\n return 2\n }\n\n const runDir = resolve(args.runDir)\n // A path that is not a directory is a caller error, not a run with every\n // measurement missing: reporting an absent-shaped run for a typo would read\n // as evidence about a run that never existed.\n const entry = await stat(runDir).catch(() => null)\n if (entry === null \|\| !entry.isDirectory()) {\n io.stderr(`[agent-eval] supervisor-run report: ${runDir} is not a directory\\n`)\n return 1\n }\n\n try {\n const report = await analyzeSupervisorRun(runDir)\n switch (args.format) {\n case 'headline':\n io.stdout(`${renderSupervisorRunHeadline(report)}\\n`)\n break\n case 'markdown':\n io.stdout(renderSupervisorRunMarkdown(report))\n break\n case 'json':\n io.stdout(`${JSON.stringify(report, null, 2)}\\n`)\n break\n }\n return 0\n } catch (error) {\n io.stderr(`[agent-eval] supervisor-run report failed for ${runDir}: ${errorMessage(error)}\\n`)\n return 1\n }\n}\n","#!/usr/bin/env node\n/\n agent-eval CLI.\n \n agent-eval serve [--port 5005] [--host 127.0.0.1]\n * agent-eval rpc <method> # one request from stdin → one response on stdout\n * agent-eval rpc-batch <method> # JSONL stdin → JSONL stdout\n * agent-eval analyst-benchmark ... # benchmark a real-model trace analyst\n * agent-eval supervisor-run report <runDir> [--format headline\|markdown\|json]\n * agent-eval search ship <search-ledger.jsonl> --run-kind optimization\|eval\n * agent-eval search show <search-ledger.jsonl>\n * agent-eval openapi [--out path] # write OpenAPI spec\n * agent-eval version\n \n <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n /\nimport { writeFileSync } from 'node:fs'\nimport { runAnalystBenchmarkCommand } from './analyst/benchmark-command'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { runRolloutReleaseCli } from './rollout/release/hf-dataset'\nimport { runSearchCommand } from './search-command'\nimport { runSupervisorRunCommand } from './supervisor-run/report-command'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n command: string\n positional: string[]\n flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n const [command, ...rest] = argv\n const positional: string[] = []\n const flags: Record<string, string> = {}\n for (let i = 0; i < rest.length; i++) {\n const tok = rest[i]!\n if (tok.startsWith('--')) {\n const raw = tok.slice(2)\n const equalsAt = raw.indexOf('=')\n if (equalsAt >= 0) {\n flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n continue\n }\n const key = raw\n if (key === 'help') {\n flags[key] = 'true'\n continue\n }\n const next = rest[i + 1]\n if (next != null && !next.startsWith('--')) {\n flags[key] = next\n i++\n } else {\n flags[key] = 'true'\n }\n } else if (tok === '-h') {\n flags.help = 'true'\n } else {\n positional.push(tok)\n }\n }\n return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n serve [--port 5005] [--host 127.0.0.1]\n Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n rpc <method>\n Read one JSON object from stdin (the params for <method>), write one\n JSON object to stdout. Methods: judge, listRubrics, version.\n rpc-batch <method>\n Like 'rpc' but JSONL in / JSONL out.\n openapi [--out openapi.json]\n Write the OpenAPI 3.1 spec.\n rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]\n Build a HuggingFace-ready dataset dir from tangle.rollout.v1 ledgers:\n validate, fail-closed split filter, scrub, export formats + card.\n analyst-benchmark --dataset <agentrx\|codetracebench> --labels <path> --trace-dir <path> [--artifact-dir <path>] --out <dir> ...\n Compare an empty baseline with a real-model trace analyst on public labels.\n Run with --help for all required inputs and controls.\n supervisor-run report <runDir> [--format headline\|markdown\|json]\n Report one Runtime or loops supervisor run directory. The status comes\n from Runtime's own result.json / failure.json; every missing measurement\n stays named. Exits 1 when the directory cannot be read.\n search ship <search-ledger.jsonl> --run-kind optimization\|eval [--content full\|digests]\n Ship a search ledger and its blobs to the hosted store in TANGLE_INGEST_URL,\n starting from the store's head. Run with --help for the environment it reads.\n search show <search-ledger.jsonl>\n Verify a search ledger and print its summary: leading nodes, recently\n discarded nodes and recent proposals, the same text a proposer reads.\n version\n Print server + wire-protocol version JSON.\n\nJudge provider:\n Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n OPENAI_ and TANGLE_* equivalents are also accepted. This binary is the only\n place that reads a provider credential; the library never does. Without both\n a base URL and a key, /v1/judge refuses with llm_not_configured.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n const { command, positional, flags } = parseArgs(process.argv.slice(2))\n assertKnownFlags(command, flags)\n\n if (command === 'analyst-benchmark' && flags.help === 'true') {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n if (command === 'search') return await runSearchCommand(process.argv.slice(3))\n if (flags.help === 'true') {\n process.stdout.write(`${HELP}\\n`)\n return 0\n }\n\n switch (command) {\n case 'serve': {\n const port = parsePort(flags.port ?? '5005')\n const host = flags.host ?? '127.0.0.1'\n const llm = resolveCliLlmConfig()\n const { server } = await startServerAsync({\n port,\n host,\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n // Keep process alive on SIGINT/SIGTERM\n const shutdown = (sig: string) => {\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] received ${sig}, shutting down`)\n server.close(() => process.exit(0))\n // Force exit after 5s if close hangs\n setTimeout(() => process.exit(1), 5000).unref()\n }\n process.on('SIGINT', () => shutdown('SIGINT'))\n process.on('SIGTERM', () => shutdown('SIGTERM'))\n // Block forever\n await new Promise(() => {})\n return 0\n }\n case 'rpc': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcOnce(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'rpc-batch': {\n const [method] = positional\n const llm = resolveCliLlmConfig()\n return await runRpcBatch(method, {\n ...(llm.chat ? { chat: llm.chat } : {}),\n ...(llm.model ? { judgeModel: llm.model } : {}),\n })\n }\n case 'openapi': {\n const out = flags.out ?? 'openapi.json'\n const spec = buildOpenApi(handleVersion().version)\n writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n return 0\n }\n case 'rollout-release': {\n // The subcommand owns its own flag grammar (multi-value --formats,\n // boolean --include-proposers) — pass raw argv through untouched.\n return await runRolloutReleaseCli(process.argv.slice(3))\n }\n case 'analyst-benchmark': {\n return await runAnalystBenchmarkCommand(process.argv.slice(3))\n }\n case 'supervisor-run': {\n // The subcommand owns its own flag grammar; pass raw argv through.\n return await runSupervisorRunCommand(process.argv.slice(3))\n }\n case 'version': {\n process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n return 0\n }\n case '--version': {\n process.stdout.write(`${handleVersion().version}\\n`)\n return 0\n }\n case 'help':\n case '--help':\n case '-h':\n case '':\n process.stdout.write(`${HELP}\\n`)\n return 0\n default:\n process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n return 1\n }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n serve: new Set(['help', 'host', 'port']),\n rpc: new Set(['help']),\n 'rpc-batch': new Set(['help']),\n openapi: new Set(['help', 'out']),\n version: new Set(['help']),\n help: new Set(),\n '--help': new Set(),\n '-h': new Set(),\n '--version': new Set(),\n '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n // These subcommands parse their own argv.\n if (\n command === 'rollout-release' \|\|\n command === 'analyst-benchmark' \|\|\n command === 'supervisor-run' \|\|\n command === 'search'\n ) {\n return\n }\n const allowed = FLAGS_BY_COMMAND[command]\n if (!allowed) return\n const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n if (unknown.length > 0) {\n throw new Error(`unknown flag for ${command \|\| 'help'}: --${unknown[0]}`)\n }\n}\n\nfunction parsePort(raw: string): number {\n const port = Number(raw)\n if (!Number.isInteger(port) \|\| port < 0 \|\| port > 65_535) {\n throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n }\n return port\n}\n\nmain()\n .then((code) => {\n process.exitCode = code\n })\n .catch((err) => {\n // eslint-disable-next-line no-console\n console.error('[agent-eval] cli error:', err)\n process.exitCode = 1\n })\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,wBACP,MAAyB,QAAQ,KACH;CAC9B,MAAM,kBAAkB,SAAS,IAAI,uBAAuB;CAC5D,MAAM,iBAAiB,SAAS,IAAI,sBAAsB;CAC1D,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,eAAe,SAAS,IAAI,cAAc;CAChD,MAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,KAAA,OAC7C,eAAe,mCAAmC,KAAA;CACrD,MAAM,SAAS,kBAAkB,gBAAgB;CACjD,MAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;CAE/F,IAAI,CAAC,WAAW,CAAC,QAAQ,OAAO,KAAA;CAChC,OAAO;EAAE;EAAS;EAAQ,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;CAAG;AACxD;AAEA,SAAgB,oBAAoB,MAAyB,QAAQ,KAAmB;CACtF,MAAM,QAAQ,wBAAwB,GAAG;CACzC,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,OAAO;EACL,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,QAAQ,MAAM;EAAO,GAAG,MAAM,KAAK;EACjF,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;CAC9C;AACF;;;;;;;;;AAUA,SAAS,cACP,OACA,cACY;CACZ,MAAM,SAAS,iBAAiB;EAC9B,WAAW;EACX,SAAS,MAAM;EACf,QAAQ,MAAM;EACd,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;CACzC,CAAC;CACD,OAAO;EACL,GAAG;EACH,OAAO,KAAK,aAAa;GACvB,IAAI,CAAC,IAAI,SAAS,CAAC,cACjB,MAAM,IAAI,MACR,4EACF;GAEF,OAAO,OAAO,KAAK,KAAK,QAAQ;EAClC;CACF;AACF;AAEA,SAAS,SAAS,OAA+C;CAC/D,MAAM,UAAU,OAAO,KAAK;CAC5B,OAAO,UAAU,UAAU,KAAA;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC7CA,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0Cd,eAAsB,iBAAiB,MAAiC;CACtE,MAAM,CAAC,YAAY,GAAG,QAAQ;CAC9B,IAAI,eAAe,KAAA,KAAa,eAAe,YAAY,eAAe,MAAM;EAC9E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,eAAe,QAAQ,OAAO,MAAM,eAAe,IAAI;CAC3D,IAAI,eAAe,QAAQ;EACzB,QAAQ,OAAO,MAAM,8BAA8B,WAAW,IAAI,MAAM,GAAG;EAC3E,OAAO;CACT;CACA,IAAI,KAAK,MAAM,QAAQ,QAAQ,KAAA,KAAa,QAAQ,YAAY,QAAQ,IAAI,GAAG;EAC7E,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,EAAE,MAAM,UAAU,cAAc,IAAI;CAC1C,MAAM,UAAU,oBAAoB,UAAU,MAAM,WAAW;CAC/D,IAAI,CAAC,QAAQ,SAAS,MAAM,IAAI,MAAM,4CAA4C,OAAO;CACzF,MAAM,UAAU,MAAM,WAAW;CACjC,IAAI,YAAY,UAAU,YAAY,WACpC,MAAM,IAAI,MAAM,sCAAsC,OAAO;CAE/D,MAAM,SAAS,oBAAoB;CACnC,IAAI,CAAC,QACH,MAAM,IAAI,MACR,qFACF;CAEF,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,IAAI;EACF,MAAM,UAAU,MAAM,iBAAiB;GACrC;GACA,QAAQ;IAAE;IAAM;GAAS;GACzB,SAAS,QAAQ;GACjB;EACF,CAAC;EACD,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,EAAE,GAAG;EAC5D,OAAO,QAAQ,KAAK,iBAAiB,QAAQ,aAAa,IAAI;CAChE,SAAS,OAAO;EACd,IAAI,EAAE,iBAAiB,0BAA0B,MAAM;EACvD,QAAQ,OAAO,MAAM,GAAG,MAAM,QAAQ,GAAG;EACzC,OAAO;CACT;AACF;AAEA,eAAe,eAAe,MAAiC;CAC7D,IAAI,KAAK,SAAS,QAAQ,KAAK,KAAK,SAAS,IAAI,GAAG;EAClD,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,IAAI,KAAK,SAAS,QAAQ,GAAG,OAAO,MAAM,mBAAmB,IAAI;CACjE,MAAM,aAAa,KAAK,QAAQ,QAAQ,CAAC,IAAI,WAAW,IAAI,CAAC;CAC7D,MAAM,QAAQ,IAAI,IAAI,KAAK,QAAQ,QAAQ,IAAI,WAAW,IAAI,CAAC,CAAC;CAChE,MAAM,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC;CACjE,IAAI,WAAW,WAAW,KAAK,QAAQ,SAAS,GAAG;EACjD,IAAI,QAAQ,SAAS,GAAG,QAAQ,OAAO,MAAM,gBAAgB,QAAQ,KAAK,IAAI,EAAE,GAAG;EACnF,QAAQ,OAAO,MAAM,GAAG,MAAM,GAAG;EACjC,OAAO;CACT;CACA,MAAM,OAAO,WAAW;CACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;CAC7C,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM;CACxC,MAAM,QAAQ,uBAAuB,MAAM,UAAU,IAAI;CACzD,MAAM,QACJ,MAAM,UAAU,MAAM,OAAO,OAAO,UAAU,MAAM,SAAS,IAAI,cAAc;CACjF,MAAM,SAAS,WAAW,OAAO,MAAM,KAAK;CAC5C,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,IAAI,OAAO,WAAW,GAAG;GACvB,QAAQ,OAAO,MAAM,4BAA4B;GACjD,OAAO;EACT;EACA,MAAM,OAAO,OAAO,YAAY,OAAO,KAAK,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;EAC9E,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,GAAG;EACzD,OAAO;CACT;CACA,MAAM,WAAW,CAAC,oBAAoB,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,OAAO,KAAK,SAAS,KAAK,IAAI,CAAC;CAC3F,QAAQ,OAAO,MAAM,GAAG,SAAS,KAAK,MAAM,EAAE,GAAG;CACjD,OAAO;AACT;AAEA,eAAe,mBAAmB,MAAiC;CACjE,MAAM,QAAkB,CAAC;CACzB,IAAI;CACJ,IAAI,OAAO;CACX,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,UAAU;EACxB,IAAI,UAAU,UAAU;GACtB,OAAO;GACP;EACF;EACA,IAAI,UAAU,eAAe;GAC3B,YAAY,KAAK,EAAE;GACnB,IAAI,cAAc,KAAA,GAAW,MAAM,IAAI,MAAM,8BAA8B,OAAO;GAClF;EACF;EACA,IAAI,MAAM,WAAW,IAAI,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,IAAI,OAAO;EAC7E,MAAM,KAAK,KAAK;CAClB;CACA,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,0CAA0C,OAAO;CACzF,MAAM,SAAS,CAAC;CAChB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,MAAM,kBAAkB,IAAI;EAC7C,OAAO,KAAK,uBAAuB,MAAM,SAAS,MAAM,MAAM,GAAG,UAAU,IAAI,CAAC;CAClF;CACA,MAAM,OAAO,WAAW,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,CAAC;CAC5E,QAAQ,OAAO,MACb,OAAO,GAAG,KAAK,UAAU,EAAE,YAAY,KAAK,GAAG,MAAM,CAAC,EAAE,MAAM,GAAG,qBAAqB,IAAI,EAAE,GAC9F;CACA,OAAO;AACT;AAEA,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,SAAS,WACP,OACA,YACA,OACwD;CACxD,MAAM,SAAiE,CAAC;CACxE,IAAI,MAAM,IAAI,QAAQ,GAAG;EACvB,MAAM,SAAS,KAAK,KAAK;EACzB,OAAO,KAAK;GAAE,MAAM;GAAQ;GAAQ,MAAM,eAAe,OAAO,IAAI;EAAE,CAAC;CACzE;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,wBAAwB,OAAO,IAAI;EAAE,CAAC;CAC3F;CACA,IAAI,MAAM,IAAI,SAAS,GAAG;EACxB,MAAM,SAAS,MAAM,KAAK;EAC1B,OAAO,KAAK;GAAE,MAAM;GAAS;GAAQ,MAAM,gBAAgB,OAAO,IAAI;EAAE,CAAC;CAC3E;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,KAAK;EAC/B,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,qBAAqB,OAAO,IAAI;EAAE,CAAC;CACrF;CACA,IAAI,MAAM,IAAI,eAAe,GAAG;EAC9B,MAAM,SAAS,WAAW,OAAO,EAAE,cAAc,iBAAiB,UAAU,EAAE,CAAC;EAC/E,OAAO,KAAK;GAAE,MAAM;GAAc;GAAQ,MAAM,eAAe,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,aAAa,GAAG;EAC5B,MAAM,OAAO,iBAAiB,UAAU;EACxC,MAAM,SAAS,UACb,OACA,kBAAkB,YAAY,KAAK,QAAQ,QAAQ,KAAK,IAAI,CAC9D;EACA,OAAO,KAAK;GAAE,MAAM;GAAa;GAAQ,MAAM,gBAAgB,MAAM;EAAE,CAAC;CAC1E;CACA,IAAI,MAAM,IAAI,kBAAkB,GAAG;EACjC,MAAM,SAAS,cAAc,KAAK;EAClC,OAAO,KAAK;GAAE,MAAM;GAAiB;GAAQ,MAAM,oBAAoB,MAAM;EAAE,CAAC;CAClF;CACA,OAAO;AACT;;;;;;AAOA,SAAS,iBAAiB,YAAqE;CAC7F,MAAM,wBAAQ,IAAI,IAAqB;CACvC,QAAQ,QAAQ;EACd,IAAI,MAAM,IAAI,IAAI,MAAM,GAAG,OAAO,MAAM,IAAI,IAAI,MAAM;EACtD,MAAM,MAAM,IAAI,OAAO,MAAM,CAAgB;EAC7C,MAAM,aAAa,CAAC,KAAK,QAAQ,UAAU,GAAG,SAAS,GAAG,IAAI,MAAM,CAAC;EACrE,IAAI,IAAI,IAAI,WAAW,OAAO,GAAG,WAAW,QAAQ,cAAc,IAAI,GAAG,CAAC;EAC1E,IAAI;EACJ,KAAK,MAAM,aAAa,YAAY;GAClC,IAAI,CAAC,WAAW,SAAS,GAAG;GAC5B,MAAM,QAAQ,aAAa,SAAS;GAEpC,IADe,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAChD,MAAM,OAAO,MAAM,eAAe,IAAI,YAAY;GAC3D,IAAI;IACF,QAAQ,KAAK,MAAM,MAAM,SAAS,MAAM,CAAC;GAC3C,QAAQ,CAER;GACA;EACF;EACA,MAAM,IAAI,IAAI,QAAQ,KAAK;EAC3B,OAAO;CACT;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,MAAM,QAAQ,CACZ,SAAS,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,IAAI,IAAI,KAAK,UAAU,OAAO,KAAK,cAAc,IAAI,KAAK,KACvH;CACA,MAAM,QAAQ,MAAgB,QAAgB,WAA0B;EACtE,MAAM,SAAS,WAAW,KAAK,KAAK,SAAS,QAAQ;EACrD,MAAM,KAAK,KAAK,WAAW,GAAG,KAAK,SAAS,OAAO;EACnD,MAAM,OAAO,KAAK,SAAS,OAAO,KAAK,SAAS,KAAK;EACrD,MAAM,QAAQ,KAAK,UAAU,OAAO,mBAAmB;EACvD,MAAM,OACJ,KAAK,qBAAqB,IACtB,IAAI,KAAK,aAAa,QAAQ,CAAC,MAC/B,IAAI,KAAK,aAAa,QAAQ,CAAC,EAAE,qBAAqB,KAAK,SAAS,QAAQ,CAAC,EAAE,QAAQ,KAAK,iBAAiB,oBAAoB,KAAK,qBAAqB,IAAI,KAAK;EAC1K,MAAM,KACJ,GAAG,SAAS,SAAS,KAAK,KAAK,OAAO,IAAI,KAAK,UAAU,cAAc,OAAO,MAAM,IAAI,MAC1F;EACA,MAAM,cAAc,UAAU,WAAW,KAAK,KAAK,SAAS,QAAQ;EACpE,KAAK,SAAS,SAAS,OAAO,UAAU;GACtC,KAAK,OAAO,aAAa,UAAU,KAAK,SAAS,SAAS,CAAC;EAC7D,CAAC;CACH;CACA,KAAK,MAAM,SAAS,MAAM,UAAU;EAClC,KAAK,MAAM,IAAI,UAAU,KAAK,MAAM,SAAS,CAAC;CAChD,CAAC;CACD,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,wBAAwB,MAAiC;CAChE,MAAM,QAAQ,CAAC,mBAAmB,KAAK,MAAM,gDAAgD;CAC7F,IAAI,KAAK,KAAK,WAAW,GAAG,MAAM,KAAK,gBAAgB;CACvD,KAAK,MAAM,OAAO,KAAK,MAAM;EAC3B,MAAM,IAAI,IAAI;EACd,MAAM,WAAW,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAC1C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,GAAG;EACX,MAAM,YACJ,EAAE,WAAW,SACT,EAAE,MAAM,IACN,gCACA,kBAAkB,EAAE,EAAE,MAAM,kBAAkB,WAChD,EAAE,WAAW,iBACX,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,kBAAkB,mBAC9D,SAAS,EAAE,KAAM,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,SAAU,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,eAAe,MAAM,EAAE,EAAE;EAC/I,MAAM,QAAQ,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;EAC/E,MAAM,cAAc,IAAI,cAAc,IAAI,KAAK,IAAI,YAAY,iBAAiB;EAChF,MAAM,KACJ,KAAK,IAAI,SAAS,IAAI,IAAI,UAAU,YAAY,YAAY,KAAK,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,YAAY,mBAAmB,KAAK,UAAU,IAAI,IAAI,SAAS,UAC1K;CACF;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,gBAAgB,MAAyB;CAChD,MAAM,UAAU,KAAK,KAAK,QAAQ,QAAQ,IAAI,OAAO;CACrD,MAAM,UAAU,OAAO,QAAQ,KAAK,UAAU,CAAC,CAC5C,QAAQ,GAAG,WAAW,QAAQ,CAAC,CAAC,CAChC,KAAK,CAAC,QAAQ,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC,CAC9C,KAAK,IAAI;CACZ,MAAM,QAAQ,CACZ,UAAU,KAAK,MAAM,gBAAgB,KAAK,KAAK,KAAK,IAAI,EAAE,KAAK,KAAK,aAAa,MAAM,KAAK,KAAK,OAAO,4BAA4B,KAAK,cAAc,WAAW,UAAU,KAAK,QAAQ,KAAK,IAChM;CACA,KAAK,MAAM,OAAO,SAChB,MAAM,KACJ,KAAK,IAAI,OAAO,UAAU,IAAI,MAAO,QAAQ,CAAC,EAAE,MAAM,IAAI,MAAM,UAAU,IAAI,OAAO,iBAAiB,IAAI,eAAgB,QAAQ,CAAC,GACrI;CAEF,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,qBAAqB,MAA8B;CAC1D,MAAM,QAAQ,CACZ,gBAAgB,KAAK,MAAM,UAAU,KAAK,UAAU,KAAK,KAAK,QAAQ,OAAO,aAAa,KAAK,QAAQ,OAAO,YAAY,KAAK,aAAa,OAAO,oBAAoB,KAAK,aAAa,OAAO,iBAClM;CACA,IAAI,KAAK,UACP,MAAM,KACJ,eAAe,KAAK,SAAS,eAAe,MAAM,KAAK,SAAS,WAAW,cAAc,KAAK,SAAS,IAAI,+FAC7G;CAEF,KAAK,MAAM,OAAO,KAAK,gBAAgB;EACrC,MAAM,OACJ,IAAI,SAAS,OACT,iBAAiB,IAAI,aAAa,KAClC,GAAG,IAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,IAAI,WAAW,IAAI,IAAI,kBAAkB,YAAY,IAAI,QAAQ,OAAO;EAC9G,MAAM,KAAK,aAAa,IAAI,UAAU,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,qBAAqB,MAAM;CAC9F;CACA,OAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,cAAc,MAAiE;CACtF,MAAM,QAAgC,CAAC;CACvC,MAAM,aAAuB,CAAC;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;EAChD,MAAM,QAAQ,KAAK;EACnB,IAAI,CAAC,MAAM,WAAW,IAAI,GAAG;GAC3B,WAAW,KAAK,KAAK;GACrB;EACF;EACA,MAAM,CAAC,MAAM,UAAU,MAAM,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC;EAClD,IAAI,SAAS,cAAc,SAAS,WAAW,MAAM,IAAI,MAAM,kBAAkB,MAAM;EACvF,MAAM,QAAQ,UAAU,KAAK,EAAE;EAC/B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,KAAK,KAAK,eAAe;EAClE,MAAM,QAAQ;CAChB;CACA,IAAI,WAAW,WAAW,GAAG,MAAM,IAAI,MAAM,KAAK;CAClD,OAAO;EAAE,MAAM,WAAW;EAAK;CAAM;AACvC;;AAGA,eAAe,kBAAkB,MAA+B;CAC9D,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;CACnC,IAAI;EAEF,MAAM,QAAQ,OAAO,MAAM,gCAAgC,CAAC;EAC5D,MAAM,EAAE,cAAc,MAAM,OAAO,KAAK,OAAO,GAAG,MAAM,YAAY,CAAC;EACrE,MAAM,UAAU,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ,EAAI;EACzD,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,GAAG,KAAK,+BAA+B;EACxE,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,QAAQ,GAAG,OAAO,CAAC;EAC3D,IAAI,OAAO,MAAM,aAAa,UAAU,MAAM,IAAI,MAAM,GAAG,KAAK,0BAA0B;EAC1F,OAAO,MAAM;CACf,UAAU;EACR,MAAM,OAAO,MAAM;CACrB;AACF;;;;;;;;;;;;;ACrYA,MAAM,UAAgD;CAAC;CAAY;CAAY;AAAM;AAErF,MAAM,uBACJ;AAQF,MAAM,aAAqC;CACzC,SAAS,SAAS;EAChB,QAAQ,OAAO,MAAM,IAAI;CAC3B;CACA,SAAS,SAAS;EAChB,QAAQ,OAAO,MAAM,IAAI;CAC3B;AACF;AAOA,SAAS,SAAS,OAAmD;CACnE,OAAQ,QAA8B,SAAS,KAAK;AACtD;AAEA,SAAS,gBAAgB,MAAqC;CAC5D,IAAI,SAAwB;CAC5B,IAAI,SAAoC;CACxC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,QAAQ,YAAY;GACtB,MAAM,MAAM,KAAK,EAAE;GACnB,IAAI,QAAQ,KAAA,KAAa,CAAC,SAAS,GAAG,GACpC,MAAM,IAAI,MACR,2BAA2B,QAAQ,KAAK,GAAG,EAAE,QAAQ,KAAK,UAAU,OAAO,EAAE,GAC/E;GAEF,SAAS;EACX,OAAO,IAAI,IAAI,WAAW,WAAW,GAAG;GACtC,MAAM,MAAM,IAAI,MAAM,CAAkB;GACxC,IAAI,CAAC,SAAS,GAAG,GACf,MAAM,IAAI,MAAM,2BAA2B,QAAQ,KAAK,GAAG,EAAE,QAAQ,KAAK,UAAU,GAAG,GAAG;GAE5F,SAAS;EACX,OAAO,IAAI,IAAI,WAAW,IAAI,GAC5B,MAAM,IAAI,MAAM,iBAAiB,IAAI,EAAE;OAClC,IAAI,WAAW,MACpB,SAAS;OAET,MAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;CAElD;CACA,IAAI,WAAW,MAAM,MAAM,IAAI,MAAM,oBAAoB;CACzD,OAAO;EAAE;EAAQ;CAAO;AAC1B;;;;;AAMA,eAAsB,wBACpB,MACA,KAA6B,YACZ;CACjB,MAAM,CAAC,MAAM,GAAG,QAAQ;CACxB,IAAI,SAAS,KAAA,KAAa,SAAS,YAAY,SAAS,QAAQ,SAAS,QAAQ;EAC/E,GAAG,OAAO,GAAG,qBAAqB,GAAG;EACrC,OAAO,SAAS,KAAA,IAAY,IAAI;CAClC;CACA,IAAI,SAAS,UAAU;EACrB,GAAG,OAAO,sCAAsC,KAAK,KAAK,qBAAqB,GAAG;EAClF,OAAO;CACT;CACA,IAAI,KAAK,SAAS,QAAQ,KAAK,KAAK,SAAS,IAAI,GAAG;EAClD,GAAG,OAAO,GAAG,qBAAqB,GAAG;EACrC,OAAO;CACT;CAEA,IAAI;CACJ,IAAI;EACF,OAAO,gBAAgB,IAAI;CAC7B,SAAS,OAAO;EACd,GAAG,OAAO,GAAG,aAAa,KAAK,EAAE,GAAG;EACpC,OAAO;CACT;CAEA,MAAM,SAAS,QAAQ,KAAK,MAAM;CAIlC,MAAM,QAAQ,MAAM,KAAK,MAAM,CAAC,CAAC,YAAY,IAAI;CACjD,IAAI,UAAU,QAAQ,CAAC,MAAM,YAAY,GAAG;EAC1C,GAAG,OAAO,uCAAuC,OAAO,sBAAsB;EAC9E,OAAO;CACT;CAEA,IAAI;EACF,MAAM,SAAS,MAAM,qBAAqB,MAAM;EAChD,QAAQ,KAAK,QAAb;GACE,KAAK;IACH,GAAG,OAAO,GAAG,4BAA4B,MAAM,EAAE,GAAG;IACpD;GACF,KAAK;IACH,GAAG,OAAO,4BAA4B,MAAM,CAAC;IAC7C;GACF,KAAK,QACH,GAAG,OAAO,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,EAAE,GAAG;EAEpD;EACA,OAAO;CACT,SAAS,OAAO;EACd,GAAG,OAAO,iDAAiD,OAAO,IAAI,aAAa,KAAK,EAAE,GAAG;EAC7F,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;ACvGA,SAAS,UAAU,MAAsB;CACvC,MAAM,CAAC,SAAS,GAAG,QAAQ;CAC3B,MAAM,aAAuB,CAAC;CAC9B,MAAM,QAAgC,CAAC;CACvC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,IAAI,WAAW,IAAI,GAAG;GACxB,MAAM,MAAM,IAAI,MAAM,CAAC;GACvB,MAAM,WAAW,IAAI,QAAQ,GAAG;GAChC,IAAI,YAAY,GAAG;IACjB,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,IAAI,MAAM,WAAW,CAAC;IACtD;GACF;GACA,MAAM,MAAM;GACZ,IAAI,QAAQ,QAAQ;IAClB,MAAM,OAAO;IACb;GACF;GACA,MAAM,OAAO,KAAK,IAAI;GACtB,IAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;IAC1C,MAAM,OAAO;IACb;GACF,OACE,MAAM,OAAO;EAEjB,OAAO,IAAI,QAAQ,MACjB,MAAM,OAAO;OAEb,WAAW,KAAK,GAAG;CAEvB;CACA,OAAO;EAAE,SAAS,WAAW;EAAQ;EAAY;CAAM;AACzD;AAEA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCb,eAAe,OAAwB;CACrC,MAAM,EAAE,SAAS,YAAY,UAAU,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;CACtE,iBAAiB,SAAS,KAAK;CAE/B,IAAI,YAAY,uBAAuB,MAAM,SAAS,QACpD,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;CAE/D,IAAI,YAAY,UAAU,OAAO,MAAM,iBAAiB,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7E,IAAI,MAAM,SAAS,QAAQ;EACzB,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;EAChC,OAAO;CACT;CAEA,QAAQ,SAAR;EACE,KAAK,SAAS;GACZ,MAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;GAC3C,MAAM,OAAO,MAAM,QAAQ;GAC3B,MAAM,MAAM,oBAAoB;GAChC,MAAM,EAAE,WAAW,MAAM,iBAAiB;IACxC;IACA;IACA,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;GAED,MAAM,YAAY,QAAgB;IAEhC,QAAQ,IAAI,yBAAyB,IAAI,gBAAgB;IACzD,OAAO,YAAY,QAAQ,KAAK,CAAC,CAAC;IAElC,iBAAiB,QAAQ,KAAK,CAAC,GAAG,GAAI,CAAC,CAAC,MAAM;GAChD;GACA,QAAQ,GAAG,gBAAgB,SAAS,QAAQ,CAAC;GAC7C,QAAQ,GAAG,iBAAiB,SAAS,SAAS,CAAC;GAE/C,MAAM,IAAI,cAAc,CAAC,CAAC;GAC1B,OAAO;EACT;EACA,KAAK,OAAO;GACV,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,WAAW,QAAQ;IAC9B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,aAAa;GAChB,MAAM,CAAC,UAAU;GACjB,MAAM,MAAM,oBAAoB;GAChC,OAAO,MAAM,YAAY,QAAQ;IAC/B,GAAI,IAAI,OAAO,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IACrC,GAAI,IAAI,QAAQ,EAAE,YAAY,IAAI,MAAM,IAAI,CAAC;GAC/C,CAAC;EACH;EACA,KAAK,WAAW;GACd,MAAM,MAAM,MAAM,OAAO;GACzB,MAAM,OAAO,aAAa,cAAc,CAAC,CAAC,OAAO;GACjD,cAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,EAAE,KAAK,OAAO;GAEhE,QAAQ,IAAI,0CAA0C,KAAK;GAC3D,OAAO;EACT;EACA,KAAK,mBAGH,OAAO,MAAM,qBAAqB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAEzD,KAAK,qBACH,OAAO,MAAM,2BAA2B,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE/D,KAAK,kBAEH,OAAO,MAAM,wBAAwB,QAAQ,KAAK,MAAM,CAAC,CAAC;EAE5D,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,EAAE,GAAG;GACpE,OAAO;EAET,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,cAAc,CAAC,CAAC,QAAQ,GAAG;GACnD,OAAO;EAET,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK;GACH,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;GAChC,OAAO;EACT;GACE,QAAQ,OAAO,MAAM,oBAAoB,QAAQ,IAAI,KAAK,GAAG;GAC7D,OAAO;CACX;AACF;AAEA,MAAM,mBAAwD;CAC5D,uBAAO,IAAI,IAAI;EAAC;EAAQ;EAAQ;CAAM,CAAC;CACvC,qBAAK,IAAI,IAAI,CAAC,MAAM,CAAC;CACrB,6BAAa,IAAI,IAAI,CAAC,MAAM,CAAC;CAC7B,yBAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;CAChC,yBAAS,IAAI,IAAI,CAAC,MAAM,CAAC;CACzB,sBAAM,IAAI,IAAI;CACd,0BAAU,IAAI,IAAI;CAClB,sBAAM,IAAI,IAAI;CACd,6BAAa,IAAI,IAAI;CACrB,oBAAI,IAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;CAE9E,IACE,YAAY,qBACZ,YAAY,uBACZ,YAAY,oBACZ,YAAY,UAEZ;CAEF,MAAM,UAAU,iBAAiB;CACjC,IAAI,CAAC,SAAS;CACd,MAAM,UAAU,OAAO,KAAK,KAAK,CAAC,CAAC,QAAQ,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;CACtE,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MAAM,oBAAoB,WAAW,OAAO,MAAM,QAAQ,IAAI;AAE5E;AAEA,SAAS,UAAU,KAAqB;CACtC,MAAM,OAAO,OAAO,GAAG;CACvB,IAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAChD,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,GAAG;CAE9F,OAAO;AACT;AAEA,KAAK,CAAC,CACH,MAAM,SAAS;CACd,QAAQ,WAAW;AACrB,CAAC,CAAC,CACD,OAAO,QAAQ;CAEd,QAAQ,MAAM,2BAA2B,GAAG;CAC5C,QAAQ,WAAW;AACrB,CAAC"}

package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} RENAMED Viewed

@@ -1,15 +1,10 @@
-import { A as SearchLedgerHash, O as SearchLedgerEntry, i as SearchArtifactRef, k as SearchLedgerEvent } from "./search-ledger-types-p2xKfiDJ.js";
+import { A as SearchLedgerHash, O as SearchLedgerEntry } from "./search-ledger-types-p2xKfiDJ.js";
 import { z } from "zod";
 //#region src/hosted/search-ledger-wire.d.ts
-/** At most this many lines in one ingest request. */
-declare const SEARCH_LEDGER_BATCH_MAX_LINES = 1000;
 /** At most this many UTF-8 bytes across one request's lines. A single line
  * larger than this cannot be shipped. */
 declare const SEARCH_LEDGER_BATCH_MAX_BYTES = 1048576;
 declare const SEARCH_LEDGER_INGEST_PATH = "/v1/ingest/search-ledger";
-declare function searchLedgerHeadPath(searchId: string): string;
-/** Blobs are addressed by the hex digest alone. */
-declare function searchBlobPath(sha256: SearchLedgerHash): string;
 /** What the run is for: an optimization, or an eval (a one-node search). */
 declare const SearchRunKindSchema: z.ZodEnum<{
   eval: "eval";
@@ -61,8 +56,6 @@ declare const SearchBlobPutResponseSchema: z.ZodObject<{
   }>;
 }, z.core.$strict>;
 type SearchBlobPutResponse = z.infer<typeof SearchBlobPutResponseSchema>;
-/** Every blob an event names, once each. Ship these before the event. */
-declare function searchLedgerArtifactRefs(event: SearchLedgerEvent): SearchArtifactRef[];
 type SearchLedgerBatchAdmission = {
   status: 'accepted';
   /** Entries past the stored head, in order, with their exact lines. */
@@ -207,12 +200,6 @@ interface HostedClient {
   readonly tenant: HostedTenant;
   readonly wireVersion: HostedWireVersion;
 }
-/** A request the server refused, after any retries. */
-declare class HostedRequestError extends Error {
-  readonly status: number;
-  readonly body: string;
-  constructor(url: string, status: number, body: string);
-}
 declare function createHostedClient(tenant: HostedTenant): HostedClient;
 /**
  * Build a {@link HostedTenant} from env, or `undefined` when ingest is not
@@ -232,5 +219,5 @@ declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
   env?: Record<string, string | undefined>;
 }): HostedClient | undefined;
 //#endregion
-export { admitSearchLedgerBatch as A, SearchLedgerBatchAdmission as C, SearchLedgerHeadSchema as D, SearchLedgerHead as E, searchLedgerArtifactRefs as M, searchLedgerHeadPath as N, SearchRunKind as O, SearchBlobPutResponseSchema as S, SearchLedgerConflictSchema as T, IngestSearchLedgerRequestSchema as _, createHostedClient as a, SEARCH_LEDGER_INGEST_PATH as b, HOSTED_WIRE_VERSION as c, IngestResponse as d, IngestTracesRequest as f, IngestSearchLedgerRequest as g, AdmitSearchLedgerBatchInput as h, SearchLedgerIngestOutcome as i, searchBlobPath as j, SearchRunKindSchema as k, HostedIngestHeaders as l, UnixNanoTimestamp as m, HostedRequestError as n, hostedClientFromEnv as o, TraceSpanEvent as p, HostedTenant as r, hostedTenantFromEnv as s, HostedClient as t, HostedWireVersion as u, SEARCH_LEDGER_BATCH_MAX_BYTES as v, SearchLedgerConflict as w, SearchBlobPutResponse as x, SEARCH_LEDGER_BATCH_MAX_LINES as y };
-//# sourceMappingURL=client-BzL9niqi.d.ts.map
+export { SearchLedgerConflictSchema as C, admitSearchLedgerBatch as D, SearchRunKind as E, SearchLedgerConflict as S, SearchLedgerHeadSchema as T, SEARCH_LEDGER_BATCH_MAX_BYTES as _, hostedClientFromEnv as a, SearchBlobPutResponseSchema as b, HostedIngestHeaders as c, IngestTracesRequest as d, TraceSpanEvent as f, IngestSearchLedgerRequestSchema as g, IngestSearchLedgerRequest as h, createHostedClient as i, HostedWireVersion as l, AdmitSearchLedgerBatchInput as m, HostedTenant as n, hostedTenantFromEnv as o, UnixNanoTimestamp as p, SearchLedgerIngestOutcome as r, HOSTED_WIRE_VERSION as s, HostedClient as t, IngestResponse as u, SEARCH_LEDGER_INGEST_PATH as v, SearchLedgerHead as w, SearchLedgerBatchAdmission as x, SearchBlobPutResponse as y };
+//# sourceMappingURL=client-hwo3cJjR.d.ts.map

package/dist/client-hwo3cJjR.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"client-hwo3cJjR.d.ts","names":[],"sources":["../src/hosted/search-ledger-wire.ts","../src/hosted/types.ts","../src/hosted/client.ts"],"mappings":";;;;;cAqCa;cAEA;;cAsBA,qBAAmB,EAAA;;;;KACpB,gBAAgB,EAAE,aAAa;;;cAI9B,wBAAsB,EAAA;;;;GAM/B,EAAA,KAAA;KACQ,mBAAmB,EAAE,aAAa;cAEjC,iCAA+B,EAAA;;;;;;;;;GAkBxC,EAAA,KAAA;KACQ,4BAA4B,EAAE,aAAa;;cAG1C,4BAA0B,EAAA;;;;;;;;;;;GAM5B,EAAA,KAAA;KACC,uBAAuB,EAAE,aAAa;;;cAIrC,6BAA2B,EAAA;;;;;;;;GAM7B,EAAA,KAAA;KACC,wBAAwB,EAAE,aAAa;KA+BvC;EAEN;;EAEA,SAAS;EACT;;EAEA,MAAM;;EAEN;EAAoB,UAAU;;UAEnB;EACf,SAAS;;EAET,MAAM;;EAEN,gBAAgB,mBAAmB;;;;;;;;iBASrB,uBACd,OAAO,8BACN;;;;;;;;;;;;;;;;;;;;;;;;;cCtJU;KACD,2BAA2B;;UAKtB;;EAEf;;EAEA;;EAEA,yBAAyB;;EAEzB;;;;;;KASU;;;;;;UAOK;EACf;EACA;EACA;EACA;EACA,mBAAmB;EACnB,iBAAiB;EACjB,YAAY;EACZ,SAAS;IACP,cAAc;IACd;IACA,aAAa;;EAEf;IAAW;IAAgC;;;EAE3C;;EAEA;;EAEA;;EAEA;;UAKe;EACf,aAAa;EACb,OAAO;;UAGQ;;EAEf;;EAEA,UAAU;IAAQ;IAAe;;;;;UC7ClB;;EAEf;;EAEA;;EAEA;;EAEA,mBAAmB;;EAEnB;;EAEA;;;KAIU;EACN;EAAoB,MAAM;;EAC1B;EAAoB,UAAU;;UAEnB;EACf,aAAa,OAAO,kBAAkB,0BAA0B,QAAQ;;EAExE,iBAAiB,kBAAkB,SAAS,cAAc,QAAQ;EAClE,mBACE,SAAS,2BACT,SAAS,cACR,QAAQ;EACX,cACE,4BACA,OAAO,WAAW,cAClB,qBACA,SAAS,cACR,QAAQ;WACF,QAAQ;WACR,aAAa;;iBA8IR,mBAAmB,QAAQ,eAAe;;;;;;;;;;iBAqG1C,oBACd,YAAW,QAAQ;EAAkB,MAAM;IAC1C;;;iBAmBa,oBACd,YAAW,QAAQ;EAAkB,MAAM;IAC1C"}

package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { s as ValidationError } from "./errors-DVZiqS7K.js";
-import { i as makeRng } from "./internal-Epl-YBZW.js";
+import { s as ValidationError } from "./errors-BcOs5MKW.js";
+import { i as makeRng } from "./internal-BTK91Bqt.js";
 //#region src/meta-eval/cluster-bootstrap.ts
 /**
 * Percentile cluster bootstrap of an observation mean. Whole clusters are resampled with
@@ -63,4 +63,4 @@ function clusterBootstrapMean(observations, options = {}) {
 //#endregion
 export { clusterBootstrapMean as t };
-//# sourceMappingURL=cluster-bootstrap-qcUWYFl-.js.map
+//# sourceMappingURL=cluster-bootstrap-DEwwhl0C.js.map

package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"cluster-bootstrap-~~qcUWYFl-.~~js","names":[],"sources":["../src/meta-eval/cluster-bootstrap.ts"],"sourcesContent":["import { ValidationError } from '../errors'\nimport { makeRng } from '../statistics/internal'\n\n/** One observation and the source cluster it belongs to. /\nexport interface ClusteredObservation {\n cluster: string\n value: number\n}\n\nexport interface ClusterBootstrapOptions {\n /* Confidence level. Default 0.95. /\n confidence?: number\n /* Bootstrap resample count. Default 2000. /\n resamples?: number\n /* Deterministic seed. If omitted, derived from the values so the interval is reproducible. /\n seed?: number\n}\n\nexport interface ClusterBootstrapResult {\n n: number\n clusters: number\n /* Mean over observations, each observation weighted equally. /\n mean: number\n low: number\n high: number\n confidence: number\n resamples: number\n}\n\n/\n Percentile cluster bootstrap of an observation mean. Whole clusters are resampled with\n * replacement and each resample's statistic is Σ values / Σ observations over the drawn\n * clusters, so observations that share a source move together and a large cluster cannot\n * pose as many independent results. For a paired comparison pass the per-observation\n * difference (candidate − baseline) under the observation's cluster.\n \n With one cluster the interval collapses to the point estimate: there is no between-cluster\n * spread to measure, which callers must treat as no interval, not as certainty.\n /\nexport function clusterBootstrapMean(\n observations: readonly ClusteredObservation[],\n options: ClusterBootstrapOptions = {},\n): ClusterBootstrapResult {\n const confidence = options.confidence ?? 0.95\n const resamples = options.resamples ?? 2000\n if (!(confidence > 0 && confidence < 1)) {\n throw new ValidationError(\n `clusterBootstrapMean: confidence must be in (0,1), got ${confidence}`,\n )\n }\n if (!Number.isInteger(resamples) \|\| resamples < 1) {\n throw new ValidationError(\n `clusterBootstrapMean: resamples must be a positive integer, got ${resamples}`,\n )\n }\n if (!observations.length) {\n throw new ValidationError('clusterBootstrapMean: requires at least one observation')\n }\n\n const sums = new Map<string, { sum: number; count: number }>()\n const values: number[] = []\n for (const [index, observation] of observations.entries()) {\n if (typeof observation?.cluster !== 'string' \|\| observation.cluster === '') {\n throw new ValidationError(`clusterBootstrapMean: observation ${index} needs a cluster id`)\n }\n if (!Number.isFinite(observation.value)) {\n throw new ValidationError(`clusterBootstrapMean: observation ${index} value must be finite`)\n }\n const cell = sums.get(observation.cluster) ?? { sum: 0, count: 0 }\n cell.sum += observation.value\n cell.count += 1\n sums.set(observation.cluster, cell)\n values.push(observation.value)\n }\n\n const cells = [...sums.values()]\n const total = cells.reduce((acc, cell) => acc + cell.sum, 0)\n const mean = total / observations.length\n const k = cells.length\n const rng = makeRng(options.seed, values)\n const samples = new Array<number>(resamples)\n for (let b = 0; b < resamples; b++) {\n let sum = 0\n let count = 0\n for (let draw = 0; draw < k; draw++) {\n const cell = cells[Math.floor(rng() k)]!\n sum += cell.sum\n count += cell.count\n }\n samples[b] = sum / count\n }\n samples.sort((a, b) => a - b)\n const alpha = 1 - confidence\n const lowIdx = Math.floor((alpha / 2) * resamples)\n const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)\n return {\n n: observations.length,\n clusters: k,\n mean,\n low: samples[lowIdx]!,\n high: samples[Math.max(highIdx, lowIdx)]!,\n confidence,\n resamples,\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAuCA,SAAgB,qBACd,cACA,UAAmC,CAAC,GACZ;CACxB,MAAM,aAAa,QAAQ,cAAc;CACzC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI,EAAE,aAAa,KAAK,aAAa,IACnC,MAAM,IAAI,gBACR,0DAA0D,YAC5D;CAEF,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,gBACR,mEAAmE,WACrE;CAEF,IAAI,CAAC,aAAa,QAChB,MAAM,IAAI,gBAAgB,yDAAyD;CAGrF,MAAM,uBAAO,IAAI,IAA4C;CAC7D,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,CAAC,OAAO,gBAAgB,aAAa,QAAQ,GAAG;EACzD,IAAI,OAAO,aAAa,YAAY,YAAY,YAAY,YAAY,IACtE,MAAM,IAAI,gBAAgB,qCAAqC,MAAM,oBAAoB;EAE3F,IAAI,CAAC,OAAO,SAAS,YAAY,KAAK,GACpC,MAAM,IAAI,gBAAgB,qCAAqC,MAAM,sBAAsB;EAE7F,MAAM,OAAO,KAAK,IAAI,YAAY,OAAO,KAAK;GAAE,KAAK;GAAG,OAAO;EAAE;EACjE,KAAK,OAAO,YAAY;EACxB,KAAK,SAAS;EACd,KAAK,IAAI,YAAY,SAAS,IAAI;EAClC,OAAO,KAAK,YAAY,KAAK;CAC/B;CAEA,MAAM,QAAQ,CAAC,GAAG,KAAK,OAAO,CAAC;CAE/B,MAAM,OADQ,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,KAAK,CACzC,IAAI,aAAa;CAClC,MAAM,IAAI,MAAM;CAChB,MAAM,MAAM,QAAQ,QAAQ,MAAM,MAAM;CACxC,MAAM,UAAU,IAAI,MAAc,SAAS;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;EAClC,IAAI,MAAM;EACV,IAAI,QAAQ;EACZ,KAAK,IAAI,OAAO,GAAG,OAAO,GAAG,QAAQ;GACnC,MAAM,OAAO,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC;GACvC,OAAO,KAAK;GACZ,SAAS,KAAK;EAChB;EACA,QAAQ,KAAK,MAAM;CACrB;CACA,QAAQ,MAAM,GAAG,MAAM,IAAI,CAAC;CAC5B,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,KAAK,MAAO,QAAQ,IAAK,SAAS;CACjD,MAAM,UAAU,KAAK,IAAI,YAAY,GAAG,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC;CAClF,OAAO;EACL,GAAG,aAAa;EAChB,UAAU;EACV;EACA,KAAK,QAAQ;EACb,MAAM,QAAQ,KAAK,IAAI,SAAS,MAAM;EACtC;EACA;CACF;AACF"}
1	+ {"version":3,"file":"cluster-bootstrap-DEwwhl0C.js","names":[],"sources":["../src/meta-eval/cluster-bootstrap.ts"],"sourcesContent":["import { ValidationError } from '../errors'\nimport { makeRng } from '../statistics/internal'\n\n/** One observation and the source cluster it belongs to. /\nexport interface ClusteredObservation {\n cluster: string\n value: number\n}\n\nexport interface ClusterBootstrapOptions {\n /* Confidence level. Default 0.95. /\n confidence?: number\n /* Bootstrap resample count. Default 2000. /\n resamples?: number\n /* Deterministic seed. If omitted, derived from the values so the interval is reproducible. /\n seed?: number\n}\n\nexport interface ClusterBootstrapResult {\n n: number\n clusters: number\n /* Mean over observations, each observation weighted equally. /\n mean: number\n low: number\n high: number\n confidence: number\n resamples: number\n}\n\n/\n Percentile cluster bootstrap of an observation mean. Whole clusters are resampled with\n * replacement and each resample's statistic is Σ values / Σ observations over the drawn\n * clusters, so observations that share a source move together and a large cluster cannot\n * pose as many independent results. For a paired comparison pass the per-observation\n * difference (candidate − baseline) under the observation's cluster.\n \n With one cluster the interval collapses to the point estimate: there is no between-cluster\n * spread to measure, which callers must treat as no interval, not as certainty.\n /\nexport function clusterBootstrapMean(\n observations: readonly ClusteredObservation[],\n options: ClusterBootstrapOptions = {},\n): ClusterBootstrapResult {\n const confidence = options.confidence ?? 0.95\n const resamples = options.resamples ?? 2000\n if (!(confidence > 0 && confidence < 1)) {\n throw new ValidationError(\n `clusterBootstrapMean: confidence must be in (0,1), got ${confidence}`,\n )\n }\n if (!Number.isInteger(resamples) \|\| resamples < 1) {\n throw new ValidationError(\n `clusterBootstrapMean: resamples must be a positive integer, got ${resamples}`,\n )\n }\n if (!observations.length) {\n throw new ValidationError('clusterBootstrapMean: requires at least one observation')\n }\n\n const sums = new Map<string, { sum: number; count: number }>()\n const values: number[] = []\n for (const [index, observation] of observations.entries()) {\n if (typeof observation?.cluster !== 'string' \|\| observation.cluster === '') {\n throw new ValidationError(`clusterBootstrapMean: observation ${index} needs a cluster id`)\n }\n if (!Number.isFinite(observation.value)) {\n throw new ValidationError(`clusterBootstrapMean: observation ${index} value must be finite`)\n }\n const cell = sums.get(observation.cluster) ?? { sum: 0, count: 0 }\n cell.sum += observation.value\n cell.count += 1\n sums.set(observation.cluster, cell)\n values.push(observation.value)\n }\n\n const cells = [...sums.values()]\n const total = cells.reduce((acc, cell) => acc + cell.sum, 0)\n const mean = total / observations.length\n const k = cells.length\n const rng = makeRng(options.seed, values)\n const samples = new Array<number>(resamples)\n for (let b = 0; b < resamples; b++) {\n let sum = 0\n let count = 0\n for (let draw = 0; draw < k; draw++) {\n const cell = cells[Math.floor(rng() k)]!\n sum += cell.sum\n count += cell.count\n }\n samples[b] = sum / count\n }\n samples.sort((a, b) => a - b)\n const alpha = 1 - confidence\n const lowIdx = Math.floor((alpha / 2) * resamples)\n const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)\n return {\n n: observations.length,\n clusters: k,\n mean,\n low: samples[lowIdx]!,\n high: samples[Math.max(highIdx, lowIdx)]!,\n confidence,\n resamples,\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAuCA,SAAgB,qBACd,cACA,UAAmC,CAAC,GACZ;CACxB,MAAM,aAAa,QAAQ,cAAc;CACzC,MAAM,YAAY,QAAQ,aAAa;CACvC,IAAI,EAAE,aAAa,KAAK,aAAa,IACnC,MAAM,IAAI,gBACR,0DAA0D,YAC5D;CAEF,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,gBACR,mEAAmE,WACrE;CAEF,IAAI,CAAC,aAAa,QAChB,MAAM,IAAI,gBAAgB,yDAAyD;CAGrF,MAAM,uBAAO,IAAI,IAA4C;CAC7D,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,CAAC,OAAO,gBAAgB,aAAa,QAAQ,GAAG;EACzD,IAAI,OAAO,aAAa,YAAY,YAAY,YAAY,YAAY,IACtE,MAAM,IAAI,gBAAgB,qCAAqC,MAAM,oBAAoB;EAE3F,IAAI,CAAC,OAAO,SAAS,YAAY,KAAK,GACpC,MAAM,IAAI,gBAAgB,qCAAqC,MAAM,sBAAsB;EAE7F,MAAM,OAAO,KAAK,IAAI,YAAY,OAAO,KAAK;GAAE,KAAK;GAAG,OAAO;EAAE;EACjE,KAAK,OAAO,YAAY;EACxB,KAAK,SAAS;EACd,KAAK,IAAI,YAAY,SAAS,IAAI;EAClC,OAAO,KAAK,YAAY,KAAK;CAC/B;CAEA,MAAM,QAAQ,CAAC,GAAG,KAAK,OAAO,CAAC;CAE/B,MAAM,OADQ,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,KAAK,CACzC,IAAI,aAAa;CAClC,MAAM,IAAI,MAAM;CAChB,MAAM,MAAM,QAAQ,QAAQ,MAAM,MAAM;CACxC,MAAM,UAAU,IAAI,MAAc,SAAS;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;EAClC,IAAI,MAAM;EACV,IAAI,QAAQ;EACZ,KAAK,IAAI,OAAO,GAAG,OAAO,GAAG,QAAQ;GACnC,MAAM,OAAO,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC;GACvC,OAAO,KAAK;GACZ,SAAS,KAAK;EAChB;EACA,QAAQ,KAAK,MAAM;CACrB;CACA,QAAQ,MAAM,GAAG,MAAM,IAAI,CAAC;CAC5B,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,KAAK,MAAO,QAAQ,IAAK,SAAS;CACjD,MAAM,UAAU,KAAK,IAAI,YAAY,GAAG,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC;CAClF,OAAO;EACL,GAAG,aAAa;EAChB,UAAU;EACV;EACA,KAAK,QAAQ;EACb,MAAM,QAAQ,KAAK,IAAI,SAAS,MAAM;EACtC;EACA;CACF;AACF"}

package/dist/{produced-state-BsDmnwxc.js → completion-verifier-B_jw6C9p.js} RENAMED Viewed

@@ -1,10 +1,10 @@
-import { s as ValidationError } from "./errors-DVZiqS7K.js";
-import { r as canonicalString } from "./canonical-B2RqLofv.js";
-import { n as JudgeParseError } from "./llm-judge-CG12SQJq.js";
-import { t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
-import { w as recoverTruncatedJson } from "./campaign-evidence-Clrn4hBV.js";
-import { t as certificationEvidenceDigest } from "./verdict-Dl0tiT1X.js";
-import { a as maximumChargeForLlmRequest, c as ModelSubstitutionError, d as assertServedModel, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-CyBAoYiX.js";
+import { s as ValidationError } from "./errors-BcOs5MKW.js";
+import { r as canonicalString } from "./canonical-Dq2hOQ8L.js";
+import { n as JudgeParseError } from "./llm-judge-C8jMTNNG.js";
+import { t as CostLedger } from "./cost-ledger-D0352Z4d.js";
+import { v as recoverTruncatedJson } from "./campaign-evidence-CQYw8_0H.js";
+import { t as certificationEvidenceDigest } from "./verdict-BGQmxQ8A.js";
+import { a as maximumChargeForLlmRequest, c as ModelSubstitutionError, i as costReceiptFromLlmError, r as costReceiptFromLlm, u as assertServedModel } from "./llm-client-BshCREn6.js";
 import { createHash, randomUUID } from "node:crypto";
 import { harnessSupportsModel } from "@tangle-network/agent-interface";
 //#region src/agent-profile.ts
@@ -534,61 +534,6 @@ function createLlmCorrectnessChecker(chat, opts = {}) {
 	return checker;
 }
 //#endregion
-//#region src/produced-state.ts
-function artifactKind(mimeType) {
-	if (!mimeType) return "file";
-	if (mimeType.includes("json")) return "json";
-	if (mimeType.startsWith("text/")) return "text";
-	return "file";
-}
-/**
-* Normalize an emission-ordered stream into its latest observed produced state.
-* Artifacts are keyed by their exact output path (name, then URI, then id);
-* proposals by id. Later observations replace earlier ones, including missing
-* content or a rejected status. An obsolete version cannot prove completion.
-* Distinct paths remain distinct; no path normalization or aliasing is inferred.
-* Results retain first-seen identity order. Tool names describe invocation,
-* not success, and are deduplicated in first-seen order.
-*
-* An artifact without observed content yields empty content, which the
-* completion oracle rejects. This projection does not verify external effects,
-* recover dropped events, or establish freshness beyond the supplied stream.
-*/
-function extractProducedState(events) {
-	const artifacts = /* @__PURE__ */ new Map();
-	const proposals = /* @__PURE__ */ new Map();
-	const toolCalls = [];
-	const seenTools = /* @__PURE__ */ new Set();
-	for (const ev of events) if (ev.type === "tool_call") {
-		const name = ev.toolName;
-		if (name && !seenTools.has(name)) {
-			seenTools.add(name);
-			toolCalls.push(name);
-		}
-	} else if (ev.type === "artifact") {
-		const a = ev;
-		const path = a.name ?? a.uri ?? a.artifactId;
-		artifacts.set(path, {
-			kind: artifactKind(a.mimeType),
-			path,
-			content: a.content ?? ""
-		});
-	} else if (ev.type === "proposal_created") {
-		const p = ev;
-		proposals.set(p.proposalId, {
-			id: p.proposalId,
-			title: p.title,
-			status: p.status ?? "pending",
-			...p.content !== void 0 ? { content: p.content } : {}
-		});
-	}
-	return {
-		artifacts: [...artifacts.values()],
-		proposals: [...proposals.values()],
-		toolCalls
-	};
-}
-//#endregion
-export { CODING_HARNESSES as a, agentProfileId as c, harnessAxisOf as d, verifyCompletion as i, agentProfileModelId as l, completionVerdict as n, HARNESS_NATIVE_MODEL as o, createLlmCorrectnessChecker as r, agentProfileHash as s, extractProducedState as t, expandProfileAxes as u };
+export { HARNESS_NATIVE_MODEL as a, agentProfileModelId as c, CODING_HARNESSES as i, expandProfileAxes as l, createLlmCorrectnessChecker as n, agentProfileHash as o, verifyCompletion as r, agentProfileId as s, completionVerdict as t, harnessAxisOf as u };
-//# sourceMappingURL=produced-state-BsDmnwxc.js.map
+//# sourceMappingURL=completion-verifier-B_jw6C9p.js.map

package/dist/completion-verifier-B_jw6C9p.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"completion-verifier-B_jw6C9p.js","names":[],"sources":["../src/agent-profile.ts","../src/completion-verifier.ts"],"sourcesContent":["import { createHash } from 'node:crypto'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { type HarnessType, harnessSupportsModel } from '@tangle-network/agent-interface'\nimport { ValidationError } from './errors'\nimport { canonicalString } from './ledger-core/canonical'\n\nexport type { AgentProfile, HarnessType } from '@tangle-network/agent-interface'\n\n/**\n * The agentic coding harnesses an eval sweeps by default — the ones we care about\n * ranking. This is the SINGLE source of that list; consumers import it instead of\n * re-declaring their own (a re-declared list is how the fleet drifts). Pass an\n * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known\n * harness) to widen beyond these.\n */\nexport const CODING_HARNESSES: readonly HarnessType[] = [\n 'opencode',\n 'claude-code',\n 'codex',\n 'kimi-code',\n]\n\nexport interface ProfileAxisSpec {\n /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the\n * harness and model vary. `model.default` is the fallback model. */\n base: AgentProfile\n /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */\n harnesses?: readonly HarnessType[]\n /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's\n * single-model behaviour, so omitting this never changes an existing run. */\n models?: readonly string[]\n /** Force every (harness, model) pair verbatim, even ones the harness can't run —\n * for deliberately testing failure modes. Default (false): SNAP instead — a\n * vendor-locked harness runs only the swept models in its family, or its native\n * default when it supports none, so no harness is dropped and none gets a\n * guaranteed-failing foreign-model cell. */\n keepIncompatible?: boolean\n}\n\n/** Model sentinel for a vendor-locked harness that supports none of the swept models:\n * it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness\n * resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi\n * model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship\n * table that would rot as router catalogs change. */\nexport const HARNESS_NATIVE_MODEL = 'default'\n\n/**\n * Expand a base profile across the harness × model matrix into the `AgentProfile[]`\n * that `runProfileMatrix` / `selfImprove` score — the ONE place \"which harnesses ×\n * which models do we evaluate\" lives, so no product hand-rolls its own harness list\n * or column→profile mapping (the pattern that let those copies drift and silently\n * break the harness pivot).\n *\n * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,\n * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both\n * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row\n * and results join back by harness/model via {@link harnessAxisOf} with no\n * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or\n * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every\n * requested harness runs; `keepIncompatible` forces every pair verbatim.\n *\n * Omit `harnesses`/`models` to sweep the full default set — the \"turn it on for\n * everything we care about\" switch, identical in shape whether one harness or all.\n */\nexport function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[] {\n const harnesses = spec.harnesses ?? CODING_HARNESSES\n if (harnesses.length === 0) throw new ValidationError('expandProfileAxes: no harnesses to sweep')\n const baseModel = spec.base.model?.default\n const models = spec.models ?? (baseModel ? [baseModel] : [])\n if (models.length === 0) {\n throw new ValidationError(\n 'expandProfileAxes: no models to sweep — base profile has no model.default and none were supplied',\n )\n }\n const out: AgentProfile[] = []\n const seen = new Set<string>()\n for (const harness of harnesses) {\n // A universal (router-backed) harness — opencode/pi/claudish — runs every swept\n // model. A vendor-locked harness — codex/claude-code/kimi-code — runs only the\n // swept models in its own family; when it supports NONE of them it snaps to its\n // native default (the `HARNESS_NATIVE_MODEL` sentinel it resolves at runtime)\n // rather than being dropped, so every requested harness still appears in the\n // sweep on a model it can actually run — e.g. sweeping `deepseek/x` puts opencode\n // on deepseek and kimi-code on its own Kimi model, a real head-to-head.\n // `keepIncompatible` forces every (harness, model) pair verbatim (failure-mode runs).\n const supported = spec.keepIncompatible\n ? models\n : models.filter((model) => harnessSupportsModel(harness, model))\n const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL]\n for (const model of effective) {\n const profile: AgentProfile = {\n ...spec.base,\n harness,\n name: `${spec.base.name ?? 'agent'}/${harness}/${model}`,\n model: { ...spec.base.model, default: model },\n metadata: { ...(spec.base.metadata ?? {}), harness, harnessModel: model },\n }\n const id = agentProfileId(profile)\n if (seen.has(id)) continue\n seen.add(id)\n out.push(profile)\n }\n }\n if (out.length === 0) {\n // Unreachable in normal use — snapping guarantees ≥1 cell per harness — but keep a\n // fail-closed guard so a future refactor can't silently produce an empty sweep.\n throw new ValidationError(\n `expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(', ')}], models=[${models.join(', ')}]).`,\n )\n }\n return out\n}\n\n/**\n * Read the (harness, model) a matrix cell ran under, off a profile or a result row's\n * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile\n * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by\n * this instead of recomputing an id (recomputing the wrong key is what broke the pivot\n * in the hand-rolled copies).\n */\nexport function harnessAxisOf(\n profile: Pick<AgentProfile, 'metadata'>,\n): { harness: HarnessType; model: string } | undefined {\n const m = profile.metadata as Record<string, unknown> | undefined\n const harness = m?.harness\n const model = m?.harnessModel\n if (typeof harness === 'string' && typeof model === 'string') {\n return { harness: harness as HarnessType, model }\n }\n return undefined\n}\n\n/**\n * Collision-resistant, path-safe, human-readable profile id for eval artifacts.\n * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix\n * keys, and directory names where two profiles must not collapse onto one row.\n * The suffix is the first 64 bits of the behaviour hash, enough for ordinary\n * eval matrices while keeping filenames readable.\n */\nexport function agentProfileId(profile: AgentProfile): string {\n const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? 'profile'\n return `${label}-${agentProfileHash(profile).slice(0, 16)}`\n}\n\n/**\n * Model snapshot used for `RunRecord.model`. Eval surfaces require a concrete\n * model id because run records reject bare/missing model aliases.\n */\nexport function agentProfileModelId(profile: AgentProfile): string {\n const model = profile.model?.default?.trim()\n if (!model) {\n const label = agentProfileDisplayLabel(profile) ?? 'unnamed profile'\n throw new ValidationError(\n `AgentProfile \"${label}\" has no model.default — cannot record eval run`,\n )\n }\n return model\n}\n\nfunction agentProfileDisplayLabel(profile: AgentProfile): string | undefined {\n return profile.name?.trim() || profile.version?.trim() || undefined\n}\n\nfunction pathSafeProfileLabel(label: string | undefined): string | undefined {\n const safe = label\n ?.trim()\n .replace(/[^A-Za-z0-9._-]+/g, '-')\n .replace(/-+/g, '-')\n .replace(/^-|-$/g, '')\n return safe || undefined\n}\n\nfunction compact<T extends Record<string, unknown>>(input: T): Partial<T> {\n const out: Record<string, unknown> = {}\n for (const [key, value] of Object.entries(input)) {\n if (value !== undefined) out[key] = value\n }\n return out as Partial<T>\n}\n\n/**\n * Deterministic behaviour identity for the canonical\n * `@tangle-network/agent-interface` AgentProfile.\n *\n * `name` and `description` are labels and do not affect the hash. Profile\n * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,\n * and extensions do affect the hash. Resource array order is hash-bearing\n * because mount order can change agent behaviour. Undefined fields are treated\n * as absent; explicit `null` fields remain hash-bearing.\n */\nexport function agentProfileHash(profile: AgentProfile): string {\n const model = agentProfileModelId(profile)\n const behaviour = compact({\n ...profile,\n name: undefined,\n description: undefined,\n tags: profile.tags ? [...profile.tags].sort() : undefined,\n model: compact({ ...profile.model, default: model }),\n })\n return createHash('sha256').update(canonicalString(behaviour)).digest('hex')\n}\n","/**\n * Completion verifier — the task-completion oracle.\n *\n * Answers the only eval question that is not a proxy: did the agent actually\n * COMPLETE the task — produce every required deliverable, persisted and\n * correct — rather than describe what should be done. A fluent transcript\n * that never produces the artifact scores zero here.\n *\n * Per requirement, a two-stage check:\n * 1. Structural — a produced item (vault artifact / approved proposal /\n * tool call) of the right kind is matched against the requirement and\n * carries non-empty content. Deterministic; no LLM.\n * 2. Correctness — only if structurally present AND the matched item\n * carries content, one targeted check decides whether that item\n * actually fulfils the requirement. A hallucinated artifact fails here;\n * an absent one already failed stage 1.\n *\n * `completionRate` is satisfied / MEASURABLE requirements (unmeasured rows —\n * checker failures — are excluded from the denominator, never scored as\n * zeros). Quality dimensions are meaningless on an incomplete task — callers\n * gate on `fullyComplete` / `completionRate` before scoring quality.\n */\n\nimport { randomUUID } from 'node:crypto'\nimport type { ChatClient, ChatRequest } from './analyst/chat-client'\nimport type { ValidatedArtifact } from './artifact-validator'\nimport { CostLedger, type CostLedgerHandle } from './cost-ledger'\nimport { assertServedModel, ModelSubstitutionError } from './integrity/served-model'\nimport { recoverTruncatedJson } from './json-recovery'\nimport { JudgeParseError } from './judges'\nimport {\n costReceiptFromLlm,\n costReceiptFromLlmError,\n maximumChargeForLlmRequest,\n} from './llm-client'\nimport type { RawProviderEvent, RawProviderSink } from './trace/raw-provider-sink'\nimport {\n certificationEvidenceDigest,\n type DefaultVerdict,\n type VerdictCertification,\n} from './verdict'\nimport type { CheckerIdentity, VerificationStrategySource } from './verification-strategy'\n\n/** What kind of produced state can satisfy a requirement structurally. */\nexport type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any'\n\nexport interface CompletionRequirement {\n /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */\n reqId: string\n /** Human-readable description of the required deliverable. */\n title: string\n /** Optional kind/category hint, matched against a produced item's kind. */\n category?: string\n /** What produced state satisfies this requirement. Defaults to 'any'. */\n satisfiedBy?: SatisfiedBy\n}\n\nexport interface TaskGold {\n taskId: string\n requirements: CompletionRequirement[]\n}\n\nexport interface ProducedProposal {\n id: string\n title: string\n status: 'pending' | 'approved' | 'rejected'\n /** Optional persisted body — when present, enables a correctness check. */\n content?: string\n}\n\n/** Everything observable about what a run actually produced. */\nexport interface ProducedState {\n /** Persisted vault artifacts. Reuses the shared `ValidatedArtifact` shape. */\n artifacts: ValidatedArtifact[]\n /** Proposals / filings the agent created. */\n proposals: ProducedProposal[]\n /** Names of tools the agent invoked. */\n toolCalls: string[]\n}\n\nexport interface RequirementCheck {\n reqId: string\n title: string\n /** A produced item of the right kind matched the requirement, non-empty. */\n structurallyPresent: boolean\n /**\n * Whether the matched item actually fulfils the requirement. `null` when\n * not structurally present, when the matched item carries no content\n * to assess, or when the correctness check itself failed (`unmeasured`).\n */\n correct: boolean | null\n /** structurallyPresent && !unmeasured && correct !== false. */\n satisfied: boolean\n /**\n * Set when the correctness check itself errored (LLM call failure or an\n * unparseable response after retry). The requirement's fulfilment is\n * UNKNOWN — `correct` stays null, `satisfied` is false, and\n * `completionVerdict` excludes the row from `completionRate`'s\n * denominator. Never folded into a zero: a synthetic zero is\n * indistinguishable from a real failure (see `JudgeParseError`).\n */\n unmeasured?: true\n /** Why the correctness check could not be measured (present iff `unmeasured`). */\n unmeasuredReason?: string\n /** Human-readable evidence for the verdict. */\n evidence: string[]\n}\n\n/** Extends the substrate verdict spine: `valid` = `fullyComplete` and\n * `score` = `completionRate` — derived in `completionVerdict()`, the one\n * place those equalities hold by construction. */\nexport interface CompletionVerdict extends DefaultVerdict {\n taskId: string\n requirements: RequirementCheck[]\n /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */\n completionRate: number\n /** Every measurable requirement satisfied (false when anything is unmeasured). */\n fullyComplete: boolean\n /** Requirements whose correctness check errored — reported, never scored as zero. */\n unmeasuredCount: number\n}\n\n/**\n * Construct a `CompletionVerdict` from the per-requirement checks, deriving\n * `completionRate` / `fullyComplete` and the spine fields (`valid` =\n * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero\n * requirements — a verdict over nothing is a misconfiguration, mirroring\n * `verifyCompletion`'s gold-spec guard.\n */\nexport function completionVerdict(input: {\n taskId: string\n requirements: RequirementCheck[]\n /** What certified the correctness stage, when anything did. Omitted =\n * an uncertified verdict — the honest default for a bare checker. */\n certification?: VerdictCertification\n}): CompletionVerdict {\n if (input.requirements.length === 0) {\n throw new Error(\n `completionVerdict: task '${input.taskId}' has no requirement checks — nothing to derive a verdict from`,\n )\n }\n const measurable = input.requirements.filter((r) => !r.unmeasured)\n const unmeasuredCount = input.requirements.length - measurable.length\n if (measurable.length === 0) {\n // Every check errored: this is an infrastructure failure, not a scored\n // run. A 0-rate verdict here would be a fabricated measurement.\n throw new Error(\n `completionVerdict: task '${input.taskId}' has no measurable requirements — all ${input.requirements.length} correctness checks failed (${input.requirements[0]?.unmeasuredReason ?? 'unknown reason'})`,\n )\n }\n const satisfiedCount = measurable.filter((r) => r.satisfied).length\n const completionRate = satisfiedCount / measurable.length\n // A run with unmeasured rows can still report a rate over what WAS\n // measured, but must not claim full completion.\n const fullyComplete = unmeasuredCount === 0 && satisfiedCount === measurable.length\n return {\n taskId: input.taskId,\n requirements: input.requirements,\n completionRate,\n fullyComplete,\n unmeasuredCount,\n valid: fullyComplete,\n score: completionRate,\n ...(input.certification === undefined ? {} : { certification: input.certification }),\n }\n}\n\n/**\n * What a correctness checker declares about itself so the completion\n * verdict can carry a certification: which strategy member it discharges,\n * its exact identity, and the steps its answers rest on unverified.\n */\nexport interface CorrectnessCheckerAttestation {\n strategy: VerificationStrategySource\n checker: CheckerIdentity\n assumptions: string[]\n}\n\n/**\n * Decides whether a produced item's content actually fulfils a requirement.\n * Injected so the structural verifier stays pure and unit-testable; the\n * production implementation is `createLlmCorrectnessChecker`.\n *\n * `attestation` is optional metadata on the function value: a checker that\n * carries one yields certified completion verdicts; a bare function yields\n * the same verdict uncertified. A plain arrow function remains a valid\n * checker.\n */\nexport interface CorrectnessChecker {\n (\n requirement: CompletionRequirement,\n content: string,\n ): Promise<{ correct: boolean; reason: string }>\n attestation?: CorrectnessCheckerAttestation\n}\n\nconst STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'of',\n 'for',\n 'and',\n 'or',\n 'to',\n 'in',\n 'on',\n 'with',\n 'by',\n])\n\n// Deliverable-FORM vocabulary — words that name the SHAPE of an output, not its\n// domain content. A correct \"swap-comparison view persisted as a ui/** artifact\"\n// is an OpenUI JSON whose body says nothing about \"artifact\" / \"persisted\" /\n// \"view\"; the discriminative tokens are the domain nouns (swap, comparison).\n// Stripped from the REQUIREMENT side of structural recall so a deliverable is\n// matched on what it IS about, not on the boilerplate describing its form. The\n// correctness checker strips the same class via TITLE_STOPWORDS. Anti-game holds:\n// the distinctive domain tokens remain, so an off-topic item still fails.\nconst REQUIREMENT_FORM_STOPWORDS = new Set([\n 'generated',\n 'generate',\n 'view',\n 'render',\n 'rendered',\n 'persisted',\n 'persist',\n 'artifact',\n 'file',\n 'document',\n 'note',\n 'proposal',\n 'deliverable',\n 'output',\n 'created',\n 'create',\n 'produce',\n 'produced',\n 'flag',\n])\n\nconst MATCH_THRESHOLD = 0.5\nconst MIN_CONTENT_CHARS = 50\n\nfunction tokens(s: string, extraStop?: Set<string>): Set<string> {\n return new Set(\n s\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t)),\n )\n}\n\n/**\n * Recall of the requirement's tokens within a candidate's identifying text.\n * Recall, not Jaccard — a candidate's path/id legitimately carries extra\n * tokens the requirement does not name. The requirement side drops\n * deliverable-FORM vocabulary so recall keys on the distinctive domain tokens.\n */\nfunction tokenRecall(requirementText: string, candidateText: string): number {\n const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS)\n if (req.size === 0) return 0\n const cand = tokens(candidateText)\n let hit = 0\n for (const t of req) if (cand.has(t)) hit++\n return hit / req.size\n}\n\ninterface Candidate {\n reqIndex: number\n /** Unique key for a produced item — each item satisfies at most one requirement. */\n itemKey: string\n score: number\n evidence: string\n /** Content to correctness-check, or null when the matched item has none. */\n content: string | null\n}\n\nfunction artifactCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n artifacts: ValidatedArtifact[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n artifacts.forEach((a, i) => {\n if ((a.content ?? '').trim().length < MIN_CONTENT_CHARS) return\n // Match against the artifact CONTENT too, not just its path + kind — a\n // generated view / note whose path is generic still satisfies a requirement\n // when its body covers it (e.g. an OpenUI comparison grounded in the on-file\n // figures). Bounded slice keeps the recall text cheap; MATCH_THRESHOLD holds.\n let score = tokenRecall(\n reqText,\n `${a.path ?? ''} ${a.kind} ${(a.content ?? '').slice(0, 4000)}`,\n )\n if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {\n score = Math.max(score, 1)\n }\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `artifact:${i}`,\n score,\n evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,\n content: a.content ?? null,\n })\n })\n return out\n}\n\nfunction proposalCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n proposals: ProducedProposal[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n for (const p of proposals) {\n // Pending or rejected work is not a completed deliverable.\n if (p.status !== 'approved') continue\n // A proposal needs an assessable BODY to be a deliverable. A bare title is\n // not completion: correctness cannot be judged on it, so a title-only match\n // would auto-pass the oracle (structurallyPresent && correct===null →\n // satisfied) with no verifiable content. Tool calls are the only\n // legitimately content-less deliverable (`toolCallCandidates`).\n const body = (p.content ?? '').trim()\n if (body.length < MIN_CONTENT_CHARS) continue\n // Match against the body as well as the (often short) title — a refusal /\n // flag / analysis proposal whose title is a label still satisfies a\n // descriptively-worded requirement when its content covers it. MATCH_THRESHOLD\n // + the requirement's distinctive tokens keep an off-topic proposal out;\n // correctness (a SEMANTIC checker, NOT this lexical pass) then judges\n // polarity/fulfilment, so a negation that merely contains the tokens fails.\n // Structural and correctness must use different evidence or the two-stage\n // check collapses to one lexical gate.\n const score = tokenRecall(reqText, `${p.title} ${body}`)\n if (score < MATCH_THRESHOLD) continue\n out.push({\n reqIndex,\n itemKey: `proposal:${p.id}`,\n score,\n evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,\n content: body,\n })\n }\n return out\n}\n\nfunction toolCallCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n toolCalls: string[],\n): Candidate[] {\n const out: Candidate[] = []\n toolCalls.forEach((name, i) => {\n const score = tokenRecall(req.title, name)\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `tool:${i}`,\n score,\n evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,\n content: null,\n })\n })\n return out\n}\n\n/**\n * Verify whether a run completed the task. `checkCorrectness` is injected —\n * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.\n *\n * Throws on a gold spec with no requirements: an eval task that requires\n * nothing is a misconfiguration, not a vacuously-complete task.\n */\nexport async function verifyCompletion(\n gold: TaskGold,\n state: ProducedState,\n checkCorrectness: CorrectnessChecker,\n): Promise<CompletionVerdict> {\n if (gold.requirements.length === 0) {\n throw new Error(\n `verifyCompletion: task '${gold.taskId}' has no requirements — malformed gold spec`,\n )\n }\n\n // Collect every above-threshold (requirement, produced-item) candidate, then\n // assign greedily by descending score: each requirement and each produced\n // item is used at most once. One deliverable fulfils one requirement.\n const candidates: Candidate[] = []\n gold.requirements.forEach((req, i) => {\n const by = req.satisfiedBy ?? 'any'\n if (by === 'artifact' || by === 'any') {\n candidates.push(...artifactCandidates(req, i, state.artifacts))\n }\n if (by === 'proposal' || by === 'any') {\n candidates.push(...proposalCandidates(req, i, state.proposals))\n }\n if (by === 'tool-call' || by === 'any') {\n candidates.push(...toolCallCandidates(req, i, state.toolCalls))\n }\n })\n candidates.sort((a, b) => b.score - a.score)\n\n const assigned = new Map<number, Candidate>()\n const itemTaken = new Set<string>()\n for (const c of candidates) {\n if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue\n assigned.set(c.reqIndex, c)\n itemTaken.add(c.itemKey)\n }\n\n const requirements: RequirementCheck[] = []\n for (let i = 0; i < gold.requirements.length; i++) {\n const req = gold.requirements[i]!\n const match = assigned.get(i)\n const evidence: string[] = []\n let correct: boolean | null = null\n let unmeasuredReason: string | undefined\n\n if (match) {\n evidence.push(match.evidence)\n if (match.content !== null) {\n try {\n const r = await checkCorrectness(req, match.content)\n correct = r.correct\n evidence.push(`correctness: ${r.correct ? 'pass' : 'fail'} — ${r.reason}`)\n } catch (err) {\n // The CHECKER failed, not the requirement. Recording this as a\n // zero would fabricate a model failure out of an infrastructure\n // one; the requirement is unmeasured and leaves the denominator.\n unmeasuredReason =\n err instanceof JudgeParseError\n ? `checker response unparseable after retry: ${err.raw.slice(0, 200)}`\n : `checker call failed: ${err instanceof Error ? err.message : String(err)}`\n evidence.push(`correctness: UNMEASURED — ${unmeasuredReason}`)\n }\n } else {\n evidence.push('correctness: not assessed — matched item carries no content')\n }\n } else {\n const by = req.satisfiedBy ?? 'any'\n const kind = by === 'any' ? 'artifact/proposal/tool-call' : by\n evidence.push(`no produced ${kind} matched this requirement`)\n }\n\n const structurallyPresent = match !== undefined\n const unmeasured = unmeasuredReason !== undefined\n const satisfied = structurallyPresent && !unmeasured && correct !== false\n requirements.push({\n reqId: req.reqId,\n title: req.title,\n structurallyPresent,\n correct,\n satisfied,\n ...(unmeasured ? { unmeasured: true as const, unmeasuredReason } : {}),\n evidence,\n })\n }\n\n // The verdict blends the deterministic structural stage with the injected\n // correctness stage, so the certification names the checker's own member\n // and carries the structural stage's lexical nature as an assumption. A\n // checker without an attestation yields an uncertified verdict.\n const attestation = checkCorrectness.attestation\n const certification: VerdictCertification | undefined = attestation\n ? {\n strategy: attestation.strategy,\n checker: attestation.checker,\n assumptions: [\n 'structural matching is lexical token recall over produced items — the correctness stage only sees items it matched',\n ...attestation.assumptions,\n ],\n evidenceDigest: certificationEvidenceDigest({ taskId: gold.taskId, requirements }),\n }\n : undefined\n\n return completionVerdict({\n taskId: gold.taskId,\n requirements,\n ...(certification === undefined ? {} : { certification }),\n })\n}\n\nexport interface LlmCorrectnessCheckerOpts {\n model?: string\n /** Optional ledger for direct use. */\n costLedger?: CostLedgerHandle\n costPhase?: string\n costTags?: Record<string, string>\n signal?: AbortSignal\n /** Max chars of artifact content sent to the checker. */\n maxContentChars?: number\n /**\n * Checker LLM calls per requirement before giving up (parse failures and\n * call errors both consume attempts). The failure then surfaces as an\n * `unmeasured` requirement, never a zero.\n */\n maxAttempts?: number\n /**\n * Forensic capture of every checker request/response/error — without it a\n * checker failure is unauditable (the agent-turn raws never contain the\n * checker's own calls). Same sink contract as `LlmClient`.\n */\n rawSink?: RawProviderSink\n}\n\n/**\n * Parse the correctness checker's model response. Tolerates a response\n * truncated mid-JSON (max_tokens cap) by auto-closing the prefix — the\n * verdict boolean usually lands in the first few tokens, so a recovered\n * prefix with a boolean `correct` is a real measurement, not a guess.\n * Fails loud (JudgeParseError) when no boolean verdict is recoverable.\n */\nfunction parseCorrectnessResponse(raw: string): { correct: boolean; reason: string } {\n const readVerdict = (candidate: unknown): { correct: boolean; reason: string } | null => {\n if (candidate === null || typeof candidate !== 'object') return null\n const { correct, reason } = candidate as { correct?: unknown; reason?: unknown }\n if (typeof correct !== 'boolean') return null\n return { correct, reason: typeof reason === 'string' ? reason : '' }\n }\n\n const match = raw.match(/\\{[\\s\\S]*\\}/)\n if (match) {\n try {\n const strict = readVerdict(JSON.parse(match[0]))\n if (strict) return strict\n } catch {\n // fall through to truncation recovery\n }\n }\n // The strict path needs a closing `}`; a cap-hit response has none. Take\n // everything from the first `{` and auto-close it.\n const start = raw.indexOf('{')\n if (start !== -1) {\n const recovered = readVerdict(recoverTruncatedJson(raw.slice(start)))\n if (recovered) return recovered\n }\n throw new JudgeParseError('correctness-checker', raw)\n}\n\n/**\n * Production `CorrectnessChecker` — one LLM call per matched artifact,\n * deterministic (temperature 0), structured JSON out. Judges fulfilment\n * only: a plan, a gesture, or a description of what should be done does not\n * fulfil a requirement — the artifact must BE the deliverable.\n */\nexport function createLlmCorrectnessChecker(\n chat: ChatClient,\n opts: LlmCorrectnessCheckerOpts = {},\n): CorrectnessChecker {\n const model = opts.model ?? 'claude-sonnet-4-6'\n const maxContentChars = opts.maxContentChars ?? 8000\n const maxAttempts = opts.maxAttempts ?? 2\n const costLedger = opts.costLedger ?? new CostLedger()\n const sink = opts.rawSink\n const record = async (event: RawProviderEvent): Promise<void> => {\n // Forensic capture is best-effort; the verdict is the system of record.\n try {\n await sink?.record(event)\n } catch {\n // Intentionally swallowed.\n }\n }\n const checker: CorrectnessChecker = async (requirement, content) => {\n const request = {\n model,\n messages: [\n {\n role: 'system' as const,\n content:\n 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only — is the deliverable substantively present and on-point — not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {\"correct\": boolean, \"reason\": string (<= 30 words)}.',\n },\n {\n role: 'user' as const,\n content: `Requirement: ${requirement.title}\\n${\n requirement.category ? `Category: ${requirement.category}\\n` : ''\n }\\nProduced artifact:\\n${content.slice(0, maxContentChars)}`,\n },\n ],\n temperature: 0,\n maxTokens: 200,\n } satisfies ChatRequest\n let lastErr: unknown\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n const started = Date.now()\n await record({\n eventId: randomUUID(),\n provider: chat.transport,\n model,\n endpoint: '/chat',\n baseUrl: '',\n attemptIndex: attempt,\n direction: 'request',\n timestamp: started,\n requestBody: request,\n redactedFields: [],\n })\n try {\n const paid = await costLedger.runPaidCall({\n channel: 'verifier',\n phase: opts.costPhase ?? 'completion.correctness',\n actor: 'correctness-checker',\n model,\n maximumCharge:\n chat.maximumAttempts === undefined\n ? undefined\n : maximumChargeForLlmRequest(request, { maximumAttempts: chat.maximumAttempts }),\n tags: {\n ...opts.costTags,\n requirementId: requirement.reqId,\n attempt: String(attempt),\n },\n signal: opts.signal,\n execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),\n receipt: costReceiptFromLlm,\n receiptFromError: costReceiptFromLlmError,\n })\n if (!paid.succeeded) throw paid.error\n const resp = paid.value\n // Hold the transport to its own word: a verdict produced by a model\n // other than the one requested is not this model's verdict. A\n // transport that echoes no id cannot be made to prove identity here —\n // callers needing that proof enable it at the client\n // (`LlmClientOptions.assertServedModel`) or gate on `assertModelsServed`.\n assertServedModel(model, resp.servedModel, {\n allowUnreported: true,\n context: `correctness checker for requirement ${requirement.reqId}`,\n })\n const raw = resp.content\n await record({\n eventId: randomUUID(),\n provider: chat.transport,\n model,\n endpoint: '/chat',\n baseUrl: '',\n attemptIndex: attempt,\n direction: 'response',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n responseBody: resp,\n redactedFields: [],\n })\n return parseCorrectnessResponse(raw)\n } catch (err) {\n lastErr = err\n await record({\n eventId: randomUUID(),\n provider: chat.transport,\n model,\n endpoint: '/chat',\n baseUrl: '',\n attemptIndex: attempt,\n direction: 'error',\n timestamp: Date.now(),\n durationMs: Date.now() - started,\n errorMessage: err instanceof Error ? err.message : String(err),\n redactedFields: [],\n })\n // Substitution is a verdict about identity, not a transient fault:\n // another attempt reaches the same wrong model and spends more tokens.\n if (err instanceof ModelSubstitutionError) throw err\n }\n }\n throw lastErr instanceof Error ? lastErr : new Error(String(lastErr))\n }\n checker.attestation = {\n strategy: 'judge',\n checker: { name: 'llm-correctness-checker', version: model },\n assumptions: [\n 'served-model identity is accepted when the transport reports none (assertServedModel allowUnreported)',\n ],\n }\n return checker\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAeA,MAAa,mBAA2C;CACtD;CACA;CACA;CACA;AACF;;;;;;AAwBA,MAAa,uBAAuB;;;;;;;;;;;;;;;;;;;AAoBpC,SAAgB,kBAAkB,MAAuC;CACvE,MAAM,YAAY,KAAK,aAAa;CACpC,IAAI,UAAU,WAAW,GAAG,MAAM,IAAI,gBAAgB,0CAA0C;CAChG,MAAM,YAAY,KAAK,KAAK,OAAO;CACnC,MAAM,SAAS,KAAK,WAAW,YAAY,CAAC,SAAS,IAAI,CAAC;CAC1D,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBACR,kGACF;CAEF,MAAM,MAAsB,CAAC;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,WAAW,WAAW;EAS/B,MAAM,YAAY,KAAK,mBACnB,SACA,OAAO,QAAQ,UAAU,qBAAqB,SAAS,KAAK,CAAC;EACjE,MAAM,YAAY,UAAU,SAAS,IAAI,YAAY,CAAC,oBAAoB;EAC1E,KAAK,MAAM,SAAS,WAAW;GAC7B,MAAM,UAAwB;IAC5B,GAAG,KAAK;IACR;IACA,MAAM,GAAG,KAAK,KAAK,QAAQ,QAAQ,GAAG,QAAQ,GAAG;IACjD,OAAO;KAAE,GAAG,KAAK,KAAK;KAAO,SAAS;IAAM;IAC5C,UAAU;KAAE,GAAI,KAAK,KAAK,YAAY,CAAC;KAAI;KAAS,cAAc;IAAM;GAC1E;GACA,MAAM,KAAK,eAAe,OAAO;GACjC,IAAI,KAAK,IAAI,EAAE,GAAG;GAClB,KAAK,IAAI,EAAE;GACX,IAAI,KAAK,OAAO;EAClB;CACF;CACA,IAAI,IAAI,WAAW,GAGjB,MAAM,IAAI,gBACR,uDAAuD,UAAU,KAAK,IAAI,EAAE,aAAa,OAAO,KAAK,IAAI,EAAE,IAC7G;CAEF,OAAO;AACT;;;;;;;;AASA,SAAgB,cACd,SACqD;CACrD,MAAM,IAAI,QAAQ;CAClB,MAAM,UAAU,GAAG;CACnB,MAAM,QAAQ,GAAG;CACjB,IAAI,OAAO,YAAY,YAAY,OAAO,UAAU,UAClD,OAAO;EAAW;EAAwB;CAAM;AAGpD;;;;;;;;AASA,SAAgB,eAAe,SAA+B;CAE5D,OAAO,GADO,qBAAqB,yBAAyB,OAAO,CAAC,KAAK,UACzD,GAAG,iBAAiB,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1D;;;;;AAMA,SAAgB,oBAAoB,SAA+B;CACjE,MAAM,QAAQ,QAAQ,OAAO,SAAS,KAAK;CAC3C,IAAI,CAAC,OAAO;EACV,MAAM,QAAQ,yBAAyB,OAAO,KAAK;EACnD,MAAM,IAAI,gBACR,iBAAiB,MAAM,gDACzB;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBAAyB,SAA2C;CAC3E,OAAO,QAAQ,MAAM,KAAK,KAAK,QAAQ,SAAS,KAAK,KAAK,KAAA;AAC5D;AAEA,SAAS,qBAAqB,OAA+C;CAM3E,OALa,OACT,KAAK,CAAC,CACP,QAAQ,qBAAqB,GAAG,CAAC,CACjC,QAAQ,OAAO,GAAG,CAAC,CACnB,QAAQ,UAAU,EAAE,KACR,KAAA;AACjB;AAEA,SAAS,QAA2C,OAAsB;CACxE,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,KAAK,GAC7C,IAAI,UAAU,KAAA,GAAW,IAAI,OAAO;CAEtC,OAAO;AACT;;;;;;;;;;;AAYA,SAAgB,iBAAiB,SAA+B;CAC9D,MAAM,QAAQ,oBAAoB,OAAO;CACzC,MAAM,YAAY,QAAQ;EACxB,GAAG;EACH,MAAM,KAAA;EACN,aAAa,KAAA;EACb,MAAM,QAAQ,OAAO,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,KAAA;EAChD,OAAO,QAAQ;GAAE,GAAG,QAAQ;GAAO,SAAS;EAAM,CAAC;CACrD,CAAC;CACD,OAAO,WAAW,QAAQ,CAAC,CAAC,OAAO,gBAAgB,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK;AAC7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACvEA,SAAgB,kBAAkB,OAMZ;CACpB,IAAI,MAAM,aAAa,WAAW,GAChC,MAAM,IAAI,MACR,4BAA4B,MAAM,OAAO,+DAC3C;CAEF,MAAM,aAAa,MAAM,aAAa,QAAQ,MAAM,CAAC,EAAE,UAAU;CACjE,MAAM,kBAAkB,MAAM,aAAa,SAAS,WAAW;CAC/D,IAAI,WAAW,WAAW,GAGxB,MAAM,IAAI,MACR,4BAA4B,MAAM,OAAO,yCAAyC,MAAM,aAAa,OAAO,8BAA8B,MAAM,aAAa,EAAE,EAAE,oBAAoB,iBAAiB,EACxM;CAEF,MAAM,iBAAiB,WAAW,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC;CAC7D,MAAM,iBAAiB,iBAAiB,WAAW;CAGnD,MAAM,gBAAgB,oBAAoB,KAAK,mBAAmB,WAAW;CAC7E,OAAO;EACL,QAAQ,MAAM;EACd,cAAc,MAAM;EACpB;EACA;EACA;EACA,OAAO;EACP,OAAO;EACP,GAAI,MAAM,kBAAkB,KAAA,IAAY,CAAC,IAAI,EAAE,eAAe,MAAM,cAAc;CACpF;AACF;AA+BA,MAAM,4BAAY,IAAI,IAAI;CACxB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAUD,MAAM,6CAA6B,IAAI,IAAI;CACzC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAAkB;AACxB,MAAM,oBAAoB;AAE1B,SAAS,OAAO,GAAW,WAAsC;CAC/D,OAAO,IAAI,IACT,EACG,YAAY,CAAC,CACb,MAAM,YAAY,CAAC,CACnB,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAC1E;AACF;;;;;;;AAQA,SAAS,YAAY,iBAAyB,eAA+B;CAC3E,MAAM,MAAM,OAAO,iBAAiB,0BAA0B;CAC9D,IAAI,IAAI,SAAS,GAAG,OAAO;CAC3B,MAAM,OAAO,OAAO,aAAa;CACjC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,KAAK,IAAI,KAAK,IAAI,CAAC,GAAG;CACtC,OAAO,MAAM,IAAI;AACnB;AAYA,SAAS,mBACP,KACA,UACA,WACa;CACb,MAAM,UAAU,GAAG,IAAI,MAAM,GAAG,IAAI,YAAY;CAChD,MAAM,MAAmB,CAAC;CAC1B,UAAU,SAAS,GAAG,MAAM;EAC1B,KAAK,EAAE,WAAW,GAAA,CAAI,KAAK,CAAC,CAAC,SAAS,mBAAmB;EAKzD,IAAI,QAAQ,YACV,SACA,GAAG,EAAE,QAAQ,GAAG,GAAG,EAAE,KAAK,IAAI,EAAE,WAAW,GAAA,CAAI,MAAM,GAAG,GAAI,GAC9D;EACA,IAAI,IAAI,YAAY,EAAE,QAAQ,IAAI,SAAS,YAAY,MAAM,EAAE,KAAK,YAAY,GAC9E,QAAQ,KAAK,IAAI,OAAO,CAAC;EAE3B,IAAI,QAAQ,iBAAiB;EAC7B,IAAI,KAAK;GACP;GACA,SAAS,YAAY;GACrB;GACA,UAAU,aAAa,EAAE,QAAQ,EAAE,KAAK,0BAA0B,MAAM,QAAQ,CAAC,EAAE;GACnF,SAAS,EAAE,WAAW;EACxB,CAAC;CACH,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;CACb,MAAM,UAAU,GAAG,IAAI,MAAM,GAAG,IAAI,YAAY;CAChD,MAAM,MAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,WAAW;EAEzB,IAAI,EAAE,WAAW,YAAY;EAM7B,MAAM,QAAQ,EAAE,WAAW,GAAA,CAAI,KAAK;EACpC,IAAI,KAAK,SAAS,mBAAmB;EASrC,MAAM,QAAQ,YAAY,SAAS,GAAG,EAAE,MAAM,GAAG,MAAM;EACvD,IAAI,QAAQ,iBAAiB;EAC7B,IAAI,KAAK;GACP;GACA,SAAS,YAAY,EAAE;GACvB;GACA,UAAU,sBAAsB,EAAE,MAAM,0BAA0B,MAAM,QAAQ,CAAC,EAAE;GACnF,SAAS;EACX,CAAC;CACH;CACA,OAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;CACb,MAAM,MAAmB,CAAC;CAC1B,UAAU,SAAS,MAAM,MAAM;EAC7B,MAAM,QAAQ,YAAY,IAAI,OAAO,IAAI;EACzC,IAAI,QAAQ,iBAAiB;EAC7B,IAAI,KAAK;GACP;GACA,SAAS,QAAQ;GACjB;GACA,UAAU,cAAc,KAAK,0BAA0B,MAAM,QAAQ,CAAC,EAAE;GACxE,SAAS;EACX,CAAC;CACH,CAAC;CACD,OAAO;AACT;;;;;;;;AASA,eAAsB,iBACpB,MACA,OACA,kBAC4B;CAC5B,IAAI,KAAK,aAAa,WAAW,GAC/B,MAAM,IAAI,MACR,2BAA2B,KAAK,OAAO,4CACzC;CAMF,MAAM,aAA0B,CAAC;CACjC,KAAK,aAAa,SAAS,KAAK,MAAM;EACpC,MAAM,KAAK,IAAI,eAAe;EAC9B,IAAI,OAAO,cAAc,OAAO,OAC9B,WAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;EAEhE,IAAI,OAAO,cAAc,OAAO,OAC9B,WAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;EAEhE,IAAI,OAAO,eAAe,OAAO,OAC/B,WAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;CAElE,CAAC;CACD,WAAW,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;CAE3C,MAAM,2BAAW,IAAI,IAAuB;CAC5C,MAAM,4BAAY,IAAI,IAAY;CAClC,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI,SAAS,IAAI,EAAE,QAAQ,KAAK,UAAU,IAAI,EAAE,OAAO,GAAG;EAC1D,SAAS,IAAI,EAAE,UAAU,CAAC;EAC1B,UAAU,IAAI,EAAE,OAAO;CACzB;CAEA,MAAM,eAAmC,CAAC;CAC1C,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,aAAa,QAAQ,KAAK;EACjD,MAAM,MAAM,KAAK,aAAa;EAC9B,MAAM,QAAQ,SAAS,IAAI,CAAC;EAC5B,MAAM,WAAqB,CAAC;EAC5B,IAAI,UAA0B;EAC9B,IAAI;EAEJ,IAAI,OAAO;GACT,SAAS,KAAK,MAAM,QAAQ;GAC5B,IAAI,MAAM,YAAY,MACpB,IAAI;IACF,MAAM,IAAI,MAAM,iBAAiB,KAAK,MAAM,OAAO;IACnD,UAAU,EAAE;IACZ,SAAS,KAAK,gBAAgB,EAAE,UAAU,SAAS,OAAO,KAAK,EAAE,QAAQ;GAC3E,SAAS,KAAK;IAIZ,mBACE,eAAe,kBACX,6CAA6C,IAAI,IAAI,MAAM,GAAG,GAAG,MACjE,wBAAwB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC7E,SAAS,KAAK,6BAA6B,kBAAkB;GAC/D;QAEA,SAAS,KAAK,6DAA6D;EAE/E,OAAO;GACL,MAAM,KAAK,IAAI,eAAe;GAC9B,MAAM,OAAO,OAAO,QAAQ,gCAAgC;GAC5D,SAAS,KAAK,eAAe,KAAK,0BAA0B;EAC9D;EAEA,MAAM,sBAAsB,UAAU,KAAA;EACtC,MAAM,aAAa,qBAAqB,KAAA;EACxC,MAAM,YAAY,uBAAuB,CAAC,cAAc,YAAY;EACpE,aAAa,KAAK;GAChB,OAAO,IAAI;GACX,OAAO,IAAI;GACX;GACA;GACA;GACA,GAAI,aAAa;IAAE,YAAY;IAAe;GAAiB,IAAI,CAAC;GACpE;EACF,CAAC;CACH;CAMA,MAAM,cAAc,iBAAiB;CACrC,MAAM,gBAAkD,cACpD;EACE,UAAU,YAAY;EACtB,SAAS,YAAY;EACrB,aAAa,CACX,sHACA,GAAG,YAAY,WACjB;EACA,gBAAgB,4BAA4B;GAAE,QAAQ,KAAK;GAAQ;EAAa,CAAC;CACnF,IACA,KAAA;CAEJ,OAAO,kBAAkB;EACvB,QAAQ,KAAK;EACb;EACA,GAAI,kBAAkB,KAAA,IAAY,CAAC,IAAI,EAAE,cAAc;CACzD,CAAC;AACH;;;;;;;;AAgCA,SAAS,yBAAyB,KAAmD;CACnF,MAAM,eAAe,cAAoE;EACvF,IAAI,cAAc,QAAQ,OAAO,cAAc,UAAU,OAAO;EAChE,MAAM,EAAE,SAAS,WAAW;EAC5B,IAAI,OAAO,YAAY,WAAW,OAAO;EACzC,OAAO;GAAE;GAAS,QAAQ,OAAO,WAAW,WAAW,SAAS;EAAG;CACrE;CAEA,MAAM,QAAQ,IAAI,MAAM,aAAa;CACrC,IAAI,OACF,IAAI;EACF,MAAM,SAAS,YAAY,KAAK,MAAM,MAAM,EAAE,CAAC;EAC/C,IAAI,QAAQ,OAAO;CACrB,QAAQ,CAER;CAIF,MAAM,QAAQ,IAAI,QAAQ,GAAG;CAC7B,IAAI,UAAU,IAAI;EAChB,MAAM,YAAY,YAAY,qBAAqB,IAAI,MAAM,KAAK,CAAC,CAAC;EACpE,IAAI,WAAW,OAAO;CACxB;CACA,MAAM,IAAI,gBAAgB,uBAAuB,GAAG;AACtD;;;;;;;AAQA,SAAgB,4BACd,MACA,OAAkC,CAAC,GACf;CACpB,MAAM,QAAQ,KAAK,SAAS;CAC5B,MAAM,kBAAkB,KAAK,mBAAmB;CAChD,MAAM,cAAc,KAAK,eAAe;CACxC,MAAM,aAAa,KAAK,cAAc,IAAI,WAAW;CACrD,MAAM,OAAO,KAAK;CAClB,MAAM,SAAS,OAAO,UAA2C;EAE/D,IAAI;GACF,MAAM,MAAM,OAAO,KAAK;EAC1B,QAAQ,CAER;CACF;CACA,MAAM,UAA8B,OAAO,aAAa,YAAY;EAClE,MAAM,UAAU;GACd;GACA,UAAU,CACR;IACE,MAAM;IACN,SACE;GACJ,GACA;IACE,MAAM;IACN,SAAS,gBAAgB,YAAY,MAAM,IACzC,YAAY,WAAW,aAAa,YAAY,SAAS,MAAM,GAChE,wBAAwB,QAAQ,MAAM,GAAG,eAAe;GAC3D,CACF;GACA,aAAa;GACb,WAAW;EACb;EACA,IAAI;EACJ,KAAK,IAAI,UAAU,GAAG,UAAU,aAAa,WAAW;GACtD,MAAM,UAAU,KAAK,IAAI;GACzB,MAAM,OAAO;IACX,SAAS,WAAW;IACpB,UAAU,KAAK;IACf;IACA,UAAU;IACV,SAAS;IACT,cAAc;IACd,WAAW;IACX,WAAW;IACX,aAAa;IACb,gBAAgB,CAAC;GACnB,CAAC;GACD,IAAI;IACF,MAAM,OAAO,MAAM,WAAW,YAAY;KACxC,SAAS;KACT,OAAO,KAAK,aAAa;KACzB,OAAO;KACP;KACA,eACE,KAAK,oBAAoB,KAAA,IACrB,KAAA,IACA,2BAA2B,SAAS,EAAE,iBAAiB,KAAK,gBAAgB,CAAC;KACnF,MAAM;MACJ,GAAG,KAAK;MACR,eAAe,YAAY;MAC3B,SAAS,OAAO,OAAO;KACzB;KACA,QAAQ,KAAK;KACb,UAAU,QAAQ,WAAW,KAAK,KAAK,SAAS;MAAE;MAAQ,gBAAgB;KAAO,CAAC;KAClF,SAAS;KACT,kBAAkB;IACpB,CAAC;IACD,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK;IAChC,MAAM,OAAO,KAAK;IAMlB,kBAAkB,OAAO,KAAK,aAAa;KACzC,iBAAiB;KACjB,SAAS,uCAAuC,YAAY;IAC9D,CAAC;IACD,MAAM,MAAM,KAAK;IACjB,MAAM,OAAO;KACX,SAAS,WAAW;KACpB,UAAU,KAAK;KACf;KACA,UAAU;KACV,SAAS;KACT,cAAc;KACd,WAAW;KACX,WAAW,KAAK,IAAI;KACpB,YAAY,KAAK,IAAI,IAAI;KACzB,cAAc;KACd,gBAAgB,CAAC;IACnB,CAAC;IACD,OAAO,yBAAyB,GAAG;GACrC,SAAS,KAAK;IACZ,UAAU;IACV,MAAM,OAAO;KACX,SAAS,WAAW;KACpB,UAAU,KAAK;KACf;KACA,UAAU;KACV,SAAS;KACT,cAAc;KACd,WAAW;KACX,WAAW,KAAK,IAAI;KACpB,YAAY,KAAK,IAAI,IAAI;KACzB,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;KAC7D,gBAAgB,CAAC;IACnB,CAAC;IAGD,IAAI,eAAe,wBAAwB,MAAM;GACnD;EACF;EACA,MAAM,mBAAmB,QAAQ,UAAU,IAAI,MAAM,OAAO,OAAO,CAAC;CACtE;CACA,QAAQ,cAAc;EACpB,UAAU;EACV,SAAS;GAAE,MAAM;GAA2B,SAAS;EAAM;EAC3D,aAAa,CACX,uGACF;CACF;CACA,OAAO;AACT"}