@x-otto/eval 0.0.1-alpha.2 → 0.1.0-alpha.9

package/README.md

@@ -77,6 +77,9 @@ One case = one `evals/cases/<id>/` directory:
 evals/cases/my-case/
   case.yaml      # prompt, scoring method, run count, timeout, token budget
   fixture/       # optional: initial workspace state (copied to a temp dir on each run)
+  setup.sh       # optional: pre-run hook (RFC-417 M2) — runs in the workspace after fixture
+                 #   copy, before the agent starts (git init / npm install / start a service).
+                 #   exit≠0 → the run is recorded as infra_error (not a capability fail).
   verify.sh      # scoring script when judge=script, exit 0 = pass
 ```
 
@@ -91,7 +94,16 @@ golden:                # can coexist with script; both criteria must pass
   toolSequence: [read, edit]
 ```
 
-## Scoring Priority (Honest Boundaries)
+## Judging is an extension point
+
+`judge` values are resolved against a **verifier registry** (see `src/verify.ts`), not a hard-coded
+enum. Three verifiers are built in — `script`, `golden`, `liveness` — and new ones can be added with
+`registerVerifier()` without touching core code. `case-loader` rejects unknown judges loudly instead
+of silently degrading to liveness. (The registry is currently a thin skeleton with only the three
+built-ins; per RFC-417 D2 it will be reassessed — and possibly folded back into a hard-coded
+whitelist — if no second judging method emerges.)
+
+Scoring priority (honest boundaries) stays:
 
 ```
 executable criteria (verify.sh exit 0)  >  golden assertions  >  liveness (ran without crashing)

package/dist/cli.js

@@ -1,12 +1,12 @@
 #!/usr/bin/env node
-import{C as e,S as t,T as n,a as r,c as i,l as a,o,p as s,r as c,s as l,t as u}from"./extract-CJuoP2bV.js";import{mkdirSync as d,writeFileSync as f}from"node:fs";import{join as p,resolve as m}from"node:path";import{flushEnvWarnings as h}from"@x-otto/env";function g(e){let t=[],n={evalsRoot:m(process.cwd(),`evals`),reason:``,saveResult:!0};for(let r=0;r<e.length;r++){let i=e[r];i===`--evals-root`?n.evalsRoot=m(e[++r]??``):i===`--runs`?n.runs=Number(e[++r]):i===`--reason`?n.reason=e[++r]??``:i===`--no-save`?n.saveResult=!1:i!==void 0&&t.push(i)}return{command:t.join(` `)||`help`,options:n}}function _(e,t){let n=p(e,`results`);d(n,{recursive:!0});let r=p(n,`${t.startedAt.replace(/[:.]/g,`-`)}-${t.modelId.replace(/[^\w.-]/g,`_`)}.json`);return f(r,`${JSON.stringify(t,null,2)}\n`,`utf8`),r}async function v(){h(e=>process.stderr.write(`${e}\n`));let{command:d,options:f}=g(process.argv.slice(2)),m=p(f.evalsRoot,`cases`),v=p(f.evalsRoot,`baseline.json`);if(d===`baseline list`){let e=l(v);if(e.entries.length===0)return process.stdout.write(`no baselines recorded
+import{E as e,a as t,b as n,c as r,l as i,o as a,p as o,r as s,s as c,t as l,w as u}from"./extract-BmkiRXQY.js";import{mkdirSync as d,writeFileSync as f}from"node:fs";import{join as p,resolve as m}from"node:path";import{flushEnvWarnings as h}from"@x-otto/env";function g(e){let t=[],n={evalsRoot:m(process.cwd(),`evals`),reason:``,saveResult:!0};for(let r=0;r<e.length;r++){let i=e[r];i===`--evals-root`?n.evalsRoot=m(e[++r]??``):i===`--runs`?n.runs=Number(e[++r]):i===`--reason`?n.reason=e[++r]??``:i===`--no-save`?n.saveResult=!1:i!==void 0&&t.push(i)}return{command:t.join(` `)||`help`,options:n}}function _(e,t){let n=p(e,`results`);d(n,{recursive:!0});let r=p(n,`${t.startedAt.replace(/[:.]/g,`-`)}-${t.modelId.replace(/[^\w.-]/g,`_`)}.json`);return f(r,`${JSON.stringify(t,null,2)}\n`,`utf8`),r}async function v(){h(e=>process.stderr.write(`${e}\n`));let{command:d,options:f}=g(process.argv.slice(2)),m=p(f.evalsRoot,`cases`),v=p(f.evalsRoot,`baseline.json`);if(d===`baseline list`){let e=c(v);if(e.entries.length===0)return process.stdout.write(`no baselines recorded
 `),0;for(let t of e.entries)process.stdout.write(`${t.modelId}  suite=${t.suiteVersion}  pass=${t.overallPassRate===void 0?`—`:`${(t.overallPassRate*100).toFixed(0)}%`}  ${t.recordedAt}  (${t.reason})\n`);return 0}if(d.startsWith(`extract`)){let e=d.split(/\s+/)[1];if(!e)return process.stderr.write(`usage: otto-eval extract <sessionId> [--evals-root <dir>]
-`),2;let{caseDir:t,skeleton:n}=u(e,m);return process.stdout.write(`case skeleton created: ${t}\n  turns=${n.turns} toolCalls=${n.toolSequence.length} touchedPaths=${n.touchedPaths.length}\nnext: read ${t}/EXTRACTION.md — the case is NOT usable until you add a judge and pass the sanitisation checklist.\n`),0}if(d!==`run`&&d!==`baseline update`)return process.stdout.write(`usage:
+`),2;let{caseDir:t,skeleton:n}=l(e,m);return process.stdout.write(`case skeleton created: ${t}\n  turns=${n.turns} toolCalls=${n.toolSequence.length} touchedPaths=${n.touchedPaths.length}\nnext: read ${t}/EXTRACTION.md — the case is NOT usable until you add a judge and pass the sanitisation checklist.\n`),0}if(d!==`run`&&d!==`baseline update`)return process.stdout.write(`usage:
   otto-eval run [--evals-root <dir>] [--runs <n>] [--no-save]
   otto-eval baseline update --reason "<why>" [--evals-root <dir>] [--runs <n>]
   otto-eval baseline list [--evals-root <dir>]
   otto-eval extract <sessionId> [--evals-root <dir>]
-`),d===`help`?0:2;let y=n(m).map(e=>f.runs===void 0?e:{...e,runs:f.runs});if(y.length===0)return process.stderr.write(`no cases found under ${m}\n`),2;let b=e(y);process.stderr.write(`running ${y.length} case(s), suite=${b}\n`);let x=await s(y,b,{invoker:t(),onRunComplete:(e,t,n)=>{process.stderr.write(`  ${e} #${t+1}: ${n.outcome}\n`)}});if(d===`baseline update`)return f.reason===``?(process.stderr.write(`baseline update requires --reason "<why>"
+`),d===`help`?0:2;let y=e(m).map(e=>f.runs===void 0?e:{...e,runs:f.runs});if(y.length===0)return process.stderr.write(`no cases found under ${m}\n`),2;let b=u(y);process.stderr.write(`running ${y.length} case(s), suite=${b}\n`);let x=await o(y,b,{invoker:n(),onRunComplete:(e,t,n)=>{process.stderr.write(`  ${e} #${t+1}: ${n.outcome}\n`)}});if(d===`baseline update`)return f.reason===``?(process.stderr.write(`baseline update requires --reason "<why>"
 `),2):x.invalid?(process.stderr.write(`refusing to record baseline from an invalid run (too many infra errors)
-`),1):(i(v,a(l(v),x,f.reason)),process.stdout.write(`baseline recorded: model=${x.modelId} suite=${x.suiteVersion}\n`),0);let S=o(x,r(l(v),x.modelId,b));return process.stdout.write(`${c(x,S)}\n`),f.saveResult&&process.stderr.write(`result saved: ${_(f.evalsRoot,x)}\n`),S.status===`regressed`?1:0}v().then(e=>process.exit(e)).catch(e=>{process.stderr.write(`otto-eval failed: ${e instanceof Error?e.message:String(e)}\n`),process.exit(2)});export{};
+`),1):(r(v,i(c(v),x,f.reason)),process.stdout.write(`baseline recorded: model=${x.modelId} suite=${x.suiteVersion}\n`),0);let S=a(x,t(c(v),x.modelId,b));return process.stdout.write(`${s(x,S)}\n`),f.saveResult&&process.stderr.write(`result saved: ${_(f.evalsRoot,x)}\n`),S.status===`regressed`?1:0}v().then(e=>process.exit(e)).catch(e=>{process.stderr.write(`otto-eval failed: ${e instanceof Error?e.message:String(e)}\n`),process.exit(2)});export{};
 //# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","names":[],"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * cli.ts —— `otto-eval` 入口（RFC-316 D6 消费回路的触发点）。\n *\n * 三个子命令刚好对应消费回路的三个动作：\n *   run              跑套件 → 报告 → 与基线比 → 回归即非零退出（可直接当门禁用）\n *   baseline update  显式固化基线（带理由，RFC-316 D6 规则 3）\n *   baseline list    查看已有基线键（模型×套件）\n */\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join, resolve } from 'node:path'\n\nimport { flushEnvWarnings } from '@x-otto/env'\n\nimport { computeSuiteVersion, loadSuite } from './case-loader'\nimport { createSpawnInvoker } from './otto-invoker'\nimport { runSuite } from './runner'\nimport {\n  findBaseline,\n  judgeRegression,\n  loadBaselines,\n  saveBaselines,\n  upsertBaseline,\n} from './baseline'\nimport { renderReport } from './report'\nimport { extractCase } from './extract'\n\nimport type { Scorecard } from './types'\n\ninterface CliOptions {\n  evalsRoot: string\n  runs?: number\n  reason: string\n  saveResult: boolean\n}\n\nfunction parseArgs(argv: readonly string[]): { command: string; options: CliOptions } {\n  const positional: string[] = []\n  const options: CliOptions = {\n    evalsRoot: resolve(process.cwd(), 'evals'),\n    reason: '',\n    saveResult: true,\n  }\n  for (let i = 0; i < argv.length; i++) {\n    const arg = argv[i]\n    if (arg === '--evals-root') {\n      options.evalsRoot = resolve(argv[++i] ?? '')\n    } else if (arg === '--runs') {\n      options.runs = Number(argv[++i])\n    } else if (arg === '--reason') {\n      options.reason = argv[++i] ?? ''\n    } else if (arg === '--no-save') {\n      options.saveResult = false\n    } else if (arg !== undefined) {\n      positional.push(arg)\n    }\n  }\n  return { command: positional.join(' ') || 'help', options }\n}\n\nfunction persistResult(evalsRoot: string, scorecard: Scorecard): string {\n  const dir = join(evalsRoot, 'results')\n  mkdirSync(dir, { recursive: true })\n  const stamp = scorecard.startedAt.replace(/[:.]/g, '-')\n  const path = join(dir, `${stamp}-${scorecard.modelId.replace(/[^\\w.-]/g, '_')}.json`)\n  writeFileSync(path, `${JSON.stringify(scorecard, null, 2)}\\n`, 'utf8')\n  return path\n}\n\nasync function execute(): Promise<number> {\n  // RFC-324：eval 是独立 headless 入口，flush env 非法值 warning 到 stderr。\n  flushEnvWarnings((msg) => process.stderr.write(`${msg}\\n`))\n\n  const { command, options } = parseArgs(process.argv.slice(2))\n  const casesRoot = join(options.evalsRoot, 'cases')\n  const baselinePath = join(options.evalsRoot, 'baseline.json')\n\n  if (command === 'baseline list') {\n    const file = loadBaselines(baselinePath)\n    if (file.entries.length === 0) {\n      process.stdout.write('no baselines recorded\\n')\n      return 0\n    }\n    for (const e of file.entries) {\n      process.stdout.write(\n        `${e.modelId}  suite=${e.suiteVersion}  pass=${e.overallPassRate === undefined ? '—' : `${(e.overallPassRate * 100).toFixed(0)}%`}  ${e.recordedAt}  (${e.reason})\\n`,\n      )\n    }\n    return 0\n  }\n\n  if (command.startsWith('extract')) {\n    const sessionId = command.split(/\\s+/)[1]\n    if (!sessionId) {\n      process.stderr.write('usage: otto-eval extract <sessionId> [--evals-root <dir>]\\n')\n      return 2\n    }\n    const { caseDir, skeleton } = extractCase(sessionId, casesRoot)\n    process.stdout.write(\n      `case skeleton created: ${caseDir}\\n` +\n        `  turns=${skeleton.turns} toolCalls=${skeleton.toolSequence.length}` +\n        ` touchedPaths=${skeleton.touchedPaths.length}\\n` +\n        `next: read ${caseDir}/EXTRACTION.md — the case is NOT usable until you add a judge and pass the sanitisation checklist.\\n`,\n    )\n    return 0\n  }\n\n  if (command !== 'run' && command !== 'baseline update') {\n    process.stdout.write(\n      'usage:\\n' +\n        '  otto-eval run [--evals-root <dir>] [--runs <n>] [--no-save]\\n' +\n        '  otto-eval baseline update --reason \"<why>\" [--evals-root <dir>] [--runs <n>]\\n' +\n        '  otto-eval baseline list [--evals-root <dir>]\\n' +\n        '  otto-eval extract <sessionId> [--evals-root <dir>]\\n',\n    )\n    return command === 'help' ? 0 : 2\n  }\n\n  const cases = loadSuite(casesRoot).map((spec) =>\n    options.runs === undefined ? spec : { ...spec, runs: options.runs },\n  )\n  if (cases.length === 0) {\n    process.stderr.write(`no cases found under ${casesRoot}\\n`)\n    return 2\n  }\n  const suiteVersion = computeSuiteVersion(cases)\n\n  process.stderr.write(`running ${cases.length} case(s), suite=${suiteVersion}\\n`)\n  const scorecard = await runSuite(cases, suiteVersion, {\n    invoker: createSpawnInvoker(),\n    onRunComplete: (caseId, index, result) => {\n      process.stderr.write(`  ${caseId} #${index + 1}: ${result.outcome}\\n`)\n    },\n  })\n\n  if (command === 'baseline update') {\n    if (options.reason === '') {\n      process.stderr.write('baseline update requires --reason \"<why>\"\\n')\n      return 2\n    }\n    if (scorecard.invalid) {\n      process.stderr.write(\n        'refusing to record baseline from an invalid run (too many infra errors)\\n',\n      )\n      return 1\n    }\n    const updated = upsertBaseline(loadBaselines(baselinePath), scorecard, options.reason)\n    saveBaselines(baselinePath, updated)\n    process.stdout.write(\n      `baseline recorded: model=${scorecard.modelId} suite=${scorecard.suiteVersion}\\n`,\n    )\n    return 0\n  }\n\n  const baseline = findBaseline(loadBaselines(baselinePath), scorecard.modelId, suiteVersion)\n  const verdict = judgeRegression(scorecard, baseline)\n  process.stdout.write(`${renderReport(scorecard, verdict)}\\n`)\n  if (options.saveResult) {\n    process.stderr.write(`result saved: ${persistResult(options.evalsRoot, scorecard)}\\n`)\n  }\n  return verdict.status === 'regressed' ? 1 : 0\n}\n\nexecute()\n  .then((code) => process.exit(code))\n  .catch((error: unknown) => {\n    process.stderr.write(\n      `otto-eval failed: ${error instanceof Error ? error.message : String(error)}\\n`,\n    )\n    process.exit(2)\n  })\n"],"mappings":";+PAqCA,SAAS,EAAU,EAAmE,CACpF,IAAM,EAAuB,EAAE,CACzB,EAAsB,CAC1B,UAAW,EAAQ,QAAQ,KAAK,CAAE,QAAQ,CAC1C,OAAQ,GACR,WAAY,GACb,CACD,IAAK,IAAI,EAAI,EAAG,EAAI,EAAK,OAAQ,IAAK,CACpC,IAAM,EAAM,EAAK,GACb,IAAQ,eACV,EAAQ,UAAY,EAAQ,EAAK,EAAE,IAAM,GAAG,CACnC,IAAQ,SACjB,EAAQ,KAAO,OAAO,EAAK,EAAE,GAAG,CACvB,IAAQ,WACjB,EAAQ,OAAS,EAAK,EAAE,IAAM,GACrB,IAAQ,YACjB,EAAQ,WAAa,GACZ,IAAQ,IAAA,IACjB,EAAW,KAAK,EAAI,CAGxB,MAAO,CAAE,QAAS,EAAW,KAAK,IAAI,EAAI,OAAQ,UAAS,CAG7D,SAAS,EAAc,EAAmB,EAA8B,CACtE,IAAM,EAAM,EAAK,EAAW,UAAU,CACtC,EAAU,EAAK,CAAE,UAAW,GAAM,CAAC,CAEnC,IAAM,EAAO,EAAK,EAAK,GADT,EAAU,UAAU,QAAQ,QAAS,IAAI,CACvB,GAAG,EAAU,QAAQ,QAAQ,WAAY,IAAI,CAAC,OAAO,CAErF,OADA,EAAc,EAAM,GAAG,KAAK,UAAU,EAAW,KAAM,EAAE,CAAC,IAAK,OAAO,CAC/D,EAGT,eAAe,GAA2B,CAExC,EAAkB,GAAQ,QAAQ,OAAO,MAAM,GAAG,EAAI,IAAI,CAAC,CAE3D,GAAM,CAAE,UAAS,WAAY,EAAU,QAAQ,KAAK,MAAM,EAAE,CAAC,CACvD,EAAY,EAAK,EAAQ,UAAW,QAAQ,CAC5C,EAAe,EAAK,EAAQ,UAAW,gBAAgB,CAE7D,GAAI,IAAY,gBAAiB,CAC/B,IAAM,EAAO,EAAc,EAAa,CACxC,GAAI,EAAK,QAAQ,SAAW,EAE1B,OADA,QAAQ,OAAO,MAAM;EAA0B,CACxC,EAET,IAAK,IAAM,KAAK,EAAK,QACnB,QAAQ,OAAO,MACb,GAAG,EAAE,QAAQ,UAAU,EAAE,aAAa,SAAS,EAAE,kBAAoB,IAAA,GAAY,IAAM,IAAI,EAAE,gBAAkB,KAAK,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,WAAW,KAAK,EAAE,OAAO,KAClK,CAEH,MAAO,GAGT,GAAI,EAAQ,WAAW,UAAU,CAAE,CACjC,IAAM,EAAY,EAAQ,MAAM,MAAM,CAAC,GACvC,GAAI,CAAC,EAEH,OADA,QAAQ,OAAO,MAAM;EAA8D,CAC5E,EAET,GAAM,CAAE,UAAS,YAAa,EAAY,EAAW,EAAU,CAO/D,OANA,QAAQ,OAAO,MACb,0BAA0B,EAAQ,YACrB,EAAS,MAAM,aAAa,EAAS,aAAa,OAAA,gBAC5C,EAAS,aAAa,OAAO,eAChC,EAAQ,sGACzB,CACM,EAGT,GAAI,IAAY,OAAS,IAAY,kBAQnC,OAPA,QAAQ,OAAO,MACb;;;;;EAKD,CACM,IAAY,OAAS,EAAI,EAGlC,IAAM,EAAQ,EAAU,EAAU,CAAC,IAAK,GACtC,EAAQ,OAAS,IAAA,GAAY,EAAO,CAAE,GAAG,EAAM,KAAM,EAAQ,KAAM,CACpE,CACD,GAAI,EAAM,SAAW,EAEnB,OADA,QAAQ,OAAO,MAAM,wBAAwB,EAAU,IAAI,CACpD,EAET,IAAM,EAAe,EAAoB,EAAM,CAE/C,QAAQ,OAAO,MAAM,WAAW,EAAM,OAAO,kBAAkB,EAAa,IAAI,CAChF,IAAM,EAAY,MAAM,EAAS,EAAO,EAAc,CACpD,QAAS,GAAoB,CAC7B,eAAgB,EAAQ,EAAO,IAAW,CACxC,QAAQ,OAAO,MAAM,KAAK,EAAO,IAAI,EAAQ,EAAE,IAAI,EAAO,QAAQ,IAAI,EAEzE,CAAC,CAEF,GAAI,IAAY,kBAgBd,OAfI,EAAQ,SAAW,IACrB,QAAQ,OAAO,MAAM;EAA8C,CAC5D,GAEL,EAAU,SACZ,QAAQ,OAAO,MACb;EACD,CACM,IAGT,EAAc,EADE,EAAe,EAAc,EAAa,CAAE,EAAW,EAAQ,OAAO,CAClD,CACpC,QAAQ,OAAO,MACb,4BAA4B,EAAU,QAAQ,SAAS,EAAU,aAAa,IAC/E,CACM,GAIT,IAAM,EAAU,EAAgB,EADf,EAAa,EAAc,EAAa,CAAE,EAAU,QAAS,EAAa,CACvC,CAKpD,OAJA,QAAQ,OAAO,MAAM,GAAG,EAAa,EAAW,EAAQ,CAAC,IAAI,CACzD,EAAQ,YACV,QAAQ,OAAO,MAAM,iBAAiB,EAAc,EAAQ,UAAW,EAAU,CAAC,IAAI,CAEjF,EAAQ,SAAW,YAAc,EAAI,EAG9C,GAAS,CACN,KAAM,GAAS,QAAQ,KAAK,EAAK,CAAC,CAClC,MAAO,GAAmB,CACzB,QAAQ,OAAO,MACb,qBAAqB,aAAiB,MAAQ,EAAM,QAAU,OAAO,EAAM,CAAC,IAC7E,CACD,QAAQ,KAAK,EAAE,EACf"}
+{"version":3,"file":"cli.js","names":[],"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * cli.ts —— `otto-eval` 入口（RFC-316 D6 消费回路的触发点）。\n *\n * 三个子命令刚好对应消费回路的三个动作：\n *   run              跑套件 → 报告 → 与基线比 → 回归即非零退出（可直接当门禁用）\n *   baseline update  显式固化基线（带理由，RFC-316 D6 规则 3）\n *   baseline list    查看已有基线键（模型×套件）\n */\n\nimport { mkdirSync, writeFileSync } from 'node:fs'\nimport { join, resolve } from 'node:path'\n\nimport { flushEnvWarnings } from '@x-otto/env'\n\nimport { computeSuiteVersion, loadSuite } from './case-loader'\nimport { createSpawnInvoker } from './otto-invoker'\nimport { runSuite } from './runner'\nimport {\n  findBaseline,\n  judgeRegression,\n  loadBaselines,\n  saveBaselines,\n  upsertBaseline,\n} from './baseline'\nimport { renderReport } from './report'\nimport { extractCase } from './extract'\n\nimport type { Scorecard } from './types'\n\ninterface CliOptions {\n  evalsRoot: string\n  runs?: number\n  reason: string\n  saveResult: boolean\n}\n\nfunction parseArgs(argv: readonly string[]): { command: string; options: CliOptions } {\n  const positional: string[] = []\n  const options: CliOptions = {\n    evalsRoot: resolve(process.cwd(), 'evals'),\n    reason: '',\n    saveResult: true,\n  }\n  for (let i = 0; i < argv.length; i++) {\n    const arg = argv[i]\n    if (arg === '--evals-root') {\n      options.evalsRoot = resolve(argv[++i] ?? '')\n    } else if (arg === '--runs') {\n      options.runs = Number(argv[++i])\n    } else if (arg === '--reason') {\n      options.reason = argv[++i] ?? ''\n    } else if (arg === '--no-save') {\n      options.saveResult = false\n    } else if (arg !== undefined) {\n      positional.push(arg)\n    }\n  }\n  return { command: positional.join(' ') || 'help', options }\n}\n\nfunction persistResult(evalsRoot: string, scorecard: Scorecard): string {\n  const dir = join(evalsRoot, 'results')\n  mkdirSync(dir, { recursive: true })\n  const stamp = scorecard.startedAt.replace(/[:.]/g, '-')\n  const path = join(dir, `${stamp}-${scorecard.modelId.replace(/[^\\w.-]/g, '_')}.json`)\n  writeFileSync(path, `${JSON.stringify(scorecard, null, 2)}\\n`, 'utf8')\n  return path\n}\n\nasync function execute(): Promise<number> {\n  // RFC-324：eval 是独立 headless 入口，flush env 非法值 warning 到 stderr。\n  flushEnvWarnings((msg) => process.stderr.write(`${msg}\\n`))\n\n  const { command, options } = parseArgs(process.argv.slice(2))\n  const casesRoot = join(options.evalsRoot, 'cases')\n  const baselinePath = join(options.evalsRoot, 'baseline.json')\n\n  if (command === 'baseline list') {\n    const file = loadBaselines(baselinePath)\n    if (file.entries.length === 0) {\n      process.stdout.write('no baselines recorded\\n')\n      return 0\n    }\n    for (const e of file.entries) {\n      process.stdout.write(\n        `${e.modelId}  suite=${e.suiteVersion}  pass=${e.overallPassRate === undefined ? '—' : `${(e.overallPassRate * 100).toFixed(0)}%`}  ${e.recordedAt}  (${e.reason})\\n`,\n      )\n    }\n    return 0\n  }\n\n  if (command.startsWith('extract')) {\n    const sessionId = command.split(/\\s+/)[1]\n    if (!sessionId) {\n      process.stderr.write('usage: otto-eval extract <sessionId> [--evals-root <dir>]\\n')\n      return 2\n    }\n    const { caseDir, skeleton } = extractCase(sessionId, casesRoot)\n    process.stdout.write(\n      `case skeleton created: ${caseDir}\\n` +\n        `  turns=${skeleton.turns} toolCalls=${skeleton.toolSequence.length}` +\n        ` touchedPaths=${skeleton.touchedPaths.length}\\n` +\n        `next: read ${caseDir}/EXTRACTION.md — the case is NOT usable until you add a judge and pass the sanitisation checklist.\\n`,\n    )\n    return 0\n  }\n\n  if (command !== 'run' && command !== 'baseline update') {\n    process.stdout.write(\n      'usage:\\n' +\n        '  otto-eval run [--evals-root <dir>] [--runs <n>] [--no-save]\\n' +\n        '  otto-eval baseline update --reason \"<why>\" [--evals-root <dir>] [--runs <n>]\\n' +\n        '  otto-eval baseline list [--evals-root <dir>]\\n' +\n        '  otto-eval extract <sessionId> [--evals-root <dir>]\\n',\n    )\n    return command === 'help' ? 0 : 2\n  }\n\n  const cases = loadSuite(casesRoot).map((spec) =>\n    options.runs === undefined ? spec : { ...spec, runs: options.runs },\n  )\n  if (cases.length === 0) {\n    process.stderr.write(`no cases found under ${casesRoot}\\n`)\n    return 2\n  }\n  const suiteVersion = computeSuiteVersion(cases)\n\n  process.stderr.write(`running ${cases.length} case(s), suite=${suiteVersion}\\n`)\n  const scorecard = await runSuite(cases, suiteVersion, {\n    invoker: createSpawnInvoker(),\n    onRunComplete: (caseId, index, result) => {\n      process.stderr.write(`  ${caseId} #${index + 1}: ${result.outcome}\\n`)\n    },\n  })\n\n  if (command === 'baseline update') {\n    if (options.reason === '') {\n      process.stderr.write('baseline update requires --reason \"<why>\"\\n')\n      return 2\n    }\n    if (scorecard.invalid) {\n      process.stderr.write(\n        'refusing to record baseline from an invalid run (too many infra errors)\\n',\n      )\n      return 1\n    }\n    const updated = upsertBaseline(loadBaselines(baselinePath), scorecard, options.reason)\n    saveBaselines(baselinePath, updated)\n    process.stdout.write(\n      `baseline recorded: model=${scorecard.modelId} suite=${scorecard.suiteVersion}\\n`,\n    )\n    return 0\n  }\n\n  const baseline = findBaseline(loadBaselines(baselinePath), scorecard.modelId, suiteVersion)\n  const verdict = judgeRegression(scorecard, baseline)\n  process.stdout.write(`${renderReport(scorecard, verdict)}\\n`)\n  if (options.saveResult) {\n    process.stderr.write(`result saved: ${persistResult(options.evalsRoot, scorecard)}\\n`)\n  }\n  return verdict.status === 'regressed' ? 1 : 0\n}\n\nexecute()\n  .then((code) => process.exit(code))\n  .catch((error: unknown) => {\n    process.stderr.write(\n      `otto-eval failed: ${error instanceof Error ? error.message : String(error)}\\n`,\n    )\n    process.exit(2)\n  })\n"],"mappings":";oQAqCA,SAAS,EAAU,EAAmE,CACpF,IAAM,EAAuB,EAAE,CACzB,EAAsB,CAC1B,UAAW,EAAQ,QAAQ,KAAK,CAAE,QAAQ,CAC1C,OAAQ,GACR,WAAY,GACb,CACD,IAAK,IAAI,EAAI,EAAG,EAAI,EAAK,OAAQ,IAAK,CACpC,IAAM,EAAM,EAAK,GACb,IAAQ,eACV,EAAQ,UAAY,EAAQ,EAAK,EAAE,IAAM,GAAG,CACnC,IAAQ,SACjB,EAAQ,KAAO,OAAO,EAAK,EAAE,GAAG,CACvB,IAAQ,WACjB,EAAQ,OAAS,EAAK,EAAE,IAAM,GACrB,IAAQ,YACjB,EAAQ,WAAa,GACZ,IAAQ,IAAA,IACjB,EAAW,KAAK,EAAI,CAGxB,MAAO,CAAE,QAAS,EAAW,KAAK,IAAI,EAAI,OAAQ,UAAS,CAG7D,SAAS,EAAc,EAAmB,EAA8B,CACtE,IAAM,EAAM,EAAK,EAAW,UAAU,CACtC,EAAU,EAAK,CAAE,UAAW,GAAM,CAAC,CAEnC,IAAM,EAAO,EAAK,EAAK,GADT,EAAU,UAAU,QAAQ,QAAS,IAAI,CACvB,GAAG,EAAU,QAAQ,QAAQ,WAAY,IAAI,CAAC,OAAO,CAErF,OADA,EAAc,EAAM,GAAG,KAAK,UAAU,EAAW,KAAM,EAAE,CAAC,IAAK,OAAO,CAC/D,EAGT,eAAe,GAA2B,CAExC,EAAkB,GAAQ,QAAQ,OAAO,MAAM,GAAG,EAAI,IAAI,CAAC,CAE3D,GAAM,CAAE,UAAS,WAAY,EAAU,QAAQ,KAAK,MAAM,EAAE,CAAC,CACvD,EAAY,EAAK,EAAQ,UAAW,QAAQ,CAC5C,EAAe,EAAK,EAAQ,UAAW,gBAAgB,CAE7D,GAAI,IAAY,gBAAiB,CAC/B,IAAM,EAAO,EAAc,EAAa,CACxC,GAAI,EAAK,QAAQ,SAAW,EAE1B,OADA,QAAQ,OAAO,MAAM;EAA0B,CACxC,EAET,IAAK,IAAM,KAAK,EAAK,QACnB,QAAQ,OAAO,MACb,GAAG,EAAE,QAAQ,UAAU,EAAE,aAAa,SAAS,EAAE,kBAAoB,IAAA,GAAY,IAAM,IAAI,EAAE,gBAAkB,KAAK,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,WAAW,KAAK,EAAE,OAAO,KAClK,CAEH,MAAO,GAGT,GAAI,EAAQ,WAAW,UAAU,CAAE,CACjC,IAAM,EAAY,EAAQ,MAAM,MAAM,CAAC,GACvC,GAAI,CAAC,EAEH,OADA,QAAQ,OAAO,MAAM;EAA8D,CAC5E,EAET,GAAM,CAAE,UAAS,YAAa,EAAY,EAAW,EAAU,CAO/D,OANA,QAAQ,OAAO,MACb,0BAA0B,EAAQ,YACrB,EAAS,MAAM,aAAa,EAAS,aAAa,OAAA,gBAC5C,EAAS,aAAa,OAAO,eAChC,EAAQ,sGACzB,CACM,EAGT,GAAI,IAAY,OAAS,IAAY,kBAQnC,OAPA,QAAQ,OAAO,MACb;;;;;EAKD,CACM,IAAY,OAAS,EAAI,EAGlC,IAAM,EAAQ,EAAU,EAAU,CAAC,IAAK,GACtC,EAAQ,OAAS,IAAA,GAAY,EAAO,CAAE,GAAG,EAAM,KAAM,EAAQ,KAAM,CACpE,CACD,GAAI,EAAM,SAAW,EAEnB,OADA,QAAQ,OAAO,MAAM,wBAAwB,EAAU,IAAI,CACpD,EAET,IAAM,EAAe,EAAoB,EAAM,CAE/C,QAAQ,OAAO,MAAM,WAAW,EAAM,OAAO,kBAAkB,EAAa,IAAI,CAChF,IAAM,EAAY,MAAM,EAAS,EAAO,EAAc,CACpD,QAAS,GAAoB,CAC7B,eAAgB,EAAQ,EAAO,IAAW,CACxC,QAAQ,OAAO,MAAM,KAAK,EAAO,IAAI,EAAQ,EAAE,IAAI,EAAO,QAAQ,IAAI,EAEzE,CAAC,CAEF,GAAI,IAAY,kBAgBd,OAfI,EAAQ,SAAW,IACrB,QAAQ,OAAO,MAAM;EAA8C,CAC5D,GAEL,EAAU,SACZ,QAAQ,OAAO,MACb;EACD,CACM,IAGT,EAAc,EADE,EAAe,EAAc,EAAa,CAAE,EAAW,EAAQ,OAAO,CAClD,CACpC,QAAQ,OAAO,MACb,4BAA4B,EAAU,QAAQ,SAAS,EAAU,aAAa,IAC/E,CACM,GAIT,IAAM,EAAU,EAAgB,EADf,EAAa,EAAc,EAAa,CAAE,EAAU,QAAS,EAAa,CACvC,CAKpD,OAJA,QAAQ,OAAO,MAAM,GAAG,EAAa,EAAW,EAAQ,CAAC,IAAI,CACzD,EAAQ,YACV,QAAQ,OAAO,MAAM,iBAAiB,EAAc,EAAQ,UAAW,EAAU,CAAC,IAAI,CAEjF,EAAQ,SAAW,YAAc,EAAI,EAG9C,GAAS,CACN,KAAM,GAAS,QAAQ,KAAK,EAAK,CAAC,CAClC,MAAO,GAAmB,CACzB,QAAQ,OAAO,MACb,qBAAqB,aAAiB,MAAQ,EAAM,QAAU,OAAO,EAAM,CAAC,IAC7E,CACD,QAAQ,KAAK,EAAE,EACf"}

package/dist/extract-BmkiRXQY.js

@@ -0,0 +1,39 @@
+import{t as e}from"./workspace-C_kTWfF8.js";import{createHash as t}from"node:crypto";import{existsSync as n,mkdirSync as r,readFileSync as i,readdirSync as a,statSync as o,writeFileSync as s}from"node:fs";import{dirname as c,join as l}from"node:path";import{parse as u}from"yaml";import{spawn as d,spawnSync as f}from"node:child_process";import{OTTO_SESSIONS_DIR as p,normalizeEnv as m,normalizeEnvFraction as h}from"@x-otto/env";const g=new Map;function _(e){g.set(e.name,e)}function v(e){return g.get(e)}function y(){return[...g.keys()]}function b(e,t){let n=0;for(let r of e)n<t.length&&r===t[n]&&n++;return n===t.length}function x(e,t,n){let r=[];if(e.answerContains){let n=t.toLowerCase();for(let t of e.answerContains)n.includes(t.toLowerCase())||r.push(`answer missing '${t}'`)}return e.toolSequence&&!b(n,e.toolSequence)&&r.push(`tool sequence [${e.toolSequence.join(`, `)}] not satisfied by [${n.join(`, `)}]`),r.length===0?{passed:!0,detail:`golden assertions satisfied`}:{passed:!1,detail:r.join(`; `)}}function S(e,t){let n=f(`bash`,[t],{cwd:e.workspaceDir,encoding:`utf8`,timeout:12e4,env:{...process.env,OTTO_EVAL_RESPONSE:e.response}});if(n.error)return{passed:!1,detail:`verify script error: ${n.error.message}`};let r=`${(n.stdout??``).trim()}${n.stderr?` | ${n.stderr.trim()}`:``}`.slice(0,500);return{passed:n.status===0,detail:r||`exit ${n.status}`}}const C={name:`script`,async verify(e){let t=e.spec;return t.judge===`script`?S(e,l(t.dir,t.verifyScript)):{passed:!1,detail:`script verifier applied to a non-script case`}}},w={name:`golden`,async verify(e){return x(e.spec.golden??{},e.response,e.toolNames)}},T={name:`liveness`,async verify(){return{passed:!0,detail:`liveness only (no quality judgement)`}}};_(C),_(w),_(T);async function E(e){let{spec:t}=e,n=[],r=v(t.judge);if(!r)return{passed:!1,detail:`no verifier registered for judge '${t.judge}' (known: ${y().join(`, `)})`};if(n.push(await r.verify(e)),t.golden&&n.push(x(t.golden,e.response,e.toolNames)),n.length===0)return{passed:!0,detail:`liveness only (no quality judgement)`};let i=n.filter(e=>!e.passed);return i.length===0?{passed:!0,detail:n.map(e=>e.detail).join(`; `)}:{passed:!1,detail:i.map(e=>e.detail).join(`; `)}}function D(e,t,n){let r=e[t];if(typeof r!=`string`||r.trim()===``)throw Error(`case '${n}': field '${t}' must be a non-empty string`);return r}function O(e,t,n,r){let i=e[t];if(i===void 0)return r;if(typeof i!=`number`||!Number.isInteger(i)||i<=0)throw Error(`case '${n}': field '${t}' must be a positive integer`);return i}function ee(e,t){if(typeof e!=`object`||!e)throw Error(`case '${t}': judge='golden' requires a 'golden' mapping`);let n=e,r={},i=n.answerContains;if(i!==void 0){if(!Array.isArray(i)||i.some(e=>typeof e!=`string`))throw Error(`case '${t}': golden.answerContains must be a string array`);Object.assign(r,{answerContains:i})}let a=n.toolSequence;if(a!==void 0){if(!Array.isArray(a)||a.some(e=>typeof e!=`string`))throw Error(`case '${t}': golden.toolSequence must be a string array`);Object.assign(r,{toolSequence:a})}if(!r.answerContains&&!r.toolSequence)throw Error(`case '${t}': judge='golden' requires at least one of golden.answerContains / golden.toolSequence`);return r}function k(e){let t=e.split(`/`).filter(Boolean).at(-1)??e,r=l(e,`case.yaml`);if(!n(r))throw Error(`case '${t}': missing case.yaml at ${r}`);let a=u(i(r,`utf8`));if(typeof a!=`object`||!a)throw Error(`case '${t}': case.yaml must contain a mapping`);let o=a,s=D(o,`prompt`,t),c=D(o,`judge`,t);if(v(c)===void 0)throw Error(`case '${t}': judge '${c}' is not a registered verifier (known: ${y().join(`, `)})`);let d=c,f={id:t,prompt:s,runs:O(o,`runs`,t,3),timeoutMs:O(o,`timeoutMs`,t,3e5),dir:e,hasFixture:n(l(e,`fixture`)),hasSetup:n(l(e,`setup.sh`))};o.maxTokens!==void 0&&Object.assign(f,{maxTokens:O(o,`maxTokens`,t,0)});let p=d===`golden`||o.golden!==void 0?ee(o.golden,t):void 0;if(d===`script`){let r=o.verifyScript===void 0?`verify.sh`:D(o,`verifyScript`,t);if(!n(l(e,r)))throw Error(`case '${t}': judge='script' but verify script '${r}' not found`);return{...f,judge:d,verifyScript:r,...p===void 0?{}:{golden:p}}}return d===`golden`?{...f,judge:d,golden:p}:{...f,judge:d,...p===void 0?{}:{golden:p}}}function A(e){if(!n(e))throw Error(`cases root not found: ${e}`);return a(e).filter(e=>!e.startsWith(`.`)).map(t=>l(e,t)).filter(e=>o(e).isDirectory()).sort().map(k)}function j(e){let n=e.map(e=>({id:e.id,prompt:e.prompt,judge:e.judge,verifyScript:e.judge===`script`?e.verifyScript:void 0,golden:e.golden??null}));return t(`sha256`).update(JSON.stringify(n)).digest(`hex`).slice(0,12)}const M=`setup.sh`,N=6e4;function P(e,t){if(!e.hasSetup)return{ok:!0,detail:`no setup`};let n=f(`bash`,[l(e.dir,M)],{cwd:t,encoding:`utf8`,timeout:N});if(n.error)return{ok:!1,detail:`setup script error: ${n.error.message}`};if(n.status!==0){let e=(n.stderr??``).trim().slice(0,500);return{ok:!1,detail:`setup exit ${n.status}${e?`: ${e}`:``}`}}return{ok:!0,detail:`setup ok`}}function F(e){let t=e.toLowerCase();return[`rate limit`,`rate_limit`,`429`,`overloaded`,`econnreset`,`etimedout`,`enotfound`,`socket hang up`,`service unavailable`,`internal server error`,`bad gateway`,`no model configured`,`command not found`,`enoent`].some(e=>t.includes(e))?`infra`:`failed`}function te(e={}){let t=e.bin??`otto`;return{invoke(n){return new Promise(r=>{let i=d(t,[`--json`,n.prompt],{cwd:n.cwd,env:{...process.env,...e.env},stdio:[`ignore`,`pipe`,`pipe`]}),a=``,o=``,s=!1,c=e=>{s||(s=!0,clearTimeout(l),r(e))},l=setTimeout(()=>{i.kill(`SIGKILL`),c({kind:`failed`,detail:`timeout after ${n.timeoutMs}ms`})},n.timeoutMs);i.stdout.on(`data`,e=>{a+=e.toString()}),i.stderr.on(`data`,e=>{o+=e.toString()}),i.on(`error`,e=>{c({kind:`infra`,detail:`spawn failed: ${e.message}`})}),i.on(`close`,e=>{if(e!==0){c({kind:F(o),detail:`exit ${e}: ${o.trim().slice(0,500)}`});return}try{let e=JSON.parse(a);if(typeof e.sessionId!=`string`){c({kind:`failed`,detail:`json output missing sessionId`});return}c({kind:`ok`,result:e})}catch{c({kind:`failed`,detail:`unparsable json output: ${a.slice(0,300)}`})}})})}}}function I(e,t){let n=e?.[t];return typeof n==`number`&&Number.isFinite(n)?n:0}function L(e){return l(p,`trace`,`${e}.jsonl`)}function R(e){let t=[];return{metrics:z(e,t),toolNames:t}}function z(e,t){let n={turns:0,toolCalls:0,toolErrors:0,inputTokens:0,outputTokens:0,cacheReadTokens:0,toolDurationMs:0};for(let r of e.split(`
+`)){if(r.trim()===``)continue;let e;try{e=JSON.parse(r)}catch{continue}if(e.node===`tool.call.end`){n.toolCalls++,n.toolDurationMs+=I(e.payload,`durationMs`),e.payload?.errorCategory!==void 0&&n.toolErrors++;let r=e.payload?.name;typeof r==`string`&&t?.push(r);continue}if(e.node===`turn.end`){n.turns++;let t=e.payload?.usage;if(typeof t==`object`&&t){let e=t;n.inputTokens+=I(e,`inputTokens`),n.outputTokens+=I(e,`outputTokens`),n.cacheReadTokens+=I(e,`cacheReadTokens`)}let r=e.payload?.model;typeof r==`string`&&r!==``&&(n.modelId=r)}}return n}function B(e){let t=L(e);if(n(t))return R(i(t,`utf8`))}function V(e){if(e.length===0)return;let t=e.reduce((e,t)=>({turns:e.turns+t.turns,toolCalls:e.toolCalls+t.toolCalls,toolErrors:e.toolErrors+t.toolErrors,inputTokens:e.inputTokens+t.inputTokens,outputTokens:e.outputTokens+t.outputTokens,cacheReadTokens:e.cacheReadTokens+t.cacheReadTokens,toolDurationMs:e.toolDurationMs+t.toolDurationMs,modelId:t.modelId??e.modelId}),{turns:0,toolCalls:0,toolErrors:0,inputTokens:0,outputTokens:0,cacheReadTokens:0,toolDurationMs:0}),n=e.length,r={turns:t.turns/n,toolCalls:t.toolCalls/n,toolErrors:t.toolErrors/n,inputTokens:t.inputTokens/n,outputTokens:t.outputTokens/n,cacheReadTokens:t.cacheReadTokens/n,toolDurationMs:t.toolDurationMs/n};return t.modelId!==void 0&&(r.modelId=t.modelId),r}const H=h(process.env.OTTO_EVAL_INFRA_ERROR_THRESHOLD,.2);function U(e){return e?e.inputTokens+e.outputTokens:0}async function W(t,n){let r=n.now??(()=>Date.now()),i=n.readTrace??B,a=[],o=0,s=0,c=0,l=()=>{if(t.maxTokens===void 0)return!1;let e=s>0?o/s:t.maxTokens/t.runs;return o+c*e>=t.maxTokens};for(let u=0;u<t.runs;u++){if(l()){a.push({outcome:`skipped_budget`,wallMs:0,detail:`budget exhausted: ${o} measured tokens${c>0?` + ${c} unmeasured run(s)`:``} vs limit ${t.maxTokens}, remaining runs skipped`});continue}let d=e(t),f=r();try{let e=P(t,d.dir);if(!e.ok){c++,a.push({outcome:`infra_error`,wallMs:r()-f,detail:e.detail});continue}let l=await n.invoker.invoke({prompt:t.prompt,cwd:d.dir,timeoutMs:t.timeoutMs}),u=r()-f;if(l.kind===`infra`){c++,a.push({outcome:`infra_error`,wallMs:u,detail:l.detail});continue}if(l.kind===`failed`){c++,a.push({outcome:`fail`,wallMs:u,detail:l.detail});continue}let p=i(l.result.sessionId),m=await E({spec:t,workspaceDir:d.dir,response:l.result.response,toolNames:p?.toolNames??[]});p?(o+=U(p.metrics),s++):c++;let h={outcome:m.passed?`pass`:`fail`,wallMs:u,sessionId:l.result.sessionId,detail:m.detail};p&&(h.metrics=p.metrics),h.ottoVersion=l.result.ottoVersion??`unknown`,a.push(h)}finally{d.dispose()}let p=a.at(-1);p&&n.onRunComplete?.(t.id,u,p)}return G(t,a)}function G(e,t){let n=t.filter(e=>e.outcome===`infra_error`).length,r=t.filter(e=>e.outcome===`skipped_budget`).length,i=t.length-n-r,a=t.filter(e=>e.outcome===`pass`).length,o=e.judge!==`liveness`,s={caseId:e.id,judge:e.judge,runs:t,passed:a,effective:i,infraErrors:n,skippedBudget:r,qualityScored:o};return i>0&&Object.assign(s,{passRate:a/i}),s}async function K(e,t,n){let r=n.now??(()=>Date.now()),i=new Date(r()).toISOString(),a=[];for(let t of e)a.push(await W(t,n));return q(a,t,i)}function q(e,t,n){let r=e.filter(e=>e.qualityScored&&e.passRate!==void 0),i=e.reduce((e,t)=>e+t.skippedBudget,0),a=e.reduce((e,t)=>e+t.runs.length,0)-i,o=e.reduce((e,t)=>e+t.infraErrors,0),s=a===0?0:o/a,c=V(e.flatMap(e=>e.runs.map(e=>e.metrics).filter(e=>e!==void 0))),l=e.flatMap(e=>e.runs).find(e=>e.ottoVersion!==void 0)?.ottoVersion??`unknown`,u={suiteVersion:t,modelId:c?.modelId??`unknown`,ottoVersion:l,startedAt:n,cases:e,scoredCases:e.filter(e=>e.qualityScored).length,unscoredCases:e.filter(e=>!e.qualityScored).length,infraErrorRate:s,skippedBudgetTotal:i,invalid:s>H};if(r.length>0){let e=r.reduce((e,t)=>e+(t.passRate??0),0);Object.assign(u,{overallPassRate:e/r.length})}return c&&Object.assign(u,{aggregateMetrics:c}),u}const J=m(process.env.OTTO_EVAL_REGRESSION_PP,15);function Y(e){return n(e)?{entries:JSON.parse(i(e,`utf8`)).entries??[]}:{entries:[]}}function X(e,t){r(c(e),{recursive:!0}),s(e,`${JSON.stringify(t,null,2)}\n`,`utf8`)}function Z(e,t,n){return e.entries.find(e=>e.modelId===t&&e.suiteVersion===n)}function ne(e,t,n){let r={};for(let e of t.cases)r[e.caseId]=e.passRate;let i={modelId:t.modelId,suiteVersion:t.suiteVersion,recordedAt:t.startedAt,reason:n,ottoVersion:t.ottoVersion,casePassRates:r};return t.overallPassRate!==void 0&&Object.assign(i,{overallPassRate:t.overallPassRate}),{entries:[...e.entries.filter(e=>!(e.modelId===i.modelId&&e.suiteVersion===i.suiteVersion)),i]}}function re(e,t){if(e.invalid)return{status:`invalid_run`,findings:[`infra_error rate ${(e.infraErrorRate*100).toFixed(1)}% exceeds threshold — run not comparable`],flippedCases:[]};if(!t)return{status:`no_baseline`,findings:[`no baseline for (model=${e.modelId}, suite=${e.suiteVersion}) — record one with \`baseline update\``],flippedCases:[]};let n=[],r=[];t.ottoVersion!==void 0&&e.ottoVersion!==`unknown`&&t.ottoVersion!==e.ottoVersion&&n.push(`otto version differs from baseline: baseline recorded by '${t.ottoVersion}', this run '${e.ottoVersion}' — comparison spans engine versions`);for(let i of e.cases){if(!i.qualityScored)continue;let e=t.casePassRates[i.caseId];if(e===1&&i.passRate===0){r.push(i.caseId),n.push(`case flip: '${i.caseId}' was 100% pass, now 0%`);continue}i.passRate===void 0&&e!==void 0&&n.push(`'${i.caseId}' produced no comparable runs (all infra errors) — excluded from the suite average`)}let i;e.overallPassRate!==void 0&&t.overallPassRate!==void 0&&(i=(e.overallPassRate-t.overallPassRate)*100,i<-J&&n.push(`suite pass rate dropped ${Math.abs(i).toFixed(1)}pp (${(t.overallPassRate*100).toFixed(1)}% → ${(e.overallPassRate*100).toFixed(1)}%)`));let a={status:r.length>0||i!==void 0&&i<-J?`regressed`:`ok`,findings:n.length>0?n:[`no regression detected`],flippedCases:r};return i!==void 0&&Object.assign(a,{passRateDeltaPp:i}),a}function Q(e){return e===void 0?`—`:`${(e*100).toFixed(0)}%`}function ie(e,t){let n=[],r=t.status===`regressed`?`REGRESSED`:t.status===`ok`?`OK`:t.status===`invalid_run`?`INVALID (infra errors)`:`NO BASELINE`;n.push(`otto eval — ${r}`),n.push(`model=${e.modelId}  suite=${e.suiteVersion}  otto=${e.ottoVersion}  at=${e.startedAt}`),n.push(``),n.push(`case                              judge     pass   runs  infra  skip`);for(let t of e.cases){let e=t.caseId.padEnd(33).slice(0,33),r=t.judge.padEnd(9),i=(t.qualityScored?Q(t.passRate):`n/a`).padStart(5),a=`${t.passed}/${t.effective}`.padStart(6),o=String(t.infraErrors).padStart(6),s=String(t.skippedBudget).padStart(5);n.push(`${e} ${r} ${i} ${a} ${o} ${s}`)}if(n.push(``),n.push(`suite pass rate: ${Q(e.overallPassRate)}  (scored ${e.scoredCases}, liveness-only ${e.unscoredCases})`),t.passRateDeltaPp!==void 0){let e=t.passRateDeltaPp>=0?`+`:``;n.push(`vs baseline:     ${e}${t.passRateDeltaPp.toFixed(1)}pp`)}n.push(`infra errors:    ${(e.infraErrorRate*100).toFixed(1)}% of runs`),e.skippedBudgetTotal>0&&n.push(`skipped (budget): ${e.skippedBudgetTotal} run(s) — suite incomplete for those cases`);let i=e.aggregateMetrics;i&&n.push(`avg per run:     ${i.turns.toFixed(1)} turns · ${i.toolCalls.toFixed(1)} tools (${i.toolErrors.toFixed(1)} err) · ${Math.round(i.inputTokens+i.outputTokens)} tokens`),n.push(``);for(let e of t.findings)n.push(`- ${e}`);return e.unscoredCases>0&&(n.push(``),n.push(`note: ${e.unscoredCases} case(s) have no machine judgement (liveness only) and do not affect the suite pass rate.`)),n.join(`
+`)}const ae=[`path`,`filePath`,`file_path`];function $(e,t){let n,r=[],i=new Set,a=0;for(let e of t.split(`
+`)){if(e.trim()===``)continue;let t;try{t=JSON.parse(e)}catch{continue}if(t.node===`prompt.before`&&n===void 0){let e=t.payload?.text;typeof e==`string`&&e.trim()!==``&&(n=e);continue}if(t.node===`tool.call.start`){let e=t.payload?.name;typeof e==`string`&&r.push(e);let n=t.payload?.arguments;if(typeof n==`object`&&n)for(let e of ae){let t=n[e];typeof t==`string`&&t!==``&&i.add(t)}continue}t.node===`turn.end`&&a++}return{sessionId:e,toolSequence:r,touchedPaths:[...i].sort(),turns:a,...n===void 0?{}:{prompt:n}}}function oe(e){let t=e.prompt===void 0?`prompt: |
+  TODO: trace 里没有 prompt.before 文本，请手工填写`:`prompt: |\n${e.prompt.split(`
+`).map(e=>`  ${e}`).join(`
+`)}`,n=e.toolSequence.length>0?`# 实际会话的工具序列（golden.toolSequence 候选，按需裁剪）：\n# ${e.toolSequence.join(` → `)}\n`:``;return`# 由 otto-eval extract 生成的骨架（会话 ${e.sessionId}）。
+# TODO：本 case 尚不可用——judge 是 liveness 占位，必须人工补判据后才能进套件质量分。
+${t}
+judge: liveness  # TODO: 改成 script（写 verify.sh）或 golden（写断言）
+${n}runs: 3
+timeoutMs: 240000
+maxTokens: 200000
+`}function se(e){let t=e.touchedPaths.length>0?e.touchedPaths.map(e=>`- \`${e}\``).join(`
+`):`- （无 —— 会话没有文件类工具调用）`;return`# 提炼报告 — 会话 ${e.sessionId}
+
+- LLM 往返：${e.turns}
+- 工具调用：${e.toolSequence.length} 次
+
+## 会话触碰过的文件（fixture 需要复现哪些初始状态的线索）
+
+${t}
+
+## 下一步（按序完成后删除本文件）
+
+1. 在 fixture/ 里构造这些文件的**任务开始前**状态（trace 只有调用记录，初始内容需要你还原）
+2. 写判据：verify.sh（首选，可执行判据）或 case.yaml 里的 golden 断言
+3. 把 judge 从 liveness 改成 script/golden
+4. 过一遍下面的脱敏清单
+
+## 脱敏检查清单（入库前必须逐项人工核对 — RFC-316 规则 7）
+
+- [ ] prompt 里没有 API key / token / 密码 / 内部主机名
+- [ ] prompt 里没有真实用户数据（邮箱/手机号/姓名）
+- [ ] fixture 文件里没有上述任何内容
+- [ ] 路径不含本机用户名等机器指纹（用相对路径）
+- [ ] verify 判据已由人工编写（不要用"输出像上次一样"的快照当判据）
+`}function ce(e,t,i=le){let a=i(e);if(a===void 0)throw Error(`no trace found for session ${e} (looked at ${L(e)})`);let o=$(e,a),c=l(t,`extracted-${e.slice(0,8)}`);if(n(c))throw Error(`${c} already exists — refusing to overwrite (delete it first if intended)`);return r(l(c,`fixture`),{recursive:!0}),s(l(c,`case.yaml`),oe(o),`utf8`),s(l(c,`EXTRACTION.md`),se(o),`utf8`),{caseDir:c,skeleton:o}}function le(e){let t=L(e);return n(t)?i(t,`utf8`):void 0}export{T as A,P as C,v as D,A as E,_ as M,C as N,w as O,E as P,N as S,k as T,B as _,Z as a,te as b,X as c,q as d,W as f,R as g,V as h,J as i,b as j,y as k,ne as l,G as m,$ as n,re as o,K as p,ie as r,Y as s,ce as t,H as u,L as v,j as w,M as x,F as y};
+//# sourceMappingURL=extract-BmkiRXQY.js.map

package/dist/extract-BmkiRXQY.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-BmkiRXQY.js","names":["parseYaml","renderReport"],"sources":["../src/verify.ts","../src/case-loader.ts","../src/setup.ts","../src/otto-invoker.ts","../src/trace-metrics.ts","../src/runner.ts","../src/baseline.ts","../src/report.ts","../src/extract.ts"],"sourcesContent":["/**\n * verify.ts —— 判分（RFC-316 D3 判分优先级的执行处）。\n *\n * 优先级：可执行判据（script exit 0） > golden 断言 > liveness（跑完没崩）。\n * **liveness 永不产生质量分**——这是继承已删 @x-otto/eval scorecard 的诚实边界：\n * 没有判据就老实说没有，不许用「跑完了」冒充「做对了」。\n *\n * 扩展点：判分方式是**注册表驱动**的。内置 script/golden/liveness 三个 Verifier 在\n * 模块加载时注册；新判分方式经 `registerVerifier` 注册后即可被 case.yaml 的\n * `judge` 字段引用——case-loader 以注册表为唯一权威校验 judge 值，未知判分直接报错，\n * 不会静默降级成 liveness。\n */\n\nimport { spawnSync } from 'node:child_process'\nimport { join } from 'node:path'\n\nimport type { CaseSpec, GoldenSpec } from './types'\n\nexport interface VerifyInput {\n  spec: CaseSpec\n  /** agent 运行后的工作区（判分脚本的 cwd）。 */\n  workspaceDir: string\n  /** headless 输出的最终回答。 */\n  response: string\n  /** 本次运行的工具调用序列（按发生顺序），无 trace 时为空。 */\n  toolNames: readonly string[]\n}\n\nexport interface VerifyOutput {\n  passed: boolean\n  detail: string\n}\n\n/** 一种判分方式的端口。 */\nexport interface Verifier {\n  /** 判分方式名（case.yaml 的 judge 字段值）。 */\n  readonly name: string\n  verify(input: VerifyInput): Promise<VerifyOutput>\n}\n\nconst verifiers = new Map<string, Verifier>()\n\n/**\n * 注册一种判分方式（同名覆盖）。case-loader 会按注册表校验 judge 值。\n *\n * RFC-417 D2：本注册表目前是**薄骨架**，仅内置 script/golden/liveness 三种，无外部消费方。\n * 保留它的理由是让\"判分方式集合\"只定义一处（case-loader 校验与 verifyRun 分派共用单一权威），\n * 而非纯转发层。反悔条件：若结案 +6 个月内无第二种判分方式的真实 case 需求（见 T417-followup），\n * 应评估把注册表压回硬编码白名单 + sync 分派，消除无消费的扩展点——不让骨架永久驻留。\n */\nexport function registerVerifier(verifier: Verifier): void {\n  verifiers.set(verifier.name, verifier)\n}\n\n/** 按名字取判分器；未注册返回 undefined。 */\nexport function getVerifier(name: string): Verifier | undefined {\n  return verifiers.get(name)\n}\n\n/** 当前已注册的全部判分方式名（错误信息用）。 */\nexport function knownVerifiers(): readonly string[] {\n  return [...verifiers.keys()]\n}\n\n/** 子序列匹配：期望序列的每个工具名按顺序出现即可，允许中间夹杂其他调用。 */\nexport function matchesToolSubsequence(\n  actual: readonly string[],\n  expected: readonly string[],\n): boolean {\n  let cursor = 0\n  for (const name of actual) {\n    if (cursor < expected.length && name === expected[cursor]) {\n      cursor++\n    }\n  }\n  return cursor === expected.length\n}\n\nexport function verifyGolden(\n  golden: GoldenSpec,\n  response: string,\n  toolNames: readonly string[],\n): VerifyOutput {\n  const misses: string[] = []\n  if (golden.answerContains) {\n    const haystack = response.toLowerCase()\n    for (const needle of golden.answerContains) {\n      if (!haystack.includes(needle.toLowerCase())) {\n        misses.push(`answer missing '${needle}'`)\n      }\n    }\n  }\n  if (golden.toolSequence && !matchesToolSubsequence(toolNames, golden.toolSequence)) {\n    misses.push(\n      `tool sequence [${golden.toolSequence.join(', ')}] not satisfied by [${toolNames.join(', ')}]`,\n    )\n  }\n  return misses.length === 0\n    ? { passed: true, detail: 'golden assertions satisfied' }\n    : { passed: false, detail: misses.join('; ') }\n}\n\nfunction runVerifyScript(input: VerifyInput, scriptPath: string): VerifyOutput {\n  const proc = spawnSync('bash', [scriptPath], {\n    cwd: input.workspaceDir,\n    encoding: 'utf8',\n    timeout: 120_000,\n    env: { ...process.env, OTTO_EVAL_RESPONSE: input.response },\n  })\n  if (proc.error) {\n    return { passed: false, detail: `verify script error: ${proc.error.message}` }\n  }\n  const detail =\n    `${(proc.stdout ?? '').trim()}${proc.stderr ? ` | ${proc.stderr.trim()}` : ''}`.slice(0, 500)\n  return { passed: proc.status === 0, detail: detail || `exit ${proc.status}` }\n}\n\n/** 内置判分器：可执行判据（script）。 */\nexport const scriptVerifier: Verifier = {\n  name: 'script',\n  async verify(input) {\n    const spec = input.spec\n    if (spec.judge !== 'script') {\n      return { passed: false, detail: 'script verifier applied to a non-script case' }\n    }\n    return runVerifyScript(input, join(spec.dir, spec.verifyScript))\n  },\n}\n\n/** 内置判分器：golden 断言（answerContains / toolSequence）。 */\nexport const goldenVerifier: Verifier = {\n  name: 'golden',\n  async verify(input) {\n    return verifyGolden(input.spec.golden ?? {}, input.response, input.toolNames)\n  },\n}\n\n/** 内置判分器：liveness——只评「跑完没崩」，永不产生质量分。 */\nexport const livenessVerifier: Verifier = {\n  name: 'liveness',\n  async verify() {\n    return { passed: true, detail: 'liveness only (no quality judgement)' }\n  },\n}\n\nregisterVerifier(scriptVerifier)\nregisterVerifier(goldenVerifier)\nregisterVerifier(livenessVerifier)\n\n/**\n * 执行判分：按 case 的 judge 取注册的判分器，golden 断言作为**正交判据**可同时存在\n * 且必须都满足——判分方式表达的是\"这个 case 靠什么下结论\"，而工具纪律\n * （golden.toolSequence）与产物正确性（script）是**两个正交的判据**——\n * 只有前者会让\"读了但改错\"蒙混过关，只有后者会让\"盲改碰对\"蒙混过关。\n */\nexport async function verifyRun(input: VerifyInput): Promise<VerifyOutput> {\n  const { spec } = input\n  const parts: VerifyOutput[] = []\n\n  const verifier = getVerifier(spec.judge)\n  if (!verifier) {\n    return {\n      passed: false,\n      detail: `no verifier registered for judge '${spec.judge}' (known: ${knownVerifiers().join(', ')})`,\n    }\n  }\n  parts.push(await verifier.verify(input))\n  if (spec.golden) {\n    parts.push(verifyGolden(spec.golden, input.response, input.toolNames))\n  }\n\n  if (parts.length === 0) {\n    return { passed: true, detail: 'liveness only (no quality judgement)' }\n  }\n\n  const failed = parts.filter((p) => !p.passed)\n  return failed.length === 0\n    ? { passed: true, detail: parts.map((p) => p.detail).join('; ') }\n    : { passed: false, detail: failed.map((p) => p.detail).join('; ') }\n}\n","/**\n * case-loader.ts —— 读取并校验 `evals/cases/<id>/case.yaml`（RFC-316 D4）。\n *\n * 校验失败一律抛错而非静默降级：一个格式错的 case 静默变成 liveness case，\n * 会让 scorecard 悄悄失去质量信号——这正是 RFC-316 规则 2 要防的「假质量分」。\n * judge 字段以 verify.ts 的**判分器注册表**为唯一权威：注册过的判分方式才能用，\n * 未注册的一律报错（新判分方式先 registerVerifier，后写 case）。\n */\n\nimport { createHash } from 'node:crypto'\nimport { readFileSync, readdirSync, existsSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { parse as parseYaml } from 'yaml'\n\nimport { getVerifier, knownVerifiers } from './verify'\n\nimport type { CaseSpec, GoldenSpec } from './types'\n\nconst DEFAULT_RUNS = 3\nconst DEFAULT_TIMEOUT_MS = 300_000\nconst DEFAULT_VERIFY_SCRIPT = 'verify.sh'\n\nfunction requireString(raw: Record<string, unknown>, key: string, caseId: string): string {\n  const value = raw[key]\n  if (typeof value !== 'string' || value.trim() === '') {\n    throw new Error(`case '${caseId}': field '${key}' must be a non-empty string`)\n  }\n  return value\n}\n\nfunction optionalPositiveInt(\n  raw: Record<string, unknown>,\n  key: string,\n  caseId: string,\n  fallback: number,\n): number {\n  const value = raw[key]\n  if (value === undefined) {\n    return fallback\n  }\n  if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {\n    throw new Error(`case '${caseId}': field '${key}' must be a positive integer`)\n  }\n  return value\n}\n\nfunction parseGolden(raw: unknown, caseId: string): GoldenSpec {\n  if (typeof raw !== 'object' || raw === null) {\n    throw new Error(`case '${caseId}': judge='golden' requires a 'golden' mapping`)\n  }\n  const record = raw as Record<string, unknown>\n  const golden: GoldenSpec = {}\n  const answerContains = record['answerContains']\n  if (answerContains !== undefined) {\n    if (!Array.isArray(answerContains) || answerContains.some((x) => typeof x !== 'string')) {\n      throw new Error(`case '${caseId}': golden.answerContains must be a string array`)\n    }\n    Object.assign(golden, { answerContains: answerContains as string[] })\n  }\n  const toolSequence = record['toolSequence']\n  if (toolSequence !== undefined) {\n    if (!Array.isArray(toolSequence) || toolSequence.some((x) => typeof x !== 'string')) {\n      throw new Error(`case '${caseId}': golden.toolSequence must be a string array`)\n    }\n    Object.assign(golden, { toolSequence: toolSequence as string[] })\n  }\n  if (!golden.answerContains && !golden.toolSequence) {\n    throw new Error(\n      `case '${caseId}': judge='golden' requires at least one of golden.answerContains / golden.toolSequence`,\n    )\n  }\n  return golden\n}\n\n/** 解析单个 case 目录。`dir` 必须已存在且含 case.yaml。 */\nexport function loadCase(dir: string): CaseSpec {\n  const caseId = dir.split('/').filter(Boolean).at(-1) ?? dir\n  const yamlPath = join(dir, 'case.yaml')\n  if (!existsSync(yamlPath)) {\n    throw new Error(`case '${caseId}': missing case.yaml at ${yamlPath}`)\n  }\n\n  const parsed: unknown = parseYaml(readFileSync(yamlPath, 'utf8'))\n  if (typeof parsed !== 'object' || parsed === null) {\n    throw new Error(`case '${caseId}': case.yaml must contain a mapping`)\n  }\n  const raw = parsed as Record<string, unknown>\n\n  const prompt = requireString(raw, 'prompt', caseId)\n  const judgeRaw = requireString(raw, 'judge', caseId)\n  if (getVerifier(judgeRaw) === undefined) {\n    throw new Error(\n      `case '${caseId}': judge '${judgeRaw}' is not a registered verifier (known: ${knownVerifiers().join(', ')})`,\n    )\n  }\n  const judge = judgeRaw as CaseSpec['judge']\n\n  const common = {\n    id: caseId,\n    prompt,\n    runs: optionalPositiveInt(raw, 'runs', caseId, DEFAULT_RUNS),\n    timeoutMs: optionalPositiveInt(raw, 'timeoutMs', caseId, DEFAULT_TIMEOUT_MS),\n    dir,\n    hasFixture: existsSync(join(dir, 'fixture')),\n    hasSetup: existsSync(join(dir, 'setup.sh')),\n  }\n\n  if (raw['maxTokens'] !== undefined) {\n    Object.assign(common, { maxTokens: optionalPositiveInt(raw, 'maxTokens', caseId, 0) })\n  }\n\n  // golden 断言与判分方式正交：script/liveness 判分的 case 也可附加 golden\n  // （工具纪律 + 产物正确性两个判据都要满足）。judge='golden' 时 golden 必填。\n  const golden =\n    judge === 'golden' || raw['golden'] !== undefined ? parseGolden(raw['golden'], caseId) : undefined\n\n  if (judge === 'script') {\n    const script =\n      raw['verifyScript'] === undefined\n        ? DEFAULT_VERIFY_SCRIPT\n        : requireString(raw, 'verifyScript', caseId)\n    if (!existsSync(join(dir, script))) {\n      throw new Error(`case '${caseId}': judge='script' but verify script '${script}' not found`)\n    }\n    return { ...common, judge, verifyScript: script, ...(golden !== undefined ? { golden } : {}) }\n  }\n  if (judge === 'golden') {\n    // judge='golden' 分支的 golden 必填：上面的 parseGolden 在该分支无条件执行。\n    return { ...common, judge, golden: golden as GoldenSpec }\n  }\n  // 其余（liveness 及未来注册的判分方式）：无专属字段，仅公共字段 + 可选 golden。\n  return { ...common, judge, ...(golden !== undefined ? { golden } : {}) }\n}\n\n/** 扫描 case 根目录（`evals/cases/`），按 id 排序返回全部 case。 */\nexport function loadSuite(casesRoot: string): readonly CaseSpec[] {\n  if (!existsSync(casesRoot)) {\n    throw new Error(`cases root not found: ${casesRoot}`)\n  }\n  const dirs = readdirSync(casesRoot)\n    .filter((name) => !name.startsWith('.'))\n    .map((name) => join(casesRoot, name))\n    .filter((path) => statSync(path).isDirectory())\n    .sort()\n  return dirs.map(loadCase)\n}\n\n/**\n * 套件版本 = 全部 case 判分相关字段的内容指纹（基线分键用）。\n *\n * 只纳入**影响判分口径**的字段：case 集合、prompt、judge、judge 专属配置\n * （verifyScript）、golden。改超时/runs 不算换套件（不影响可比性），\n * 改 prompt 或判据则必须换基线键——否则会把两套不同题目的分数当同一条曲线看。\n */\nexport function computeSuiteVersion(cases: readonly CaseSpec[]): string {\n  const canonical = cases.map((c) => ({\n    id: c.id,\n    prompt: c.prompt,\n    judge: c.judge,\n    verifyScript: c.judge === 'script' ? c.verifyScript : undefined,\n    golden: c.golden ?? null,\n  }))\n  return createHash('sha256').update(JSON.stringify(canonical)).digest('hex').slice(0, 12)\n}\n","/**\n * setup.ts —— case 前置钩子（RFC-417 M2 / D5）。\n *\n * fixture 复制后、agent 运行前，在隔离工作区内执行可选的 `setup.sh`——用于 git init /\n * npm install / 起服务等静态 fixture 无法表达的准备步骤。扩大 case 表达能力，是 case 库\n * 能持续生长（评测\"第二次不死\"）的关键。\n *\n * 与 verify.sh 对称：都是工作区内的 bash 脚本，setup 在前（准备）、verify 在后（判分）。\n *\n * 失败语义（RFC-417 规则 2，对齐四态纪律）：setup exit≠0 或超时一律记 `infra_error`——\n * 准备环境失败不是 agent 能力问题，不进 pass 率。**不区分\"环境失败 vs 脚本 bug\"**：\n * eval 无法可靠分辨 `npm install` 网络失败与脚本 typo（都是 exit 1），强行分类即猜测。\n * 诚实边界 = 记 infra_error + 保留 exit code/stderr 证据；一个恒失败的 setup 会把\n * infraErrorRate 顶上去触发整套 run invalid（RFC-316 既有机制），而非静默。\n */\n\nimport { spawnSync } from 'node:child_process'\nimport { join } from 'node:path'\n\nimport type { CaseSpec } from './types'\n\n/** setup 脚本相对 case 目录的固定文件名。 */\nexport const SETUP_SCRIPT = 'setup.sh'\n\n/** setup 执行的墙钟上限（毫秒）。准备步骤（装依赖/起服务）可能偏慢，独立于判分脚本超时。 */\nexport const SETUP_TIMEOUT_MS = 60_000\n\nexport interface SetupResult {\n  ok: boolean\n  /** 失败时携带 exit code + stderr 摘要（成功时为简短 ok 说明）。 */\n  detail: string\n}\n\n/**\n * 在工作区内执行 case 的 setup.sh（若存在）。无 setup 直接返回 ok。\n * 脚本路径取自 case 目录，cwd 为隔离工作区——与 verify.sh 同一执行模型。\n */\nexport function runSetup(spec: CaseSpec, workspaceDir: string): SetupResult {\n  if (!spec.hasSetup) {\n    return { ok: true, detail: 'no setup' }\n  }\n  const scriptPath = join(spec.dir, SETUP_SCRIPT)\n  const proc = spawnSync('bash', [scriptPath], {\n    cwd: workspaceDir,\n    encoding: 'utf8',\n    timeout: SETUP_TIMEOUT_MS,\n  })\n  if (proc.error) {\n    // spawn 层错误（含超时 ETIMEDOUT）：记失败，带原因。\n    return { ok: false, detail: `setup script error: ${proc.error.message}` }\n  }\n  if (proc.status !== 0) {\n    const stderr = (proc.stderr ?? '').trim().slice(0, 500)\n    return {\n      ok: false,\n      detail: `setup exit ${proc.status}${stderr ? `: ${stderr}` : ''}`,\n    }\n  }\n  return { ok: true, detail: 'setup ok' }\n}\n","/**\n * otto-invoker.ts —— 调用 headless otto 的唯一出口（RFC-316 规则 6：只走公开入口）。\n *\n * 端口化的理由：runner 的编排逻辑（n 次运行 / 预算护栏 / 三态统计）必须能在**不起真实模型**\n * 的情况下被单测，否则评测体系自身没有回归保护。真实实现 spawn `otto --json`，测试注入 fake。\n */\n\nimport { spawn } from 'node:child_process'\n\n/** headless `otto --json` 的输出契约（packages/cli/src/modes/run-modes.ts JsonModeResult）。 */\nexport interface OttoJsonResult {\n  sessionId: string\n  status: string\n  messageCount: number\n  response: string\n  /** 产出本结果的 otto 版本指纹（含 build id）。RFC-417 M1：旧 otto 无此字段时兜底 'unknown'。 */\n  ottoVersion?: string\n}\n\nexport interface InvokeRequest {\n  prompt: string\n  /** agent 的工作目录。 */\n  cwd: string\n  timeoutMs: number\n}\n\nexport type InvokeResponse =\n  | { kind: 'ok'; result: OttoJsonResult }\n  /** 进程跑起来了但任务失败（非零退出、输出不可解析）——算 fail。 */\n  | { kind: 'failed'; detail: string }\n  /**\n   * 基础设施问题（限流、网络、供应商 5xx、二进制缺失）——算 infra_error，不进 pass 率\n   * （RFC-316 D6 规则 5：防限流假回归）。\n   */\n  | { kind: 'infra'; detail: string }\n\n/** 调用 otto 的端口。 */\nexport interface OttoInvoker {\n  invoke(request: InvokeRequest): Promise<InvokeResponse>\n}\n\n/**\n * 判定一段 stderr 是否为基础设施故障而非任务失败。\n *\n * 保守取向：只认明确的传输层/额度层信号。误判成 infra 会让真实回归被藏起来，\n * 比误判成 fail 更危险——所以这里宁可漏放。\n */\nexport function classifyFailure(stderr: string): 'infra' | 'failed' {\n  const text = stderr.toLowerCase()\n  const infraSignals = [\n    'rate limit',\n    'rate_limit',\n    '429',\n    'overloaded',\n    'econnreset',\n    'etimedout',\n    'enotfound',\n    'socket hang up',\n    'service unavailable',\n    'internal server error',\n    'bad gateway',\n    'no model configured',\n    'command not found',\n    'enoent',\n  ]\n  return infraSignals.some((signal) => text.includes(signal)) ? 'infra' : 'failed'\n}\n\nexport interface SpawnInvokerOptions {\n  /** otto 可执行文件（默认 'otto'）。 */\n  bin?: string\n  /** 附加环境变量（如指定模型）。 */\n  env?: Readonly<Record<string, string>>\n}\n\n/** 真实实现：spawn `otto --json \"<prompt>\"`。 */\nexport function createSpawnInvoker(options: SpawnInvokerOptions = {}): OttoInvoker {\n  const bin = options.bin ?? 'otto'\n  return {\n    invoke(request) {\n      return new Promise<InvokeResponse>((resolve) => {\n        const child = spawn(bin, ['--json', request.prompt], {\n          cwd: request.cwd,\n          env: { ...process.env, ...options.env },\n          stdio: ['ignore', 'pipe', 'pipe'],\n        })\n        let stdout = ''\n        let stderr = ''\n        let settled = false\n        const finish = (response: InvokeResponse): void => {\n          if (settled) {\n            return\n          }\n          settled = true\n          clearTimeout(timer)\n          resolve(response)\n        }\n        const timer = setTimeout(() => {\n          child.kill('SIGKILL')\n          finish({ kind: 'failed', detail: `timeout after ${request.timeoutMs}ms` })\n        }, request.timeoutMs)\n\n        child.stdout.on('data', (chunk: Buffer) => {\n          stdout += chunk.toString()\n        })\n        child.stderr.on('data', (chunk: Buffer) => {\n          stderr += chunk.toString()\n        })\n        child.on('error', (error) => {\n          finish({ kind: 'infra', detail: `spawn failed: ${error.message}` })\n        })\n        child.on('close', (code) => {\n          if (code !== 0) {\n            const kind = classifyFailure(stderr)\n            finish({ kind, detail: `exit ${code}: ${stderr.trim().slice(0, 500)}` })\n            return\n          }\n          try {\n            const parsed = JSON.parse(stdout) as OttoJsonResult\n            if (typeof parsed.sessionId !== 'string') {\n              finish({ kind: 'failed', detail: 'json output missing sessionId' })\n              return\n            }\n            finish({ kind: 'ok', result: parsed })\n          } catch {\n            finish({ kind: 'failed', detail: `unparsable json output: ${stdout.slice(0, 300)}` })\n          }\n        })\n      })\n    },\n  }\n}\n","/**\n * trace-metrics.ts —— 从 trace JSONL 聚合运行指标（RFC-316 G3 / L3 层）。\n *\n * 数据源是 append-only 的 `OTTO_SESSIONS_DIR/trace/<sessionId>.jsonl`，otto 每个会话都会写。\n * 这一层是「零成本指标」：不额外跑任何东西，评测跑完顺手把成本/步数/工具成功率读出来。\n *\n * 只读两类锚点（其余事件忽略，容忍跨版本 trace）：\n * - `tool.call.end`：工具名 / 耗时 / errorCategory\n * - `turn.end`：usage（token）/ model\n */\n\nimport { readFileSync, existsSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { OTTO_SESSIONS_DIR } from '@x-otto/env'\n\nimport type { RunMetrics } from './types'\n\ninterface RawTraceEvent {\n  kind?: string\n  node?: string\n  payload?: Record<string, unknown>\n}\n\nfunction readNumber(payload: Record<string, unknown> | undefined, key: string): number {\n  const value = payload?.[key]\n  return typeof value === 'number' && Number.isFinite(value) ? value : 0\n}\n\n/** trace 文件的默认路径。 */\nexport function traceFilePath(sessionId: string): string {\n  return join(OTTO_SESSIONS_DIR, 'trace', `${sessionId}.jsonl`)\n}\n\n/** 一次运行从 trace 还原出的全部可判分信息。 */\nexport interface TraceRun {\n  metrics: RunMetrics\n  /** 工具调用名序列（按 tool.call.end 发生顺序），golden.toolSequence 判据的输入。 */\n  toolNames: readonly string[]\n}\n\n/**\n * 解析 trace 内容为指标 + 工具序列。传内容而非路径，便于单测与远端 trace 复用。\n * 坏行直接跳过——trace 是 append-only 的，进程被杀时最后一行可能截断，这是正常状态。\n */\nexport function parseTraceRun(content: string): TraceRun {\n  const toolNames: string[] = []\n  const metrics = parseTraceMetrics(content, toolNames)\n  return { metrics, toolNames }\n}\n\nfunction parseTraceMetrics(content: string, toolNamesOut?: string[]): RunMetrics {\n  const metrics: RunMetrics = {\n    turns: 0,\n    toolCalls: 0,\n    toolErrors: 0,\n    inputTokens: 0,\n    outputTokens: 0,\n    cacheReadTokens: 0,\n    toolDurationMs: 0,\n  }\n\n  for (const line of content.split('\\n')) {\n    if (line.trim() === '') {\n      continue\n    }\n    let event: RawTraceEvent\n    try {\n      event = JSON.parse(line) as RawTraceEvent\n    } catch {\n      continue\n    }\n\n    if (event.node === 'tool.call.end') {\n      metrics.toolCalls++\n      metrics.toolDurationMs += readNumber(event.payload, 'durationMs')\n      if (event.payload?.['errorCategory'] !== undefined) {\n        metrics.toolErrors++\n      }\n      const name = event.payload?.['name']\n      if (typeof name === 'string') {\n        toolNamesOut?.push(name)\n      }\n      continue\n    }\n\n    if (event.node === 'turn.end') {\n      metrics.turns++\n      const usage = event.payload?.['usage']\n      if (typeof usage === 'object' && usage !== null) {\n        const u = usage as Record<string, unknown>\n        metrics.inputTokens += readNumber(u, 'inputTokens')\n        metrics.outputTokens += readNumber(u, 'outputTokens')\n        metrics.cacheReadTokens += readNumber(u, 'cacheReadTokens')\n      }\n      const model = event.payload?.['model']\n      if (typeof model === 'string' && model !== '') {\n        metrics.modelId = model\n      }\n    }\n  }\n\n  return metrics\n}\n\n/** 读某个会话的 trace 并聚合；文件不存在返回 undefined（trace 可能被关闭）。 */\nexport function readTraceRun(sessionId: string): TraceRun | undefined {\n  const path = traceFilePath(sessionId)\n  if (!existsSync(path)) {\n    return undefined\n  }\n  return parseTraceRun(readFileSync(path, 'utf8'))\n}\n\n/** 多次运行的指标均值（scorecard 汇总用）。 */\nexport function averageMetrics(samples: readonly RunMetrics[]): RunMetrics | undefined {\n  if (samples.length === 0) {\n    return undefined\n  }\n  const sum = samples.reduce<RunMetrics>(\n    (acc, m) => ({\n      turns: acc.turns + m.turns,\n      toolCalls: acc.toolCalls + m.toolCalls,\n      toolErrors: acc.toolErrors + m.toolErrors,\n      inputTokens: acc.inputTokens + m.inputTokens,\n      outputTokens: acc.outputTokens + m.outputTokens,\n      cacheReadTokens: acc.cacheReadTokens + m.cacheReadTokens,\n      toolDurationMs: acc.toolDurationMs + m.toolDurationMs,\n      modelId: m.modelId ?? acc.modelId,\n    }),\n    {\n      turns: 0,\n      toolCalls: 0,\n      toolErrors: 0,\n      inputTokens: 0,\n      outputTokens: 0,\n      cacheReadTokens: 0,\n      toolDurationMs: 0,\n    },\n  )\n  const n = samples.length\n  const averaged: RunMetrics = {\n    turns: sum.turns / n,\n    toolCalls: sum.toolCalls / n,\n    toolErrors: sum.toolErrors / n,\n    inputTokens: sum.inputTokens / n,\n    outputTokens: sum.outputTokens / n,\n    cacheReadTokens: sum.cacheReadTokens / n,\n    toolDurationMs: sum.toolDurationMs / n,\n  }\n  if (sum.modelId !== undefined) {\n    averaged.modelId = sum.modelId\n  }\n  return averaged\n}\n","/**\n * runner.ts —— 评测编排（RFC-316 M1 核心链路）。\n *\n * 一个 case 跑 n 次 → 四态统计（pass/fail/infra_error/skipped_budget）→ 汇总成 CaseResult。\n * 三条不可削弱的纪律都落在这里：\n * - **infra_error 不进 pass 率**（规则 5）：限流/网络故障不许伪装成能力回归。\n * - **预算护栏**（规则 4）：单 case token 超限即停后续运行并标 incomplete，不静默烧钱。\n * - **liveness 不产生质量分**（规则 2）：qualityScored=false 的 case 不进套件 pass 率。\n */\n\nimport { normalizeEnvFraction } from '@x-otto/env'\n\nimport { createWorkspace } from './workspace'\nimport { runSetup } from './setup'\nimport { verifyRun } from './verify'\nimport { readTraceRun, averageMetrics } from './trace-metrics'\n\nimport type { OttoInvoker } from './otto-invoker'\nimport type { TraceRun } from './trace-metrics'\nimport type { CaseResult, CaseSpec, RunMetrics, RunResult, Scorecard } from './types'\n\n/**\n * infra_error 占比超过此阈值时整次运行标 invalid（RFC-316 D6 规则 5）。\n * 硬编码审计 P1：不同 CI 环境对限流/网络故障的容忍度不同，经 `OTTO_EVAL_INFRA_ERROR_THRESHOLD`\n * 覆盖（比例语义，(0,1] 区间，normalizeEnvFraction 越界回退默认）。\n */\nexport const INFRA_ERROR_INVALID_THRESHOLD = normalizeEnvFraction(\n  process.env['OTTO_EVAL_INFRA_ERROR_THRESHOLD'],\n  0.2,\n)\n\nexport interface RunnerDeps {\n  invoker: OttoInvoker\n  /** 读 trace 的端口（默认读真实 trace 文件；测试注入 fake）。 */\n  readTrace?: (sessionId: string) => TraceRun | undefined\n  /** 当前时刻（毫秒），确定性测试用。 */\n  now?: () => number\n  /** 每次运行结束的进度回调（CLI 打点用）。 */\n  onRunComplete?: (caseId: string, index: number, result: RunResult) => void\n}\n\nfunction totalTokens(metrics: RunMetrics | undefined): number {\n  if (!metrics) {\n    return 0\n  }\n  return metrics.inputTokens + metrics.outputTokens\n}\n\n/** 跑单个 case 的 n 次运行。 */\nexport async function runCase(spec: CaseSpec, deps: RunnerDeps): Promise<CaseResult> {\n  const now = deps.now ?? (() => Date.now())\n  const readTrace = deps.readTrace ?? readTraceRun\n  const runs: RunResult[] = []\n  let spentTokens = 0\n  /** 已成功计量的运行次数（用于估算不可计量运行的开销）。 */\n  let measuredRuns = 0\n  /**\n   * 无法计量 token 的运行次数：trace 缺失，或 fail/timeout 路径拿不到 usage。\n   *\n   * 审计修订：早期实现只在「成功且有 trace」时累加 spentTokens，于是失败运行被当作\n   * **零消耗**——但它们同样调用了模型、同样烧钱。一个\"每次都超时\"的 case 因此可以\n   * 无限重试而预算护栏一次都不触发。护栏若只在一切正常时生效，就不是护栏。\n   *\n   * 估算口径：有实测样本时按**已观测均值**计价（最贴近现实）；一个样本都没有时按\n   * `maxTokens/runs` 均摊——那正是用户声明 `runs` 次时的预期单价，此时跑满 runs 次\n   * 属于预期内，不该被拦。\n   */\n  let unmeasuredRuns = 0\n\n  const budgetExceeded = (): boolean => {\n    if (spec.maxTokens === undefined) {\n      return false\n    }\n    const assumedPerRun = measuredRuns > 0 ? spentTokens / measuredRuns : spec.maxTokens / spec.runs\n    return spentTokens + unmeasuredRuns * assumedPerRun >= spec.maxTokens\n  }\n\n  for (let index = 0; index < spec.runs; index++) {\n    if (budgetExceeded()) {\n      // RFC-353 S9: 预算耗尽是本地跳过决策，不是 infra 故障——用 skipped_budget 终态\n      // 区分\"未尝试\"与\"provider/网络不可靠\"，避免膨胀 infraErrorRate 误判套件失效。\n      runs.push({\n        outcome: 'skipped_budget',\n        wallMs: 0,\n        detail: `budget exhausted: ${spentTokens} measured tokens${\n          unmeasuredRuns > 0 ? ` + ${unmeasuredRuns} unmeasured run(s)` : ''\n        } vs limit ${spec.maxTokens}, remaining runs skipped`,\n      })\n      continue\n    }\n\n    const workspace = createWorkspace(spec)\n    const startedAt = now()\n    try {\n      // RFC-417 M2：fixture 复制后、agent 运行前执行 setup.sh（若有）。setup 失败记\n      // infra_error（准备环境失败非 agent 能力问题，不进 pass 率；带 exit code/stderr 证据）。\n      const setup = runSetup(spec, workspace.dir)\n      if (!setup.ok) {\n        unmeasuredRuns++\n        runs.push({ outcome: 'infra_error', wallMs: now() - startedAt, detail: setup.detail })\n        continue\n      }\n      const response = await deps.invoker.invoke({\n        prompt: spec.prompt,\n        cwd: workspace.dir,\n        timeoutMs: spec.timeoutMs,\n      })\n      const wallMs = now() - startedAt\n\n      if (response.kind === 'infra') {\n        // infra 故障（限流/网络）通常在模型真正产出前就断了，不计入已花费；\n        // 但也拿不到用量证据，故记为不可计量，让护栏保守对待。\n        unmeasuredRuns++\n        runs.push({ outcome: 'infra_error', wallMs, detail: response.detail })\n        continue\n      }\n      if (response.kind === 'failed') {\n        // 失败/超时同样烧了 token（模型已被调用），只是拿不到 usage —— 必须计入护栏，\n        // 否则\"每次都失败\"的 case 会无限重试烧钱。\n        unmeasuredRuns++\n        runs.push({ outcome: 'fail', wallMs, detail: response.detail })\n        continue\n      }\n\n      const trace = readTrace(response.result.sessionId)\n      const verdict = await verifyRun({\n        spec,\n        workspaceDir: workspace.dir,\n        response: response.result.response,\n        toolNames: trace?.toolNames ?? [],\n      })\n      if (trace) {\n        spentTokens += totalTokens(trace.metrics)\n        measuredRuns++\n      } else {\n        // 成功但无 trace（trace 被关闭/文件缺失）：同样计量不到，不能当作零消耗。\n        unmeasuredRuns++\n      }\n\n      const result: RunResult = {\n        outcome: verdict.passed ? 'pass' : 'fail',\n        wallMs,\n        sessionId: response.result.sessionId,\n        detail: verdict.detail,\n      }\n      if (trace) {\n        result.metrics = trace.metrics\n      }\n      // RFC-417 M1：记录本次运行的 otto 版本指纹（旧 otto 无字段 → 'unknown'）。\n      result.ottoVersion = response.result.ottoVersion ?? 'unknown'\n      runs.push(result)\n    } finally {\n      workspace.dispose()\n    }\n\n    const last = runs.at(-1)\n    if (last) {\n      deps.onRunComplete?.(spec.id, index, last)\n    }\n  }\n\n  return summarizeCase(spec, runs)\n}\n\n/** 把 n 次运行汇总成 CaseResult（四态统计的唯一实现处）。 */\nexport function summarizeCase(spec: CaseSpec, runs: readonly RunResult[]): CaseResult {\n  const infraErrors = runs.filter((r) => r.outcome === 'infra_error').length\n  const skippedBudget = runs.filter((r) => r.outcome === 'skipped_budget').length\n  const effective = runs.length - infraErrors - skippedBudget\n  const passed = runs.filter((r) => r.outcome === 'pass').length\n  const qualityScored = spec.judge !== 'liveness'\n\n  const result: CaseResult = {\n    caseId: spec.id,\n    judge: spec.judge,\n    runs,\n    passed,\n    effective,\n    infraErrors,\n    skippedBudget,\n    qualityScored,\n  }\n  if (effective > 0) {\n    Object.assign(result, { passRate: passed / effective })\n  }\n  return result\n}\n\n/** 跑整套 case，产出 scorecard。 */\nexport async function runSuite(\n  cases: readonly CaseSpec[],\n  suiteVersion: string,\n  deps: RunnerDeps,\n): Promise<Scorecard> {\n  const now = deps.now ?? (() => Date.now())\n  const startedAt = new Date(now()).toISOString()\n  const results: CaseResult[] = []\n  for (const spec of cases) {\n    results.push(await runCase(spec, deps))\n  }\n  return buildScorecard(results, suiteVersion, startedAt)\n}\n\n/** 聚合 case 结果为 scorecard（纯函数，便于对拍手算值）。 */\nexport function buildScorecard(\n  cases: readonly CaseResult[],\n  suiteVersion: string,\n  startedAt: string,\n): Scorecard {\n  const scored = cases.filter((c) => c.qualityScored && c.passRate !== undefined)\n  const totalSkippedBudget = cases.reduce((sum, c) => sum + c.skippedBudget, 0)\n  // RFC-353 S9: infraErrorRate 分母排除 skipped_budget——它们不是\"发起的运行\"（本地\n  // 预算护栏主动跳过，未真正调用模型/触达 provider），混入分母会稀释 infraErrorRate\n  // 掩盖真实的 infra 不稳定信号。\n  const totalRuns = cases.reduce((sum, c) => sum + c.runs.length, 0) - totalSkippedBudget\n  const totalInfra = cases.reduce((sum, c) => sum + c.infraErrors, 0)\n  const infraErrorRate = totalRuns === 0 ? 0 : totalInfra / totalRuns\n\n  const allMetrics = cases.flatMap((c) =>\n    c.runs.map((r) => r.metrics).filter((m): m is RunMetrics => m !== undefined),\n  )\n  const aggregate = averageMetrics(allMetrics)\n\n  // RFC-417 M1：取任一运行的 otto 版本指纹（同一次评测全程用同一个 otto bin，取首个非空即可）。\n  const ottoVersion =\n    cases.flatMap((c) => c.runs).find((r) => r.ottoVersion !== undefined)?.ottoVersion ?? 'unknown'\n\n  const scorecard: Scorecard = {\n    suiteVersion,\n    modelId: aggregate?.modelId ?? 'unknown',\n    ottoVersion,\n    startedAt,\n    cases,\n    scoredCases: cases.filter((c) => c.qualityScored).length,\n    unscoredCases: cases.filter((c) => !c.qualityScored).length,\n    infraErrorRate,\n    skippedBudgetTotal: totalSkippedBudget,\n    invalid: infraErrorRate > INFRA_ERROR_INVALID_THRESHOLD,\n  }\n  if (scored.length > 0) {\n    const sum = scored.reduce((acc, c) => acc + (c.passRate ?? 0), 0)\n    Object.assign(scorecard, { overallPassRate: sum / scored.length })\n  }\n  if (aggregate) {\n    Object.assign(scorecard, { aggregateMetrics: aggregate })\n  }\n  return scorecard\n}\n","/**\n * baseline.ts —— 基线存储与回归判定（RFC-316 D6 规则 3/4）。\n *\n * 两条承重规则：\n * - **基线按 (modelId, suiteVersion) 分键**：模型升级产生新条目而非覆盖，跨模型对比必须显式。\n *   没有这条，换一次模型就会把曲线接错——之后所有\"变强/变弱\"结论都是噪声。\n * - **阈值显式且诚实**：n=3 的统计功效只够抓大回归，故只用两个粗信号（套件 pass 率跌幅、\n *   case flip），不假装有统计显著性。\n */\n\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'\nimport { dirname } from 'node:path'\n\nimport { normalizeEnv } from '@x-otto/env'\n\nimport type { BaselineEntry, BaselineFile, RegressionVerdict, Scorecard } from './types'\n\n/**\n * 套件 pass 率相对基线的下降超过此百分点即判回归（RFC-316 D6 规则 4）。\n * 硬编码审计 P1：不同 CI 环境的回归门禁松紧不同，经 `OTTO_EVAL_REGRESSION_PP` 覆盖\n * （百分点整数语义，normalizeEnv 非正数回退默认）。\n */\nexport const PASS_RATE_REGRESSION_PP = normalizeEnv(process.env['OTTO_EVAL_REGRESSION_PP'], 15)\n\nexport function loadBaselines(path: string): BaselineFile {\n  if (!existsSync(path)) {\n    return { entries: [] }\n  }\n  const parsed = JSON.parse(readFileSync(path, 'utf8')) as BaselineFile\n  return { entries: parsed.entries ?? [] }\n}\n\nexport function saveBaselines(path: string, file: BaselineFile): void {\n  mkdirSync(dirname(path), { recursive: true })\n  writeFileSync(path, `${JSON.stringify(file, null, 2)}\\n`, 'utf8')\n}\n\n/** 按 (modelId, suiteVersion) 查基线。 */\nexport function findBaseline(\n  file: BaselineFile,\n  modelId: string,\n  suiteVersion: string,\n): BaselineEntry | undefined {\n  return file.entries.find((e) => e.modelId === modelId && e.suiteVersion === suiteVersion)\n}\n\n/** 把一次运行固化成基线条目（同键覆盖，异键新增）。 */\nexport function upsertBaseline(\n  file: BaselineFile,\n  scorecard: Scorecard,\n  reason: string,\n): BaselineFile {\n  const casePassRates: Record<string, number | undefined> = {}\n  for (const c of scorecard.cases) {\n    casePassRates[c.caseId] = c.passRate\n  }\n\n  const entry: BaselineEntry = {\n    modelId: scorecard.modelId,\n    suiteVersion: scorecard.suiteVersion,\n    recordedAt: scorecard.startedAt,\n    reason,\n    ottoVersion: scorecard.ottoVersion,\n    casePassRates,\n  }\n  if (scorecard.overallPassRate !== undefined) {\n    Object.assign(entry, { overallPassRate: scorecard.overallPassRate })\n  }\n\n  const kept = file.entries.filter(\n    (e) => !(e.modelId === entry.modelId && e.suiteVersion === entry.suiteVersion),\n  )\n  return { entries: [...kept, entry] }\n}\n\n/** 回归判定：套件 pass 率跌幅 + case flip 双信号。 */\nexport function judgeRegression(\n  scorecard: Scorecard,\n  baseline: BaselineEntry | undefined,\n): RegressionVerdict {\n  if (scorecard.invalid) {\n    return {\n      status: 'invalid_run',\n      findings: [\n        `infra_error rate ${(scorecard.infraErrorRate * 100).toFixed(1)}% exceeds threshold — run not comparable`,\n      ],\n      flippedCases: [],\n    }\n  }\n  if (!baseline) {\n    return {\n      status: 'no_baseline',\n      findings: [\n        `no baseline for (model=${scorecard.modelId}, suite=${scorecard.suiteVersion}) — record one with \\`baseline update\\``,\n      ],\n      flippedCases: [],\n    }\n  }\n\n  const findings: string[] = []\n  const flipped: string[] = []\n\n  // RFC-417 M1：otto 版本指纹不一致时显式标注——基线由旧引擎记录、本次为新引擎（或反之），\n  // 提示读者跨引擎版本对比。不参与 regressed 判定（版本不同不等于回归），只防\"拿旧 otto 跑新基线\n  // 却当同引擎看\"的误读。ottoVersion 不进分键，故此处是唯一暴露版本漂移的地方。\n  if (\n    baseline.ottoVersion !== undefined &&\n    scorecard.ottoVersion !== 'unknown' &&\n    baseline.ottoVersion !== scorecard.ottoVersion\n  ) {\n    findings.push(\n      `otto version differs from baseline: baseline recorded by '${baseline.ottoVersion}', this run '${scorecard.ottoVersion}' — comparison spans engine versions`,\n    )\n  }\n\n  for (const c of scorecard.cases) {\n    if (!c.qualityScored) {\n      continue\n    }\n    const before = baseline.casePassRates[c.caseId]\n    if (before === 1 && c.passRate === 0) {\n      flipped.push(c.caseId)\n      findings.push(`case flip: '${c.caseId}' was 100% pass, now 0%`)\n      continue\n    }\n    // 一个 case 的运行全是 infra_error 时它没有 pass 率，会**静默退出**套件均值——\n    // 于是\"某题目彻底跑不起来\"看上去和\"套件表现不变\"一模一样。这里显式点名，\n    // 不判回归（确实无从判断），但绝不让它无声消失。\n    if (c.passRate === undefined && before !== undefined) {\n      findings.push(\n        `'${c.caseId}' produced no comparable runs (all infra errors) — excluded from the suite average`,\n      )\n    }\n  }\n\n  let deltaPp: number | undefined\n  if (scorecard.overallPassRate !== undefined && baseline.overallPassRate !== undefined) {\n    deltaPp = (scorecard.overallPassRate - baseline.overallPassRate) * 100\n    if (deltaPp < -PASS_RATE_REGRESSION_PP) {\n      findings.push(\n        `suite pass rate dropped ${Math.abs(deltaPp).toFixed(1)}pp ` +\n          `(${(baseline.overallPassRate * 100).toFixed(1)}% → ${(scorecard.overallPassRate * 100).toFixed(1)}%)`,\n      )\n    }\n  }\n\n  const regressed =\n    flipped.length > 0 || (deltaPp !== undefined && deltaPp < -PASS_RATE_REGRESSION_PP)\n\n  const verdict: RegressionVerdict = {\n    status: regressed ? 'regressed' : 'ok',\n    findings: findings.length > 0 ? findings : ['no regression detected'],\n    flippedCases: flipped,\n  }\n  if (deltaPp !== undefined) {\n    Object.assign(verdict, { passRateDeltaPp: deltaPp })\n  }\n  return verdict\n}\n","/**\n * report.ts —— scorecard + 回归判定的人类可读渲染（RFC-316 D6 消费回路的出口）。\n *\n * 报告是评测体系唯一被人真正阅读的产物——@x-otto/eval 上一次死于「没人读」，\n * 所以这里刻意把三件事放在最显眼处：回归结论、逐 case pass 率、诚实边界（未评质量的 case 数）。\n */\n\nimport type { RegressionVerdict, Scorecard } from './types'\n\nfunction pct(value: number | undefined): string {\n  return value === undefined ? '—' : `${(value * 100).toFixed(0)}%`\n}\n\nexport function renderReport(scorecard: Scorecard, verdict: RegressionVerdict): string {\n  const lines: string[] = []\n\n  const headline =\n    verdict.status === 'regressed'\n      ? 'REGRESSED'\n      : verdict.status === 'ok'\n        ? 'OK'\n        : verdict.status === 'invalid_run'\n          ? 'INVALID (infra errors)'\n          : 'NO BASELINE'\n\n  lines.push(`otto eval — ${headline}`)\n  lines.push(\n    `model=${scorecard.modelId}  suite=${scorecard.suiteVersion}  otto=${scorecard.ottoVersion}  at=${scorecard.startedAt}`,\n  )\n  lines.push('')\n\n  lines.push('case                              judge     pass   runs  infra  skip')\n  for (const c of scorecard.cases) {\n    const id = c.caseId.padEnd(33).slice(0, 33)\n    const judge = c.judge.padEnd(9)\n    const rate = (c.qualityScored ? pct(c.passRate) : 'n/a').padStart(5)\n    const runs = `${c.passed}/${c.effective}`.padStart(6)\n    const infra = String(c.infraErrors).padStart(6)\n    const skip = String(c.skippedBudget).padStart(5)\n    lines.push(`${id} ${judge} ${rate} ${runs} ${infra} ${skip}`)\n  }\n  lines.push('')\n\n  lines.push(\n    `suite pass rate: ${pct(scorecard.overallPassRate)}  (scored ${scorecard.scoredCases}, liveness-only ${scorecard.unscoredCases})`,\n  )\n  if (verdict.passRateDeltaPp !== undefined) {\n    const sign = verdict.passRateDeltaPp >= 0 ? '+' : ''\n    lines.push(`vs baseline:     ${sign}${verdict.passRateDeltaPp.toFixed(1)}pp`)\n  }\n  lines.push(`infra errors:    ${(scorecard.infraErrorRate * 100).toFixed(1)}% of runs`)\n  if (scorecard.skippedBudgetTotal > 0) {\n    // RFC-353 S9: 预算耗尽跳过的运行不是 infra 故障，单独一行呈现——提醒读者本次\n    // 评测有 case 未完整跑满 runs 次，结果基于更少样本量。\n    lines.push(`skipped (budget): ${scorecard.skippedBudgetTotal} run(s) — suite incomplete for those cases`)\n  }\n\n  const m = scorecard.aggregateMetrics\n  if (m) {\n    lines.push(\n      `avg per run:     ${m.turns.toFixed(1)} turns · ${m.toolCalls.toFixed(1)} tools ` +\n        `(${m.toolErrors.toFixed(1)} err) · ${Math.round(m.inputTokens + m.outputTokens)} tokens`,\n    )\n  }\n\n  lines.push('')\n  for (const finding of verdict.findings) {\n    lines.push(`- ${finding}`)\n  }\n\n  if (scorecard.unscoredCases > 0) {\n    lines.push('')\n    lines.push(\n      `note: ${scorecard.unscoredCases} case(s) have no machine judgement (liveness only) and do not affect the suite pass rate.`,\n    )\n  }\n\n  return lines.join('\\n')\n}\n","/**\n * extract.ts —— trace 提炼器（RFC-316 D4 来源②：真实会话 → case 骨架）。\n *\n * case 库能否持续生长，决定这套评测第二次会不会死。真实事故/真实任务是最好的 case 来源，\n * 但从零手写 case 的摩擦太大——本模块把一个真实会话一键转成**骨架**：prompt 原文、\n * 工具调用序列（golden.toolSequence 的现成候选）、触碰过的文件清单（fixture 提示）。\n *\n * 两条刻意的\"不做\"：\n * - **不自动生成判据**：判据是 case 的灵魂，必须人写（RFC-316 D3 诚实边界——自动生成的\n *   判据只会是\"输出像上次一样\"，那是快照不是判据）。骨架里留 TODO 与建议。\n * - **不自动脱敏**：不可靠的脱敏比明示风险更危险（对齐 RFC-200 录制红线）。骨架自带\n *   脱敏检查清单，入库前人工过一遍是硬规则（RFC-316 规则 7）。\n */\n\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\n\nimport { traceFilePath } from './trace-metrics'\n\ninterface RawTraceEvent {\n  node?: string\n  payload?: Record<string, unknown>\n}\n\n/** 从 trace 还原的会话骨架素材。 */\nexport interface SessionSkeleton {\n  sessionId: string\n  /** 首条用户 prompt 原文（prompt.before 的 text）。 */\n  prompt?: string\n  /** 工具调用序列（名称，按发生顺序）。 */\n  toolSequence: readonly string[]\n  /** 工具参数里出现过的文件路径（write/edit/read 的 path 类参数）——fixture 需要什么的线索。 */\n  touchedPaths: readonly string[]\n  /** LLM 往返数（turn.end 计数）。 */\n  turns: number\n}\n\nconst PATH_ARGUMENT_KEYS = ['path', 'filePath', 'file_path'] as const\n\n/** 解析 trace 内容为骨架素材（纯函数，可测）。 */\nexport function parseSessionSkeleton(sessionId: string, content: string): SessionSkeleton {\n  let prompt: string | undefined\n  const toolSequence: string[] = []\n  const touchedPaths = new Set<string>()\n  let turns = 0\n\n  for (const line of content.split('\\n')) {\n    if (line.trim() === '') {\n      continue\n    }\n    let event: RawTraceEvent\n    try {\n      event = JSON.parse(line) as RawTraceEvent\n    } catch {\n      continue\n    }\n\n    if (event.node === 'prompt.before' && prompt === undefined) {\n      const text = event.payload?.['text']\n      if (typeof text === 'string' && text.trim() !== '') {\n        prompt = text\n      }\n      continue\n    }\n\n    if (event.node === 'tool.call.start') {\n      const name = event.payload?.['name']\n      if (typeof name === 'string') {\n        toolSequence.push(name)\n      }\n      const args = event.payload?.['arguments']\n      if (typeof args === 'object' && args !== null) {\n        for (const key of PATH_ARGUMENT_KEYS) {\n          const value = (args as Record<string, unknown>)[key]\n          if (typeof value === 'string' && value !== '') {\n            touchedPaths.add(value)\n          }\n        }\n      }\n      continue\n    }\n\n    if (event.node === 'turn.end') {\n      turns++\n    }\n  }\n\n  return {\n    sessionId,\n    toolSequence,\n    touchedPaths: [...touchedPaths].sort(),\n    turns,\n    ...(prompt !== undefined ? { prompt } : {}),\n  }\n}\n\nconst SANITIZE_CHECKLIST = `## 脱敏检查清单（入库前必须逐项人工核对 — RFC-316 规则 7）\n\n- [ ] prompt 里没有 API key / token / 密码 / 内部主机名\n- [ ] prompt 里没有真实用户数据（邮箱/手机号/姓名）\n- [ ] fixture 文件里没有上述任何内容\n- [ ] 路径不含本机用户名等机器指纹（用相对路径）\n- [ ] verify 判据已由人工编写（不要用\"输出像上次一样\"的快照当判据）\n`\n\nfunction renderCaseYaml(skeleton: SessionSkeleton): string {\n  const promptBlock =\n    skeleton.prompt !== undefined\n      ? `prompt: |\\n${skeleton.prompt\n          .split('\\n')\n          .map((line) => `  ${line}`)\n          .join('\\n')}`\n      : 'prompt: |\\n  TODO: trace 里没有 prompt.before 文本，请手工填写'\n\n  const toolHint =\n    skeleton.toolSequence.length > 0\n      ? `# 实际会话的工具序列（golden.toolSequence 候选，按需裁剪）：\\n# ${skeleton.toolSequence.join(' → ')}\\n`\n      : ''\n\n  return `# 由 otto-eval extract 生成的骨架（会话 ${skeleton.sessionId}）。\n# TODO：本 case 尚不可用——judge 是 liveness 占位，必须人工补判据后才能进套件质量分。\n${promptBlock}\njudge: liveness  # TODO: 改成 script（写 verify.sh）或 golden（写断言）\n${toolHint}runs: 3\ntimeoutMs: 240000\nmaxTokens: 200000\n`\n}\n\nfunction renderReport(skeleton: SessionSkeleton): string {\n  const paths =\n    skeleton.touchedPaths.length > 0\n      ? skeleton.touchedPaths.map((p) => `- \\`${p}\\``).join('\\n')\n      : '- （无 —— 会话没有文件类工具调用）'\n\n  return `# 提炼报告 — 会话 ${skeleton.sessionId}\n\n- LLM 往返：${skeleton.turns}\n- 工具调用：${skeleton.toolSequence.length} 次\n\n## 会话触碰过的文件（fixture 需要复现哪些初始状态的线索）\n\n${paths}\n\n## 下一步（按序完成后删除本文件）\n\n1. 在 fixture/ 里构造这些文件的**任务开始前**状态（trace 只有调用记录，初始内容需要你还原）\n2. 写判据：verify.sh（首选，可执行判据）或 case.yaml 里的 golden 断言\n3. 把 judge 从 liveness 改成 script/golden\n4. 过一遍下面的脱敏清单\n\n${SANITIZE_CHECKLIST}`\n}\n\nexport interface ExtractResult {\n  caseDir: string\n  skeleton: SessionSkeleton\n}\n\n/** 从会话 trace 生成 case 骨架目录。目标已存在时拒绝（不覆盖人工编辑过的 case）。 */\nexport function extractCase(\n  sessionId: string,\n  casesRoot: string,\n  readTraceContent: (sessionId: string) => string | undefined = defaultReadTrace,\n): ExtractResult {\n  const content = readTraceContent(sessionId)\n  if (content === undefined) {\n    throw new Error(\n      `no trace found for session ${sessionId} (looked at ${traceFilePath(sessionId)})`,\n    )\n  }\n\n  const skeleton = parseSessionSkeleton(sessionId, content)\n  const caseDir = join(casesRoot, `extracted-${sessionId.slice(0, 8)}`)\n  if (existsSync(caseDir)) {\n    throw new Error(\n      `${caseDir} already exists — refusing to overwrite (delete it first if intended)`,\n    )\n  }\n\n  mkdirSync(join(caseDir, 'fixture'), { recursive: true })\n  writeFileSync(join(caseDir, 'case.yaml'), renderCaseYaml(skeleton), 'utf8')\n  writeFileSync(join(caseDir, 'EXTRACTION.md'), renderReport(skeleton), 'utf8')\n\n  return { caseDir, skeleton }\n}\n\nfunction defaultReadTrace(sessionId: string): string | undefined {\n  const path = traceFilePath(sessionId)\n  return existsSync(path) ? readFileSync(path, 'utf8') : undefined\n}\n"],"mappings":"8aAwCA,MAAM,EAAY,IAAI,IAUtB,SAAgB,EAAiB,EAA0B,CACzD,EAAU,IAAI,EAAS,KAAM,EAAS,CAIxC,SAAgB,EAAY,EAAoC,CAC9D,OAAO,EAAU,IAAI,EAAK,CAI5B,SAAgB,GAAoC,CAClD,MAAO,CAAC,GAAG,EAAU,MAAM,CAAC,CAI9B,SAAgB,EACd,EACA,EACS,CACT,IAAI,EAAS,EACb,IAAK,IAAM,KAAQ,EACb,EAAS,EAAS,QAAU,IAAS,EAAS,IAChD,IAGJ,OAAO,IAAW,EAAS,OAG7B,SAAgB,EACd,EACA,EACA,EACc,CACd,IAAM,EAAmB,EAAE,CAC3B,GAAI,EAAO,eAAgB,CACzB,IAAM,EAAW,EAAS,aAAa,CACvC,IAAK,IAAM,KAAU,EAAO,eACrB,EAAS,SAAS,EAAO,aAAa,CAAC,EAC1C,EAAO,KAAK,mBAAmB,EAAO,GAAG,CAS/C,OALI,EAAO,cAAgB,CAAC,EAAuB,EAAW,EAAO,aAAa,EAChF,EAAO,KACL,kBAAkB,EAAO,aAAa,KAAK,KAAK,CAAC,sBAAsB,EAAU,KAAK,KAAK,CAAC,GAC7F,CAEI,EAAO,SAAW,EACrB,CAAE,OAAQ,GAAM,OAAQ,8BAA+B,CACvD,CAAE,OAAQ,GAAO,OAAQ,EAAO,KAAK,KAAK,CAAE,CAGlD,SAAS,EAAgB,EAAoB,EAAkC,CAC7E,IAAM,EAAO,EAAU,OAAQ,CAAC,EAAW,CAAE,CAC3C,IAAK,EAAM,aACX,SAAU,OACV,QAAS,KACT,IAAK,CAAE,GAAG,QAAQ,IAAK,mBAAoB,EAAM,SAAU,CAC5D,CAAC,CACF,GAAI,EAAK,MACP,MAAO,CAAE,OAAQ,GAAO,OAAQ,wBAAwB,EAAK,MAAM,UAAW,CAEhF,IAAM,EACJ,IAAI,EAAK,QAAU,IAAI,MAAM,GAAG,EAAK,OAAS,MAAM,EAAK,OAAO,MAAM,GAAK,KAAK,MAAM,EAAG,IAAI,CAC/F,MAAO,CAAE,OAAQ,EAAK,SAAW,EAAG,OAAQ,GAAU,QAAQ,EAAK,SAAU,CAI/E,MAAa,EAA2B,CACtC,KAAM,SACN,MAAM,OAAO,EAAO,CAClB,IAAM,EAAO,EAAM,KAInB,OAHI,EAAK,QAAU,SAGZ,EAAgB,EAAO,EAAK,EAAK,IAAK,EAAK,aAAa,CAAC,CAFvD,CAAE,OAAQ,GAAO,OAAQ,+CAAgD,EAIrF,CAGY,EAA2B,CACtC,KAAM,SACN,MAAM,OAAO,EAAO,CAClB,OAAO,EAAa,EAAM,KAAK,QAAU,EAAE,CAAE,EAAM,SAAU,EAAM,UAAU,EAEhF,CAGY,EAA6B,CACxC,KAAM,WACN,MAAM,QAAS,CACb,MAAO,CAAE,OAAQ,GAAM,OAAQ,uCAAwC,EAE1E,CAED,EAAiB,EAAe,CAChC,EAAiB,EAAe,CAChC,EAAiB,EAAiB,CAQlC,eAAsB,EAAU,EAA2C,CACzE,GAAM,CAAE,QAAS,EACX,EAAwB,EAAE,CAE1B,EAAW,EAAY,EAAK,MAAM,CACxC,GAAI,CAAC,EACH,MAAO,CACL,OAAQ,GACR,OAAQ,qCAAqC,EAAK,MAAM,YAAY,GAAgB,CAAC,KAAK,KAAK,CAAC,GACjG,CAOH,GALA,EAAM,KAAK,MAAM,EAAS,OAAO,EAAM,CAAC,CACpC,EAAK,QACP,EAAM,KAAK,EAAa,EAAK,OAAQ,EAAM,SAAU,EAAM,UAAU,CAAC,CAGpE,EAAM,SAAW,EACnB,MAAO,CAAE,OAAQ,GAAM,OAAQ,uCAAwC,CAGzE,IAAM,EAAS,EAAM,OAAQ,GAAM,CAAC,EAAE,OAAO,CAC7C,OAAO,EAAO,SAAW,EACrB,CAAE,OAAQ,GAAM,OAAQ,EAAM,IAAK,GAAM,EAAE,OAAO,CAAC,KAAK,KAAK,CAAE,CAC/D,CAAE,OAAQ,GAAO,OAAQ,EAAO,IAAK,GAAM,EAAE,OAAO,CAAC,KAAK,KAAK,CAAE,CC5JvE,SAAS,EAAc,EAA8B,EAAa,EAAwB,CACxF,IAAM,EAAQ,EAAI,GAClB,GAAI,OAAO,GAAU,UAAY,EAAM,MAAM,GAAK,GAChD,MAAU,MAAM,SAAS,EAAO,YAAY,EAAI,8BAA8B,CAEhF,OAAO,EAGT,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,IAAM,EAAQ,EAAI,GAClB,GAAI,IAAU,IAAA,GACZ,OAAO,EAET,GAAI,OAAO,GAAU,UAAY,CAAC,OAAO,UAAU,EAAM,EAAI,GAAS,EACpE,MAAU,MAAM,SAAS,EAAO,YAAY,EAAI,8BAA8B,CAEhF,OAAO,EAGT,SAAS,GAAY,EAAc,EAA4B,CAC7D,GAAI,OAAO,GAAQ,WAAY,EAC7B,MAAU,MAAM,SAAS,EAAO,+CAA+C,CAEjF,IAAM,EAAS,EACT,EAAqB,EAAE,CACvB,EAAiB,EAAO,eAC9B,GAAI,IAAmB,IAAA,GAAW,CAChC,GAAI,CAAC,MAAM,QAAQ,EAAe,EAAI,EAAe,KAAM,GAAM,OAAO,GAAM,SAAS,CACrF,MAAU,MAAM,SAAS,EAAO,iDAAiD,CAEnF,OAAO,OAAO,EAAQ,CAAkB,iBAA4B,CAAC,CAEvE,IAAM,EAAe,EAAO,aAC5B,GAAI,IAAiB,IAAA,GAAW,CAC9B,GAAI,CAAC,MAAM,QAAQ,EAAa,EAAI,EAAa,KAAM,GAAM,OAAO,GAAM,SAAS,CACjF,MAAU,MAAM,SAAS,EAAO,+CAA+C,CAEjF,OAAO,OAAO,EAAQ,CAAgB,eAA0B,CAAC,CAEnE,GAAI,CAAC,EAAO,gBAAkB,CAAC,EAAO,aACpC,MAAU,MACR,SAAS,EAAO,wFACjB,CAEH,OAAO,EAIT,SAAgB,EAAS,EAAuB,CAC9C,IAAM,EAAS,EAAI,MAAM,IAAI,CAAC,OAAO,QAAQ,CAAC,GAAG,GAAG,EAAI,EAClD,EAAW,EAAK,EAAK,YAAY,CACvC,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,SAAS,EAAO,0BAA0B,IAAW,CAGvE,IAAM,EAAkBA,EAAU,EAAa,EAAU,OAAO,CAAC,CACjE,GAAI,OAAO,GAAW,WAAY,EAChC,MAAU,MAAM,SAAS,EAAO,qCAAqC,CAEvE,IAAM,EAAM,EAEN,EAAS,EAAc,EAAK,SAAU,EAAO,CAC7C,EAAW,EAAc,EAAK,QAAS,EAAO,CACpD,GAAI,EAAY,EAAS,GAAK,IAAA,GAC5B,MAAU,MACR,SAAS,EAAO,YAAY,EAAS,yCAAyC,GAAgB,CAAC,KAAK,KAAK,CAAC,GAC3G,CAEH,IAAM,EAAQ,EAER,EAAS,CACb,GAAI,EACJ,SACA,KAAM,EAAoB,EAAK,OAAQ,EAAQ,EAAa,CAC5D,UAAW,EAAoB,EAAK,YAAa,EAAQ,IAAmB,CAC5E,MACA,WAAY,EAAW,EAAK,EAAK,UAAU,CAAC,CAC5C,SAAU,EAAW,EAAK,EAAK,WAAW,CAAC,CAC5C,CAEG,EAAI,YAAiB,IAAA,IACvB,OAAO,OAAO,EAAQ,CAAE,UAAW,EAAoB,EAAK,YAAa,EAAQ,EAAE,CAAE,CAAC,CAKxF,IAAM,EACJ,IAAU,UAAY,EAAI,SAAc,IAAA,GAAY,GAAY,EAAI,OAAW,EAAO,CAAG,IAAA,GAE3F,GAAI,IAAU,SAAU,CACtB,IAAM,EACJ,EAAI,eAAoB,IAAA,GACpB,YACA,EAAc,EAAK,eAAgB,EAAO,CAChD,GAAI,CAAC,EAAW,EAAK,EAAK,EAAO,CAAC,CAChC,MAAU,MAAM,SAAS,EAAO,uCAAuC,EAAO,aAAa,CAE7F,MAAO,CAAE,GAAG,EAAQ,QAAO,aAAc,EAAQ,GAAI,IAAW,IAAA,GAAyB,EAAE,CAAf,CAAE,SAAQ,CAAQ,CAOhG,OALI,IAAU,SAEL,CAAE,GAAG,EAAQ,QAAe,SAAsB,CAGpD,CAAE,GAAG,EAAQ,QAAO,GAAI,IAAW,IAAA,GAAyB,EAAE,CAAf,CAAE,SAAQ,CAAQ,CAI1E,SAAgB,EAAU,EAAwC,CAChE,GAAI,CAAC,EAAW,EAAU,CACxB,MAAU,MAAM,yBAAyB,IAAY,CAOvD,OALa,EAAY,EAAU,CAChC,OAAQ,GAAS,CAAC,EAAK,WAAW,IAAI,CAAC,CACvC,IAAK,GAAS,EAAK,EAAW,EAAK,CAAC,CACpC,OAAQ,GAAS,EAAS,EAAK,CAAC,aAAa,CAAC,CAC9C,MAAM,CACG,IAAI,EAAS,CAU3B,SAAgB,EAAoB,EAAoC,CACtE,IAAM,EAAY,EAAM,IAAK,IAAO,CAClC,GAAI,EAAE,GACN,OAAQ,EAAE,OACV,MAAO,EAAE,MACT,aAAc,EAAE,QAAU,SAAW,EAAE,aAAe,IAAA,GACtD,OAAQ,EAAE,QAAU,KACrB,EAAE,CACH,OAAO,EAAW,SAAS,CAAC,OAAO,KAAK,UAAU,EAAU,CAAC,CAAC,OAAO,MAAM,CAAC,MAAM,EAAG,GAAG,CC5I1F,MAAa,EAAe,WAGf,EAAmB,IAYhC,SAAgB,EAAS,EAAgB,EAAmC,CAC1E,GAAI,CAAC,EAAK,SACR,MAAO,CAAE,GAAI,GAAM,OAAQ,WAAY,CAGzC,IAAM,EAAO,EAAU,OAAQ,CADZ,EAAK,EAAK,IAAK,EAAa,CACJ,CAAE,CAC3C,IAAK,EACL,SAAU,OACV,QAAS,EACV,CAAC,CACF,GAAI,EAAK,MAEP,MAAO,CAAE,GAAI,GAAO,OAAQ,uBAAuB,EAAK,MAAM,UAAW,CAE3E,GAAI,EAAK,SAAW,EAAG,CACrB,IAAM,GAAU,EAAK,QAAU,IAAI,MAAM,CAAC,MAAM,EAAG,IAAI,CACvD,MAAO,CACL,GAAI,GACJ,OAAQ,cAAc,EAAK,SAAS,EAAS,KAAK,IAAW,KAC9D,CAEH,MAAO,CAAE,GAAI,GAAM,OAAQ,WAAY,CCXzC,SAAgB,EAAgB,EAAoC,CAClE,IAAM,EAAO,EAAO,aAAa,CAiBjC,MAhBqB,CACnB,aACA,aACA,MACA,aACA,aACA,YACA,YACA,iBACA,sBACA,wBACA,cACA,sBACA,oBACA,SACD,CACmB,KAAM,GAAW,EAAK,SAAS,EAAO,CAAC,CAAG,QAAU,SAW1E,SAAgB,GAAmB,EAA+B,EAAE,CAAe,CACjF,IAAM,EAAM,EAAQ,KAAO,OAC3B,MAAO,CACL,OAAO,EAAS,CACd,OAAO,IAAI,QAAyB,GAAY,CAC9C,IAAM,EAAQ,EAAM,EAAK,CAAC,SAAU,EAAQ,OAAO,CAAE,CACnD,IAAK,EAAQ,IACb,IAAK,CAAE,GAAG,QAAQ,IAAK,GAAG,EAAQ,IAAK,CACvC,MAAO,CAAC,SAAU,OAAQ,OAAO,CAClC,CAAC,CACE,EAAS,GACT,EAAS,GACT,EAAU,GACR,EAAU,GAAmC,CAC7C,IAGJ,EAAU,GACV,aAAa,EAAM,CACnB,EAAQ,EAAS,GAEb,EAAQ,eAAiB,CAC7B,EAAM,KAAK,UAAU,CACrB,EAAO,CAAE,KAAM,SAAU,OAAQ,iBAAiB,EAAQ,UAAU,IAAK,CAAC,EACzE,EAAQ,UAAU,CAErB,EAAM,OAAO,GAAG,OAAS,GAAkB,CACzC,GAAU,EAAM,UAAU,EAC1B,CACF,EAAM,OAAO,GAAG,OAAS,GAAkB,CACzC,GAAU,EAAM,UAAU,EAC1B,CACF,EAAM,GAAG,QAAU,GAAU,CAC3B,EAAO,CAAE,KAAM,QAAS,OAAQ,iBAAiB,EAAM,UAAW,CAAC,EACnE,CACF,EAAM,GAAG,QAAU,GAAS,CAC1B,GAAI,IAAS,EAAG,CAEd,EAAO,CAAE,KADI,EAAgB,EAAO,CACrB,OAAQ,QAAQ,EAAK,IAAI,EAAO,MAAM,CAAC,MAAM,EAAG,IAAI,GAAI,CAAC,CACxE,OAEF,GAAI,CACF,IAAM,EAAS,KAAK,MAAM,EAAO,CACjC,GAAI,OAAO,EAAO,WAAc,SAAU,CACxC,EAAO,CAAE,KAAM,SAAU,OAAQ,gCAAiC,CAAC,CACnE,OAEF,EAAO,CAAE,KAAM,KAAM,OAAQ,EAAQ,CAAC,MAChC,CACN,EAAO,CAAE,KAAM,SAAU,OAAQ,2BAA2B,EAAO,MAAM,EAAG,IAAI,GAAI,CAAC,GAEvF,EACF,EAEL,CC3GH,SAAS,EAAW,EAA8C,EAAqB,CACrF,IAAM,EAAQ,IAAU,GACxB,OAAO,OAAO,GAAU,UAAY,OAAO,SAAS,EAAM,CAAG,EAAQ,EAIvE,SAAgB,EAAc,EAA2B,CACvD,OAAO,EAAK,EAAmB,QAAS,GAAG,EAAU,QAAQ,CAc/D,SAAgB,EAAc,EAA2B,CACvD,IAAM,EAAsB,EAAE,CAE9B,MAAO,CAAE,QADO,EAAkB,EAAS,EAAU,CACnC,YAAW,CAG/B,SAAS,EAAkB,EAAiB,EAAqC,CAC/E,IAAM,EAAsB,CAC1B,MAAO,EACP,UAAW,EACX,WAAY,EACZ,YAAa,EACb,aAAc,EACd,gBAAiB,EACjB,eAAgB,EACjB,CAED,IAAK,IAAM,KAAQ,EAAQ,MAAM;EAAK,CAAE,CACtC,GAAI,EAAK,MAAM,GAAK,GAClB,SAEF,IAAI,EACJ,GAAI,CACF,EAAQ,KAAK,MAAM,EAAK,MAClB,CACN,SAGF,GAAI,EAAM,OAAS,gBAAiB,CAClC,EAAQ,YACR,EAAQ,gBAAkB,EAAW,EAAM,QAAS,aAAa,CAC7D,EAAM,SAAU,gBAAqB,IAAA,IACvC,EAAQ,aAEV,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAClB,GAAc,KAAK,EAAK,CAE1B,SAGF,GAAI,EAAM,OAAS,WAAY,CAC7B,EAAQ,QACR,IAAM,EAAQ,EAAM,SAAU,MAC9B,GAAI,OAAO,GAAU,UAAY,EAAgB,CAC/C,IAAM,EAAI,EACV,EAAQ,aAAe,EAAW,EAAG,cAAc,CACnD,EAAQ,cAAgB,EAAW,EAAG,eAAe,CACrD,EAAQ,iBAAmB,EAAW,EAAG,kBAAkB,CAE7D,IAAM,EAAQ,EAAM,SAAU,MAC1B,OAAO,GAAU,UAAY,IAAU,KACzC,EAAQ,QAAU,IAKxB,OAAO,EAIT,SAAgB,EAAa,EAAyC,CACpE,IAAM,EAAO,EAAc,EAAU,CAChC,KAAW,EAAK,CAGrB,OAAO,EAAc,EAAa,EAAM,OAAO,CAAC,CAIlD,SAAgB,EAAe,EAAwD,CACrF,GAAI,EAAQ,SAAW,EACrB,OAEF,IAAM,EAAM,EAAQ,QACjB,EAAK,KAAO,CACX,MAAO,EAAI,MAAQ,EAAE,MACrB,UAAW,EAAI,UAAY,EAAE,UAC7B,WAAY,EAAI,WAAa,EAAE,WAC/B,YAAa,EAAI,YAAc,EAAE,YACjC,aAAc,EAAI,aAAe,EAAE,aACnC,gBAAiB,EAAI,gBAAkB,EAAE,gBACzC,eAAgB,EAAI,eAAiB,EAAE,eACvC,QAAS,EAAE,SAAW,EAAI,QAC3B,EACD,CACE,MAAO,EACP,UAAW,EACX,WAAY,EACZ,YAAa,EACb,aAAc,EACd,gBAAiB,EACjB,eAAgB,EACjB,CACF,CACK,EAAI,EAAQ,OACZ,EAAuB,CAC3B,MAAO,EAAI,MAAQ,EACnB,UAAW,EAAI,UAAY,EAC3B,WAAY,EAAI,WAAa,EAC7B,YAAa,EAAI,YAAc,EAC/B,aAAc,EAAI,aAAe,EACjC,gBAAiB,EAAI,gBAAkB,EACvC,eAAgB,EAAI,eAAiB,EACtC,CAID,OAHI,EAAI,UAAY,IAAA,KAClB,EAAS,QAAU,EAAI,SAElB,EC9HT,MAAa,EAAgC,EAC3C,QAAQ,IAAI,gCACZ,GACD,CAYD,SAAS,EAAY,EAAyC,CAI5D,OAHK,EAGE,EAAQ,YAAc,EAAQ,aAF5B,EAMX,eAAsB,EAAQ,EAAgB,EAAuC,CACnF,IAAM,EAAM,EAAK,UAAc,KAAK,KAAK,EACnC,EAAY,EAAK,WAAa,EAC9B,EAAoB,EAAE,CACxB,EAAc,EAEd,EAAe,EAYf,EAAiB,EAEf,MAAgC,CACpC,GAAI,EAAK,YAAc,IAAA,GACrB,MAAO,GAET,IAAM,EAAgB,EAAe,EAAI,EAAc,EAAe,EAAK,UAAY,EAAK,KAC5F,OAAO,EAAc,EAAiB,GAAiB,EAAK,WAG9D,IAAK,IAAI,EAAQ,EAAG,EAAQ,EAAK,KAAM,IAAS,CAC9C,GAAI,GAAgB,CAAE,CAGpB,EAAK,KAAK,CACR,QAAS,iBACT,OAAQ,EACR,OAAQ,qBAAqB,EAAY,kBACvC,EAAiB,EAAI,MAAM,EAAe,oBAAsB,GACjE,YAAY,EAAK,UAAU,0BAC7B,CAAC,CACF,SAGF,IAAM,EAAY,EAAgB,EAAK,CACjC,EAAY,GAAK,CACvB,GAAI,CAGF,IAAM,EAAQ,EAAS,EAAM,EAAU,IAAI,CAC3C,GAAI,CAAC,EAAM,GAAI,CACb,IACA,EAAK,KAAK,CAAE,QAAS,cAAe,OAAQ,GAAK,CAAG,EAAW,OAAQ,EAAM,OAAQ,CAAC,CACtF,SAEF,IAAM,EAAW,MAAM,EAAK,QAAQ,OAAO,CACzC,OAAQ,EAAK,OACb,IAAK,EAAU,IACf,UAAW,EAAK,UACjB,CAAC,CACI,EAAS,GAAK,CAAG,EAEvB,GAAI,EAAS,OAAS,QAAS,CAG7B,IACA,EAAK,KAAK,CAAE,QAAS,cAAe,SAAQ,OAAQ,EAAS,OAAQ,CAAC,CACtE,SAEF,GAAI,EAAS,OAAS,SAAU,CAG9B,IACA,EAAK,KAAK,CAAE,QAAS,OAAQ,SAAQ,OAAQ,EAAS,OAAQ,CAAC,CAC/D,SAGF,IAAM,EAAQ,EAAU,EAAS,OAAO,UAAU,CAC5C,EAAU,MAAM,EAAU,CAC9B,OACA,aAAc,EAAU,IACxB,SAAU,EAAS,OAAO,SAC1B,UAAW,GAAO,WAAa,EAAE,CAClC,CAAC,CACE,GACF,GAAe,EAAY,EAAM,QAAQ,CACzC,KAGA,IAGF,IAAM,EAAoB,CACxB,QAAS,EAAQ,OAAS,OAAS,OACnC,SACA,UAAW,EAAS,OAAO,UAC3B,OAAQ,EAAQ,OACjB,CACG,IACF,EAAO,QAAU,EAAM,SAGzB,EAAO,YAAc,EAAS,OAAO,aAAe,UACpD,EAAK,KAAK,EAAO,QACT,CACR,EAAU,SAAS,CAGrB,IAAM,EAAO,EAAK,GAAG,GAAG,CACpB,GACF,EAAK,gBAAgB,EAAK,GAAI,EAAO,EAAK,CAI9C,OAAO,EAAc,EAAM,EAAK,CAIlC,SAAgB,EAAc,EAAgB,EAAwC,CACpF,IAAM,EAAc,EAAK,OAAQ,GAAM,EAAE,UAAY,cAAc,CAAC,OAC9D,EAAgB,EAAK,OAAQ,GAAM,EAAE,UAAY,iBAAiB,CAAC,OACnE,EAAY,EAAK,OAAS,EAAc,EACxC,EAAS,EAAK,OAAQ,GAAM,EAAE,UAAY,OAAO,CAAC,OAClD,EAAgB,EAAK,QAAU,WAE/B,EAAqB,CACzB,OAAQ,EAAK,GACb,MAAO,EAAK,MACZ,OACA,SACA,YACA,cACA,gBACA,gBACD,CAID,OAHI,EAAY,GACd,OAAO,OAAO,EAAQ,CAAE,SAAU,EAAS,EAAW,CAAC,CAElD,EAIT,eAAsB,EACpB,EACA,EACA,EACoB,CACpB,IAAM,EAAM,EAAK,UAAc,KAAK,KAAK,EACnC,EAAY,IAAI,KAAK,GAAK,CAAC,CAAC,aAAa,CACzC,EAAwB,EAAE,CAChC,IAAK,IAAM,KAAQ,EACjB,EAAQ,KAAK,MAAM,EAAQ,EAAM,EAAK,CAAC,CAEzC,OAAO,EAAe,EAAS,EAAc,EAAU,CAIzD,SAAgB,EACd,EACA,EACA,EACW,CACX,IAAM,EAAS,EAAM,OAAQ,GAAM,EAAE,eAAiB,EAAE,WAAa,IAAA,GAAU,CACzE,EAAqB,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,cAAe,EAAE,CAIvE,EAAY,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,KAAK,OAAQ,EAAE,CAAG,EAC/D,EAAa,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,YAAa,EAAE,CAC7D,EAAiB,IAAc,EAAI,EAAI,EAAa,EAKpD,EAAY,EAHC,EAAM,QAAS,GAChC,EAAE,KAAK,IAAK,GAAM,EAAE,QAAQ,CAAC,OAAQ,GAAuB,IAAM,IAAA,GAAU,CAC7E,CAC2C,CAGtC,EACJ,EAAM,QAAS,GAAM,EAAE,KAAK,CAAC,KAAM,GAAM,EAAE,cAAgB,IAAA,GAAU,EAAE,aAAe,UAElF,EAAuB,CAC3B,eACA,QAAS,GAAW,SAAW,UAC/B,cACA,YACA,QACA,YAAa,EAAM,OAAQ,GAAM,EAAE,cAAc,CAAC,OAClD,cAAe,EAAM,OAAQ,GAAM,CAAC,EAAE,cAAc,CAAC,OACrD,iBACA,mBAAoB,EACpB,QAAS,EAAiB,EAC3B,CACD,GAAI,EAAO,OAAS,EAAG,CACrB,IAAM,EAAM,EAAO,QAAQ,EAAK,IAAM,GAAO,EAAE,UAAY,GAAI,EAAE,CACjE,OAAO,OAAO,EAAW,CAAE,gBAAiB,EAAM,EAAO,OAAQ,CAAC,CAKpE,OAHI,GACF,OAAO,OAAO,EAAW,CAAE,iBAAkB,EAAW,CAAC,CAEpD,EChOT,MAAa,EAA0B,EAAa,QAAQ,IAAI,wBAA4B,GAAG,CAE/F,SAAgB,EAAc,EAA4B,CAKxD,OAJK,EAAW,EAAK,CAId,CAAE,QADM,KAAK,MAAM,EAAa,EAAM,OAAO,CAAC,CAC5B,SAAW,EAAE,CAAE,CAH/B,CAAE,QAAS,EAAE,CAAE,CAM1B,SAAgB,EAAc,EAAc,EAA0B,CACpE,EAAU,EAAQ,EAAK,CAAE,CAAE,UAAW,GAAM,CAAC,CAC7C,EAAc,EAAM,GAAG,KAAK,UAAU,EAAM,KAAM,EAAE,CAAC,IAAK,OAAO,CAInE,SAAgB,EACd,EACA,EACA,EAC2B,CAC3B,OAAO,EAAK,QAAQ,KAAM,GAAM,EAAE,UAAY,GAAW,EAAE,eAAiB,EAAa,CAI3F,SAAgB,GACd,EACA,EACA,EACc,CACd,IAAM,EAAoD,EAAE,CAC5D,IAAK,IAAM,KAAK,EAAU,MACxB,EAAc,EAAE,QAAU,EAAE,SAG9B,IAAM,EAAuB,CAC3B,QAAS,EAAU,QACnB,aAAc,EAAU,aACxB,WAAY,EAAU,UACtB,SACA,YAAa,EAAU,YACvB,gBACD,CAQD,OAPI,EAAU,kBAAoB,IAAA,IAChC,OAAO,OAAO,EAAO,CAAE,gBAAiB,EAAU,gBAAiB,CAAC,CAM/D,CAAE,QAAS,CAAC,GAHN,EAAK,QAAQ,OACvB,GAAM,EAAE,EAAE,UAAY,EAAM,SAAW,EAAE,eAAiB,EAAM,cAClE,CAC2B,EAAM,CAAE,CAItC,SAAgB,GACd,EACA,EACmB,CACnB,GAAI,EAAU,QACZ,MAAO,CACL,OAAQ,cACR,SAAU,CACR,qBAAqB,EAAU,eAAiB,KAAK,QAAQ,EAAE,CAAC,0CACjE,CACD,aAAc,EAAE,CACjB,CAEH,GAAI,CAAC,EACH,MAAO,CACL,OAAQ,cACR,SAAU,CACR,0BAA0B,EAAU,QAAQ,UAAU,EAAU,aAAa,yCAC9E,CACD,aAAc,EAAE,CACjB,CAGH,IAAM,EAAqB,EAAE,CACvB,EAAoB,EAAE,CAM1B,EAAS,cAAgB,IAAA,IACzB,EAAU,cAAgB,WAC1B,EAAS,cAAgB,EAAU,aAEnC,EAAS,KACP,6DAA6D,EAAS,YAAY,eAAe,EAAU,YAAY,sCACxH,CAGH,IAAK,IAAM,KAAK,EAAU,MAAO,CAC/B,GAAI,CAAC,EAAE,cACL,SAEF,IAAM,EAAS,EAAS,cAAc,EAAE,QACxC,GAAI,IAAW,GAAK,EAAE,WAAa,EAAG,CACpC,EAAQ,KAAK,EAAE,OAAO,CACtB,EAAS,KAAK,eAAe,EAAE,OAAO,yBAAyB,CAC/D,SAKE,EAAE,WAAa,IAAA,IAAa,IAAW,IAAA,IACzC,EAAS,KACP,IAAI,EAAE,OAAO,oFACd,CAIL,IAAI,EACA,EAAU,kBAAoB,IAAA,IAAa,EAAS,kBAAoB,IAAA,KAC1E,GAAW,EAAU,gBAAkB,EAAS,iBAAmB,IAC/D,EAAU,CAAC,GACb,EAAS,KACP,2BAA2B,KAAK,IAAI,EAAQ,CAAC,QAAQ,EAAE,CAAC,OACjD,EAAS,gBAAkB,KAAK,QAAQ,EAAE,CAAC,OAAO,EAAU,gBAAkB,KAAK,QAAQ,EAAE,CAAC,IACtG,EAOL,IAAM,EAA6B,CACjC,OAHA,EAAQ,OAAS,GAAM,IAAY,IAAA,IAAa,EAAU,CAAC,EAGvC,YAAc,KAClC,SAAU,EAAS,OAAS,EAAI,EAAW,CAAC,yBAAyB,CACrE,aAAc,EACf,CAID,OAHI,IAAY,IAAA,IACd,OAAO,OAAO,EAAS,CAAE,gBAAiB,EAAS,CAAC,CAE/C,ECpJT,SAAS,EAAI,EAAmC,CAC9C,OAAO,IAAU,IAAA,GAAY,IAAM,IAAI,EAAQ,KAAK,QAAQ,EAAE,CAAC,GAGjE,SAAgBC,GAAa,EAAsB,EAAoC,CACrF,IAAM,EAAkB,EAAE,CAEpB,EACJ,EAAQ,SAAW,YACf,YACA,EAAQ,SAAW,KACjB,KACA,EAAQ,SAAW,cACjB,yBACA,cAEV,EAAM,KAAK,eAAe,IAAW,CACrC,EAAM,KACJ,SAAS,EAAU,QAAQ,UAAU,EAAU,aAAa,SAAS,EAAU,YAAY,OAAO,EAAU,YAC7G,CACD,EAAM,KAAK,GAAG,CAEd,EAAM,KAAK,uEAAuE,CAClF,IAAK,IAAM,KAAK,EAAU,MAAO,CAC/B,IAAM,EAAK,EAAE,OAAO,OAAO,GAAG,CAAC,MAAM,EAAG,GAAG,CACrC,EAAQ,EAAE,MAAM,OAAO,EAAE,CACzB,GAAQ,EAAE,cAAgB,EAAI,EAAE,SAAS,CAAG,OAAO,SAAS,EAAE,CAC9D,EAAO,GAAG,EAAE,OAAO,GAAG,EAAE,YAAY,SAAS,EAAE,CAC/C,EAAQ,OAAO,EAAE,YAAY,CAAC,SAAS,EAAE,CACzC,EAAO,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,CAChD,EAAM,KAAK,GAAG,EAAG,GAAG,EAAM,GAAG,EAAK,GAAG,EAAK,GAAG,EAAM,GAAG,IAAO,CAO/D,GALA,EAAM,KAAK,GAAG,CAEd,EAAM,KACJ,oBAAoB,EAAI,EAAU,gBAAgB,CAAC,YAAY,EAAU,YAAY,kBAAkB,EAAU,cAAc,GAChI,CACG,EAAQ,kBAAoB,IAAA,GAAW,CACzC,IAAM,EAAO,EAAQ,iBAAmB,EAAI,IAAM,GAClD,EAAM,KAAK,oBAAoB,IAAO,EAAQ,gBAAgB,QAAQ,EAAE,CAAC,IAAI,CAE/E,EAAM,KAAK,qBAAqB,EAAU,eAAiB,KAAK,QAAQ,EAAE,CAAC,WAAW,CAClF,EAAU,mBAAqB,GAGjC,EAAM,KAAK,qBAAqB,EAAU,mBAAmB,4CAA4C,CAG3G,IAAM,EAAI,EAAU,iBAChB,GACF,EAAM,KACJ,oBAAoB,EAAE,MAAM,QAAQ,EAAE,CAAC,WAAW,EAAE,UAAU,QAAQ,EAAE,CAAC,UACnE,EAAE,WAAW,QAAQ,EAAE,CAAC,UAAU,KAAK,MAAM,EAAE,YAAc,EAAE,aAAa,CAAC,SACpF,CAGH,EAAM,KAAK,GAAG,CACd,IAAK,IAAM,KAAW,EAAQ,SAC5B,EAAM,KAAK,KAAK,IAAU,CAU5B,OAPI,EAAU,cAAgB,IAC5B,EAAM,KAAK,GAAG,CACd,EAAM,KACJ,SAAS,EAAU,cAAc,2FAClC,EAGI,EAAM,KAAK;EAAK,CCxCzB,MAAM,GAAqB,CAAC,OAAQ,WAAY,YAAY,CAG5D,SAAgB,EAAqB,EAAmB,EAAkC,CACxF,IAAI,EACE,EAAyB,EAAE,CAC3B,EAAe,IAAI,IACrB,EAAQ,EAEZ,IAAK,IAAM,KAAQ,EAAQ,MAAM;EAAK,CAAE,CACtC,GAAI,EAAK,MAAM,GAAK,GAClB,SAEF,IAAI,EACJ,GAAI,CACF,EAAQ,KAAK,MAAM,EAAK,MAClB,CACN,SAGF,GAAI,EAAM,OAAS,iBAAmB,IAAW,IAAA,GAAW,CAC1D,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAAY,EAAK,MAAM,GAAK,KAC9C,EAAS,GAEX,SAGF,GAAI,EAAM,OAAS,kBAAmB,CACpC,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAClB,EAAa,KAAK,EAAK,CAEzB,IAAM,EAAO,EAAM,SAAU,UAC7B,GAAI,OAAO,GAAS,UAAY,EAC9B,IAAK,IAAM,KAAO,GAAoB,CACpC,IAAM,EAAS,EAAiC,GAC5C,OAAO,GAAU,UAAY,IAAU,IACzC,EAAa,IAAI,EAAM,CAI7B,SAGE,EAAM,OAAS,YACjB,IAIJ,MAAO,CACL,YACA,eACA,aAAc,CAAC,GAAG,EAAa,CAAC,MAAM,CACtC,QACA,GAAI,IAAW,IAAA,GAAyB,EAAE,CAAf,CAAE,SAAQ,CACtC,CAYH,SAAS,GAAe,EAAmC,CACzD,IAAM,EACJ,EAAS,SAAW,IAAA,GAKhB;0CAJA,cAAc,EAAS,OACpB,MAAM;EAAK,CACX,IAAK,GAAS,KAAK,IAAO,CAC1B,KAAK;EAAK,GAGb,EACJ,EAAS,aAAa,OAAS,EAC3B,gDAAgD,EAAS,aAAa,KAAK,MAAM,CAAC,IAClF,GAEN,MAAO,kCAAkC,EAAS,UAAU;;EAE5D,EAAY;;EAEZ,EAAS;;;EAMX,SAAS,GAAa,EAAmC,CACvD,IAAM,EACJ,EAAS,aAAa,OAAS,EAC3B,EAAS,aAAa,IAAK,GAAM,OAAO,EAAE,IAAI,CAAC,KAAK;EAAK,CACzD,uBAEN,MAAO,eAAe,EAAS,UAAU;;WAEhC,EAAS,MAAM;SACjB,EAAS,aAAa,OAAO;;;;EAIpC,EAAM;;;;;;;;;;;;;;;;EAkBR,SAAgB,GACd,EACA,EACA,EAA8D,GAC/C,CACf,IAAM,EAAU,EAAiB,EAAU,CAC3C,GAAI,IAAY,IAAA,GACd,MAAU,MACR,8BAA8B,EAAU,cAAc,EAAc,EAAU,CAAC,GAChF,CAGH,IAAM,EAAW,EAAqB,EAAW,EAAQ,CACnD,EAAU,EAAK,EAAW,aAAa,EAAU,MAAM,EAAG,EAAE,GAAG,CACrE,GAAI,EAAW,EAAQ,CACrB,MAAU,MACR,GAAG,EAAQ,uEACZ,CAOH,OAJA,EAAU,EAAK,EAAS,UAAU,CAAE,CAAE,UAAW,GAAM,CAAC,CACxD,EAAc,EAAK,EAAS,YAAY,CAAE,GAAe,EAAS,CAAE,OAAO,CAC3E,EAAc,EAAK,EAAS,gBAAgB,CAAE,GAAa,EAAS,CAAE,OAAO,CAEtE,CAAE,UAAS,WAAU,CAG9B,SAAS,GAAiB,EAAuC,CAC/D,IAAM,EAAO,EAAc,EAAU,CACrC,OAAO,EAAW,EAAK,CAAG,EAAa,EAAM,OAAO,CAAG,IAAA"}

package/dist/index.d.ts

@@ -5,8 +5,14 @@
  * 设计约束（RFC-316 规则 6）：本包只消费 otto 的**公开入口**——headless CLI 的 JSON 输出、
  * trace JSONL、录制文件。类型里不出现任何引擎内部结构，保证评测层永远不需要引擎开分支。
  */
-/** 判分方式（RFC-316 D3 判分优先级的落地形态）。 */
-type JudgeKind = /** 可执行判据：verify 脚本 exit 0 = pass。coding 任务首选。 */'script' /** golden 断言：最终输出包含/等于期望，或工具调用序列匹配。 */ | 'golden' /** 无机器判据：只评 liveness（跑完没崩），不进质量分。 */ | 'liveness';
+/**
+ * 判分方式（RFC-316 D3 判分优先级的落地形态）。
+ *
+ * 这是**可扩展集合**：判分实现经 `registerVerifier` 注册（见 verify.ts），
+ * 新判分方式不改本联合即可加入。判分优先级：
+ * 可执行判据（script） > golden 断言 > liveness（不进质量分）。
+ */
+type JudgeKind = 'script' | 'golden' | 'liveness';
 /** 单次运行的四态结果（RFC-316 D6 规则 5：infra_error 不进 pass 率）。
  *  RFC-353 S9 新增 'skipped_budget'：预算耗尽跳过的运行——"未尝试"而非"infra 故障"，
  *  必须与真实 infra 故障区分（否则本地预算决策会污染 infraErrorRate、误判套件失效）。 */
@@ -18,17 +24,16 @@ interface GoldenSpec {
   /** 期望出现的工具名序列（子序列匹配：允许中间夹杂其他工具）。 */
   toolSequence?: readonly string[];
 }
-/** 单个 case 的规格（`evals/cases/<id>/case.yaml` 的解析结果）。 */
-interface CaseSpec {
+/** 所有判分方式共享的公共字段。 */
+interface CaseCommon {
   /** 稳定 case 标识，等于目录名。 */
   id: string;
   /** 喂给 otto 的提示词原文。 */
   prompt: string;
-  /** 判分方式。 */
-  judge: JudgeKind;
-  /** judge='script' 时的判分脚本相对路径（相对 case 目录），默认 `verify.sh`。 */
-  verifyScript?: string;
-  /** judge='golden' 时的断言。 */
+  /**
+   * golden 断言与任何判分方式正交：`script` 判分可附加 golden（工具纪律 +
+   * 产物正确性两个判据都要满足）；`golden` 判分时 golden 为必填主判据。
+   */
   golden?: GoldenSpec;
   /** 重复运行次数（RFC-316 规则 3：Tier B 单次不作结论）。 */
   runs: number;
@@ -40,7 +45,29 @@ interface CaseSpec {
   dir: string;
   /** 是否有 fixture/ 目录需要复制成工作区。 */
   hasFixture: boolean;
+  /** 是否有 setup.sh 前置钩子（RFC-417 M2）：fixture 复制后、agent 运行前在工作区执行。 */
+  hasSetup: boolean;
+}
+/** judge='script'：可执行判据，verify 脚本 exit 0 = pass。coding 任务首选。 */
+interface ScriptCase extends CaseCommon {
+  judge: 'script';
+  /** 判分脚本相对路径（相对 case 目录），默认 `verify.sh`。 */
+  verifyScript: string;
 }
+/** judge='golden'：最终输出包含/等于期望，或工具调用序列匹配。 */
+interface GoldenCase extends CaseCommon {
+  judge: 'golden';
+  golden: GoldenSpec;
+}
+/** judge='liveness'：无机器判据，只评 liveness（跑完没崩），不进质量分。 */
+interface LivenessCase extends CaseCommon {
+  judge: 'liveness';
+}
+/**
+ * 单个 case 的规格（`evals/cases/<id>/case.yaml` 的解析结果）。
+ * 判别联合：判分方式专属字段只在对应分支上存在，消费方无需运行时兜底。
+ */
+type CaseSpec = ScriptCase | GoldenCase | LivenessCase;
 /** 从 trace JSONL 聚合出的运行指标（RFC-316 G3 运行指标层）。 */
 interface RunMetrics {
   /** 模型往返轮数（turn.end 计数）。 */
@@ -67,6 +94,8 @@ interface RunResult {
   /** 失败/错误原因（人类可读，报告直出）。 */
   detail?: string;
   metrics?: RunMetrics;
+  /** 产出本次运行的 otto 版本指纹（RFC-417 M1）；旧 otto 无字段时 'unknown'。 */
+  ottoVersion?: string;
 }
 /** 一个 case 跑 n 次后的汇总。 */
 interface CaseResult {
@@ -93,6 +122,11 @@ interface Scorecard {
   suiteVersion: string;
   /** 本次运行的模型标识（基线分键用）；无法确定时 'unknown'。 */
   modelId: string;
+  /**
+   * 产出本次评测的 otto 版本指纹（RFC-417 M1，含 build id）；无法确定时 'unknown'。
+   * **不参与基线分键**——只用于报告展示与跨版本 finding 标注，避免 alpha 高频发版致永久 no_baseline。
+   */
+  ottoVersion: string;
   /** 运行开始时刻（ISO）。 */
   startedAt: string;
   cases: readonly CaseResult[];
@@ -119,6 +153,11 @@ interface BaselineEntry {
   recordedAt: string;
   /** 更新基线的显式理由（RFC-316 D6：基线更新是带理由的显式动作）。 */
   reason: string;
+  /**
+   * 记录本基线的 otto 版本指纹（RFC-417 M1）；不参与分键（键仍是 modelId+suiteVersion），
+   * 仅用于 judgeRegression 在版本不一致时给出 finding 提示。旧基线无此字段。
+   */
+  ottoVersion?: string;
   overallPassRate?: number;
   /** 逐 case pass 率快照，用于 case flip 判定。 */
   casePassRates: Readonly<Record<string, number | undefined>>;
@@ -145,8 +184,9 @@ declare function loadSuite(casesRoot: string): readonly CaseSpec[];
 /**
  * 套件版本 = 全部 case 判分相关字段的内容指纹（基线分键用）。
  *
- * 只纳入**影响判分口径**的字段：case 集合、prompt、judge、golden。改超时/runs 不算换套件
- * （不影响可比性），改 prompt 或判据则必须换基线键——否则会把两套不同题目的分数当同一条曲线看。
+ * 只纳入**影响判分口径**的字段：case 集合、prompt、judge、judge 专属配置
+ * （verifyScript）、golden。改超时/runs 不算换套件（不影响可比性），
+ * 改 prompt 或判据则必须换基线键——否则会把两套不同题目的分数当同一条曲线看。
  */
 declare function computeSuiteVersion(cases: readonly CaseSpec[]): string;
 //#endregion
@@ -160,6 +200,22 @@ interface EvalWorkspace {
 /** 为一次运行创建工作区：有 fixture 则复制其内容，无 fixture 则空目录。 */
 declare function createWorkspace(spec: CaseSpec): EvalWorkspace;
 //#endregion
+//#region src/setup.d.ts
+/** setup 脚本相对 case 目录的固定文件名。 */
+declare const SETUP_SCRIPT = "setup.sh";
+/** setup 执行的墙钟上限（毫秒）。准备步骤（装依赖/起服务）可能偏慢，独立于判分脚本超时。 */
+declare const SETUP_TIMEOUT_MS = 60000;
+interface SetupResult {
+  ok: boolean;
+  /** 失败时携带 exit code + stderr 摘要（成功时为简短 ok 说明）。 */
+  detail: string;
+}
+/**
+ * 在工作区内执行 case 的 setup.sh（若存在）。无 setup 直接返回 ok。
+ * 脚本路径取自 case 目录，cwd 为隔离工作区——与 verify.sh 同一执行模型。
+ */
+declare function runSetup(spec: CaseSpec, workspaceDir: string): SetupResult;
+//#endregion
 //#region src/otto-invoker.d.ts
 /**
  * otto-invoker.ts —— 调用 headless otto 的唯一出口（RFC-316 规则 6：只走公开入口）。
@@ -173,6 +229,8 @@ interface OttoJsonResult {
   status: string;
   messageCount: number;
   response: string;
+  /** 产出本结果的 otto 版本指纹（含 build id）。RFC-417 M1：旧 otto 无此字段时兜底 'unknown'。 */
+  ottoVersion?: string;
 }
 interface InvokeRequest {
   prompt: string;
@@ -248,16 +306,40 @@ interface VerifyOutput {
   passed: boolean;
   detail: string;
 }
+/** 一种判分方式的端口。 */
+interface Verifier {
+  /** 判分方式名（case.yaml 的 judge 字段值）。 */
+  readonly name: string;
+  verify(input: VerifyInput): Promise<VerifyOutput>;
+}
+/**
+ * 注册一种判分方式（同名覆盖）。case-loader 会按注册表校验 judge 值。
+ *
+ * RFC-417 D2：本注册表目前是**薄骨架**，仅内置 script/golden/liveness 三种，无外部消费方。
+ * 保留它的理由是让"判分方式集合"只定义一处（case-loader 校验与 verifyRun 分派共用单一权威），
+ * 而非纯转发层。反悔条件：若结案 +6 个月内无第二种判分方式的真实 case 需求（见 T417-followup），
+ * 应评估把注册表压回硬编码白名单 + sync 分派，消除无消费的扩展点——不让骨架永久驻留。
+ */
+declare function registerVerifier(verifier: Verifier): void;
+/** 按名字取判分器；未注册返回 undefined。 */
+declare function getVerifier(name: string): Verifier | undefined;
+/** 当前已注册的全部判分方式名（错误信息用）。 */
+declare function knownVerifiers(): readonly string[];
 /** 子序列匹配：期望序列的每个工具名按顺序出现即可，允许中间夹杂其他调用。 */
 declare function matchesToolSubsequence(actual: readonly string[], expected: readonly string[]): boolean;
+/** 内置判分器：可执行判据（script）。 */
+declare const scriptVerifier: Verifier;
+/** 内置判分器：golden 断言（answerContains / toolSequence）。 */
+declare const goldenVerifier: Verifier;
+/** 内置判分器：liveness——只评「跑完没崩」，永不产生质量分。 */
+declare const livenessVerifier: Verifier;
 /**
- * 执行判分。script 判据在工作区内跑，exit 0 = pass。
- *
- * script + golden 可**同时**存在且必须都满足：判分方式表达的是"这个 case 靠什么下结论"，
- * 而工具纪律（golden.toolSequence）与产物正确性（script）是**两个正交的判据**——
+ * 执行判分：按 case 的 judge 取注册的判分器，golden 断言作为**正交判据**可同时存在
+ * 且必须都满足——判分方式表达的是"这个 case 靠什么下结论"，而工具纪律
+ * （golden.toolSequence）与产物正确性（script）是**两个正交的判据**——
  * 只有前者会让"读了但改错"蒙混过关，只有后者会让"盲改碰对"蒙混过关。
  */
-declare function verifyRun(input: VerifyInput): VerifyOutput;
+declare function verifyRun(input: VerifyInput): Promise<VerifyOutput>;
 //#endregion
 //#region src/runner.d.ts
 /**
@@ -396,5 +478,5 @@ declare function writeGolden(scenario: TierAScenario, result: TierAResult): void
 /** 与 golden 逐面对比（断言三件套的执行处）。 */
 declare function compareToGolden(scenario: TierAScenario, result: TierAResult): TierAFinding;
 //#endregion
-export { type BaselineEntry, type BaselineFile, type CaseResult, type CaseSpec, type EvalWorkspace, type ExtractResult, type GoldenSpec, INFRA_ERROR_INVALID_THRESHOLD, type InvokeRequest, type InvokeResponse, type JudgeKind, type OttoInvoker, type OttoJsonResult, PASS_RATE_REGRESSION_PP, type RegressionVerdict, type RunMetrics, type RunOutcome, type RunResult, type RunnerDeps, type Scorecard, type SessionSkeleton, type SpawnInvokerOptions, type TierAFinding, type TierAResult, type TierAScenario, type ToolCallRecord, type TraceRun, type VerifyInput, type VerifyOutput, averageMetrics, buildScorecard, classifyFailure, compareToGolden, computeSuiteVersion, createSpawnInvoker, createWorkspace, extractCase, findBaseline, goldenPath, judgeRegression, loadBaselines, loadCase, loadScenario, loadScenarioSuite, loadSuite, matchesToolSubsequence, parseSessionSkeleton, parseTraceRun, readTraceRun, renderReport, runCase, runScenario, runSuite, saveBaselines, summarizeCase, traceFilePath, upsertBaseline, verifyRun, writeGolden };
+export { type BaselineEntry, type BaselineFile, type CaseResult, type CaseSpec, type EvalWorkspace, type ExtractResult, type GoldenSpec, INFRA_ERROR_INVALID_THRESHOLD, type InvokeRequest, type InvokeResponse, type JudgeKind, type OttoInvoker, type OttoJsonResult, PASS_RATE_REGRESSION_PP, type RegressionVerdict, type RunMetrics, type RunOutcome, type RunResult, type RunnerDeps, SETUP_SCRIPT, SETUP_TIMEOUT_MS, type Scorecard, type SessionSkeleton, type SetupResult, type SpawnInvokerOptions, type TierAFinding, type TierAResult, type TierAScenario, type ToolCallRecord, type TraceRun, type Verifier, type VerifyInput, type VerifyOutput, averageMetrics, buildScorecard, classifyFailure, compareToGolden, computeSuiteVersion, createSpawnInvoker, createWorkspace, extractCase, findBaseline, getVerifier, goldenPath, goldenVerifier, judgeRegression, knownVerifiers, livenessVerifier, loadBaselines, loadCase, loadScenario, loadScenarioSuite, loadSuite, matchesToolSubsequence, parseSessionSkeleton, parseTraceRun, readTraceRun, registerVerifier, renderReport, runCase, runScenario, runSetup, runSuite, saveBaselines, scriptVerifier, summarizeCase, traceFilePath, upsertBaseline, verifyRun, writeGolden };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/types.ts","../src/case-loader.ts","../src/workspace.ts","../src/otto-invoker.ts","../src/trace-metrics.ts","../src/verify.ts","../src/runner.ts","../src/baseline.ts","../src/report.ts","../src/extract.ts","../src/tier-a.ts"],"mappings":";;AAQA;;;;;AAWA;AAAA,KAXY,SAAA;;;AAsBZ;KAXY,UAAA;;UAGK,UAAA;EAUf;EARA,cAAA;EAYA;EAVA,YAAA;AAAA;;UAIe,QAAA;EAYf;EAVA,EAAA;EAcA;EAZA,MAAA;EAgBA;EAdA,KAAA,EAAO,SAAA;EAcG;EAZV,YAAA;EAgByB;EAdzB,MAAA,GAAS,UAAA;EAcgB;EAZzB,IAAA;EAgBA;EAdA,SAAA;EAiBA;EAfA,SAAA;EAiBA;EAfA,GAAA;EAmBA;EAjBA,UAAA;AAAA;AAqBF;AAAA,UAjBiB,UAAA;;EAEf,KAAA;EAgBA;EAdA,SAAA;EAgBA;EAdA,UAAA;EACA,WAAA;EACA,YAAA;EACA,eAAA;EAgBoB;EAdpB,OAAA;EAkBe;EAhBf,cAAA;AAAA;;UAIe,SAAA;EACf,OAAA,EAAS,UAAA;EAaF;EAXP,MAAA;EAYe;EAVf,SAAA;EAcA;EAZA,MAAA;EACA,OAAA,GAAU,UAAA;AAAA;;UAIK,UAAA;EACf,MAAA;EACA,KAAA,EAAO,SAAA;EACP,IAAA,WAAe,SAAA;;EAEf,MAAA;EAiBA;EAfA,SAAA;EAmBA;EAjBA,WAAA;EAkBgB;;EAfhB,aAAA;EAqBA;EAnBA,QAAA;EAwBA;EAtBA,aAAA;AAAA;;UAIe,SAAA;EAsBc;EApB7B,YAAA;EAwB4B;EAtB5B,OAAA;EA8BuB;EA5BvB,SAAA;EACA,KAAA,WAAgB,UAAA;EAsBhB;EApBA,WAAA;EAuBA;EArBA,aAAA;EAuBe;EArBf,eAAA;EAqB8B;EAnB9B,cAAA;EAsBe;;EAnBf,kBAAA;EAoBA;EAlBA,OAAA;EAsBe;EApBf,gBAAA,GAAmB,UAAA;AAAA;;UAIJ,aAAA;EACf,OAAA;EACA,YAAA;EACA,UAAA;EAoBY;EAlBZ,MAAA;EACA,eAAA;;EAEA,aAAA,EAAe,QAAA,CAAS,MAAA;AAAA;AAAA,UAGT,YAAA;EACf,OAAA,WAAkB,aAAA;AAAA;;UAIH,iBAAA;EACf,MAAA;;EAEA,QAAA;ECpB6D;EDsB7D,eAAA;ECJiC;EDMjC,YAAA;AAAA;;;;iBC/Ec,QAAA,CAAS,GAAA,WAAc,QAAA;;iBAuDvB,SAAA,CAAU,SAAA,oBAA6B,QAAA;AD1GvD;;;;;AAQA;AARA,iBC4HgB,mBAAA,CAAoB,KAAA,WAAgB,QAAA;;;UCpInC,aAAA;EFKK;EEHpB,GAAA;EFMe;EEJf,OAAA;AAAA;;iBAIc,eAAA,CAAgB,IAAA,EAAM,QAAA,GAAW,aAAA;;;;AFdjD;;;;;AAWA;AAAA,UGTiB,cAAA;EACf,SAAA;EACA,MAAA;EACA,YAAA;EACA,QAAA;AAAA;AAAA,UAGe,aAAA;EACf,MAAA;EHQY;EGNZ,GAAA;EACA,SAAA;AAAA;AAAA,KAGU,cAAA;EACN,IAAA;EAAY,MAAA,EAAQ,cAAA;AAAA;EAEpB,IAAA;EAAgB,MAAA;AAAA;;;;;;EAKhB,IAAA;EAAe,MAAA;AAAA;;UAGJ,WAAA;EACf,MAAA,CAAO,OAAA,EAAS,aAAA,GAAgB,OAAA,CAAQ,cAAA;AAAA;;;;;;;iBAS1B,eAAA,CAAgB,MAAA;AAAA,UAqBf,mBAAA;EHKA;EGHf,GAAA;;EAEA,GAAA,GAAM,QAAA,CAAS,MAAA;AAAA;;iBAID,kBAAA,CAAmB,OAAA,GAAS,mBAAA,GAA2B,WAAA;;;AHpDvE;AAAA,iBIOgB,aAAA,CAAc,SAAA;;UAKb,QAAA;EACf,OAAA,EAAS,UAAA;EJLM;EIOf,SAAA;AAAA;;;;;iBAOc,aAAA,CAAc,OAAA,WAAkB,QAAA;;iBA6DhC,YAAA,CAAa,SAAA,WAAoB,QAAA;;iBASjC,cAAA,CAAe,OAAA,WAAkB,UAAA,KAAe,UAAA;;;UCrG/C,WAAA;EACf,IAAA,EAAM,QAAA;ELKc;EKHpB,YAAA;ELMyB;EKJzB,QAAA;ELMA;EKJA,SAAA;AAAA;AAAA,UAGe,YAAA;EACf,MAAA;EACA,MAAA;AAAA;;iBAIc,sBAAA,CACd,MAAA,qBACA,QAAA;;;;;;;;iBAyDc,SAAA,CAAU,KAAA,EAAO,WAAA,GAAc,YAAA;;;;;;;AL1D/C;cMLa,6BAAA;AAAA,UAKI,UAAA;EACf,OAAA,EAAS,WAAA;ENCT;EMCA,SAAA,IAAa,SAAA,aAAsB,QAAA;ENGnC;EMDA,GAAA;ENGA;EMDA,aAAA,IAAiB,MAAA,UAAgB,KAAA,UAAe,MAAA,EAAQ,SAAA;AAAA;;iBAWpC,OAAA,CAAQ,IAAA,EAAM,QAAA,EAAU,IAAA,EAAM,UAAA,GAAa,OAAA,CAAQ,UAAA;;iBA0GzD,aAAA,CAAc,IAAA,EAAM,QAAA,EAAU,IAAA,WAAe,SAAA,KAAc,UAAA;;iBAwBrD,QAAA,CACpB,KAAA,WAAgB,QAAA,IAChB,YAAA,UACA,IAAA,EAAM,UAAA,GACL,OAAA,CAAQ,SAAA;;iBAWK,cAAA,CACd,KAAA,WAAgB,UAAA,IAChB,YAAA,UACA,SAAA,WACC,SAAA;;;;AN/KH;;;;cOAa,uBAAA;AAAA,iBAEG,aAAA,CAAc,IAAA,WAAe,YAAA;AAAA,iBAQ7B,aAAA,CAAc,IAAA,UAAc,IAAA,EAAM,YAAA;;iBAMlC,YAAA,CACd,IAAA,EAAM,YAAA,EACN,OAAA,UACA,YAAA,WACC,aAAA;;iBAKa,cAAA,CACd,IAAA,EAAM,YAAA,EACN,SAAA,EAAW,SAAA,EACX,MAAA,WACC,YAAA;;iBAwBa,eAAA,CACd,SAAA,EAAW,SAAA,EACX,QAAA,EAAU,aAAA,eACT,iBAAA;;;iBCjEa,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAA,EAAS,iBAAA;;;;ARL5D;;;;;AAWA;;;;;AAGA;;;USGiB,eAAA;EACf,SAAA;ETIe;ESFf,MAAA;;EAEA,YAAA;ETEA;ESAA,YAAA;ETIA;ESFA,KAAA;AAAA;;iBAMc,oBAAA,CAAqB,SAAA,UAAmB,OAAA,WAAkB,eAAA;AAAA,UAkHzD,aAAA;EACf,OAAA;EACA,QAAA,EAAU,eAAA;AAAA;;iBAII,WAAA,CACd,SAAA,UACA,SAAA,UACA,gBAAA,IAAmB,SAAA,kCAClB,aAAA;;;;AT5JH;;;;;AAWA;;;;;AAGA;;;;;UUaiB,aAAA;EACf,EAAA;;EAEA,MAAA;EVNA;EUQA,KAAA;EVJA;EUMA,SAAA;EACA,GAAA;EACA,UAAA;AAAA;AAAA,UAGe,cAAA;EACf,IAAA;EACA,SAAA,EAAW,MAAA;AAAA;;UAII,WAAA;EACf,UAAA;EACA,SAAA,WAAoB,cAAA;EVDK;EUGzB,SAAA,EAAW,QAAA,CAAS,MAAA;EACpB,OAAA;IACE,UAAA;IACA,QAAA;EAAA;AAAA;AAAA,UAIa,YAAA;EACf,UAAA;EACA,OAAA;EACA,OAAA;AAAA;;iBAMc,YAAA,CAAa,GAAA,WAAc,aAAA;AAAA,iBAmD3B,iBAAA,CAAkB,cAAA,oBAAkC,aAAA;;iBAyG9C,WAAA,CAAY,QAAA,EAAU,aAAA,GAAgB,OAAA,CAAQ,WAAA;AAAA,iBA+DpD,UAAA,CAAW,QAAA,EAAU,aAAA;AAAA,iBAIrB,WAAA,CAAY,QAAA,EAAU,aAAA,EAAe,MAAA,EAAQ,WAAA;;iBAK7C,eAAA,CAAgB,QAAA,EAAU,aAAA,EAAe,MAAA,EAAQ,WAAA,GAAc,YAAA"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/types.ts","../src/case-loader.ts","../src/workspace.ts","../src/setup.ts","../src/otto-invoker.ts","../src/trace-metrics.ts","../src/verify.ts","../src/runner.ts","../src/baseline.ts","../src/report.ts","../src/extract.ts","../src/tier-a.ts"],"mappings":";;AAcA;;;;;AAKA;;;;;AAGA;;AAHA,KALY,SAAA;;;AAaX;KARW,UAAA;;UAGK,UAAA;EAUf;EARA,cAAA;EAeA;EAbA,YAAA;AAAA;;UAIQ,UAAA;EAiBR;EAfA,EAAA;EAmBA;EAjBA,MAAA;EAiBQ;AAIV;;;EAhBE,MAAA,GAAS,UAAA;EAgByB;EAdlC,IAAA;EAiBA;EAfA,SAAA;EAeY;EAbZ,SAAA;EAiB0B;EAf1B,GAAA;EAe4C;EAb5C,UAAA;EAcA;EAZA,QAAA;AAAA;;UAIe,UAAA,SAAmB,UAAA;EAClC,KAAA;EAY4B;EAV5B,YAAA;AAAA;;UAIe,UAAA,SAAmB,UAAA;EAClC,KAAA;EACA,MAAA,EAAQ,UAAA;AAAA;;UAIO,YAAA,SAAqB,UAAA;EACpC,KAAA;AAAA;;;;;KAOU,QAAA,GAAW,UAAA,GAAa,UAAA,GAAa,YAAA;AAGjD;AAAA,UAAiB,UAAA;;EAEf,KAAA;EAAA;EAEA,SAAA;EAEA;EAAA,UAAA;EACA,WAAA;EACA,YAAA;EACA,eAAA;EAIA;EAFA,OAAA;EAEc;EAAd,cAAA;AAAA;;UAIe,SAAA;EACf,OAAA,EAAS,UAAA;EAAA;EAET,MAAA;EAEA;EAAA,SAAA;EAGA;EADA,MAAA;EACA,OAAA,GAAU,UAAA;EAEC;EAAX,WAAA;AAAA;;UAIe,UAAA;EACf,MAAA;EACA,KAAA,EAAO,SAAA;EACP,IAAA,WAAe,SAAA;EADR;EAGP,MAAA;EAFe;EAIf,SAAA;EAAA;EAEA,WAAA;EAGA;;EAAA,aAAA;EAIa;EAFb,QAAA;EAMe;EAJf,aAAA;AAAA;;UAIe,SAAA;EAIf;EAFA,YAAA;EASA;EAPA,OAAA;EAQgB;;;;EAHhB,WAAA;EAcA;EAZA,SAAA;EACA,KAAA,WAAgB,UAAA;EAeG;EAbnB,WAAA;EAa6B;EAX7B,aAAA;EAe4B;EAb5B,eAAA;EA0BuB;EAxBvB,cAAA;EAaA;;EAVA,kBAAA;EAkBA;EAhBA,OAAA;EAmBA;EAjBA,gBAAA,GAAmB,UAAA;AAAA;;UAIJ,aAAA;EACf,OAAA;EACA,YAAA;EACA,UAAA;EAcA;EAZA,MAAA;EAgBe;;;;EAXf,WAAA;EACA,eAAA;EAeA;EAbA,aAAA,EAAe,QAAA,CAAS,MAAA;AAAA;AAAA,UAGT,YAAA;EACf,OAAA,WAAkB,aAAA;AAAA;;UAIH,iBAAA;EACf,MAAA;;EAEA,QAAA;EC9G6C;EDgH7C,eAAA;ECpDuB;EDsDvB,YAAA;AAAA;;;;iBClHc,QAAA,CAAS,GAAA,WAAc,QAAA;ADrDvC;AAAA,iBCiHgB,SAAA,CAAU,SAAA,oBAA6B,QAAA;;;;AD5GtD;;;;iBC+He,mBAAA,CAAoB,KAAA,WAAgB,QAAA;;;UC5InC,aAAA;EFKK;EEHpB,GAAA;EFMe;EEJf,OAAA;AAAA;;iBAIc,eAAA,CAAgB,IAAA,EAAM,QAAA,GAAW,aAAA;;;AFKhD;AAAA,cGLY,YAAA;;cAGA,gBAAA;AAAA,UAEI,WAAA;EACf,EAAA;EHWA;EGTA,MAAA;AAAA;;;;;iBAOc,QAAA,CAAS,IAAA,EAAM,QAAA,EAAU,YAAA,WAAuB,WAAA;;;;AHvBhE;;;;;AAKA;AAAA,UITiB,cAAA;EACf,SAAA;EACA,MAAA;EACA,YAAA;EACA,QAAA;EJQyB;EINzB,WAAA;AAAA;AAAA,UAGe,aAAA;EACf,MAAA;EJUkB;EIRlB,GAAA;EACA,SAAA;AAAA;AAAA,KAGU,cAAA;EACN,IAAA;EAAY,MAAA,EAAQ,cAAA;AAAA;EAEpB,IAAA;EAAgB,MAAA;AAAA;;;AJ0BtB;;;EIrBM,IAAA;EAAe,MAAA;AAAA;;UAGJ,WAAA;EACf,MAAA,CAAO,OAAA,EAAS,aAAA,GAAgB,OAAA,CAAQ,cAAA;AAAA;;;;;;;iBAS1B,eAAA,CAAgB,MAAA;AAAA,UAqBf,mBAAA;EJJG;EIMlB,GAAA;EJF4B;EII5B,GAAA,GAAM,QAAA,CAAS,MAAA;AAAA;;iBAID,kBAAA,CAAmB,OAAA,GAAS,mBAAA,GAA2B,WAAA;;;AJtDvE;AAAA,iBKOgB,aAAA,CAAc,SAAA;;UAKb,QAAA;EACf,OAAA,EAAS,UAAA;ELLD;EKOR,SAAA;AAAA;;;;;iBAOc,aAAA,CAAc,OAAA,WAAkB,QAAA;;iBA6DhC,YAAA,CAAa,SAAA,WAAoB,QAAA;;iBASjC,cAAA,CAAe,OAAA,WAAkB,UAAA,KAAe,UAAA;;;UChG/C,WAAA;EACf,IAAA,EAAM,QAAA;ENOM;EMLZ,YAAA;ENSkB;EMPlB,QAAA;ENgBmB;EMdnB,SAAA;AAAA;AAAA,UAGe,YAAA;EACf,MAAA;EACA,MAAA;AAAA;;UAIe,QAAA;ENef;EAAA,SMbS,IAAA;EACT,MAAA,CAAO,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;AAAA;ANkBtC;;;;;;;;AAAA,iBMLgB,gBAAA,CAAiB,QAAA,EAAU,QAAA;ANY3C;AAAA,iBMPgB,WAAA,CAAY,IAAA,WAAe,QAAA;;iBAK3B,cAAA,CAAA;;iBAKA,sBAAA,CACd,MAAA,qBACA,QAAA;;cAmDW,cAAA,EAAgB,QAAA;;cAYhB,cAAA,EAAgB,QAAA;;cAQhB,gBAAA,EAAkB,QAAA;;;AN9D/B;;;;iBM+EsB,SAAA,CAAU,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;;;;;;;ANhI5D;cODY,6BAAA;AAAA,UAKI,UAAA;EACf,OAAA,EAAS,WAAA;EPAT;EOEA,SAAA,IAAa,SAAA,aAAsB,QAAA;EPKnC;EOHA,GAAA;EPKA;EOHA,aAAA,IAAiB,MAAA,UAAgB,KAAA,UAAe,MAAA,EAAQ,SAAA;AAAA;;iBAWpC,OAAA,CAAQ,IAAA,EAAM,QAAA,EAAU,IAAA,EAAM,UAAA,GAAa,OAAA,CAAQ,UAAA;;iBAoHzD,aAAA,CAAc,IAAA,EAAM,QAAA,EAAU,IAAA,WAAe,SAAA,KAAc,UAAA;;iBAwBrD,QAAA,CACpB,KAAA,WAAgB,QAAA,IAChB,YAAA,UACA,IAAA,EAAM,UAAA,GACL,OAAA,CAAQ,SAAA;;iBAWK,cAAA,CACd,KAAA,WAAgB,UAAA,IAChB,YAAA,UACA,SAAA,WACC,SAAA;;;;AP1LH;;;;cQAa,uBAAA;AAAA,iBAEG,aAAA,CAAc,IAAA,WAAe,YAAA;AAAA,iBAQ7B,aAAA,CAAc,IAAA,UAAc,IAAA,EAAM,YAAA;;iBAMlC,YAAA,CACd,IAAA,EAAM,YAAA,EACN,OAAA,UACA,YAAA,WACC,aAAA;;iBAKa,cAAA,CACd,IAAA,EAAM,YAAA,EACN,SAAA,EAAW,SAAA,EACX,MAAA,WACC,YAAA;;iBAyBa,eAAA,CACd,SAAA,EAAW,SAAA,EACX,QAAA,EAAU,aAAA,eACT,iBAAA;;;iBClEa,YAAA,CAAa,SAAA,EAAW,SAAA,EAAW,OAAA,EAAS,iBAAA;;;;ATC5D;;;;;AAKA;;;;;AAGA;;;UUGiB,eAAA;EACf,SAAA;EVIQ;EUFR,MAAA;;EAEA,YAAA;EVEA;EUAA,YAAA;EVOA;EULA,KAAA;AAAA;;iBAMc,oBAAA,CAAqB,SAAA,UAAmB,OAAA,WAAkB,eAAA;AAAA,UAkHzD,aAAA;EACf,OAAA;EACA,QAAA,EAAU,eAAA;AAAA;;iBAII,WAAA,CACd,SAAA,UACA,SAAA,UACA,gBAAA,IAAmB,SAAA,kCAClB,aAAA;;;;AVtJH;;;;;AAKA;;;;;AAGA;;;;;UWaiB,aAAA;EACf,EAAA;;EAEA,MAAA;EXNA;EWQA,KAAA;EXDA;EWGA,SAAA;EACA,GAAA;EACA,UAAA;AAAA;AAAA,UAGe,cAAA;EACf,IAAA;EACA,SAAA,EAAW,MAAA;AAAA;;UAII,WAAA;EACf,UAAA;EACA,SAAA,WAAoB,cAAA;EXAwB;EWE5C,SAAA,EAAW,QAAA,CAAS,MAAA;EACpB,OAAA;IACE,UAAA;IACA,QAAA;EAAA;AAAA;AAAA,UAIa,YAAA;EACf,UAAA;EACA,OAAA;EACA,OAAA;AAAA;;iBAMc,YAAA,CAAa,GAAA,WAAc,aAAA;AAAA,iBAmD3B,iBAAA,CAAkB,cAAA,oBAAkC,aAAA;;iBAyG9C,WAAA,CAAY,QAAA,EAAU,aAAA,GAAgB,OAAA,CAAQ,WAAA;AAAA,iBAgEpD,UAAA,CAAW,QAAA,EAAU,aAAA;AAAA,iBAIrB,WAAA,CAAY,QAAA,EAAU,aAAA,EAAe,MAAA,EAAQ,WAAA;;iBAK7C,eAAA,CAAgB,QAAA,EAAU,aAAA,EAAe,MAAA,EAAQ,WAAA,GAAc,YAAA"}

package/dist/index.js

@@ -1 +1 @@
-import{C as e,S as t,T as n,_ as r,a as i,b as a,c as o,d as s,f as c,g as l,h as u,i as d,l as f,m as p,n as m,o as h,p as g,r as _,s as v,t as y,u as b,v as x,w as S,x as C,y as w}from"./extract-CJuoP2bV.js";import{t as T}from"./workspace-C_kTWfF8.js";import{a as E,i as D,n as O,o as k,r as A,t as j}from"./tier-a-DvMUFMGy.js";export{b as INFRA_ERROR_INVALID_THRESHOLD,d as PASS_RATE_REGRESSION_PP,r as averageMetrics,s as buildScorecard,C as classifyFailure,j as compareToGolden,e as computeSuiteVersion,t as createSpawnInvoker,T as createWorkspace,y as extractCase,i as findBaseline,O as goldenPath,h as judgeRegression,v as loadBaselines,S as loadCase,A as loadScenario,D as loadScenarioSuite,n as loadSuite,u as matchesToolSubsequence,m as parseSessionSkeleton,x as parseTraceRun,w as readTraceRun,_ as renderReport,c as runCase,E as runScenario,g as runSuite,o as saveBaselines,p as summarizeCase,a as traceFilePath,f as upsertBaseline,l as verifyRun,k as writeGolden};
+import{A as e,C as t,D as n,E as r,M as i,N as a,O as o,P as s,S as c,T as l,_ as u,a as d,b as f,c as p,d as m,f as h,g,h as _,i as v,j as y,k as b,l as x,m as S,n as C,o as w,p as T,r as E,s as D,t as O,u as k,v as A,w as j,x as M,y as N}from"./extract-BmkiRXQY.js";import{t as P}from"./workspace-C_kTWfF8.js";import{a as F,i as I,n as L,o as R,r as z,t as B}from"./tier-a-DPSonK0s.js";export{k as INFRA_ERROR_INVALID_THRESHOLD,v as PASS_RATE_REGRESSION_PP,M as SETUP_SCRIPT,c as SETUP_TIMEOUT_MS,_ as averageMetrics,m as buildScorecard,N as classifyFailure,B as compareToGolden,j as computeSuiteVersion,f as createSpawnInvoker,P as createWorkspace,O as extractCase,d as findBaseline,n as getVerifier,L as goldenPath,o as goldenVerifier,w as judgeRegression,b as knownVerifiers,e as livenessVerifier,D as loadBaselines,l as loadCase,z as loadScenario,I as loadScenarioSuite,r as loadSuite,y as matchesToolSubsequence,C as parseSessionSkeleton,g as parseTraceRun,u as readTraceRun,i as registerVerifier,E as renderReport,h as runCase,F as runScenario,t as runSetup,T as runSuite,p as saveBaselines,a as scriptVerifier,S as summarizeCase,A as traceFilePath,x as upsertBaseline,s as verifyRun,R as writeGolden};

package/dist/{tier-a-DvMUFMGy.js → tier-a-DPSonK0s.js}

@@ -1,2 +1,2 @@
-import{t as e}from"./workspace-C_kTWfF8.js";import{createHash as t}from"node:crypto";import{existsSync as n,readFileSync as r,readdirSync as i,statSync as a,writeFileSync as o}from"node:fs";import{join as s,relative as c}from"node:path";import{parse as l}from"yaml";import{createAgent as u}from"@x-otto/agent";import{createLogger as d}from"@x-otto/shared";import{loadRecording as f,replayStream as p}from"@x-otto/provider";import{createEdit as m,createRead as h,createWrite as g}from"@x-otto/tools";const _=[`read`,`edit`,`write`];function v(e){let t=e.split(`/`).filter(Boolean).at(-1)??e,a=s(e,`scenario.yaml`);if(!n(a))throw Error(`scenario '${t}': missing scenario.yaml`);let o=l(r(a,`utf8`));if(typeof o!=`object`||!o)throw Error(`scenario '${t}': scenario.yaml must be a mapping`);let c=o,u=c.prompt;if(typeof u!=`string`||u.trim()===``)throw Error(`scenario '${t}': 'prompt' must be a non-empty string`);let d=c.tools;if(!Array.isArray(d)||d.length===0)throw Error(`scenario '${t}': 'tools' must be a non-empty array`);for(let e of d)if(!_.includes(e))throw Error(`scenario '${t}': tool '${String(e)}' not allowed — Tier A only permits deterministic fs tools (${_.join(`/`)})`);let f=s(e,`turns`);if(!n(f))throw Error(`scenario '${t}': missing turns/ directory`);let p=i(f).filter(e=>e.endsWith(`.jsonl`)).sort().map(e=>s(f,e));if(p.length===0)throw Error(`scenario '${t}': turns/ has no .jsonl recordings`);return{id:t,prompt:u,tools:d,turnFiles:p,dir:e,hasFixture:n(s(e,`fixture`))}}function y(e){if(!n(e))throw Error(`recordings root not found: ${e}`);return i(e).filter(e=>!e.startsWith(`.`)).map(t=>s(e,t)).filter(e=>a(e).isDirectory()).sort().map(v)}function b(e){let t=0;return{stream:(n,r)=>{let i=t++,a=e[i];if(!a)throw Error(`engine made LLM call #${i+1} but only ${e.length} turn(s) were recorded — engine behaviour diverged from the recording (or the recording is incomplete)`);let o,s=(async function*(){for await(let e of p(a,{fast:!0},r))e.type===`done`&&(o=e.message),yield e})();return Object.assign(s,{async result(){for await(let e of s);if(!o)throw Error(`recording ended without a done event`);return o}})},calls:()=>t}}function x(e,t,n){return{...e,execute:r=>(t.push({name:e.name,arguments:JSON.parse(JSON.stringify(r.params??{}).split(n).join(`<WS>`))}),e.execute(r))}}function S(e){let n={},a=o=>{for(let l of i(o,{withFileTypes:!0})){let i=s(o,l.name);if(l.isDirectory()){a(i);continue}n[c(e,i)]=t(`sha256`).update(r(i)).digest(`hex`).slice(0,16)}};return a(e),n}function C(e,t,n){let r={read:e=>h(e),edit:e=>m(e),write:e=>g(e)};return e.tools.map(e=>x(r[e](t),n,t))}async function w(t){let n=await Promise.all(t.turnFiles.map(e=>f(e))),r=n[0]?.header;if(!r)throw Error(`scenario '${t.id}': no recordings`);let i=e({id:t.id,prompt:t.prompt,judge:`liveness`,runs:1,timeoutMs:0,dir:t.dir,hasFixture:t.hasFixture});try{let e=[],a=C(t,i.dir,e),{stream:o,calls:s}=b(n),c=u({stream:o,logger:d(`tier-a`,{level:`fatal`,destination:`stdout`})}),l={model:{id:`replay/${r.modelId}`,name:`Replay of ${r.modelId}`,api:`replay`,provider:`replay`,contextWindow:r.contextWindow},systemPrompt:`tier-a replay`,messages:[{role:`user`,content:t.prompt,timestamp:0}],tools:a,thinkingLevel:`low`},f=await c.run(l);if(s()!==n.length)throw Error(`scenario '${t.id}': engine made ${s()} LLM call(s) but ${n.length} turn(s) were recorded — the leftover recordings were never consumed, which means the engine stopped earlier than when the recording was made (or the recording has extra turns).`);return{scenarioId:t.id,toolCalls:e,artifacts:S(i.dir),outcome:{stopReason:String(f.stopReason),llmCalls:s()}}}finally{i.dispose()}}function T(e){return s(e.dir,`golden.json`)}function E(e,t){o(T(e),`${JSON.stringify(t,null,2)}\n`,`utf8`)}function D(e,t){let i=T(e);if(!n(i))return{scenarioId:e.id,verdict:`no_golden`,details:["no golden.json — record one with `tier-a run --update`"]};let a=JSON.parse(r(i,`utf8`)),o=[],s=JSON.stringify(t.toolCalls),c=JSON.stringify(a.toolCalls);s!==c&&o.push(`tool calls diverged:\n  golden: ${c}\n  actual: ${s}`);let l=new Set([...Object.keys(a.artifacts),...Object.keys(t.artifacts)]);for(let e of[...l].sort()){let n=a.artifacts[e],r=t.artifacts[e];n!==r&&(n===void 0?o.push(`unexpected new artifact: ${e}`):r===void 0?o.push(`missing artifact: ${e}`):o.push(`artifact content changed: ${e} (${n} → ${r})`))}return t.outcome.stopReason!==a.outcome.stopReason&&o.push(`stopReason changed: ${a.outcome.stopReason} → ${t.outcome.stopReason}`),t.outcome.llmCalls!==a.outcome.llmCalls&&o.push(`LLM call count changed: ${a.outcome.llmCalls} → ${t.outcome.llmCalls}`),{scenarioId:e.id,verdict:o.length===0?`match`:`mismatch`,details:o.length===0?[`identical to golden`]:o}}export{w as a,y as i,T as n,E as o,v as r,D as t};
-//# sourceMappingURL=tier-a-DvMUFMGy.js.map
+import{t as e}from"./workspace-C_kTWfF8.js";import{createHash as t}from"node:crypto";import{existsSync as n,readFileSync as r,readdirSync as i,statSync as a,writeFileSync as o}from"node:fs";import{join as s,relative as c}from"node:path";import{parse as l}from"yaml";import{createAgent as u}from"@x-otto/agent";import{createLogger as d}from"@x-otto/shared";import{loadRecording as f,replayStream as p}from"@x-otto/provider";import{createEdit as m,createRead as h,createWrite as g}from"@x-otto/tools";const _=[`read`,`edit`,`write`];function v(e){let t=e.split(`/`).filter(Boolean).at(-1)??e,a=s(e,`scenario.yaml`);if(!n(a))throw Error(`scenario '${t}': missing scenario.yaml`);let o=l(r(a,`utf8`));if(typeof o!=`object`||!o)throw Error(`scenario '${t}': scenario.yaml must be a mapping`);let c=o,u=c.prompt;if(typeof u!=`string`||u.trim()===``)throw Error(`scenario '${t}': 'prompt' must be a non-empty string`);let d=c.tools;if(!Array.isArray(d)||d.length===0)throw Error(`scenario '${t}': 'tools' must be a non-empty array`);for(let e of d)if(!_.includes(e))throw Error(`scenario '${t}': tool '${String(e)}' not allowed — Tier A only permits deterministic fs tools (${_.join(`/`)})`);let f=s(e,`turns`);if(!n(f))throw Error(`scenario '${t}': missing turns/ directory`);let p=i(f).filter(e=>e.endsWith(`.jsonl`)).sort().map(e=>s(f,e));if(p.length===0)throw Error(`scenario '${t}': turns/ has no .jsonl recordings`);return{id:t,prompt:u,tools:d,turnFiles:p,dir:e,hasFixture:n(s(e,`fixture`))}}function y(e){if(!n(e))throw Error(`recordings root not found: ${e}`);return i(e).filter(e=>!e.startsWith(`.`)).map(t=>s(e,t)).filter(e=>a(e).isDirectory()).sort().map(v)}function b(e){let t=0;return{stream:(n,r)=>{let i=t++,a=e[i];if(!a)throw Error(`engine made LLM call #${i+1} but only ${e.length} turn(s) were recorded — engine behaviour diverged from the recording (or the recording is incomplete)`);let o,s=(async function*(){for await(let e of p(a,{fast:!0},r))e.type===`done`&&(o=e.message),yield e})();return Object.assign(s,{async result(){for await(let e of s);if(!o)throw Error(`recording ended without a done event`);return o}})},calls:()=>t}}function x(e,t,n){return{...e,execute:r=>(t.push({name:e.name,arguments:JSON.parse(JSON.stringify(r.params??{}).split(n).join(`<WS>`))}),e.execute(r))}}function S(e){let n={},a=o=>{for(let l of i(o,{withFileTypes:!0})){let i=s(o,l.name);if(l.isDirectory()){a(i);continue}n[c(e,i)]=t(`sha256`).update(r(i)).digest(`hex`).slice(0,16)}};return a(e),n}function C(e,t,n){let r={read:e=>h(e),edit:e=>m(e),write:e=>g(e)};return e.tools.map(e=>x(r[e](t),n,t))}async function w(t){let n=await Promise.all(t.turnFiles.map(e=>f(e))),r=n[0]?.header;if(!r)throw Error(`scenario '${t.id}': no recordings`);let i=e({id:t.id,prompt:t.prompt,judge:`liveness`,runs:1,timeoutMs:0,dir:t.dir,hasFixture:t.hasFixture,hasSetup:!1});try{let e=[],a=C(t,i.dir,e),{stream:o,calls:s}=b(n),c=u({stream:o,logger:d(`tier-a`,{level:`fatal`,destination:`stdout`})}),l={model:{id:`replay/${r.modelId}`,name:`Replay of ${r.modelId}`,api:`replay`,provider:`replay`,contextWindow:r.contextWindow},systemPrompt:`tier-a replay`,messages:[{role:`user`,content:t.prompt,timestamp:0}],tools:a,thinkingLevel:`low`},f=await c.run(l);if(s()!==n.length)throw Error(`scenario '${t.id}': engine made ${s()} LLM call(s) but ${n.length} turn(s) were recorded — the leftover recordings were never consumed, which means the engine stopped earlier than when the recording was made (or the recording has extra turns).`);return{scenarioId:t.id,toolCalls:e,artifacts:S(i.dir),outcome:{stopReason:String(f.stopReason),llmCalls:s()}}}finally{i.dispose()}}function T(e){return s(e.dir,`golden.json`)}function E(e,t){o(T(e),`${JSON.stringify(t,null,2)}\n`,`utf8`)}function D(e,t){let i=T(e);if(!n(i))return{scenarioId:e.id,verdict:`no_golden`,details:["no golden.json — record one with `tier-a run --update`"]};let a=JSON.parse(r(i,`utf8`)),o=[],s=JSON.stringify(t.toolCalls),c=JSON.stringify(a.toolCalls);s!==c&&o.push(`tool calls diverged:\n  golden: ${c}\n  actual: ${s}`);let l=new Set([...Object.keys(a.artifacts),...Object.keys(t.artifacts)]);for(let e of[...l].sort()){let n=a.artifacts[e],r=t.artifacts[e];n!==r&&(n===void 0?o.push(`unexpected new artifact: ${e}`):r===void 0?o.push(`missing artifact: ${e}`):o.push(`artifact content changed: ${e} (${n} → ${r})`))}return t.outcome.stopReason!==a.outcome.stopReason&&o.push(`stopReason changed: ${a.outcome.stopReason} → ${t.outcome.stopReason}`),t.outcome.llmCalls!==a.outcome.llmCalls&&o.push(`LLM call count changed: ${a.outcome.llmCalls} → ${t.outcome.llmCalls}`),{scenarioId:e.id,verdict:o.length===0?`match`:`mismatch`,details:o.length===0?[`identical to golden`]:o}}export{w as a,y as i,T as n,E as o,v as r,D as t};
+//# sourceMappingURL=tier-a-DPSonK0s.js.map

package/dist/{tier-a-DvMUFMGy.js.map → tier-a-DPSonK0s.js.map}

@@ -1 +1 @@
-{"version":3,"file":"tier-a-DvMUFMGy.js","names":["parseYaml"],"sources":["../src/tier-a.ts"],"sourcesContent":["/**\n * tier-a.ts —— Tier A 引擎回归（RFC-316 D2：确定性、零模型成本）。\n *\n * 原理：把录制好的 provider 流按 LLM 往返序号回放给**真实引擎**（真 Agent + 真 fs 工具 +\n * 真工作区），模型输出被完全固定，引擎侧任何行为变化都会落在三个断言面上：\n *   ① 工具调用序列（名称 + 参数，工作区路径归一为 <WS>）\n *   ② 工作区文件产物（路径 + sha256）\n *   ③ turn 终态（stopReason + LLM 往返数）\n *\n * 为什么在进程内跑而不是 headless CLI：回放 provider 无法从 CLI 外部注入——除非给引擎加\n * 评测分支，而那恰恰违反 RFC-316 规则 6。进程内消费 @x-otto/agent、@x-otto/tools 的**公开\n * dist API** 是 RFC-200 `bench-turn-loop.mjs` 已确立的先例：引擎零改动，评测在包边界外。\n *\n * 与 Tier B 的分工：Tier B 测「agent（含模型）变强没」，本层测「引擎没被写坏」——\n * 一次录制多次回放，合法的引擎行为变更走显式重录（golden 更新同纪律，见 CLI --update）。\n */\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'\nimport { join, relative } from 'node:path'\nimport { parse as parseYaml } from 'yaml'\nimport { createAgent } from '@x-otto/agent'\nimport { createLogger } from '@x-otto/shared'\nimport { loadRecording, replayStream } from '@x-otto/provider'\nimport { createEdit, createRead, createWrite } from '@x-otto/tools'\n\nimport { createWorkspace } from './workspace'\n\nimport type { AgentRunContext, AgentTool, StreamFunction } from '@x-otto/agent'\nimport type { Recording } from '@x-otto/provider'\nimport type { AssistantMessage } from '@x-otto/interchange'\n\n/** 工作区绝对路径在 golden 里的占位符（跨机器可比的前提）。 */\nconst WORKSPACE_PLACEHOLDER = '<WS>'\n\nexport interface TierAScenario {\n  id: string\n  /** 首条用户消息（与录制时一致——engine 拿它构建首轮请求）。 */\n  prompt: string\n  /** 注册给引擎的工具名（fs 三件套子集；bash 等非确定工具不允许，D2 纯本地边界）。 */\n  tools: readonly ('read' | 'edit' | 'write')[]\n  /** 按 LLM 往返顺序排列的录制文件路径。 */\n  turnFiles: readonly string[]\n  dir: string\n  hasFixture: boolean\n}\n\nexport interface ToolCallRecord {\n  name: string\n  arguments: Record<string, unknown>\n}\n\n/** 一次 Tier A 运行的可比产物（golden 的全部内容）。 */\nexport interface TierAResult {\n  scenarioId: string\n  toolCalls: readonly ToolCallRecord[]\n  /** 相对路径 → sha256。 */\n  artifacts: Readonly<Record<string, string>>\n  outcome: {\n    stopReason: string\n    llmCalls: number\n  }\n}\n\nexport interface TierAFinding {\n  scenarioId: string\n  verdict: 'match' | 'mismatch' | 'no_golden'\n  details: readonly string[]\n}\n\nconst SUPPORTED_TOOLS = ['read', 'edit', 'write'] as const\n\n/** 解析 `evals/recordings/<id>/scenario.yaml`。 */\nexport function loadScenario(dir: string): TierAScenario {\n  const id = dir.split('/').filter(Boolean).at(-1) ?? dir\n  const yamlPath = join(dir, 'scenario.yaml')\n  if (!existsSync(yamlPath)) {\n    throw new Error(`scenario '${id}': missing scenario.yaml`)\n  }\n  const parsed: unknown = parseYaml(readFileSync(yamlPath, 'utf8'))\n  if (typeof parsed !== 'object' || parsed === null) {\n    throw new Error(`scenario '${id}': scenario.yaml must be a mapping`)\n  }\n  const raw = parsed as Record<string, unknown>\n\n  const prompt = raw['prompt']\n  if (typeof prompt !== 'string' || prompt.trim() === '') {\n    throw new Error(`scenario '${id}': 'prompt' must be a non-empty string`)\n  }\n\n  const tools = raw['tools']\n  if (!Array.isArray(tools) || tools.length === 0) {\n    throw new Error(`scenario '${id}': 'tools' must be a non-empty array`)\n  }\n  for (const tool of tools) {\n    if (!SUPPORTED_TOOLS.includes(tool as (typeof SUPPORTED_TOOLS)[number])) {\n      throw new Error(\n        `scenario '${id}': tool '${String(tool)}' not allowed — Tier A only permits deterministic fs tools (${SUPPORTED_TOOLS.join('/')})`,\n      )\n    }\n  }\n\n  const turnsDir = join(dir, 'turns')\n  if (!existsSync(turnsDir)) {\n    throw new Error(`scenario '${id}': missing turns/ directory`)\n  }\n  const turnFiles = readdirSync(turnsDir)\n    .filter((name) => name.endsWith('.jsonl'))\n    .sort()\n    .map((name) => join(turnsDir, name))\n  if (turnFiles.length === 0) {\n    throw new Error(`scenario '${id}': turns/ has no .jsonl recordings`)\n  }\n\n  return {\n    id,\n    prompt,\n    tools: tools as TierAScenario['tools'],\n    turnFiles,\n    dir,\n    hasFixture: existsSync(join(dir, 'fixture')),\n  }\n}\n\nexport function loadScenarioSuite(recordingsRoot: string): readonly TierAScenario[] {\n  if (!existsSync(recordingsRoot)) {\n    throw new Error(`recordings root not found: ${recordingsRoot}`)\n  }\n  return readdirSync(recordingsRoot)\n    .filter((name) => !name.startsWith('.'))\n    .map((name) => join(recordingsRoot, name))\n    .filter((path) => statSync(path).isDirectory())\n    .sort()\n    .map(loadScenario)\n}\n\n/**\n * 多段回放 StreamFunction：第 i 次 LLM 调用回放第 i 段录制。\n * 引擎多调（回归征兆）立即抛错——比默默循环最后一段诚实。\n */\nfunction createTurnSequenceStream(recordings: readonly Recording[]): {\n  stream: StreamFunction\n  calls: () => number\n} {\n  let call = 0\n  const stream: StreamFunction = (_context, signal) => {\n    const index = call++\n    const recording = recordings[index]\n    if (!recording) {\n      throw new Error(\n        `engine made LLM call #${index + 1} but only ${recordings.length} turn(s) were recorded — ` +\n          'engine behaviour diverged from the recording (or the recording is incomplete)',\n      )\n    }\n    let done: AssistantMessage | undefined\n    const events = (async function* () {\n      for await (const event of replayStream(recording, { fast: true }, signal)) {\n        if (event.type === 'done') {\n          done = event.message\n        }\n        yield event\n      }\n    })()\n    return Object.assign(events, {\n      async result(): Promise<AssistantMessage> {\n        for await (const _ of events) {\n          // drain（调用方未消费时兜底）\n        }\n        if (!done) {\n          throw new Error('recording ended without a done event')\n        }\n        return done\n      },\n    })\n  }\n  return { stream, calls: () => call }\n}\n\n/** 包一层参数间谍：不改工具行为，只记录调用序列（断言面①）。 */\nfunction spyTool(tool: AgentTool, log: ToolCallRecord[], workspaceDir: string): AgentTool {\n  return {\n    ...tool,\n    execute: (ctx) => {\n      log.push({\n        name: tool.name,\n        arguments: JSON.parse(\n          JSON.stringify(ctx.params ?? {})\n            .split(workspaceDir)\n            .join(WORKSPACE_PLACEHOLDER),\n        ) as Record<string, unknown>,\n      })\n      return tool.execute(ctx)\n    },\n  }\n}\n\nfunction collectArtifacts(dir: string): Record<string, string> {\n  const artifacts: Record<string, string> = {}\n  const walk = (current: string): void => {\n    for (const entry of readdirSync(current, { withFileTypes: true })) {\n      const full = join(current, entry.name)\n      if (entry.isDirectory()) {\n        walk(full)\n        continue\n      }\n      artifacts[relative(dir, full)] = createHash('sha256')\n        .update(readFileSync(full))\n        .digest('hex')\n        .slice(0, 16)\n    }\n  }\n  walk(dir)\n  return artifacts\n}\n\nfunction buildTools(\n  scenario: TierAScenario,\n  workspaceDir: string,\n  log: ToolCallRecord[],\n): AgentTool[] {\n  const factories: Record<TierAScenario['tools'][number], (root: string) => AgentTool> = {\n    read: (root) => createRead(root) as AgentTool,\n    edit: (root) => createEdit(root) as AgentTool,\n    write: (root) => createWrite(root) as AgentTool,\n  }\n  return scenario.tools.map((name) => spyTool(factories[name](workspaceDir), log, workspaceDir))\n}\n\n/** 跑单个场景：真实引擎 + 回放流 + 真实工具 + 隔离工作区。 */\nexport async function runScenario(scenario: TierAScenario): Promise<TierAResult> {\n  const recordings = await Promise.all(scenario.turnFiles.map((file) => loadRecording(file)))\n  const header = recordings[0]?.header\n  if (!header) {\n    throw new Error(`scenario '${scenario.id}': no recordings`)\n  }\n\n  const workspace = createWorkspace({\n    id: scenario.id,\n    prompt: scenario.prompt,\n    judge: 'liveness',\n    runs: 1,\n    timeoutMs: 0,\n    dir: scenario.dir,\n    hasFixture: scenario.hasFixture,\n  })\n\n  try {\n    const toolLog: ToolCallRecord[] = []\n    const tools = buildTools(scenario, workspace.dir, toolLog)\n    const { stream, calls } = createTurnSequenceStream(recordings)\n    const logger = createLogger('tier-a', { level: 'fatal', destination: 'stdout' })\n    const agent = createAgent({ stream, logger })\n\n    const context: AgentRunContext = {\n      model: {\n        id: `replay/${header.modelId}`,\n        name: `Replay of ${header.modelId}`,\n        api: 'replay',\n        provider: 'replay',\n        contextWindow: header.contextWindow,\n      } as AgentRunContext['model'],\n      systemPrompt: 'tier-a replay',\n      messages: [{ role: 'user', content: scenario.prompt, timestamp: 0 }],\n      tools,\n      thinkingLevel: 'low',\n    }\n\n    const final = await agent.run(context)\n\n    // 引擎**少调** LLM 时（提前收工），多余的录制段永远不会被消费。golden 比对能靠\n    // llmCalls 抓到这件事，但那要求 golden 已经录过正确的次数——首次 --update 时若引擎\n    // 已经是坏的，错误的次数会被直接固化成基线。这里在录制期就把它变成硬错误：\n    // 录制了 N 段就必须消费 N 段，剩余即引擎行为与录制不符。\n    if (calls() !== recordings.length) {\n      throw new Error(\n        `scenario '${scenario.id}': engine made ${calls()} LLM call(s) but ${recordings.length} ` +\n          'turn(s) were recorded — the leftover recordings were never consumed, which means the ' +\n          'engine stopped earlier than when the recording was made (or the recording has extra turns).',\n      )\n    }\n\n    return {\n      scenarioId: scenario.id,\n      toolCalls: toolLog,\n      artifacts: collectArtifacts(workspace.dir),\n      outcome: { stopReason: String(final.stopReason), llmCalls: calls() },\n    }\n  } finally {\n    workspace.dispose()\n  }\n}\n\nexport function goldenPath(scenario: TierAScenario): string {\n  return join(scenario.dir, 'golden.json')\n}\n\nexport function writeGolden(scenario: TierAScenario, result: TierAResult): void {\n  writeFileSync(goldenPath(scenario), `${JSON.stringify(result, null, 2)}\\n`, 'utf8')\n}\n\n/** 与 golden 逐面对比（断言三件套的执行处）。 */\nexport function compareToGolden(scenario: TierAScenario, result: TierAResult): TierAFinding {\n  const path = goldenPath(scenario)\n  if (!existsSync(path)) {\n    return {\n      scenarioId: scenario.id,\n      verdict: 'no_golden',\n      details: [`no golden.json — record one with \\`tier-a run --update\\``],\n    }\n  }\n  const golden = JSON.parse(readFileSync(path, 'utf8')) as TierAResult\n  const details: string[] = []\n\n  const actualCalls = JSON.stringify(result.toolCalls)\n  const goldenCalls = JSON.stringify(golden.toolCalls)\n  if (actualCalls !== goldenCalls) {\n    details.push(`tool calls diverged:\\n  golden: ${goldenCalls}\\n  actual: ${actualCalls}`)\n  }\n\n  const allPaths = new Set([...Object.keys(golden.artifacts), ...Object.keys(result.artifacts)])\n  for (const file of [...allPaths].sort()) {\n    const before = golden.artifacts[file]\n    const after = result.artifacts[file]\n    if (before === after) {\n      continue\n    }\n    if (before === undefined) {\n      details.push(`unexpected new artifact: ${file}`)\n    } else if (after === undefined) {\n      details.push(`missing artifact: ${file}`)\n    } else {\n      details.push(`artifact content changed: ${file} (${before} → ${after})`)\n    }\n  }\n\n  if (result.outcome.stopReason !== golden.outcome.stopReason) {\n    details.push(`stopReason changed: ${golden.outcome.stopReason} → ${result.outcome.stopReason}`)\n  }\n  if (result.outcome.llmCalls !== golden.outcome.llmCalls) {\n    details.push(`LLM call count changed: ${golden.outcome.llmCalls} → ${result.outcome.llmCalls}`)\n  }\n\n  return {\n    scenarioId: scenario.id,\n    verdict: details.length === 0 ? 'match' : 'mismatch',\n    details: details.length === 0 ? ['identical to golden'] : details,\n  }\n}\n"],"mappings":"mfAiCA,MAqCM,EAAkB,CAAC,OAAQ,OAAQ,QAAQ,CAGjD,SAAgB,EAAa,EAA4B,CACvD,IAAM,EAAK,EAAI,MAAM,IAAI,CAAC,OAAO,QAAQ,CAAC,GAAG,GAAG,EAAI,EAC9C,EAAW,EAAK,EAAK,gBAAgB,CAC3C,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,aAAa,EAAG,0BAA0B,CAE5D,IAAM,EAAkBA,EAAU,EAAa,EAAU,OAAO,CAAC,CACjE,GAAI,OAAO,GAAW,WAAY,EAChC,MAAU,MAAM,aAAa,EAAG,oCAAoC,CAEtE,IAAM,EAAM,EAEN,EAAS,EAAI,OACnB,GAAI,OAAO,GAAW,UAAY,EAAO,MAAM,GAAK,GAClD,MAAU,MAAM,aAAa,EAAG,wCAAwC,CAG1E,IAAM,EAAQ,EAAI,MAClB,GAAI,CAAC,MAAM,QAAQ,EAAM,EAAI,EAAM,SAAW,EAC5C,MAAU,MAAM,aAAa,EAAG,sCAAsC,CAExE,IAAK,IAAM,KAAQ,EACjB,GAAI,CAAC,EAAgB,SAAS,EAAyC,CACrE,MAAU,MACR,aAAa,EAAG,WAAW,OAAO,EAAK,CAAC,8DAA8D,EAAgB,KAAK,IAAI,CAAC,GACjI,CAIL,IAAM,EAAW,EAAK,EAAK,QAAQ,CACnC,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,aAAa,EAAG,6BAA6B,CAE/D,IAAM,EAAY,EAAY,EAAS,CACpC,OAAQ,GAAS,EAAK,SAAS,SAAS,CAAC,CACzC,MAAM,CACN,IAAK,GAAS,EAAK,EAAU,EAAK,CAAC,CACtC,GAAI,EAAU,SAAW,EACvB,MAAU,MAAM,aAAa,EAAG,oCAAoC,CAGtE,MAAO,CACL,KACA,SACO,QACP,YACA,MACA,WAAY,EAAW,EAAK,EAAK,UAAU,CAAC,CAC7C,CAGH,SAAgB,EAAkB,EAAkD,CAClF,GAAI,CAAC,EAAW,EAAe,CAC7B,MAAU,MAAM,8BAA8B,IAAiB,CAEjE,OAAO,EAAY,EAAe,CAC/B,OAAQ,GAAS,CAAC,EAAK,WAAW,IAAI,CAAC,CACvC,IAAK,GAAS,EAAK,EAAgB,EAAK,CAAC,CACzC,OAAQ,GAAS,EAAS,EAAK,CAAC,aAAa,CAAC,CAC9C,MAAM,CACN,IAAI,EAAa,CAOtB,SAAS,EAAyB,EAGhC,CACA,IAAI,EAAO,EA+BX,MAAO,CAAE,QA9BuB,EAAU,IAAW,CACnD,IAAM,EAAQ,IACR,EAAY,EAAW,GAC7B,GAAI,CAAC,EACH,MAAU,MACR,yBAAyB,EAAQ,EAAE,YAAY,EAAW,OAAO,wGAElE,CAEH,IAAI,EACE,GAAU,iBAAmB,CACjC,UAAW,IAAM,KAAS,EAAa,EAAW,CAAE,KAAM,GAAM,CAAE,EAAO,CACnE,EAAM,OAAS,SACjB,EAAO,EAAM,SAEf,MAAM,KAEN,CACJ,OAAO,OAAO,OAAO,EAAQ,CAC3B,MAAM,QAAoC,CACxC,UAAW,IAAM,KAAK,GAGtB,GAAI,CAAC,EACH,MAAU,MAAM,uCAAuC,CAEzD,OAAO,GAEV,CAAC,EAEa,UAAa,EAAM,CAItC,SAAS,EAAQ,EAAiB,EAAuB,EAAiC,CACxF,MAAO,CACL,GAAG,EACH,QAAU,IACR,EAAI,KAAK,CACP,KAAM,EAAK,KACX,UAAW,KAAK,MACd,KAAK,UAAU,EAAI,QAAU,EAAE,CAAC,CAC7B,MAAM,EAAa,CACnB,KAAK,OAAsB,CAC/B,CACF,CAAC,CACK,EAAK,QAAQ,EAAI,EAE3B,CAGH,SAAS,EAAiB,EAAqC,CAC7D,IAAM,EAAoC,EAAE,CACtC,EAAQ,GAA0B,CACtC,IAAK,IAAM,KAAS,EAAY,EAAS,CAAE,cAAe,GAAM,CAAC,CAAE,CACjE,IAAM,EAAO,EAAK,EAAS,EAAM,KAAK,CACtC,GAAI,EAAM,aAAa,CAAE,CACvB,EAAK,EAAK,CACV,SAEF,EAAU,EAAS,EAAK,EAAK,EAAI,EAAW,SAAS,CAClD,OAAO,EAAa,EAAK,CAAC,CAC1B,OAAO,MAAM,CACb,MAAM,EAAG,GAAG,GAInB,OADA,EAAK,EAAI,CACF,EAGT,SAAS,EACP,EACA,EACA,EACa,CACb,IAAM,EAAiF,CACrF,KAAO,GAAS,EAAW,EAAK,CAChC,KAAO,GAAS,EAAW,EAAK,CAChC,MAAQ,GAAS,EAAY,EAAK,CACnC,CACD,OAAO,EAAS,MAAM,IAAK,GAAS,EAAQ,EAAU,GAAM,EAAa,CAAE,EAAK,EAAa,CAAC,CAIhG,eAAsB,EAAY,EAA+C,CAC/E,IAAM,EAAa,MAAM,QAAQ,IAAI,EAAS,UAAU,IAAK,GAAS,EAAc,EAAK,CAAC,CAAC,CACrF,EAAS,EAAW,IAAI,OAC9B,GAAI,CAAC,EACH,MAAU,MAAM,aAAa,EAAS,GAAG,kBAAkB,CAG7D,IAAM,EAAY,EAAgB,CAChC,GAAI,EAAS,GACb,OAAQ,EAAS,OACjB,MAAO,WACP,KAAM,EACN,UAAW,EACX,IAAK,EAAS,IACd,WAAY,EAAS,WACtB,CAAC,CAEF,GAAI,CACF,IAAM,EAA4B,EAAE,CAC9B,EAAQ,EAAW,EAAU,EAAU,IAAK,EAAQ,CACpD,CAAE,SAAQ,SAAU,EAAyB,EAAW,CAExD,EAAQ,EAAY,CAAE,SAAQ,OADrB,EAAa,SAAU,CAAE,MAAO,QAAS,YAAa,SAAU,CAAC,CACpC,CAAC,CAEvC,EAA2B,CAC/B,MAAO,CACL,GAAI,UAAU,EAAO,UACrB,KAAM,aAAa,EAAO,UAC1B,IAAK,SACL,SAAU,SACV,cAAe,EAAO,cACvB,CACD,aAAc,gBACd,SAAU,CAAC,CAAE,KAAM,OAAQ,QAAS,EAAS,OAAQ,UAAW,EAAG,CAAC,CACpE,QACA,cAAe,MAChB,CAEK,EAAQ,MAAM,EAAM,IAAI,EAAQ,CAMtC,GAAI,GAAO,GAAK,EAAW,OACzB,MAAU,MACR,aAAa,EAAS,GAAG,iBAAiB,GAAO,CAAC,mBAAmB,EAAW,OAAO,mLAGxF,CAGH,MAAO,CACL,WAAY,EAAS,GACrB,UAAW,EACX,UAAW,EAAiB,EAAU,IAAI,CAC1C,QAAS,CAAE,WAAY,OAAO,EAAM,WAAW,CAAE,SAAU,GAAO,CAAE,CACrE,QACO,CACR,EAAU,SAAS,EAIvB,SAAgB,EAAW,EAAiC,CAC1D,OAAO,EAAK,EAAS,IAAK,cAAc,CAG1C,SAAgB,EAAY,EAAyB,EAA2B,CAC9E,EAAc,EAAW,EAAS,CAAE,GAAG,KAAK,UAAU,EAAQ,KAAM,EAAE,CAAC,IAAK,OAAO,CAIrF,SAAgB,EAAgB,EAAyB,EAAmC,CAC1F,IAAM,EAAO,EAAW,EAAS,CACjC,GAAI,CAAC,EAAW,EAAK,CACnB,MAAO,CACL,WAAY,EAAS,GACrB,QAAS,YACT,QAAS,CAAC,yDAA2D,CACtE,CAEH,IAAM,EAAS,KAAK,MAAM,EAAa,EAAM,OAAO,CAAC,CAC/C,EAAoB,EAAE,CAEtB,EAAc,KAAK,UAAU,EAAO,UAAU,CAC9C,EAAc,KAAK,UAAU,EAAO,UAAU,CAChD,IAAgB,GAClB,EAAQ,KAAK,mCAAmC,EAAY,cAAc,IAAc,CAG1F,IAAM,EAAW,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,EAAO,UAAU,CAAE,GAAG,OAAO,KAAK,EAAO,UAAU,CAAC,CAAC,CAC9F,IAAK,IAAM,IAAQ,CAAC,GAAG,EAAS,CAAC,MAAM,CAAE,CACvC,IAAM,EAAS,EAAO,UAAU,GAC1B,EAAQ,EAAO,UAAU,GAC3B,IAAW,IAGX,IAAW,IAAA,GACb,EAAQ,KAAK,4BAA4B,IAAO,CACvC,IAAU,IAAA,GACnB,EAAQ,KAAK,qBAAqB,IAAO,CAEzC,EAAQ,KAAK,6BAA6B,EAAK,IAAI,EAAO,KAAK,EAAM,GAAG,EAW5E,OAPI,EAAO,QAAQ,aAAe,EAAO,QAAQ,YAC/C,EAAQ,KAAK,uBAAuB,EAAO,QAAQ,WAAW,KAAK,EAAO,QAAQ,aAAa,CAE7F,EAAO,QAAQ,WAAa,EAAO,QAAQ,UAC7C,EAAQ,KAAK,2BAA2B,EAAO,QAAQ,SAAS,KAAK,EAAO,QAAQ,WAAW,CAG1F,CACL,WAAY,EAAS,GACrB,QAAS,EAAQ,SAAW,EAAI,QAAU,WAC1C,QAAS,EAAQ,SAAW,EAAI,CAAC,sBAAsB,CAAG,EAC3D"}
+{"version":3,"file":"tier-a-DPSonK0s.js","names":["parseYaml"],"sources":["../src/tier-a.ts"],"sourcesContent":["/**\n * tier-a.ts —— Tier A 引擎回归（RFC-316 D2：确定性、零模型成本）。\n *\n * 原理：把录制好的 provider 流按 LLM 往返序号回放给**真实引擎**（真 Agent + 真 fs 工具 +\n * 真工作区），模型输出被完全固定，引擎侧任何行为变化都会落在三个断言面上：\n *   ① 工具调用序列（名称 + 参数，工作区路径归一为 <WS>）\n *   ② 工作区文件产物（路径 + sha256）\n *   ③ turn 终态（stopReason + LLM 往返数）\n *\n * 为什么在进程内跑而不是 headless CLI：回放 provider 无法从 CLI 外部注入——除非给引擎加\n * 评测分支，而那恰恰违反 RFC-316 规则 6。进程内消费 @x-otto/agent、@x-otto/tools 的**公开\n * dist API** 是 RFC-200 `bench-turn-loop.mjs` 已确立的先例：引擎零改动，评测在包边界外。\n *\n * 与 Tier B 的分工：Tier B 测「agent（含模型）变强没」，本层测「引擎没被写坏」——\n * 一次录制多次回放，合法的引擎行为变更走显式重录（golden 更新同纪律，见 CLI --update）。\n */\n\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'\nimport { join, relative } from 'node:path'\nimport { parse as parseYaml } from 'yaml'\nimport { createAgent } from '@x-otto/agent'\nimport { createLogger } from '@x-otto/shared'\nimport { loadRecording, replayStream } from '@x-otto/provider'\nimport { createEdit, createRead, createWrite } from '@x-otto/tools'\n\nimport { createWorkspace } from './workspace'\n\nimport type { AgentRunContext, AgentTool, StreamFunction } from '@x-otto/agent'\nimport type { Recording } from '@x-otto/provider'\nimport type { AssistantMessage } from '@x-otto/interchange'\n\n/** 工作区绝对路径在 golden 里的占位符（跨机器可比的前提）。 */\nconst WORKSPACE_PLACEHOLDER = '<WS>'\n\nexport interface TierAScenario {\n  id: string\n  /** 首条用户消息（与录制时一致——engine 拿它构建首轮请求）。 */\n  prompt: string\n  /** 注册给引擎的工具名（fs 三件套子集；bash 等非确定工具不允许，D2 纯本地边界）。 */\n  tools: readonly ('read' | 'edit' | 'write')[]\n  /** 按 LLM 往返顺序排列的录制文件路径。 */\n  turnFiles: readonly string[]\n  dir: string\n  hasFixture: boolean\n}\n\nexport interface ToolCallRecord {\n  name: string\n  arguments: Record<string, unknown>\n}\n\n/** 一次 Tier A 运行的可比产物（golden 的全部内容）。 */\nexport interface TierAResult {\n  scenarioId: string\n  toolCalls: readonly ToolCallRecord[]\n  /** 相对路径 → sha256。 */\n  artifacts: Readonly<Record<string, string>>\n  outcome: {\n    stopReason: string\n    llmCalls: number\n  }\n}\n\nexport interface TierAFinding {\n  scenarioId: string\n  verdict: 'match' | 'mismatch' | 'no_golden'\n  details: readonly string[]\n}\n\nconst SUPPORTED_TOOLS = ['read', 'edit', 'write'] as const\n\n/** 解析 `evals/recordings/<id>/scenario.yaml`。 */\nexport function loadScenario(dir: string): TierAScenario {\n  const id = dir.split('/').filter(Boolean).at(-1) ?? dir\n  const yamlPath = join(dir, 'scenario.yaml')\n  if (!existsSync(yamlPath)) {\n    throw new Error(`scenario '${id}': missing scenario.yaml`)\n  }\n  const parsed: unknown = parseYaml(readFileSync(yamlPath, 'utf8'))\n  if (typeof parsed !== 'object' || parsed === null) {\n    throw new Error(`scenario '${id}': scenario.yaml must be a mapping`)\n  }\n  const raw = parsed as Record<string, unknown>\n\n  const prompt = raw['prompt']\n  if (typeof prompt !== 'string' || prompt.trim() === '') {\n    throw new Error(`scenario '${id}': 'prompt' must be a non-empty string`)\n  }\n\n  const tools = raw['tools']\n  if (!Array.isArray(tools) || tools.length === 0) {\n    throw new Error(`scenario '${id}': 'tools' must be a non-empty array`)\n  }\n  for (const tool of tools) {\n    if (!SUPPORTED_TOOLS.includes(tool as (typeof SUPPORTED_TOOLS)[number])) {\n      throw new Error(\n        `scenario '${id}': tool '${String(tool)}' not allowed — Tier A only permits deterministic fs tools (${SUPPORTED_TOOLS.join('/')})`,\n      )\n    }\n  }\n\n  const turnsDir = join(dir, 'turns')\n  if (!existsSync(turnsDir)) {\n    throw new Error(`scenario '${id}': missing turns/ directory`)\n  }\n  const turnFiles = readdirSync(turnsDir)\n    .filter((name) => name.endsWith('.jsonl'))\n    .sort()\n    .map((name) => join(turnsDir, name))\n  if (turnFiles.length === 0) {\n    throw new Error(`scenario '${id}': turns/ has no .jsonl recordings`)\n  }\n\n  return {\n    id,\n    prompt,\n    tools: tools as TierAScenario['tools'],\n    turnFiles,\n    dir,\n    hasFixture: existsSync(join(dir, 'fixture')),\n  }\n}\n\nexport function loadScenarioSuite(recordingsRoot: string): readonly TierAScenario[] {\n  if (!existsSync(recordingsRoot)) {\n    throw new Error(`recordings root not found: ${recordingsRoot}`)\n  }\n  return readdirSync(recordingsRoot)\n    .filter((name) => !name.startsWith('.'))\n    .map((name) => join(recordingsRoot, name))\n    .filter((path) => statSync(path).isDirectory())\n    .sort()\n    .map(loadScenario)\n}\n\n/**\n * 多段回放 StreamFunction：第 i 次 LLM 调用回放第 i 段录制。\n * 引擎多调（回归征兆）立即抛错——比默默循环最后一段诚实。\n */\nfunction createTurnSequenceStream(recordings: readonly Recording[]): {\n  stream: StreamFunction\n  calls: () => number\n} {\n  let call = 0\n  const stream: StreamFunction = (_context, signal) => {\n    const index = call++\n    const recording = recordings[index]\n    if (!recording) {\n      throw new Error(\n        `engine made LLM call #${index + 1} but only ${recordings.length} turn(s) were recorded — ` +\n          'engine behaviour diverged from the recording (or the recording is incomplete)',\n      )\n    }\n    let done: AssistantMessage | undefined\n    const events = (async function* () {\n      for await (const event of replayStream(recording, { fast: true }, signal)) {\n        if (event.type === 'done') {\n          done = event.message\n        }\n        yield event\n      }\n    })()\n    return Object.assign(events, {\n      async result(): Promise<AssistantMessage> {\n        for await (const _ of events) {\n          // drain（调用方未消费时兜底）\n        }\n        if (!done) {\n          throw new Error('recording ended without a done event')\n        }\n        return done\n      },\n    })\n  }\n  return { stream, calls: () => call }\n}\n\n/** 包一层参数间谍：不改工具行为，只记录调用序列（断言面①）。 */\nfunction spyTool(tool: AgentTool, log: ToolCallRecord[], workspaceDir: string): AgentTool {\n  return {\n    ...tool,\n    execute: (ctx) => {\n      log.push({\n        name: tool.name,\n        arguments: JSON.parse(\n          JSON.stringify(ctx.params ?? {})\n            .split(workspaceDir)\n            .join(WORKSPACE_PLACEHOLDER),\n        ) as Record<string, unknown>,\n      })\n      return tool.execute(ctx)\n    },\n  }\n}\n\nfunction collectArtifacts(dir: string): Record<string, string> {\n  const artifacts: Record<string, string> = {}\n  const walk = (current: string): void => {\n    for (const entry of readdirSync(current, { withFileTypes: true })) {\n      const full = join(current, entry.name)\n      if (entry.isDirectory()) {\n        walk(full)\n        continue\n      }\n      artifacts[relative(dir, full)] = createHash('sha256')\n        .update(readFileSync(full))\n        .digest('hex')\n        .slice(0, 16)\n    }\n  }\n  walk(dir)\n  return artifacts\n}\n\nfunction buildTools(\n  scenario: TierAScenario,\n  workspaceDir: string,\n  log: ToolCallRecord[],\n): AgentTool[] {\n  const factories: Record<TierAScenario['tools'][number], (root: string) => AgentTool> = {\n    read: (root) => createRead(root) as AgentTool,\n    edit: (root) => createEdit(root) as AgentTool,\n    write: (root) => createWrite(root) as AgentTool,\n  }\n  return scenario.tools.map((name) => spyTool(factories[name](workspaceDir), log, workspaceDir))\n}\n\n/** 跑单个场景：真实引擎 + 回放流 + 真实工具 + 隔离工作区。 */\nexport async function runScenario(scenario: TierAScenario): Promise<TierAResult> {\n  const recordings = await Promise.all(scenario.turnFiles.map((file) => loadRecording(file)))\n  const header = recordings[0]?.header\n  if (!header) {\n    throw new Error(`scenario '${scenario.id}': no recordings`)\n  }\n\n  const workspace = createWorkspace({\n    id: scenario.id,\n    prompt: scenario.prompt,\n    judge: 'liveness',\n    runs: 1,\n    timeoutMs: 0,\n    dir: scenario.dir,\n    hasFixture: scenario.hasFixture,\n    hasSetup: false,\n  })\n\n  try {\n    const toolLog: ToolCallRecord[] = []\n    const tools = buildTools(scenario, workspace.dir, toolLog)\n    const { stream, calls } = createTurnSequenceStream(recordings)\n    const logger = createLogger('tier-a', { level: 'fatal', destination: 'stdout' })\n    const agent = createAgent({ stream, logger })\n\n    const context: AgentRunContext = {\n      model: {\n        id: `replay/${header.modelId}`,\n        name: `Replay of ${header.modelId}`,\n        api: 'replay',\n        provider: 'replay',\n        contextWindow: header.contextWindow,\n      } as AgentRunContext['model'],\n      systemPrompt: 'tier-a replay',\n      messages: [{ role: 'user', content: scenario.prompt, timestamp: 0 }],\n      tools,\n      thinkingLevel: 'low',\n    }\n\n    const final = await agent.run(context)\n\n    // 引擎**少调** LLM 时（提前收工），多余的录制段永远不会被消费。golden 比对能靠\n    // llmCalls 抓到这件事，但那要求 golden 已经录过正确的次数——首次 --update 时若引擎\n    // 已经是坏的，错误的次数会被直接固化成基线。这里在录制期就把它变成硬错误：\n    // 录制了 N 段就必须消费 N 段，剩余即引擎行为与录制不符。\n    if (calls() !== recordings.length) {\n      throw new Error(\n        `scenario '${scenario.id}': engine made ${calls()} LLM call(s) but ${recordings.length} ` +\n          'turn(s) were recorded — the leftover recordings were never consumed, which means the ' +\n          'engine stopped earlier than when the recording was made (or the recording has extra turns).',\n      )\n    }\n\n    return {\n      scenarioId: scenario.id,\n      toolCalls: toolLog,\n      artifacts: collectArtifacts(workspace.dir),\n      outcome: { stopReason: String(final.stopReason), llmCalls: calls() },\n    }\n  } finally {\n    workspace.dispose()\n  }\n}\n\nexport function goldenPath(scenario: TierAScenario): string {\n  return join(scenario.dir, 'golden.json')\n}\n\nexport function writeGolden(scenario: TierAScenario, result: TierAResult): void {\n  writeFileSync(goldenPath(scenario), `${JSON.stringify(result, null, 2)}\\n`, 'utf8')\n}\n\n/** 与 golden 逐面对比（断言三件套的执行处）。 */\nexport function compareToGolden(scenario: TierAScenario, result: TierAResult): TierAFinding {\n  const path = goldenPath(scenario)\n  if (!existsSync(path)) {\n    return {\n      scenarioId: scenario.id,\n      verdict: 'no_golden',\n      details: [`no golden.json — record one with \\`tier-a run --update\\``],\n    }\n  }\n  const golden = JSON.parse(readFileSync(path, 'utf8')) as TierAResult\n  const details: string[] = []\n\n  const actualCalls = JSON.stringify(result.toolCalls)\n  const goldenCalls = JSON.stringify(golden.toolCalls)\n  if (actualCalls !== goldenCalls) {\n    details.push(`tool calls diverged:\\n  golden: ${goldenCalls}\\n  actual: ${actualCalls}`)\n  }\n\n  const allPaths = new Set([...Object.keys(golden.artifacts), ...Object.keys(result.artifacts)])\n  for (const file of [...allPaths].sort()) {\n    const before = golden.artifacts[file]\n    const after = result.artifacts[file]\n    if (before === after) {\n      continue\n    }\n    if (before === undefined) {\n      details.push(`unexpected new artifact: ${file}`)\n    } else if (after === undefined) {\n      details.push(`missing artifact: ${file}`)\n    } else {\n      details.push(`artifact content changed: ${file} (${before} → ${after})`)\n    }\n  }\n\n  if (result.outcome.stopReason !== golden.outcome.stopReason) {\n    details.push(`stopReason changed: ${golden.outcome.stopReason} → ${result.outcome.stopReason}`)\n  }\n  if (result.outcome.llmCalls !== golden.outcome.llmCalls) {\n    details.push(`LLM call count changed: ${golden.outcome.llmCalls} → ${result.outcome.llmCalls}`)\n  }\n\n  return {\n    scenarioId: scenario.id,\n    verdict: details.length === 0 ? 'match' : 'mismatch',\n    details: details.length === 0 ? ['identical to golden'] : details,\n  }\n}\n"],"mappings":"mfAiCA,MAqCM,EAAkB,CAAC,OAAQ,OAAQ,QAAQ,CAGjD,SAAgB,EAAa,EAA4B,CACvD,IAAM,EAAK,EAAI,MAAM,IAAI,CAAC,OAAO,QAAQ,CAAC,GAAG,GAAG,EAAI,EAC9C,EAAW,EAAK,EAAK,gBAAgB,CAC3C,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,aAAa,EAAG,0BAA0B,CAE5D,IAAM,EAAkBA,EAAU,EAAa,EAAU,OAAO,CAAC,CACjE,GAAI,OAAO,GAAW,WAAY,EAChC,MAAU,MAAM,aAAa,EAAG,oCAAoC,CAEtE,IAAM,EAAM,EAEN,EAAS,EAAI,OACnB,GAAI,OAAO,GAAW,UAAY,EAAO,MAAM,GAAK,GAClD,MAAU,MAAM,aAAa,EAAG,wCAAwC,CAG1E,IAAM,EAAQ,EAAI,MAClB,GAAI,CAAC,MAAM,QAAQ,EAAM,EAAI,EAAM,SAAW,EAC5C,MAAU,MAAM,aAAa,EAAG,sCAAsC,CAExE,IAAK,IAAM,KAAQ,EACjB,GAAI,CAAC,EAAgB,SAAS,EAAyC,CACrE,MAAU,MACR,aAAa,EAAG,WAAW,OAAO,EAAK,CAAC,8DAA8D,EAAgB,KAAK,IAAI,CAAC,GACjI,CAIL,IAAM,EAAW,EAAK,EAAK,QAAQ,CACnC,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,aAAa,EAAG,6BAA6B,CAE/D,IAAM,EAAY,EAAY,EAAS,CACpC,OAAQ,GAAS,EAAK,SAAS,SAAS,CAAC,CACzC,MAAM,CACN,IAAK,GAAS,EAAK,EAAU,EAAK,CAAC,CACtC,GAAI,EAAU,SAAW,EACvB,MAAU,MAAM,aAAa,EAAG,oCAAoC,CAGtE,MAAO,CACL,KACA,SACO,QACP,YACA,MACA,WAAY,EAAW,EAAK,EAAK,UAAU,CAAC,CAC7C,CAGH,SAAgB,EAAkB,EAAkD,CAClF,GAAI,CAAC,EAAW,EAAe,CAC7B,MAAU,MAAM,8BAA8B,IAAiB,CAEjE,OAAO,EAAY,EAAe,CAC/B,OAAQ,GAAS,CAAC,EAAK,WAAW,IAAI,CAAC,CACvC,IAAK,GAAS,EAAK,EAAgB,EAAK,CAAC,CACzC,OAAQ,GAAS,EAAS,EAAK,CAAC,aAAa,CAAC,CAC9C,MAAM,CACN,IAAI,EAAa,CAOtB,SAAS,EAAyB,EAGhC,CACA,IAAI,EAAO,EA+BX,MAAO,CAAE,QA9BuB,EAAU,IAAW,CACnD,IAAM,EAAQ,IACR,EAAY,EAAW,GAC7B,GAAI,CAAC,EACH,MAAU,MACR,yBAAyB,EAAQ,EAAE,YAAY,EAAW,OAAO,wGAElE,CAEH,IAAI,EACE,GAAU,iBAAmB,CACjC,UAAW,IAAM,KAAS,EAAa,EAAW,CAAE,KAAM,GAAM,CAAE,EAAO,CACnE,EAAM,OAAS,SACjB,EAAO,EAAM,SAEf,MAAM,KAEN,CACJ,OAAO,OAAO,OAAO,EAAQ,CAC3B,MAAM,QAAoC,CACxC,UAAW,IAAM,KAAK,GAGtB,GAAI,CAAC,EACH,MAAU,MAAM,uCAAuC,CAEzD,OAAO,GAEV,CAAC,EAEa,UAAa,EAAM,CAItC,SAAS,EAAQ,EAAiB,EAAuB,EAAiC,CACxF,MAAO,CACL,GAAG,EACH,QAAU,IACR,EAAI,KAAK,CACP,KAAM,EAAK,KACX,UAAW,KAAK,MACd,KAAK,UAAU,EAAI,QAAU,EAAE,CAAC,CAC7B,MAAM,EAAa,CACnB,KAAK,OAAsB,CAC/B,CACF,CAAC,CACK,EAAK,QAAQ,EAAI,EAE3B,CAGH,SAAS,EAAiB,EAAqC,CAC7D,IAAM,EAAoC,EAAE,CACtC,EAAQ,GAA0B,CACtC,IAAK,IAAM,KAAS,EAAY,EAAS,CAAE,cAAe,GAAM,CAAC,CAAE,CACjE,IAAM,EAAO,EAAK,EAAS,EAAM,KAAK,CACtC,GAAI,EAAM,aAAa,CAAE,CACvB,EAAK,EAAK,CACV,SAEF,EAAU,EAAS,EAAK,EAAK,EAAI,EAAW,SAAS,CAClD,OAAO,EAAa,EAAK,CAAC,CAC1B,OAAO,MAAM,CACb,MAAM,EAAG,GAAG,GAInB,OADA,EAAK,EAAI,CACF,EAGT,SAAS,EACP,EACA,EACA,EACa,CACb,IAAM,EAAiF,CACrF,KAAO,GAAS,EAAW,EAAK,CAChC,KAAO,GAAS,EAAW,EAAK,CAChC,MAAQ,GAAS,EAAY,EAAK,CACnC,CACD,OAAO,EAAS,MAAM,IAAK,GAAS,EAAQ,EAAU,GAAM,EAAa,CAAE,EAAK,EAAa,CAAC,CAIhG,eAAsB,EAAY,EAA+C,CAC/E,IAAM,EAAa,MAAM,QAAQ,IAAI,EAAS,UAAU,IAAK,GAAS,EAAc,EAAK,CAAC,CAAC,CACrF,EAAS,EAAW,IAAI,OAC9B,GAAI,CAAC,EACH,MAAU,MAAM,aAAa,EAAS,GAAG,kBAAkB,CAG7D,IAAM,EAAY,EAAgB,CAChC,GAAI,EAAS,GACb,OAAQ,EAAS,OACjB,MAAO,WACP,KAAM,EACN,UAAW,EACX,IAAK,EAAS,IACd,WAAY,EAAS,WACrB,SAAU,GACX,CAAC,CAEF,GAAI,CACF,IAAM,EAA4B,EAAE,CAC9B,EAAQ,EAAW,EAAU,EAAU,IAAK,EAAQ,CACpD,CAAE,SAAQ,SAAU,EAAyB,EAAW,CAExD,EAAQ,EAAY,CAAE,SAAQ,OADrB,EAAa,SAAU,CAAE,MAAO,QAAS,YAAa,SAAU,CAAC,CACpC,CAAC,CAEvC,EAA2B,CAC/B,MAAO,CACL,GAAI,UAAU,EAAO,UACrB,KAAM,aAAa,EAAO,UAC1B,IAAK,SACL,SAAU,SACV,cAAe,EAAO,cACvB,CACD,aAAc,gBACd,SAAU,CAAC,CAAE,KAAM,OAAQ,QAAS,EAAS,OAAQ,UAAW,EAAG,CAAC,CACpE,QACA,cAAe,MAChB,CAEK,EAAQ,MAAM,EAAM,IAAI,EAAQ,CAMtC,GAAI,GAAO,GAAK,EAAW,OACzB,MAAU,MACR,aAAa,EAAS,GAAG,iBAAiB,GAAO,CAAC,mBAAmB,EAAW,OAAO,mLAGxF,CAGH,MAAO,CACL,WAAY,EAAS,GACrB,UAAW,EACX,UAAW,EAAiB,EAAU,IAAI,CAC1C,QAAS,CAAE,WAAY,OAAO,EAAM,WAAW,CAAE,SAAU,GAAO,CAAE,CACrE,QACO,CACR,EAAU,SAAS,EAIvB,SAAgB,EAAW,EAAiC,CAC1D,OAAO,EAAK,EAAS,IAAK,cAAc,CAG1C,SAAgB,EAAY,EAAyB,EAA2B,CAC9E,EAAc,EAAW,EAAS,CAAE,GAAG,KAAK,UAAU,EAAQ,KAAM,EAAE,CAAC,IAAK,OAAO,CAIrF,SAAgB,EAAgB,EAAyB,EAAmC,CAC1F,IAAM,EAAO,EAAW,EAAS,CACjC,GAAI,CAAC,EAAW,EAAK,CACnB,MAAO,CACL,WAAY,EAAS,GACrB,QAAS,YACT,QAAS,CAAC,yDAA2D,CACtE,CAEH,IAAM,EAAS,KAAK,MAAM,EAAa,EAAM,OAAO,CAAC,CAC/C,EAAoB,EAAE,CAEtB,EAAc,KAAK,UAAU,EAAO,UAAU,CAC9C,EAAc,KAAK,UAAU,EAAO,UAAU,CAChD,IAAgB,GAClB,EAAQ,KAAK,mCAAmC,EAAY,cAAc,IAAc,CAG1F,IAAM,EAAW,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,EAAO,UAAU,CAAE,GAAG,OAAO,KAAK,EAAO,UAAU,CAAC,CAAC,CAC9F,IAAK,IAAM,IAAQ,CAAC,GAAG,EAAS,CAAC,MAAM,CAAE,CACvC,IAAM,EAAS,EAAO,UAAU,GAC1B,EAAQ,EAAO,UAAU,GAC3B,IAAW,IAGX,IAAW,IAAA,GACb,EAAQ,KAAK,4BAA4B,IAAO,CACvC,IAAU,IAAA,GACnB,EAAQ,KAAK,qBAAqB,IAAO,CAEzC,EAAQ,KAAK,6BAA6B,EAAK,IAAI,EAAO,KAAK,EAAM,GAAG,EAW5E,OAPI,EAAO,QAAQ,aAAe,EAAO,QAAQ,YAC/C,EAAQ,KAAK,uBAAuB,EAAO,QAAQ,WAAW,KAAK,EAAO,QAAQ,aAAa,CAE7F,EAAO,QAAQ,WAAa,EAAO,QAAQ,UAC7C,EAAQ,KAAK,2BAA2B,EAAO,QAAQ,SAAS,KAAK,EAAO,QAAQ,WAAW,CAG1F,CACL,WAAY,EAAS,GACrB,QAAS,EAAQ,SAAW,EAAI,QAAU,WAC1C,QAAS,EAAQ,SAAW,EAAI,CAAC,sBAAsB,CAAG,EAC3D"}

package/dist/tier-a-cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import{a as e,i as t,o as n,t as r}from"./tier-a-DvMUFMGy.js";import{join as i,resolve as a}from"node:path";async function o(){let o=process.argv.slice(2),s=o.includes(`--update`),c=o.indexOf(`--recordings-root`),l=c!==-1&&o[c+1]!==void 0?a(o[c+1]):i(process.cwd(),`evals`,`recordings`),u=o.find(e=>!e.startsWith(`--`));if(u!==`run`)return process.stdout.write(`usage:
+import{a as e,i as t,o as n,t as r}from"./tier-a-DPSonK0s.js";import{join as i,resolve as a}from"node:path";async function o(){let o=process.argv.slice(2),s=o.includes(`--update`),c=o.indexOf(`--recordings-root`),l=c!==-1&&o[c+1]!==void 0?a(o[c+1]):i(process.cwd(),`evals`,`recordings`),u=o.find(e=>!e.startsWith(`--`));if(u!==`run`)return process.stdout.write(`usage:
   otto-eval-tier-a run [--recordings-root <dir>]      # compare against golden, exit 1 on mismatch
   otto-eval-tier-a run --update                        # re-run and overwrite goldens (explicit re-record)
 `),u===void 0?0:2;let d=t(l);if(d.length===0)return process.stderr.write(`no scenarios under ${l}\n`),2;let f=0;for(let t of d){let i=await e(t);if(s){n(t,i),process.stdout.write(`updated golden: ${t.id}\n`);continue}let a=r(t,i),o=a.verdict===`match`?`ok`:a.verdict;if(process.stdout.write(`[${o}] ${t.id}\n`),a.verdict!==`match`){f++;for(let e of a.details)process.stdout.write(`    ${e}\n`)}}return s?0:f>0?(process.stdout.write(`\n${f} scenario(s) diverged from golden. If the engine change is intentional, re-record with --update (and say why in the commit).\n`),1):(process.stdout.write(`\nall ${d.length} scenario(s) match golden — engine unchanged.\n`),0)}o().then(e=>process.exit(e)).catch(e=>{process.stderr.write(`tier-a failed: ${e instanceof Error?e.message:String(e)}\n`),process.exit(2)});export{};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@x-otto/eval",
-  "version": "0.0.1-alpha.2",
+  "version": "0.1.0-alpha.9",
   "bin": {
     "otto-eval": "./dist/cli.js",
     "otto-eval-tier-a": "./dist/tier-a-cli.js"
@@ -19,12 +19,12 @@
   },
   "dependencies": {
     "yaml": "2.8.3",
-    "@x-otto/agent": "0.0.1-alpha.2",
-    "@x-otto/interchange": "0.1.0-alpha.3",
-    "@x-otto/env": "0.1.0-alpha.6",
-    "@x-otto/shared": "0.1.0-alpha.6",
-    "@x-otto/provider": "0.1.0-alpha.5",
-    "@x-otto/tools": "0.0.1-alpha.2"
+    "@x-otto/agent": "0.1.0-alpha.9",
+    "@x-otto/interchange": "0.1.0-alpha.9",
+    "@x-otto/provider": "0.1.0-alpha.9",
+    "@x-otto/shared": "0.1.0-alpha.9",
+    "@x-otto/tools": "0.1.0-alpha.9",
+    "@x-otto/env": "0.1.0-alpha.9"
   },
   "devDependencies": {
     "typescript": "6.0.2",

package/dist/extract-CJuoP2bV.js

@@ -1,39 +0,0 @@
-import{t as e}from"./workspace-C_kTWfF8.js";import{createHash as t}from"node:crypto";import{existsSync as n,mkdirSync as r,readFileSync as i,readdirSync as a,statSync as o,writeFileSync as s}from"node:fs";import{dirname as c,join as l}from"node:path";import{parse as u}from"yaml";import{spawn as d,spawnSync as f}from"node:child_process";import{OTTO_SESSIONS_DIR as p,normalizeEnv as m,normalizeEnvFraction as h}from"@x-otto/env";const g=[`script`,`golden`,`liveness`];function _(e,t,n){let r=e[t];if(typeof r!=`string`||r.trim()===``)throw Error(`case '${n}': field '${t}' must be a non-empty string`);return r}function v(e,t,n,r){let i=e[t];if(i===void 0)return r;if(typeof i!=`number`||!Number.isInteger(i)||i<=0)throw Error(`case '${n}': field '${t}' must be a positive integer`);return i}function y(e,t){if(typeof e!=`object`||!e)throw Error(`case '${t}': judge='golden' requires a 'golden' mapping`);let n=e,r={},i=n.answerContains;if(i!==void 0){if(!Array.isArray(i)||i.some(e=>typeof e!=`string`))throw Error(`case '${t}': golden.answerContains must be a string array`);Object.assign(r,{answerContains:i})}let a=n.toolSequence;if(a!==void 0){if(!Array.isArray(a)||a.some(e=>typeof e!=`string`))throw Error(`case '${t}': golden.toolSequence must be a string array`);Object.assign(r,{toolSequence:a})}if(!r.answerContains&&!r.toolSequence)throw Error(`case '${t}': judge='golden' requires at least one of golden.answerContains / golden.toolSequence`);return r}function b(e){let t=e.split(`/`).filter(Boolean).at(-1)??e,r=l(e,`case.yaml`);if(!n(r))throw Error(`case '${t}': missing case.yaml at ${r}`);let a=u(i(r,`utf8`));if(typeof a!=`object`||!a)throw Error(`case '${t}': case.yaml must contain a mapping`);let o=a,s=_(o,`prompt`,t),c=_(o,`judge`,t);if(!g.includes(c))throw Error(`case '${t}': judge must be one of ${g.join(` | `)}`);let d=c,f={id:t,prompt:s,judge:d,runs:v(o,`runs`,t,3),timeoutMs:v(o,`timeoutMs`,t,3e5),dir:e,hasFixture:n(l(e,`fixture`))};if(o.maxTokens!==void 0&&Object.assign(f,{maxTokens:v(o,`maxTokens`,t,0)}),d===`script`){let r=o.verifyScript===void 0?`verify.sh`:_(o,`verifyScript`,t);if(!n(l(e,r)))throw Error(`case '${t}': judge='script' but verify script '${r}' not found`);Object.assign(f,{verifyScript:r})}return(d===`golden`||o.golden!==void 0)&&Object.assign(f,{golden:y(o.golden,t)}),f}function x(e){if(!n(e))throw Error(`cases root not found: ${e}`);return a(e).filter(e=>!e.startsWith(`.`)).map(t=>l(e,t)).filter(e=>o(e).isDirectory()).sort().map(b)}function S(e){let n=e.map(e=>({id:e.id,prompt:e.prompt,judge:e.judge,golden:e.golden??null}));return t(`sha256`).update(JSON.stringify(n)).digest(`hex`).slice(0,12)}function C(e){let t=e.toLowerCase();return[`rate limit`,`rate_limit`,`429`,`overloaded`,`econnreset`,`etimedout`,`enotfound`,`socket hang up`,`service unavailable`,`internal server error`,`bad gateway`,`no model configured`,`command not found`,`enoent`].some(e=>t.includes(e))?`infra`:`failed`}function w(e={}){let t=e.bin??`otto`;return{invoke(n){return new Promise(r=>{let i=d(t,[`--json`,n.prompt],{cwd:n.cwd,env:{...process.env,...e.env},stdio:[`ignore`,`pipe`,`pipe`]}),a=``,o=``,s=!1,c=e=>{s||(s=!0,clearTimeout(l),r(e))},l=setTimeout(()=>{i.kill(`SIGKILL`),c({kind:`failed`,detail:`timeout after ${n.timeoutMs}ms`})},n.timeoutMs);i.stdout.on(`data`,e=>{a+=e.toString()}),i.stderr.on(`data`,e=>{o+=e.toString()}),i.on(`error`,e=>{c({kind:`infra`,detail:`spawn failed: ${e.message}`})}),i.on(`close`,e=>{if(e!==0){c({kind:C(o),detail:`exit ${e}: ${o.trim().slice(0,500)}`});return}try{let e=JSON.parse(a);if(typeof e.sessionId!=`string`){c({kind:`failed`,detail:`json output missing sessionId`});return}c({kind:`ok`,result:e})}catch{c({kind:`failed`,detail:`unparsable json output: ${a.slice(0,300)}`})}})})}}}function T(e,t){let n=e?.[t];return typeof n==`number`&&Number.isFinite(n)?n:0}function E(e){return l(p,`trace`,`${e}.jsonl`)}function D(e){let t=[];return{metrics:O(e,t),toolNames:t}}function O(e,t){let n={turns:0,toolCalls:0,toolErrors:0,inputTokens:0,outputTokens:0,cacheReadTokens:0,toolDurationMs:0};for(let r of e.split(`
-`)){if(r.trim()===``)continue;let e;try{e=JSON.parse(r)}catch{continue}if(e.node===`tool.call.end`){n.toolCalls++,n.toolDurationMs+=T(e.payload,`durationMs`),e.payload?.errorCategory!==void 0&&n.toolErrors++;let r=e.payload?.name;typeof r==`string`&&t?.push(r);continue}if(e.node===`turn.end`){n.turns++;let t=e.payload?.usage;if(typeof t==`object`&&t){let e=t;n.inputTokens+=T(e,`inputTokens`),n.outputTokens+=T(e,`outputTokens`),n.cacheReadTokens+=T(e,`cacheReadTokens`)}let r=e.payload?.model;typeof r==`string`&&r!==``&&(n.modelId=r)}}return n}function k(e){let t=E(e);if(n(t))return D(i(t,`utf8`))}function A(e){if(e.length===0)return;let t=e.reduce((e,t)=>({turns:e.turns+t.turns,toolCalls:e.toolCalls+t.toolCalls,toolErrors:e.toolErrors+t.toolErrors,inputTokens:e.inputTokens+t.inputTokens,outputTokens:e.outputTokens+t.outputTokens,cacheReadTokens:e.cacheReadTokens+t.cacheReadTokens,toolDurationMs:e.toolDurationMs+t.toolDurationMs,modelId:t.modelId??e.modelId}),{turns:0,toolCalls:0,toolErrors:0,inputTokens:0,outputTokens:0,cacheReadTokens:0,toolDurationMs:0}),n=e.length,r={turns:t.turns/n,toolCalls:t.toolCalls/n,toolErrors:t.toolErrors/n,inputTokens:t.inputTokens/n,outputTokens:t.outputTokens/n,cacheReadTokens:t.cacheReadTokens/n,toolDurationMs:t.toolDurationMs/n};return t.modelId!==void 0&&(r.modelId=t.modelId),r}function j(e,t){let n=0;for(let r of e)n<t.length&&r===t[n]&&n++;return n===t.length}function M(e,t,n){let r=[];if(e.answerContains){let n=t.toLowerCase();for(let t of e.answerContains)n.includes(t.toLowerCase())||r.push(`answer missing '${t}'`)}return e.toolSequence&&!j(n,e.toolSequence)&&r.push(`tool sequence [${e.toolSequence.join(`, `)}] not satisfied by [${n.join(`, `)}]`),r.length===0?{passed:!0,detail:`golden assertions satisfied`}:{passed:!1,detail:r.join(`; `)}}function N(e,t){let n=f(`bash`,[t],{cwd:e.workspaceDir,encoding:`utf8`,timeout:12e4,env:{...process.env,OTTO_EVAL_RESPONSE:e.response}});if(n.error)return{passed:!1,detail:`verify script error: ${n.error.message}`};let r=`${(n.stdout??``).trim()}${n.stderr?` | ${n.stderr.trim()}`:``}`.slice(0,500);return{passed:n.status===0,detail:r||`exit ${n.status}`}}function P(e){let{spec:t}=e,n=[];if(t.judge===`script`&&n.push(N(e,l(t.dir,t.verifyScript??`verify.sh`))),t.golden&&n.push(M(t.golden,e.response,e.toolNames)),n.length===0)return{passed:!0,detail:`liveness only (no quality judgement)`};let r=n.filter(e=>!e.passed);return r.length===0?{passed:!0,detail:n.map(e=>e.detail).join(`; `)}:{passed:!1,detail:r.map(e=>e.detail).join(`; `)}}const F=h(process.env.OTTO_EVAL_INFRA_ERROR_THRESHOLD,.2);function I(e){return e?e.inputTokens+e.outputTokens:0}async function L(t,n){let r=n.now??(()=>Date.now()),i=n.readTrace??k,a=[],o=0,s=0,c=0,l=()=>{if(t.maxTokens===void 0)return!1;let e=s>0?o/s:t.maxTokens/t.runs;return o+c*e>=t.maxTokens};for(let u=0;u<t.runs;u++){if(l()){a.push({outcome:`skipped_budget`,wallMs:0,detail:`budget exhausted: ${o} measured tokens${c>0?` + ${c} unmeasured run(s)`:``} vs limit ${t.maxTokens}, remaining runs skipped`});continue}let d=e(t),f=r();try{let e=await n.invoker.invoke({prompt:t.prompt,cwd:d.dir,timeoutMs:t.timeoutMs}),l=r()-f;if(e.kind===`infra`){c++,a.push({outcome:`infra_error`,wallMs:l,detail:e.detail});continue}if(e.kind===`failed`){c++,a.push({outcome:`fail`,wallMs:l,detail:e.detail});continue}let u=i(e.result.sessionId),p=P({spec:t,workspaceDir:d.dir,response:e.result.response,toolNames:u?.toolNames??[]});u?(o+=I(u.metrics),s++):c++;let m={outcome:p.passed?`pass`:`fail`,wallMs:l,sessionId:e.result.sessionId,detail:p.detail};u&&(m.metrics=u.metrics),a.push(m)}finally{d.dispose()}let p=a.at(-1);p&&n.onRunComplete?.(t.id,u,p)}return R(t,a)}function R(e,t){let n=t.filter(e=>e.outcome===`infra_error`).length,r=t.filter(e=>e.outcome===`skipped_budget`).length,i=t.length-n-r,a=t.filter(e=>e.outcome===`pass`).length,o=e.judge!==`liveness`,s={caseId:e.id,judge:e.judge,runs:t,passed:a,effective:i,infraErrors:n,skippedBudget:r,qualityScored:o};return i>0&&Object.assign(s,{passRate:a/i}),s}async function z(e,t,n){let r=n.now??(()=>Date.now()),i=new Date(r()).toISOString(),a=[];for(let t of e)a.push(await L(t,n));return B(a,t,i)}function B(e,t,n){let r=e.filter(e=>e.qualityScored&&e.passRate!==void 0),i=e.reduce((e,t)=>e+t.skippedBudget,0),a=e.reduce((e,t)=>e+t.runs.length,0)-i,o=e.reduce((e,t)=>e+t.infraErrors,0),s=a===0?0:o/a,c=A(e.flatMap(e=>e.runs.map(e=>e.metrics).filter(e=>e!==void 0))),l={suiteVersion:t,modelId:c?.modelId??`unknown`,startedAt:n,cases:e,scoredCases:e.filter(e=>e.qualityScored).length,unscoredCases:e.filter(e=>!e.qualityScored).length,infraErrorRate:s,skippedBudgetTotal:i,invalid:s>F};if(r.length>0){let e=r.reduce((e,t)=>e+(t.passRate??0),0);Object.assign(l,{overallPassRate:e/r.length})}return c&&Object.assign(l,{aggregateMetrics:c}),l}const V=m(process.env.OTTO_EVAL_REGRESSION_PP,15);function H(e){return n(e)?{entries:JSON.parse(i(e,`utf8`)).entries??[]}:{entries:[]}}function U(e,t){r(c(e),{recursive:!0}),s(e,`${JSON.stringify(t,null,2)}\n`,`utf8`)}function W(e,t,n){return e.entries.find(e=>e.modelId===t&&e.suiteVersion===n)}function G(e,t,n){let r={};for(let e of t.cases)r[e.caseId]=e.passRate;let i={modelId:t.modelId,suiteVersion:t.suiteVersion,recordedAt:t.startedAt,reason:n,casePassRates:r};return t.overallPassRate!==void 0&&Object.assign(i,{overallPassRate:t.overallPassRate}),{entries:[...e.entries.filter(e=>!(e.modelId===i.modelId&&e.suiteVersion===i.suiteVersion)),i]}}function K(e,t){if(e.invalid)return{status:`invalid_run`,findings:[`infra_error rate ${(e.infraErrorRate*100).toFixed(1)}% exceeds threshold — run not comparable`],flippedCases:[]};if(!t)return{status:`no_baseline`,findings:[`no baseline for (model=${e.modelId}, suite=${e.suiteVersion}) — record one with \`baseline update\``],flippedCases:[]};let n=[],r=[];for(let i of e.cases){if(!i.qualityScored)continue;let e=t.casePassRates[i.caseId];if(e===1&&i.passRate===0){r.push(i.caseId),n.push(`case flip: '${i.caseId}' was 100% pass, now 0%`);continue}i.passRate===void 0&&e!==void 0&&n.push(`'${i.caseId}' produced no comparable runs (all infra errors) — excluded from the suite average`)}let i;e.overallPassRate!==void 0&&t.overallPassRate!==void 0&&(i=(e.overallPassRate-t.overallPassRate)*100,i<-V&&n.push(`suite pass rate dropped ${Math.abs(i).toFixed(1)}pp (${(t.overallPassRate*100).toFixed(1)}% → ${(e.overallPassRate*100).toFixed(1)}%)`));let a={status:r.length>0||i!==void 0&&i<-V?`regressed`:`ok`,findings:n.length>0?n:[`no regression detected`],flippedCases:r};return i!==void 0&&Object.assign(a,{passRateDeltaPp:i}),a}function q(e){return e===void 0?`—`:`${(e*100).toFixed(0)}%`}function J(e,t){let n=[],r=t.status===`regressed`?`REGRESSED`:t.status===`ok`?`OK`:t.status===`invalid_run`?`INVALID (infra errors)`:`NO BASELINE`;n.push(`otto eval — ${r}`),n.push(`model=${e.modelId}  suite=${e.suiteVersion}  at=${e.startedAt}`),n.push(``),n.push(`case                              judge     pass   runs  infra  skip`);for(let t of e.cases){let e=t.caseId.padEnd(33).slice(0,33),r=t.judge.padEnd(9),i=(t.qualityScored?q(t.passRate):`n/a`).padStart(5),a=`${t.passed}/${t.effective}`.padStart(6),o=String(t.infraErrors).padStart(6),s=String(t.skippedBudget).padStart(5);n.push(`${e} ${r} ${i} ${a} ${o} ${s}`)}if(n.push(``),n.push(`suite pass rate: ${q(e.overallPassRate)}  (scored ${e.scoredCases}, liveness-only ${e.unscoredCases})`),t.passRateDeltaPp!==void 0){let e=t.passRateDeltaPp>=0?`+`:``;n.push(`vs baseline:     ${e}${t.passRateDeltaPp.toFixed(1)}pp`)}n.push(`infra errors:    ${(e.infraErrorRate*100).toFixed(1)}% of runs`),e.skippedBudgetTotal>0&&n.push(`skipped (budget): ${e.skippedBudgetTotal} run(s) — suite incomplete for those cases`);let i=e.aggregateMetrics;i&&n.push(`avg per run:     ${i.turns.toFixed(1)} turns · ${i.toolCalls.toFixed(1)} tools (${i.toolErrors.toFixed(1)} err) · ${Math.round(i.inputTokens+i.outputTokens)} tokens`),n.push(``);for(let e of t.findings)n.push(`- ${e}`);return e.unscoredCases>0&&(n.push(``),n.push(`note: ${e.unscoredCases} case(s) have no machine judgement (liveness only) and do not affect the suite pass rate.`)),n.join(`
-`)}const Y=[`path`,`filePath`,`file_path`];function X(e,t){let n,r=[],i=new Set,a=0;for(let e of t.split(`
-`)){if(e.trim()===``)continue;let t;try{t=JSON.parse(e)}catch{continue}if(t.node===`prompt.before`&&n===void 0){let e=t.payload?.text;typeof e==`string`&&e.trim()!==``&&(n=e);continue}if(t.node===`tool.call.start`){let e=t.payload?.name;typeof e==`string`&&r.push(e);let n=t.payload?.arguments;if(typeof n==`object`&&n)for(let e of Y){let t=n[e];typeof t==`string`&&t!==``&&i.add(t)}continue}t.node===`turn.end`&&a++}return{sessionId:e,toolSequence:r,touchedPaths:[...i].sort(),turns:a,...n===void 0?{}:{prompt:n}}}function Z(e){let t=e.prompt===void 0?`prompt: |
-  TODO: trace 里没有 prompt.before 文本，请手工填写`:`prompt: |\n${e.prompt.split(`
-`).map(e=>`  ${e}`).join(`
-`)}`,n=e.toolSequence.length>0?`# 实际会话的工具序列（golden.toolSequence 候选，按需裁剪）：\n# ${e.toolSequence.join(` → `)}\n`:``;return`# 由 otto-eval extract 生成的骨架（会话 ${e.sessionId}）。
-# TODO：本 case 尚不可用——judge 是 liveness 占位，必须人工补判据后才能进套件质量分。
-${t}
-judge: liveness  # TODO: 改成 script（写 verify.sh）或 golden（写断言）
-${n}runs: 3
-timeoutMs: 240000
-maxTokens: 200000
-`}function Q(e){let t=e.touchedPaths.length>0?e.touchedPaths.map(e=>`- \`${e}\``).join(`
-`):`- （无 —— 会话没有文件类工具调用）`;return`# 提炼报告 — 会话 ${e.sessionId}
-
-- LLM 往返：${e.turns}
-- 工具调用：${e.toolSequence.length} 次
-
-## 会话触碰过的文件（fixture 需要复现哪些初始状态的线索）
-
-${t}
-
-## 下一步（按序完成后删除本文件）
-
-1. 在 fixture/ 里构造这些文件的**任务开始前**状态（trace 只有调用记录，初始内容需要你还原）
-2. 写判据：verify.sh（首选，可执行判据）或 case.yaml 里的 golden 断言
-3. 把 judge 从 liveness 改成 script/golden
-4. 过一遍下面的脱敏清单
-
-## 脱敏检查清单（入库前必须逐项人工核对 — RFC-316 规则 7）
-
-- [ ] prompt 里没有 API key / token / 密码 / 内部主机名
-- [ ] prompt 里没有真实用户数据（邮箱/手机号/姓名）
-- [ ] fixture 文件里没有上述任何内容
-- [ ] 路径不含本机用户名等机器指纹（用相对路径）
-- [ ] verify 判据已由人工编写（不要用"输出像上次一样"的快照当判据）
-`}function $(e,t,i=ee){let a=i(e);if(a===void 0)throw Error(`no trace found for session ${e} (looked at ${E(e)})`);let o=X(e,a),c=l(t,`extracted-${e.slice(0,8)}`);if(n(c))throw Error(`${c} already exists — refusing to overwrite (delete it first if intended)`);return r(l(c,`fixture`),{recursive:!0}),s(l(c,`case.yaml`),Z(o),`utf8`),s(l(c,`EXTRACTION.md`),Q(o),`utf8`),{caseDir:c,skeleton:o}}function ee(e){let t=E(e);return n(t)?i(t,`utf8`):void 0}export{S as C,w as S,x as T,A as _,W as a,E as b,U as c,B as d,L as f,P as g,j as h,V as i,G as l,R as m,X as n,K as o,z as p,J as r,H as s,$ as t,F as u,D as v,b as w,C as x,k as y};
-//# sourceMappingURL=extract-CJuoP2bV.js.map

package/dist/extract-CJuoP2bV.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract-CJuoP2bV.js","names":["parseYaml","renderReport"],"sources":["../src/case-loader.ts","../src/otto-invoker.ts","../src/trace-metrics.ts","../src/verify.ts","../src/runner.ts","../src/baseline.ts","../src/report.ts","../src/extract.ts"],"sourcesContent":["/**\n * case-loader.ts —— 读取并校验 `evals/cases/<id>/case.yaml`（RFC-316 D4）。\n *\n * 校验失败一律抛错而非静默降级：一个格式错的 case 静默变成 liveness case，\n * 会让 scorecard 悄悄失去质量信号——这正是 RFC-316 规则 2 要防的「假质量分」。\n */\n\nimport { createHash } from 'node:crypto'\nimport { readFileSync, readdirSync, existsSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { parse as parseYaml } from 'yaml'\n\nimport type { CaseSpec, GoldenSpec, JudgeKind } from './types'\n\nconst DEFAULT_RUNS = 3\nconst DEFAULT_TIMEOUT_MS = 300_000\nconst DEFAULT_VERIFY_SCRIPT = 'verify.sh'\n\nconst JUDGE_KINDS: readonly JudgeKind[] = ['script', 'golden', 'liveness']\n\nfunction requireString(raw: Record<string, unknown>, key: string, caseId: string): string {\n  const value = raw[key]\n  if (typeof value !== 'string' || value.trim() === '') {\n    throw new Error(`case '${caseId}': field '${key}' must be a non-empty string`)\n  }\n  return value\n}\n\nfunction optionalPositiveInt(\n  raw: Record<string, unknown>,\n  key: string,\n  caseId: string,\n  fallback: number,\n): number {\n  const value = raw[key]\n  if (value === undefined) {\n    return fallback\n  }\n  if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {\n    throw new Error(`case '${caseId}': field '${key}' must be a positive integer`)\n  }\n  return value\n}\n\nfunction parseGolden(raw: unknown, caseId: string): GoldenSpec {\n  if (typeof raw !== 'object' || raw === null) {\n    throw new Error(`case '${caseId}': judge='golden' requires a 'golden' mapping`)\n  }\n  const record = raw as Record<string, unknown>\n  const golden: GoldenSpec = {}\n  const answerContains = record['answerContains']\n  if (answerContains !== undefined) {\n    if (!Array.isArray(answerContains) || answerContains.some((x) => typeof x !== 'string')) {\n      throw new Error(`case '${caseId}': golden.answerContains must be a string array`)\n    }\n    Object.assign(golden, { answerContains: answerContains as string[] })\n  }\n  const toolSequence = record['toolSequence']\n  if (toolSequence !== undefined) {\n    if (!Array.isArray(toolSequence) || toolSequence.some((x) => typeof x !== 'string')) {\n      throw new Error(`case '${caseId}': golden.toolSequence must be a string array`)\n    }\n    Object.assign(golden, { toolSequence: toolSequence as string[] })\n  }\n  if (!golden.answerContains && !golden.toolSequence) {\n    throw new Error(\n      `case '${caseId}': judge='golden' requires at least one of golden.answerContains / golden.toolSequence`,\n    )\n  }\n  return golden\n}\n\n/** 解析单个 case 目录。`dir` 必须已存在且含 case.yaml。 */\nexport function loadCase(dir: string): CaseSpec {\n  const caseId = dir.split('/').filter(Boolean).at(-1) ?? dir\n  const yamlPath = join(dir, 'case.yaml')\n  if (!existsSync(yamlPath)) {\n    throw new Error(`case '${caseId}': missing case.yaml at ${yamlPath}`)\n  }\n\n  const parsed: unknown = parseYaml(readFileSync(yamlPath, 'utf8'))\n  if (typeof parsed !== 'object' || parsed === null) {\n    throw new Error(`case '${caseId}': case.yaml must contain a mapping`)\n  }\n  const raw = parsed as Record<string, unknown>\n\n  const prompt = requireString(raw, 'prompt', caseId)\n  const judgeRaw = requireString(raw, 'judge', caseId)\n  if (!JUDGE_KINDS.includes(judgeRaw as JudgeKind)) {\n    throw new Error(`case '${caseId}': judge must be one of ${JUDGE_KINDS.join(' | ')}`)\n  }\n  const judge = judgeRaw as JudgeKind\n\n  const spec: CaseSpec = {\n    id: caseId,\n    prompt,\n    judge,\n    runs: optionalPositiveInt(raw, 'runs', caseId, DEFAULT_RUNS),\n    timeoutMs: optionalPositiveInt(raw, 'timeoutMs', caseId, DEFAULT_TIMEOUT_MS),\n    dir,\n    hasFixture: existsSync(join(dir, 'fixture')),\n  }\n\n  if (raw['maxTokens'] !== undefined) {\n    Object.assign(spec, { maxTokens: optionalPositiveInt(raw, 'maxTokens', caseId, 0) })\n  }\n\n  if (judge === 'script') {\n    const script =\n      raw['verifyScript'] === undefined\n        ? DEFAULT_VERIFY_SCRIPT\n        : requireString(raw, 'verifyScript', caseId)\n    if (!existsSync(join(dir, script))) {\n      throw new Error(`case '${caseId}': judge='script' but verify script '${script}' not found`)\n    }\n    Object.assign(spec, { verifyScript: script })\n  }\n\n  // golden 断言与 script 判据正交：judge='script' 的 case 也可附加 golden（工具纪律 + 产物正确性\n  // 两个判据都要满足）。judge='golden' 时 golden 必填，其余情况可选。\n  if (judge === 'golden' || raw['golden'] !== undefined) {\n    Object.assign(spec, { golden: parseGolden(raw['golden'], caseId) })\n  }\n\n  return spec\n}\n\n/** 扫描 case 根目录（`evals/cases/`），按 id 排序返回全部 case。 */\nexport function loadSuite(casesRoot: string): readonly CaseSpec[] {\n  if (!existsSync(casesRoot)) {\n    throw new Error(`cases root not found: ${casesRoot}`)\n  }\n  const dirs = readdirSync(casesRoot)\n    .filter((name) => !name.startsWith('.'))\n    .map((name) => join(casesRoot, name))\n    .filter((path) => statSync(path).isDirectory())\n    .sort()\n  return dirs.map(loadCase)\n}\n\n/**\n * 套件版本 = 全部 case 判分相关字段的内容指纹（基线分键用）。\n *\n * 只纳入**影响判分口径**的字段：case 集合、prompt、judge、golden。改超时/runs 不算换套件\n * （不影响可比性），改 prompt 或判据则必须换基线键——否则会把两套不同题目的分数当同一条曲线看。\n */\nexport function computeSuiteVersion(cases: readonly CaseSpec[]): string {\n  const canonical = cases.map((c) => ({\n    id: c.id,\n    prompt: c.prompt,\n    judge: c.judge,\n    golden: c.golden ?? null,\n  }))\n  return createHash('sha256').update(JSON.stringify(canonical)).digest('hex').slice(0, 12)\n}\n","/**\n * otto-invoker.ts —— 调用 headless otto 的唯一出口（RFC-316 规则 6：只走公开入口）。\n *\n * 端口化的理由：runner 的编排逻辑（n 次运行 / 预算护栏 / 三态统计）必须能在**不起真实模型**\n * 的情况下被单测，否则评测体系自身没有回归保护。真实实现 spawn `otto --json`，测试注入 fake。\n */\n\nimport { spawn } from 'node:child_process'\n\n/** headless `otto --json` 的输出契约（packages/cli/src/modes/run-modes.ts JsonModeResult）。 */\nexport interface OttoJsonResult {\n  sessionId: string\n  status: string\n  messageCount: number\n  response: string\n}\n\nexport interface InvokeRequest {\n  prompt: string\n  /** agent 的工作目录。 */\n  cwd: string\n  timeoutMs: number\n}\n\nexport type InvokeResponse =\n  | { kind: 'ok'; result: OttoJsonResult }\n  /** 进程跑起来了但任务失败（非零退出、输出不可解析）——算 fail。 */\n  | { kind: 'failed'; detail: string }\n  /**\n   * 基础设施问题（限流、网络、供应商 5xx、二进制缺失）——算 infra_error，不进 pass 率\n   * （RFC-316 D6 规则 5：防限流假回归）。\n   */\n  | { kind: 'infra'; detail: string }\n\n/** 调用 otto 的端口。 */\nexport interface OttoInvoker {\n  invoke(request: InvokeRequest): Promise<InvokeResponse>\n}\n\n/**\n * 判定一段 stderr 是否为基础设施故障而非任务失败。\n *\n * 保守取向：只认明确的传输层/额度层信号。误判成 infra 会让真实回归被藏起来，\n * 比误判成 fail 更危险——所以这里宁可漏放。\n */\nexport function classifyFailure(stderr: string): 'infra' | 'failed' {\n  const text = stderr.toLowerCase()\n  const infraSignals = [\n    'rate limit',\n    'rate_limit',\n    '429',\n    'overloaded',\n    'econnreset',\n    'etimedout',\n    'enotfound',\n    'socket hang up',\n    'service unavailable',\n    'internal server error',\n    'bad gateway',\n    'no model configured',\n    'command not found',\n    'enoent',\n  ]\n  return infraSignals.some((signal) => text.includes(signal)) ? 'infra' : 'failed'\n}\n\nexport interface SpawnInvokerOptions {\n  /** otto 可执行文件（默认 'otto'）。 */\n  bin?: string\n  /** 附加环境变量（如指定模型）。 */\n  env?: Readonly<Record<string, string>>\n}\n\n/** 真实实现：spawn `otto --json \"<prompt>\"`。 */\nexport function createSpawnInvoker(options: SpawnInvokerOptions = {}): OttoInvoker {\n  const bin = options.bin ?? 'otto'\n  return {\n    invoke(request) {\n      return new Promise<InvokeResponse>((resolve) => {\n        const child = spawn(bin, ['--json', request.prompt], {\n          cwd: request.cwd,\n          env: { ...process.env, ...options.env },\n          stdio: ['ignore', 'pipe', 'pipe'],\n        })\n        let stdout = ''\n        let stderr = ''\n        let settled = false\n        const finish = (response: InvokeResponse): void => {\n          if (settled) {\n            return\n          }\n          settled = true\n          clearTimeout(timer)\n          resolve(response)\n        }\n        const timer = setTimeout(() => {\n          child.kill('SIGKILL')\n          finish({ kind: 'failed', detail: `timeout after ${request.timeoutMs}ms` })\n        }, request.timeoutMs)\n\n        child.stdout.on('data', (chunk: Buffer) => {\n          stdout += chunk.toString()\n        })\n        child.stderr.on('data', (chunk: Buffer) => {\n          stderr += chunk.toString()\n        })\n        child.on('error', (error) => {\n          finish({ kind: 'infra', detail: `spawn failed: ${error.message}` })\n        })\n        child.on('close', (code) => {\n          if (code !== 0) {\n            const kind = classifyFailure(stderr)\n            finish({ kind, detail: `exit ${code}: ${stderr.trim().slice(0, 500)}` })\n            return\n          }\n          try {\n            const parsed = JSON.parse(stdout) as OttoJsonResult\n            if (typeof parsed.sessionId !== 'string') {\n              finish({ kind: 'failed', detail: 'json output missing sessionId' })\n              return\n            }\n            finish({ kind: 'ok', result: parsed })\n          } catch {\n            finish({ kind: 'failed', detail: `unparsable json output: ${stdout.slice(0, 300)}` })\n          }\n        })\n      })\n    },\n  }\n}\n","/**\n * trace-metrics.ts —— 从 trace JSONL 聚合运行指标（RFC-316 G3 / L3 层）。\n *\n * 数据源是 append-only 的 `OTTO_SESSIONS_DIR/trace/<sessionId>.jsonl`，otto 每个会话都会写。\n * 这一层是「零成本指标」：不额外跑任何东西，评测跑完顺手把成本/步数/工具成功率读出来。\n *\n * 只读两类锚点（其余事件忽略，容忍跨版本 trace）：\n * - `tool.call.end`：工具名 / 耗时 / errorCategory\n * - `turn.end`：usage（token）/ model\n */\n\nimport { readFileSync, existsSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { OTTO_SESSIONS_DIR } from '@x-otto/env'\n\nimport type { RunMetrics } from './types'\n\ninterface RawTraceEvent {\n  kind?: string\n  node?: string\n  payload?: Record<string, unknown>\n}\n\nfunction readNumber(payload: Record<string, unknown> | undefined, key: string): number {\n  const value = payload?.[key]\n  return typeof value === 'number' && Number.isFinite(value) ? value : 0\n}\n\n/** trace 文件的默认路径。 */\nexport function traceFilePath(sessionId: string): string {\n  return join(OTTO_SESSIONS_DIR, 'trace', `${sessionId}.jsonl`)\n}\n\n/** 一次运行从 trace 还原出的全部可判分信息。 */\nexport interface TraceRun {\n  metrics: RunMetrics\n  /** 工具调用名序列（按 tool.call.end 发生顺序），golden.toolSequence 判据的输入。 */\n  toolNames: readonly string[]\n}\n\n/**\n * 解析 trace 内容为指标 + 工具序列。传内容而非路径，便于单测与远端 trace 复用。\n * 坏行直接跳过——trace 是 append-only 的，进程被杀时最后一行可能截断，这是正常状态。\n */\nexport function parseTraceRun(content: string): TraceRun {\n  const toolNames: string[] = []\n  const metrics = parseTraceMetrics(content, toolNames)\n  return { metrics, toolNames }\n}\n\nfunction parseTraceMetrics(content: string, toolNamesOut?: string[]): RunMetrics {\n  const metrics: RunMetrics = {\n    turns: 0,\n    toolCalls: 0,\n    toolErrors: 0,\n    inputTokens: 0,\n    outputTokens: 0,\n    cacheReadTokens: 0,\n    toolDurationMs: 0,\n  }\n\n  for (const line of content.split('\\n')) {\n    if (line.trim() === '') {\n      continue\n    }\n    let event: RawTraceEvent\n    try {\n      event = JSON.parse(line) as RawTraceEvent\n    } catch {\n      continue\n    }\n\n    if (event.node === 'tool.call.end') {\n      metrics.toolCalls++\n      metrics.toolDurationMs += readNumber(event.payload, 'durationMs')\n      if (event.payload?.['errorCategory'] !== undefined) {\n        metrics.toolErrors++\n      }\n      const name = event.payload?.['name']\n      if (typeof name === 'string') {\n        toolNamesOut?.push(name)\n      }\n      continue\n    }\n\n    if (event.node === 'turn.end') {\n      metrics.turns++\n      const usage = event.payload?.['usage']\n      if (typeof usage === 'object' && usage !== null) {\n        const u = usage as Record<string, unknown>\n        metrics.inputTokens += readNumber(u, 'inputTokens')\n        metrics.outputTokens += readNumber(u, 'outputTokens')\n        metrics.cacheReadTokens += readNumber(u, 'cacheReadTokens')\n      }\n      const model = event.payload?.['model']\n      if (typeof model === 'string' && model !== '') {\n        metrics.modelId = model\n      }\n    }\n  }\n\n  return metrics\n}\n\n/** 读某个会话的 trace 并聚合；文件不存在返回 undefined（trace 可能被关闭）。 */\nexport function readTraceRun(sessionId: string): TraceRun | undefined {\n  const path = traceFilePath(sessionId)\n  if (!existsSync(path)) {\n    return undefined\n  }\n  return parseTraceRun(readFileSync(path, 'utf8'))\n}\n\n/** 多次运行的指标均值（scorecard 汇总用）。 */\nexport function averageMetrics(samples: readonly RunMetrics[]): RunMetrics | undefined {\n  if (samples.length === 0) {\n    return undefined\n  }\n  const sum = samples.reduce<RunMetrics>(\n    (acc, m) => ({\n      turns: acc.turns + m.turns,\n      toolCalls: acc.toolCalls + m.toolCalls,\n      toolErrors: acc.toolErrors + m.toolErrors,\n      inputTokens: acc.inputTokens + m.inputTokens,\n      outputTokens: acc.outputTokens + m.outputTokens,\n      cacheReadTokens: acc.cacheReadTokens + m.cacheReadTokens,\n      toolDurationMs: acc.toolDurationMs + m.toolDurationMs,\n      modelId: m.modelId ?? acc.modelId,\n    }),\n    {\n      turns: 0,\n      toolCalls: 0,\n      toolErrors: 0,\n      inputTokens: 0,\n      outputTokens: 0,\n      cacheReadTokens: 0,\n      toolDurationMs: 0,\n    },\n  )\n  const n = samples.length\n  const averaged: RunMetrics = {\n    turns: sum.turns / n,\n    toolCalls: sum.toolCalls / n,\n    toolErrors: sum.toolErrors / n,\n    inputTokens: sum.inputTokens / n,\n    outputTokens: sum.outputTokens / n,\n    cacheReadTokens: sum.cacheReadTokens / n,\n    toolDurationMs: sum.toolDurationMs / n,\n  }\n  if (sum.modelId !== undefined) {\n    averaged.modelId = sum.modelId\n  }\n  return averaged\n}\n","/**\n * verify.ts —— 判分（RFC-316 D3 判分优先级的执行处）。\n *\n * 优先级：可执行判据（script exit 0） > golden 断言 > liveness（跑完没崩）。\n * **liveness 永不产生质量分**——这是继承已删 @x-otto/eval scorecard 的诚实边界：\n * 没有判据就老实说没有，不许用「跑完了」冒充「做对了」。\n */\n\nimport { spawnSync } from 'node:child_process'\nimport { join } from 'node:path'\n\nimport type { CaseSpec, GoldenSpec } from './types'\n\nexport interface VerifyInput {\n  spec: CaseSpec\n  /** agent 运行后的工作区（判分脚本的 cwd）。 */\n  workspaceDir: string\n  /** headless 输出的最终回答。 */\n  response: string\n  /** 本次运行的工具调用序列（按发生顺序），无 trace 时为空。 */\n  toolNames: readonly string[]\n}\n\nexport interface VerifyOutput {\n  passed: boolean\n  detail: string\n}\n\n/** 子序列匹配：期望序列的每个工具名按顺序出现即可，允许中间夹杂其他调用。 */\nexport function matchesToolSubsequence(\n  actual: readonly string[],\n  expected: readonly string[],\n): boolean {\n  let cursor = 0\n  for (const name of actual) {\n    if (cursor < expected.length && name === expected[cursor]) {\n      cursor++\n    }\n  }\n  return cursor === expected.length\n}\n\nfunction verifyGolden(\n  golden: GoldenSpec,\n  response: string,\n  toolNames: readonly string[],\n): VerifyOutput {\n  const misses: string[] = []\n  if (golden.answerContains) {\n    const haystack = response.toLowerCase()\n    for (const needle of golden.answerContains) {\n      if (!haystack.includes(needle.toLowerCase())) {\n        misses.push(`answer missing '${needle}'`)\n      }\n    }\n  }\n  if (golden.toolSequence && !matchesToolSubsequence(toolNames, golden.toolSequence)) {\n    misses.push(\n      `tool sequence [${golden.toolSequence.join(', ')}] not satisfied by [${toolNames.join(', ')}]`,\n    )\n  }\n  return misses.length === 0\n    ? { passed: true, detail: 'golden assertions satisfied' }\n    : { passed: false, detail: misses.join('; ') }\n}\n\nfunction runVerifyScript(input: VerifyInput, script: string): VerifyOutput {\n  const proc = spawnSync('bash', [script], {\n    cwd: input.workspaceDir,\n    encoding: 'utf8',\n    timeout: 120_000,\n    env: { ...process.env, OTTO_EVAL_RESPONSE: input.response },\n  })\n  if (proc.error) {\n    return { passed: false, detail: `verify script error: ${proc.error.message}` }\n  }\n  const detail =\n    `${(proc.stdout ?? '').trim()}${proc.stderr ? ` | ${proc.stderr.trim()}` : ''}`.slice(0, 500)\n  return { passed: proc.status === 0, detail: detail || `exit ${proc.status}` }\n}\n\n/**\n * 执行判分。script 判据在工作区内跑，exit 0 = pass。\n *\n * script + golden 可**同时**存在且必须都满足：判分方式表达的是\"这个 case 靠什么下结论\"，\n * 而工具纪律（golden.toolSequence）与产物正确性（script）是**两个正交的判据**——\n * 只有前者会让\"读了但改错\"蒙混过关，只有后者会让\"盲改碰对\"蒙混过关。\n */\nexport function verifyRun(input: VerifyInput): VerifyOutput {\n  const { spec } = input\n  const parts: VerifyOutput[] = []\n\n  if (spec.judge === 'script') {\n    parts.push(runVerifyScript(input, join(spec.dir, spec.verifyScript ?? 'verify.sh')))\n  }\n  if (spec.golden) {\n    parts.push(verifyGolden(spec.golden, input.response, input.toolNames))\n  }\n\n  if (parts.length === 0) {\n    return { passed: true, detail: 'liveness only (no quality judgement)' }\n  }\n\n  const failed = parts.filter((p) => !p.passed)\n  return failed.length === 0\n    ? { passed: true, detail: parts.map((p) => p.detail).join('; ') }\n    : { passed: false, detail: failed.map((p) => p.detail).join('; ') }\n}\n","/**\n * runner.ts —— 评测编排（RFC-316 M1 核心链路）。\n *\n * 一个 case 跑 n 次 → 四态统计（pass/fail/infra_error/skipped_budget）→ 汇总成 CaseResult。\n * 三条不可削弱的纪律都落在这里：\n * - **infra_error 不进 pass 率**（规则 5）：限流/网络故障不许伪装成能力回归。\n * - **预算护栏**（规则 4）：单 case token 超限即停后续运行并标 incomplete，不静默烧钱。\n * - **liveness 不产生质量分**（规则 2）：qualityScored=false 的 case 不进套件 pass 率。\n */\n\nimport { normalizeEnvFraction } from '@x-otto/env'\n\nimport { createWorkspace } from './workspace'\nimport { verifyRun } from './verify'\nimport { readTraceRun, averageMetrics } from './trace-metrics'\n\nimport type { OttoInvoker } from './otto-invoker'\nimport type { TraceRun } from './trace-metrics'\nimport type { CaseResult, CaseSpec, RunMetrics, RunResult, Scorecard } from './types'\n\n/**\n * infra_error 占比超过此阈值时整次运行标 invalid（RFC-316 D6 规则 5）。\n * 硬编码审计 P1：不同 CI 环境对限流/网络故障的容忍度不同，经 `OTTO_EVAL_INFRA_ERROR_THRESHOLD`\n * 覆盖（比例语义，(0,1] 区间，normalizeEnvFraction 越界回退默认）。\n */\nexport const INFRA_ERROR_INVALID_THRESHOLD = normalizeEnvFraction(\n  process.env['OTTO_EVAL_INFRA_ERROR_THRESHOLD'],\n  0.2,\n)\n\nexport interface RunnerDeps {\n  invoker: OttoInvoker\n  /** 读 trace 的端口（默认读真实 trace 文件；测试注入 fake）。 */\n  readTrace?: (sessionId: string) => TraceRun | undefined\n  /** 当前时刻（毫秒），确定性测试用。 */\n  now?: () => number\n  /** 每次运行结束的进度回调（CLI 打点用）。 */\n  onRunComplete?: (caseId: string, index: number, result: RunResult) => void\n}\n\nfunction totalTokens(metrics: RunMetrics | undefined): number {\n  if (!metrics) {\n    return 0\n  }\n  return metrics.inputTokens + metrics.outputTokens\n}\n\n/** 跑单个 case 的 n 次运行。 */\nexport async function runCase(spec: CaseSpec, deps: RunnerDeps): Promise<CaseResult> {\n  const now = deps.now ?? (() => Date.now())\n  const readTrace = deps.readTrace ?? readTraceRun\n  const runs: RunResult[] = []\n  let spentTokens = 0\n  /** 已成功计量的运行次数（用于估算不可计量运行的开销）。 */\n  let measuredRuns = 0\n  /**\n   * 无法计量 token 的运行次数：trace 缺失，或 fail/timeout 路径拿不到 usage。\n   *\n   * 审计修订：早期实现只在「成功且有 trace」时累加 spentTokens，于是失败运行被当作\n   * **零消耗**——但它们同样调用了模型、同样烧钱。一个\"每次都超时\"的 case 因此可以\n   * 无限重试而预算护栏一次都不触发。护栏若只在一切正常时生效，就不是护栏。\n   *\n   * 估算口径：有实测样本时按**已观测均值**计价（最贴近现实）；一个样本都没有时按\n   * `maxTokens/runs` 均摊——那正是用户声明 `runs` 次时的预期单价，此时跑满 runs 次\n   * 属于预期内，不该被拦。\n   */\n  let unmeasuredRuns = 0\n\n  const budgetExceeded = (): boolean => {\n    if (spec.maxTokens === undefined) {\n      return false\n    }\n    const assumedPerRun = measuredRuns > 0 ? spentTokens / measuredRuns : spec.maxTokens / spec.runs\n    return spentTokens + unmeasuredRuns * assumedPerRun >= spec.maxTokens\n  }\n\n  for (let index = 0; index < spec.runs; index++) {\n    if (budgetExceeded()) {\n      // RFC-353 S9: 预算耗尽是本地跳过决策，不是 infra 故障——用 skipped_budget 终态\n      // 区分\"未尝试\"与\"provider/网络不可靠\"，避免膨胀 infraErrorRate 误判套件失效。\n      runs.push({\n        outcome: 'skipped_budget',\n        wallMs: 0,\n        detail: `budget exhausted: ${spentTokens} measured tokens${\n          unmeasuredRuns > 0 ? ` + ${unmeasuredRuns} unmeasured run(s)` : ''\n        } vs limit ${spec.maxTokens}, remaining runs skipped`,\n      })\n      continue\n    }\n\n    const workspace = createWorkspace(spec)\n    const startedAt = now()\n    try {\n      const response = await deps.invoker.invoke({\n        prompt: spec.prompt,\n        cwd: workspace.dir,\n        timeoutMs: spec.timeoutMs,\n      })\n      const wallMs = now() - startedAt\n\n      if (response.kind === 'infra') {\n        // infra 故障（限流/网络）通常在模型真正产出前就断了，不计入已花费；\n        // 但也拿不到用量证据，故记为不可计量，让护栏保守对待。\n        unmeasuredRuns++\n        runs.push({ outcome: 'infra_error', wallMs, detail: response.detail })\n        continue\n      }\n      if (response.kind === 'failed') {\n        // 失败/超时同样烧了 token（模型已被调用），只是拿不到 usage —— 必须计入护栏，\n        // 否则\"每次都失败\"的 case 会无限重试烧钱。\n        unmeasuredRuns++\n        runs.push({ outcome: 'fail', wallMs, detail: response.detail })\n        continue\n      }\n\n      const trace = readTrace(response.result.sessionId)\n      const verdict = verifyRun({\n        spec,\n        workspaceDir: workspace.dir,\n        response: response.result.response,\n        toolNames: trace?.toolNames ?? [],\n      })\n      if (trace) {\n        spentTokens += totalTokens(trace.metrics)\n        measuredRuns++\n      } else {\n        // 成功但无 trace（trace 被关闭/文件缺失）：同样计量不到，不能当作零消耗。\n        unmeasuredRuns++\n      }\n\n      const result: RunResult = {\n        outcome: verdict.passed ? 'pass' : 'fail',\n        wallMs,\n        sessionId: response.result.sessionId,\n        detail: verdict.detail,\n      }\n      if (trace) {\n        result.metrics = trace.metrics\n      }\n      runs.push(result)\n    } finally {\n      workspace.dispose()\n    }\n\n    const last = runs.at(-1)\n    if (last) {\n      deps.onRunComplete?.(spec.id, index, last)\n    }\n  }\n\n  return summarizeCase(spec, runs)\n}\n\n/** 把 n 次运行汇总成 CaseResult（四态统计的唯一实现处）。 */\nexport function summarizeCase(spec: CaseSpec, runs: readonly RunResult[]): CaseResult {\n  const infraErrors = runs.filter((r) => r.outcome === 'infra_error').length\n  const skippedBudget = runs.filter((r) => r.outcome === 'skipped_budget').length\n  const effective = runs.length - infraErrors - skippedBudget\n  const passed = runs.filter((r) => r.outcome === 'pass').length\n  const qualityScored = spec.judge !== 'liveness'\n\n  const result: CaseResult = {\n    caseId: spec.id,\n    judge: spec.judge,\n    runs,\n    passed,\n    effective,\n    infraErrors,\n    skippedBudget,\n    qualityScored,\n  }\n  if (effective > 0) {\n    Object.assign(result, { passRate: passed / effective })\n  }\n  return result\n}\n\n/** 跑整套 case，产出 scorecard。 */\nexport async function runSuite(\n  cases: readonly CaseSpec[],\n  suiteVersion: string,\n  deps: RunnerDeps,\n): Promise<Scorecard> {\n  const now = deps.now ?? (() => Date.now())\n  const startedAt = new Date(now()).toISOString()\n  const results: CaseResult[] = []\n  for (const spec of cases) {\n    results.push(await runCase(spec, deps))\n  }\n  return buildScorecard(results, suiteVersion, startedAt)\n}\n\n/** 聚合 case 结果为 scorecard（纯函数，便于对拍手算值）。 */\nexport function buildScorecard(\n  cases: readonly CaseResult[],\n  suiteVersion: string,\n  startedAt: string,\n): Scorecard {\n  const scored = cases.filter((c) => c.qualityScored && c.passRate !== undefined)\n  const totalSkippedBudget = cases.reduce((sum, c) => sum + c.skippedBudget, 0)\n  // RFC-353 S9: infraErrorRate 分母排除 skipped_budget——它们不是\"发起的运行\"（本地\n  // 预算护栏主动跳过，未真正调用模型/触达 provider），混入分母会稀释 infraErrorRate\n  // 掩盖真实的 infra 不稳定信号。\n  const totalRuns = cases.reduce((sum, c) => sum + c.runs.length, 0) - totalSkippedBudget\n  const totalInfra = cases.reduce((sum, c) => sum + c.infraErrors, 0)\n  const infraErrorRate = totalRuns === 0 ? 0 : totalInfra / totalRuns\n\n  const allMetrics = cases.flatMap((c) =>\n    c.runs.map((r) => r.metrics).filter((m): m is RunMetrics => m !== undefined),\n  )\n  const aggregate = averageMetrics(allMetrics)\n\n  const scorecard: Scorecard = {\n    suiteVersion,\n    modelId: aggregate?.modelId ?? 'unknown',\n    startedAt,\n    cases,\n    scoredCases: cases.filter((c) => c.qualityScored).length,\n    unscoredCases: cases.filter((c) => !c.qualityScored).length,\n    infraErrorRate,\n    skippedBudgetTotal: totalSkippedBudget,\n    invalid: infraErrorRate > INFRA_ERROR_INVALID_THRESHOLD,\n  }\n  if (scored.length > 0) {\n    const sum = scored.reduce((acc, c) => acc + (c.passRate ?? 0), 0)\n    Object.assign(scorecard, { overallPassRate: sum / scored.length })\n  }\n  if (aggregate) {\n    Object.assign(scorecard, { aggregateMetrics: aggregate })\n  }\n  return scorecard\n}\n","/**\n * baseline.ts —— 基线存储与回归判定（RFC-316 D6 规则 3/4）。\n *\n * 两条承重规则：\n * - **基线按 (modelId, suiteVersion) 分键**：模型升级产生新条目而非覆盖，跨模型对比必须显式。\n *   没有这条，换一次模型就会把曲线接错——之后所有\"变强/变弱\"结论都是噪声。\n * - **阈值显式且诚实**：n=3 的统计功效只够抓大回归，故只用两个粗信号（套件 pass 率跌幅、\n *   case flip），不假装有统计显著性。\n */\n\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'\nimport { dirname } from 'node:path'\n\nimport { normalizeEnv } from '@x-otto/env'\n\nimport type { BaselineEntry, BaselineFile, RegressionVerdict, Scorecard } from './types'\n\n/**\n * 套件 pass 率相对基线的下降超过此百分点即判回归（RFC-316 D6 规则 4）。\n * 硬编码审计 P1：不同 CI 环境的回归门禁松紧不同，经 `OTTO_EVAL_REGRESSION_PP` 覆盖\n * （百分点整数语义，normalizeEnv 非正数回退默认）。\n */\nexport const PASS_RATE_REGRESSION_PP = normalizeEnv(process.env['OTTO_EVAL_REGRESSION_PP'], 15)\n\nexport function loadBaselines(path: string): BaselineFile {\n  if (!existsSync(path)) {\n    return { entries: [] }\n  }\n  const parsed = JSON.parse(readFileSync(path, 'utf8')) as BaselineFile\n  return { entries: parsed.entries ?? [] }\n}\n\nexport function saveBaselines(path: string, file: BaselineFile): void {\n  mkdirSync(dirname(path), { recursive: true })\n  writeFileSync(path, `${JSON.stringify(file, null, 2)}\\n`, 'utf8')\n}\n\n/** 按 (modelId, suiteVersion) 查基线。 */\nexport function findBaseline(\n  file: BaselineFile,\n  modelId: string,\n  suiteVersion: string,\n): BaselineEntry | undefined {\n  return file.entries.find((e) => e.modelId === modelId && e.suiteVersion === suiteVersion)\n}\n\n/** 把一次运行固化成基线条目（同键覆盖，异键新增）。 */\nexport function upsertBaseline(\n  file: BaselineFile,\n  scorecard: Scorecard,\n  reason: string,\n): BaselineFile {\n  const casePassRates: Record<string, number | undefined> = {}\n  for (const c of scorecard.cases) {\n    casePassRates[c.caseId] = c.passRate\n  }\n\n  const entry: BaselineEntry = {\n    modelId: scorecard.modelId,\n    suiteVersion: scorecard.suiteVersion,\n    recordedAt: scorecard.startedAt,\n    reason,\n    casePassRates,\n  }\n  if (scorecard.overallPassRate !== undefined) {\n    Object.assign(entry, { overallPassRate: scorecard.overallPassRate })\n  }\n\n  const kept = file.entries.filter(\n    (e) => !(e.modelId === entry.modelId && e.suiteVersion === entry.suiteVersion),\n  )\n  return { entries: [...kept, entry] }\n}\n\n/** 回归判定：套件 pass 率跌幅 + case flip 双信号。 */\nexport function judgeRegression(\n  scorecard: Scorecard,\n  baseline: BaselineEntry | undefined,\n): RegressionVerdict {\n  if (scorecard.invalid) {\n    return {\n      status: 'invalid_run',\n      findings: [\n        `infra_error rate ${(scorecard.infraErrorRate * 100).toFixed(1)}% exceeds threshold — run not comparable`,\n      ],\n      flippedCases: [],\n    }\n  }\n  if (!baseline) {\n    return {\n      status: 'no_baseline',\n      findings: [\n        `no baseline for (model=${scorecard.modelId}, suite=${scorecard.suiteVersion}) — record one with \\`baseline update\\``,\n      ],\n      flippedCases: [],\n    }\n  }\n\n  const findings: string[] = []\n  const flipped: string[] = []\n\n  for (const c of scorecard.cases) {\n    if (!c.qualityScored) {\n      continue\n    }\n    const before = baseline.casePassRates[c.caseId]\n    if (before === 1 && c.passRate === 0) {\n      flipped.push(c.caseId)\n      findings.push(`case flip: '${c.caseId}' was 100% pass, now 0%`)\n      continue\n    }\n    // 一个 case 的运行全是 infra_error 时它没有 pass 率，会**静默退出**套件均值——\n    // 于是\"某题目彻底跑不起来\"看上去和\"套件表现不变\"一模一样。这里显式点名，\n    // 不判回归（确实无从判断），但绝不让它无声消失。\n    if (c.passRate === undefined && before !== undefined) {\n      findings.push(\n        `'${c.caseId}' produced no comparable runs (all infra errors) — excluded from the suite average`,\n      )\n    }\n  }\n\n  let deltaPp: number | undefined\n  if (scorecard.overallPassRate !== undefined && baseline.overallPassRate !== undefined) {\n    deltaPp = (scorecard.overallPassRate - baseline.overallPassRate) * 100\n    if (deltaPp < -PASS_RATE_REGRESSION_PP) {\n      findings.push(\n        `suite pass rate dropped ${Math.abs(deltaPp).toFixed(1)}pp ` +\n          `(${(baseline.overallPassRate * 100).toFixed(1)}% → ${(scorecard.overallPassRate * 100).toFixed(1)}%)`,\n      )\n    }\n  }\n\n  const regressed =\n    flipped.length > 0 || (deltaPp !== undefined && deltaPp < -PASS_RATE_REGRESSION_PP)\n\n  const verdict: RegressionVerdict = {\n    status: regressed ? 'regressed' : 'ok',\n    findings: findings.length > 0 ? findings : ['no regression detected'],\n    flippedCases: flipped,\n  }\n  if (deltaPp !== undefined) {\n    Object.assign(verdict, { passRateDeltaPp: deltaPp })\n  }\n  return verdict\n}\n","/**\n * report.ts —— scorecard + 回归判定的人类可读渲染（RFC-316 D6 消费回路的出口）。\n *\n * 报告是评测体系唯一被人真正阅读的产物——@x-otto/eval 上一次死于「没人读」，\n * 所以这里刻意把三件事放在最显眼处：回归结论、逐 case pass 率、诚实边界（未评质量的 case 数）。\n */\n\nimport type { RegressionVerdict, Scorecard } from './types'\n\nfunction pct(value: number | undefined): string {\n  return value === undefined ? '—' : `${(value * 100).toFixed(0)}%`\n}\n\nexport function renderReport(scorecard: Scorecard, verdict: RegressionVerdict): string {\n  const lines: string[] = []\n\n  const headline =\n    verdict.status === 'regressed'\n      ? 'REGRESSED'\n      : verdict.status === 'ok'\n        ? 'OK'\n        : verdict.status === 'invalid_run'\n          ? 'INVALID (infra errors)'\n          : 'NO BASELINE'\n\n  lines.push(`otto eval — ${headline}`)\n  lines.push(\n    `model=${scorecard.modelId}  suite=${scorecard.suiteVersion}  at=${scorecard.startedAt}`,\n  )\n  lines.push('')\n\n  lines.push('case                              judge     pass   runs  infra  skip')\n  for (const c of scorecard.cases) {\n    const id = c.caseId.padEnd(33).slice(0, 33)\n    const judge = c.judge.padEnd(9)\n    const rate = (c.qualityScored ? pct(c.passRate) : 'n/a').padStart(5)\n    const runs = `${c.passed}/${c.effective}`.padStart(6)\n    const infra = String(c.infraErrors).padStart(6)\n    const skip = String(c.skippedBudget).padStart(5)\n    lines.push(`${id} ${judge} ${rate} ${runs} ${infra} ${skip}`)\n  }\n  lines.push('')\n\n  lines.push(\n    `suite pass rate: ${pct(scorecard.overallPassRate)}  (scored ${scorecard.scoredCases}, liveness-only ${scorecard.unscoredCases})`,\n  )\n  if (verdict.passRateDeltaPp !== undefined) {\n    const sign = verdict.passRateDeltaPp >= 0 ? '+' : ''\n    lines.push(`vs baseline:     ${sign}${verdict.passRateDeltaPp.toFixed(1)}pp`)\n  }\n  lines.push(`infra errors:    ${(scorecard.infraErrorRate * 100).toFixed(1)}% of runs`)\n  if (scorecard.skippedBudgetTotal > 0) {\n    // RFC-353 S9: 预算耗尽跳过的运行不是 infra 故障，单独一行呈现——提醒读者本次\n    // 评测有 case 未完整跑满 runs 次，结果基于更少样本量。\n    lines.push(`skipped (budget): ${scorecard.skippedBudgetTotal} run(s) — suite incomplete for those cases`)\n  }\n\n  const m = scorecard.aggregateMetrics\n  if (m) {\n    lines.push(\n      `avg per run:     ${m.turns.toFixed(1)} turns · ${m.toolCalls.toFixed(1)} tools ` +\n        `(${m.toolErrors.toFixed(1)} err) · ${Math.round(m.inputTokens + m.outputTokens)} tokens`,\n    )\n  }\n\n  lines.push('')\n  for (const finding of verdict.findings) {\n    lines.push(`- ${finding}`)\n  }\n\n  if (scorecard.unscoredCases > 0) {\n    lines.push('')\n    lines.push(\n      `note: ${scorecard.unscoredCases} case(s) have no machine judgement (liveness only) and do not affect the suite pass rate.`,\n    )\n  }\n\n  return lines.join('\\n')\n}\n","/**\n * extract.ts —— trace 提炼器（RFC-316 D4 来源②：真实会话 → case 骨架）。\n *\n * case 库能否持续生长，决定这套评测第二次会不会死。真实事故/真实任务是最好的 case 来源，\n * 但从零手写 case 的摩擦太大——本模块把一个真实会话一键转成**骨架**：prompt 原文、\n * 工具调用序列（golden.toolSequence 的现成候选）、触碰过的文件清单（fixture 提示）。\n *\n * 两条刻意的\"不做\"：\n * - **不自动生成判据**：判据是 case 的灵魂，必须人写（RFC-316 D3 诚实边界——自动生成的\n *   判据只会是\"输出像上次一样\"，那是快照不是判据）。骨架里留 TODO 与建议。\n * - **不自动脱敏**：不可靠的脱敏比明示风险更危险（对齐 RFC-200 录制红线）。骨架自带\n *   脱敏检查清单，入库前人工过一遍是硬规则（RFC-316 规则 7）。\n */\n\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\n\nimport { traceFilePath } from './trace-metrics'\n\ninterface RawTraceEvent {\n  node?: string\n  payload?: Record<string, unknown>\n}\n\n/** 从 trace 还原的会话骨架素材。 */\nexport interface SessionSkeleton {\n  sessionId: string\n  /** 首条用户 prompt 原文（prompt.before 的 text）。 */\n  prompt?: string\n  /** 工具调用序列（名称，按发生顺序）。 */\n  toolSequence: readonly string[]\n  /** 工具参数里出现过的文件路径（write/edit/read 的 path 类参数）——fixture 需要什么的线索。 */\n  touchedPaths: readonly string[]\n  /** LLM 往返数（turn.end 计数）。 */\n  turns: number\n}\n\nconst PATH_ARGUMENT_KEYS = ['path', 'filePath', 'file_path'] as const\n\n/** 解析 trace 内容为骨架素材（纯函数，可测）。 */\nexport function parseSessionSkeleton(sessionId: string, content: string): SessionSkeleton {\n  let prompt: string | undefined\n  const toolSequence: string[] = []\n  const touchedPaths = new Set<string>()\n  let turns = 0\n\n  for (const line of content.split('\\n')) {\n    if (line.trim() === '') {\n      continue\n    }\n    let event: RawTraceEvent\n    try {\n      event = JSON.parse(line) as RawTraceEvent\n    } catch {\n      continue\n    }\n\n    if (event.node === 'prompt.before' && prompt === undefined) {\n      const text = event.payload?.['text']\n      if (typeof text === 'string' && text.trim() !== '') {\n        prompt = text\n      }\n      continue\n    }\n\n    if (event.node === 'tool.call.start') {\n      const name = event.payload?.['name']\n      if (typeof name === 'string') {\n        toolSequence.push(name)\n      }\n      const args = event.payload?.['arguments']\n      if (typeof args === 'object' && args !== null) {\n        for (const key of PATH_ARGUMENT_KEYS) {\n          const value = (args as Record<string, unknown>)[key]\n          if (typeof value === 'string' && value !== '') {\n            touchedPaths.add(value)\n          }\n        }\n      }\n      continue\n    }\n\n    if (event.node === 'turn.end') {\n      turns++\n    }\n  }\n\n  return {\n    sessionId,\n    toolSequence,\n    touchedPaths: [...touchedPaths].sort(),\n    turns,\n    ...(prompt !== undefined ? { prompt } : {}),\n  }\n}\n\nconst SANITIZE_CHECKLIST = `## 脱敏检查清单（入库前必须逐项人工核对 — RFC-316 规则 7）\n\n- [ ] prompt 里没有 API key / token / 密码 / 内部主机名\n- [ ] prompt 里没有真实用户数据（邮箱/手机号/姓名）\n- [ ] fixture 文件里没有上述任何内容\n- [ ] 路径不含本机用户名等机器指纹（用相对路径）\n- [ ] verify 判据已由人工编写（不要用\"输出像上次一样\"的快照当判据）\n`\n\nfunction renderCaseYaml(skeleton: SessionSkeleton): string {\n  const promptBlock =\n    skeleton.prompt !== undefined\n      ? `prompt: |\\n${skeleton.prompt\n          .split('\\n')\n          .map((line) => `  ${line}`)\n          .join('\\n')}`\n      : 'prompt: |\\n  TODO: trace 里没有 prompt.before 文本，请手工填写'\n\n  const toolHint =\n    skeleton.toolSequence.length > 0\n      ? `# 实际会话的工具序列（golden.toolSequence 候选，按需裁剪）：\\n# ${skeleton.toolSequence.join(' → ')}\\n`\n      : ''\n\n  return `# 由 otto-eval extract 生成的骨架（会话 ${skeleton.sessionId}）。\n# TODO：本 case 尚不可用——judge 是 liveness 占位，必须人工补判据后才能进套件质量分。\n${promptBlock}\njudge: liveness  # TODO: 改成 script（写 verify.sh）或 golden（写断言）\n${toolHint}runs: 3\ntimeoutMs: 240000\nmaxTokens: 200000\n`\n}\n\nfunction renderReport(skeleton: SessionSkeleton): string {\n  const paths =\n    skeleton.touchedPaths.length > 0\n      ? skeleton.touchedPaths.map((p) => `- \\`${p}\\``).join('\\n')\n      : '- （无 —— 会话没有文件类工具调用）'\n\n  return `# 提炼报告 — 会话 ${skeleton.sessionId}\n\n- LLM 往返：${skeleton.turns}\n- 工具调用：${skeleton.toolSequence.length} 次\n\n## 会话触碰过的文件（fixture 需要复现哪些初始状态的线索）\n\n${paths}\n\n## 下一步（按序完成后删除本文件）\n\n1. 在 fixture/ 里构造这些文件的**任务开始前**状态（trace 只有调用记录，初始内容需要你还原）\n2. 写判据：verify.sh（首选，可执行判据）或 case.yaml 里的 golden 断言\n3. 把 judge 从 liveness 改成 script/golden\n4. 过一遍下面的脱敏清单\n\n${SANITIZE_CHECKLIST}`\n}\n\nexport interface ExtractResult {\n  caseDir: string\n  skeleton: SessionSkeleton\n}\n\n/** 从会话 trace 生成 case 骨架目录。目标已存在时拒绝（不覆盖人工编辑过的 case）。 */\nexport function extractCase(\n  sessionId: string,\n  casesRoot: string,\n  readTraceContent: (sessionId: string) => string | undefined = defaultReadTrace,\n): ExtractResult {\n  const content = readTraceContent(sessionId)\n  if (content === undefined) {\n    throw new Error(\n      `no trace found for session ${sessionId} (looked at ${traceFilePath(sessionId)})`,\n    )\n  }\n\n  const skeleton = parseSessionSkeleton(sessionId, content)\n  const caseDir = join(casesRoot, `extracted-${sessionId.slice(0, 8)}`)\n  if (existsSync(caseDir)) {\n    throw new Error(\n      `${caseDir} already exists — refusing to overwrite (delete it first if intended)`,\n    )\n  }\n\n  mkdirSync(join(caseDir, 'fixture'), { recursive: true })\n  writeFileSync(join(caseDir, 'case.yaml'), renderCaseYaml(skeleton), 'utf8')\n  writeFileSync(join(caseDir, 'EXTRACTION.md'), renderReport(skeleton), 'utf8')\n\n  return { caseDir, skeleton }\n}\n\nfunction defaultReadTrace(sessionId: string): string | undefined {\n  const path = traceFilePath(sessionId)\n  return existsSync(path) ? readFileSync(path, 'utf8') : undefined\n}\n"],"mappings":"8aAcA,MAIM,EAAoC,CAAC,SAAU,SAAU,WAAW,CAE1E,SAAS,EAAc,EAA8B,EAAa,EAAwB,CACxF,IAAM,EAAQ,EAAI,GAClB,GAAI,OAAO,GAAU,UAAY,EAAM,MAAM,GAAK,GAChD,MAAU,MAAM,SAAS,EAAO,YAAY,EAAI,8BAA8B,CAEhF,OAAO,EAGT,SAAS,EACP,EACA,EACA,EACA,EACQ,CACR,IAAM,EAAQ,EAAI,GAClB,GAAI,IAAU,IAAA,GACZ,OAAO,EAET,GAAI,OAAO,GAAU,UAAY,CAAC,OAAO,UAAU,EAAM,EAAI,GAAS,EACpE,MAAU,MAAM,SAAS,EAAO,YAAY,EAAI,8BAA8B,CAEhF,OAAO,EAGT,SAAS,EAAY,EAAc,EAA4B,CAC7D,GAAI,OAAO,GAAQ,WAAY,EAC7B,MAAU,MAAM,SAAS,EAAO,+CAA+C,CAEjF,IAAM,EAAS,EACT,EAAqB,EAAE,CACvB,EAAiB,EAAO,eAC9B,GAAI,IAAmB,IAAA,GAAW,CAChC,GAAI,CAAC,MAAM,QAAQ,EAAe,EAAI,EAAe,KAAM,GAAM,OAAO,GAAM,SAAS,CACrF,MAAU,MAAM,SAAS,EAAO,iDAAiD,CAEnF,OAAO,OAAO,EAAQ,CAAkB,iBAA4B,CAAC,CAEvE,IAAM,EAAe,EAAO,aAC5B,GAAI,IAAiB,IAAA,GAAW,CAC9B,GAAI,CAAC,MAAM,QAAQ,EAAa,EAAI,EAAa,KAAM,GAAM,OAAO,GAAM,SAAS,CACjF,MAAU,MAAM,SAAS,EAAO,+CAA+C,CAEjF,OAAO,OAAO,EAAQ,CAAgB,eAA0B,CAAC,CAEnE,GAAI,CAAC,EAAO,gBAAkB,CAAC,EAAO,aACpC,MAAU,MACR,SAAS,EAAO,wFACjB,CAEH,OAAO,EAIT,SAAgB,EAAS,EAAuB,CAC9C,IAAM,EAAS,EAAI,MAAM,IAAI,CAAC,OAAO,QAAQ,CAAC,GAAG,GAAG,EAAI,EAClD,EAAW,EAAK,EAAK,YAAY,CACvC,GAAI,CAAC,EAAW,EAAS,CACvB,MAAU,MAAM,SAAS,EAAO,0BAA0B,IAAW,CAGvE,IAAM,EAAkBA,EAAU,EAAa,EAAU,OAAO,CAAC,CACjE,GAAI,OAAO,GAAW,WAAY,EAChC,MAAU,MAAM,SAAS,EAAO,qCAAqC,CAEvE,IAAM,EAAM,EAEN,EAAS,EAAc,EAAK,SAAU,EAAO,CAC7C,EAAW,EAAc,EAAK,QAAS,EAAO,CACpD,GAAI,CAAC,EAAY,SAAS,EAAsB,CAC9C,MAAU,MAAM,SAAS,EAAO,0BAA0B,EAAY,KAAK,MAAM,GAAG,CAEtF,IAAM,EAAQ,EAER,EAAiB,CACrB,GAAI,EACJ,SACA,QACA,KAAM,EAAoB,EAAK,OAAQ,EAAQ,EAAa,CAC5D,UAAW,EAAoB,EAAK,YAAa,EAAQ,IAAmB,CAC5E,MACA,WAAY,EAAW,EAAK,EAAK,UAAU,CAAC,CAC7C,CAMD,GAJI,EAAI,YAAiB,IAAA,IACvB,OAAO,OAAO,EAAM,CAAE,UAAW,EAAoB,EAAK,YAAa,EAAQ,EAAE,CAAE,CAAC,CAGlF,IAAU,SAAU,CACtB,IAAM,EACJ,EAAI,eAAoB,IAAA,GACpB,YACA,EAAc,EAAK,eAAgB,EAAO,CAChD,GAAI,CAAC,EAAW,EAAK,EAAK,EAAO,CAAC,CAChC,MAAU,MAAM,SAAS,EAAO,uCAAuC,EAAO,aAAa,CAE7F,OAAO,OAAO,EAAM,CAAE,aAAc,EAAQ,CAAC,CAS/C,OAJI,IAAU,UAAY,EAAI,SAAc,IAAA,KAC1C,OAAO,OAAO,EAAM,CAAE,OAAQ,EAAY,EAAI,OAAW,EAAO,CAAE,CAAC,CAG9D,EAIT,SAAgB,EAAU,EAAwC,CAChE,GAAI,CAAC,EAAW,EAAU,CACxB,MAAU,MAAM,yBAAyB,IAAY,CAOvD,OALa,EAAY,EAAU,CAChC,OAAQ,GAAS,CAAC,EAAK,WAAW,IAAI,CAAC,CACvC,IAAK,GAAS,EAAK,EAAW,EAAK,CAAC,CACpC,OAAQ,GAAS,EAAS,EAAK,CAAC,aAAa,CAAC,CAC9C,MAAM,CACG,IAAI,EAAS,CAS3B,SAAgB,EAAoB,EAAoC,CACtE,IAAM,EAAY,EAAM,IAAK,IAAO,CAClC,GAAI,EAAE,GACN,OAAQ,EAAE,OACV,MAAO,EAAE,MACT,OAAQ,EAAE,QAAU,KACrB,EAAE,CACH,OAAO,EAAW,SAAS,CAAC,OAAO,KAAK,UAAU,EAAU,CAAC,CAAC,OAAO,MAAM,CAAC,MAAM,EAAG,GAAG,CC5G1F,SAAgB,EAAgB,EAAoC,CAClE,IAAM,EAAO,EAAO,aAAa,CAiBjC,MAhBqB,CACnB,aACA,aACA,MACA,aACA,aACA,YACA,YACA,iBACA,sBACA,wBACA,cACA,sBACA,oBACA,SACD,CACmB,KAAM,GAAW,EAAK,SAAS,EAAO,CAAC,CAAG,QAAU,SAW1E,SAAgB,EAAmB,EAA+B,EAAE,CAAe,CACjF,IAAM,EAAM,EAAQ,KAAO,OAC3B,MAAO,CACL,OAAO,EAAS,CACd,OAAO,IAAI,QAAyB,GAAY,CAC9C,IAAM,EAAQ,EAAM,EAAK,CAAC,SAAU,EAAQ,OAAO,CAAE,CACnD,IAAK,EAAQ,IACb,IAAK,CAAE,GAAG,QAAQ,IAAK,GAAG,EAAQ,IAAK,CACvC,MAAO,CAAC,SAAU,OAAQ,OAAO,CAClC,CAAC,CACE,EAAS,GACT,EAAS,GACT,EAAU,GACR,EAAU,GAAmC,CAC7C,IAGJ,EAAU,GACV,aAAa,EAAM,CACnB,EAAQ,EAAS,GAEb,EAAQ,eAAiB,CAC7B,EAAM,KAAK,UAAU,CACrB,EAAO,CAAE,KAAM,SAAU,OAAQ,iBAAiB,EAAQ,UAAU,IAAK,CAAC,EACzE,EAAQ,UAAU,CAErB,EAAM,OAAO,GAAG,OAAS,GAAkB,CACzC,GAAU,EAAM,UAAU,EAC1B,CACF,EAAM,OAAO,GAAG,OAAS,GAAkB,CACzC,GAAU,EAAM,UAAU,EAC1B,CACF,EAAM,GAAG,QAAU,GAAU,CAC3B,EAAO,CAAE,KAAM,QAAS,OAAQ,iBAAiB,EAAM,UAAW,CAAC,EACnE,CACF,EAAM,GAAG,QAAU,GAAS,CAC1B,GAAI,IAAS,EAAG,CAEd,EAAO,CAAE,KADI,EAAgB,EAAO,CACrB,OAAQ,QAAQ,EAAK,IAAI,EAAO,MAAM,CAAC,MAAM,EAAG,IAAI,GAAI,CAAC,CACxE,OAEF,GAAI,CACF,IAAM,EAAS,KAAK,MAAM,EAAO,CACjC,GAAI,OAAO,EAAO,WAAc,SAAU,CACxC,EAAO,CAAE,KAAM,SAAU,OAAQ,gCAAiC,CAAC,CACnE,OAEF,EAAO,CAAE,KAAM,KAAM,OAAQ,EAAQ,CAAC,MAChC,CACN,EAAO,CAAE,KAAM,SAAU,OAAQ,2BAA2B,EAAO,MAAM,EAAG,IAAI,GAAI,CAAC,GAEvF,EACF,EAEL,CCzGH,SAAS,EAAW,EAA8C,EAAqB,CACrF,IAAM,EAAQ,IAAU,GACxB,OAAO,OAAO,GAAU,UAAY,OAAO,SAAS,EAAM,CAAG,EAAQ,EAIvE,SAAgB,EAAc,EAA2B,CACvD,OAAO,EAAK,EAAmB,QAAS,GAAG,EAAU,QAAQ,CAc/D,SAAgB,EAAc,EAA2B,CACvD,IAAM,EAAsB,EAAE,CAE9B,MAAO,CAAE,QADO,EAAkB,EAAS,EAAU,CACnC,YAAW,CAG/B,SAAS,EAAkB,EAAiB,EAAqC,CAC/E,IAAM,EAAsB,CAC1B,MAAO,EACP,UAAW,EACX,WAAY,EACZ,YAAa,EACb,aAAc,EACd,gBAAiB,EACjB,eAAgB,EACjB,CAED,IAAK,IAAM,KAAQ,EAAQ,MAAM;EAAK,CAAE,CACtC,GAAI,EAAK,MAAM,GAAK,GAClB,SAEF,IAAI,EACJ,GAAI,CACF,EAAQ,KAAK,MAAM,EAAK,MAClB,CACN,SAGF,GAAI,EAAM,OAAS,gBAAiB,CAClC,EAAQ,YACR,EAAQ,gBAAkB,EAAW,EAAM,QAAS,aAAa,CAC7D,EAAM,SAAU,gBAAqB,IAAA,IACvC,EAAQ,aAEV,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAClB,GAAc,KAAK,EAAK,CAE1B,SAGF,GAAI,EAAM,OAAS,WAAY,CAC7B,EAAQ,QACR,IAAM,EAAQ,EAAM,SAAU,MAC9B,GAAI,OAAO,GAAU,UAAY,EAAgB,CAC/C,IAAM,EAAI,EACV,EAAQ,aAAe,EAAW,EAAG,cAAc,CACnD,EAAQ,cAAgB,EAAW,EAAG,eAAe,CACrD,EAAQ,iBAAmB,EAAW,EAAG,kBAAkB,CAE7D,IAAM,EAAQ,EAAM,SAAU,MAC1B,OAAO,GAAU,UAAY,IAAU,KACzC,EAAQ,QAAU,IAKxB,OAAO,EAIT,SAAgB,EAAa,EAAyC,CACpE,IAAM,EAAO,EAAc,EAAU,CAChC,KAAW,EAAK,CAGrB,OAAO,EAAc,EAAa,EAAM,OAAO,CAAC,CAIlD,SAAgB,EAAe,EAAwD,CACrF,GAAI,EAAQ,SAAW,EACrB,OAEF,IAAM,EAAM,EAAQ,QACjB,EAAK,KAAO,CACX,MAAO,EAAI,MAAQ,EAAE,MACrB,UAAW,EAAI,UAAY,EAAE,UAC7B,WAAY,EAAI,WAAa,EAAE,WAC/B,YAAa,EAAI,YAAc,EAAE,YACjC,aAAc,EAAI,aAAe,EAAE,aACnC,gBAAiB,EAAI,gBAAkB,EAAE,gBACzC,eAAgB,EAAI,eAAiB,EAAE,eACvC,QAAS,EAAE,SAAW,EAAI,QAC3B,EACD,CACE,MAAO,EACP,UAAW,EACX,WAAY,EACZ,YAAa,EACb,aAAc,EACd,gBAAiB,EACjB,eAAgB,EACjB,CACF,CACK,EAAI,EAAQ,OACZ,EAAuB,CAC3B,MAAO,EAAI,MAAQ,EACnB,UAAW,EAAI,UAAY,EAC3B,WAAY,EAAI,WAAa,EAC7B,YAAa,EAAI,YAAc,EAC/B,aAAc,EAAI,aAAe,EACjC,gBAAiB,EAAI,gBAAkB,EACvC,eAAgB,EAAI,eAAiB,EACtC,CAID,OAHI,EAAI,UAAY,IAAA,KAClB,EAAS,QAAU,EAAI,SAElB,EC3HT,SAAgB,EACd,EACA,EACS,CACT,IAAI,EAAS,EACb,IAAK,IAAM,KAAQ,EACb,EAAS,EAAS,QAAU,IAAS,EAAS,IAChD,IAGJ,OAAO,IAAW,EAAS,OAG7B,SAAS,EACP,EACA,EACA,EACc,CACd,IAAM,EAAmB,EAAE,CAC3B,GAAI,EAAO,eAAgB,CACzB,IAAM,EAAW,EAAS,aAAa,CACvC,IAAK,IAAM,KAAU,EAAO,eACrB,EAAS,SAAS,EAAO,aAAa,CAAC,EAC1C,EAAO,KAAK,mBAAmB,EAAO,GAAG,CAS/C,OALI,EAAO,cAAgB,CAAC,EAAuB,EAAW,EAAO,aAAa,EAChF,EAAO,KACL,kBAAkB,EAAO,aAAa,KAAK,KAAK,CAAC,sBAAsB,EAAU,KAAK,KAAK,CAAC,GAC7F,CAEI,EAAO,SAAW,EACrB,CAAE,OAAQ,GAAM,OAAQ,8BAA+B,CACvD,CAAE,OAAQ,GAAO,OAAQ,EAAO,KAAK,KAAK,CAAE,CAGlD,SAAS,EAAgB,EAAoB,EAA8B,CACzE,IAAM,EAAO,EAAU,OAAQ,CAAC,EAAO,CAAE,CACvC,IAAK,EAAM,aACX,SAAU,OACV,QAAS,KACT,IAAK,CAAE,GAAG,QAAQ,IAAK,mBAAoB,EAAM,SAAU,CAC5D,CAAC,CACF,GAAI,EAAK,MACP,MAAO,CAAE,OAAQ,GAAO,OAAQ,wBAAwB,EAAK,MAAM,UAAW,CAEhF,IAAM,EACJ,IAAI,EAAK,QAAU,IAAI,MAAM,GAAG,EAAK,OAAS,MAAM,EAAK,OAAO,MAAM,GAAK,KAAK,MAAM,EAAG,IAAI,CAC/F,MAAO,CAAE,OAAQ,EAAK,SAAW,EAAG,OAAQ,GAAU,QAAQ,EAAK,SAAU,CAU/E,SAAgB,EAAU,EAAkC,CAC1D,GAAM,CAAE,QAAS,EACX,EAAwB,EAAE,CAShC,GAPI,EAAK,QAAU,UACjB,EAAM,KAAK,EAAgB,EAAO,EAAK,EAAK,IAAK,EAAK,cAAgB,YAAY,CAAC,CAAC,CAElF,EAAK,QACP,EAAM,KAAK,EAAa,EAAK,OAAQ,EAAM,SAAU,EAAM,UAAU,CAAC,CAGpE,EAAM,SAAW,EACnB,MAAO,CAAE,OAAQ,GAAM,OAAQ,uCAAwC,CAGzE,IAAM,EAAS,EAAM,OAAQ,GAAM,CAAC,EAAE,OAAO,CAC7C,OAAO,EAAO,SAAW,EACrB,CAAE,OAAQ,GAAM,OAAQ,EAAM,IAAK,GAAM,EAAE,OAAO,CAAC,KAAK,KAAK,CAAE,CAC/D,CAAE,OAAQ,GAAO,OAAQ,EAAO,IAAK,GAAM,EAAE,OAAO,CAAC,KAAK,KAAK,CAAE,CCjFvE,MAAa,EAAgC,EAC3C,QAAQ,IAAI,gCACZ,GACD,CAYD,SAAS,EAAY,EAAyC,CAI5D,OAHK,EAGE,EAAQ,YAAc,EAAQ,aAF5B,EAMX,eAAsB,EAAQ,EAAgB,EAAuC,CACnF,IAAM,EAAM,EAAK,UAAc,KAAK,KAAK,EACnC,EAAY,EAAK,WAAa,EAC9B,EAAoB,EAAE,CACxB,EAAc,EAEd,EAAe,EAYf,EAAiB,EAEf,MAAgC,CACpC,GAAI,EAAK,YAAc,IAAA,GACrB,MAAO,GAET,IAAM,EAAgB,EAAe,EAAI,EAAc,EAAe,EAAK,UAAY,EAAK,KAC5F,OAAO,EAAc,EAAiB,GAAiB,EAAK,WAG9D,IAAK,IAAI,EAAQ,EAAG,EAAQ,EAAK,KAAM,IAAS,CAC9C,GAAI,GAAgB,CAAE,CAGpB,EAAK,KAAK,CACR,QAAS,iBACT,OAAQ,EACR,OAAQ,qBAAqB,EAAY,kBACvC,EAAiB,EAAI,MAAM,EAAe,oBAAsB,GACjE,YAAY,EAAK,UAAU,0BAC7B,CAAC,CACF,SAGF,IAAM,EAAY,EAAgB,EAAK,CACjC,EAAY,GAAK,CACvB,GAAI,CACF,IAAM,EAAW,MAAM,EAAK,QAAQ,OAAO,CACzC,OAAQ,EAAK,OACb,IAAK,EAAU,IACf,UAAW,EAAK,UACjB,CAAC,CACI,EAAS,GAAK,CAAG,EAEvB,GAAI,EAAS,OAAS,QAAS,CAG7B,IACA,EAAK,KAAK,CAAE,QAAS,cAAe,SAAQ,OAAQ,EAAS,OAAQ,CAAC,CACtE,SAEF,GAAI,EAAS,OAAS,SAAU,CAG9B,IACA,EAAK,KAAK,CAAE,QAAS,OAAQ,SAAQ,OAAQ,EAAS,OAAQ,CAAC,CAC/D,SAGF,IAAM,EAAQ,EAAU,EAAS,OAAO,UAAU,CAC5C,EAAU,EAAU,CACxB,OACA,aAAc,EAAU,IACxB,SAAU,EAAS,OAAO,SAC1B,UAAW,GAAO,WAAa,EAAE,CAClC,CAAC,CACE,GACF,GAAe,EAAY,EAAM,QAAQ,CACzC,KAGA,IAGF,IAAM,EAAoB,CACxB,QAAS,EAAQ,OAAS,OAAS,OACnC,SACA,UAAW,EAAS,OAAO,UAC3B,OAAQ,EAAQ,OACjB,CACG,IACF,EAAO,QAAU,EAAM,SAEzB,EAAK,KAAK,EAAO,QACT,CACR,EAAU,SAAS,CAGrB,IAAM,EAAO,EAAK,GAAG,GAAG,CACpB,GACF,EAAK,gBAAgB,EAAK,GAAI,EAAO,EAAK,CAI9C,OAAO,EAAc,EAAM,EAAK,CAIlC,SAAgB,EAAc,EAAgB,EAAwC,CACpF,IAAM,EAAc,EAAK,OAAQ,GAAM,EAAE,UAAY,cAAc,CAAC,OAC9D,EAAgB,EAAK,OAAQ,GAAM,EAAE,UAAY,iBAAiB,CAAC,OACnE,EAAY,EAAK,OAAS,EAAc,EACxC,EAAS,EAAK,OAAQ,GAAM,EAAE,UAAY,OAAO,CAAC,OAClD,EAAgB,EAAK,QAAU,WAE/B,EAAqB,CACzB,OAAQ,EAAK,GACb,MAAO,EAAK,MACZ,OACA,SACA,YACA,cACA,gBACA,gBACD,CAID,OAHI,EAAY,GACd,OAAO,OAAO,EAAQ,CAAE,SAAU,EAAS,EAAW,CAAC,CAElD,EAIT,eAAsB,EACpB,EACA,EACA,EACoB,CACpB,IAAM,EAAM,EAAK,UAAc,KAAK,KAAK,EACnC,EAAY,IAAI,KAAK,GAAK,CAAC,CAAC,aAAa,CACzC,EAAwB,EAAE,CAChC,IAAK,IAAM,KAAQ,EACjB,EAAQ,KAAK,MAAM,EAAQ,EAAM,EAAK,CAAC,CAEzC,OAAO,EAAe,EAAS,EAAc,EAAU,CAIzD,SAAgB,EACd,EACA,EACA,EACW,CACX,IAAM,EAAS,EAAM,OAAQ,GAAM,EAAE,eAAiB,EAAE,WAAa,IAAA,GAAU,CACzE,EAAqB,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,cAAe,EAAE,CAIvE,EAAY,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,KAAK,OAAQ,EAAE,CAAG,EAC/D,EAAa,EAAM,QAAQ,EAAK,IAAM,EAAM,EAAE,YAAa,EAAE,CAC7D,EAAiB,IAAc,EAAI,EAAI,EAAa,EAKpD,EAAY,EAHC,EAAM,QAAS,GAChC,EAAE,KAAK,IAAK,GAAM,EAAE,QAAQ,CAAC,OAAQ,GAAuB,IAAM,IAAA,GAAU,CAC7E,CAC2C,CAEtC,EAAuB,CAC3B,eACA,QAAS,GAAW,SAAW,UAC/B,YACA,QACA,YAAa,EAAM,OAAQ,GAAM,EAAE,cAAc,CAAC,OAClD,cAAe,EAAM,OAAQ,GAAM,CAAC,EAAE,cAAc,CAAC,OACrD,iBACA,mBAAoB,EACpB,QAAS,EAAiB,EAC3B,CACD,GAAI,EAAO,OAAS,EAAG,CACrB,IAAM,EAAM,EAAO,QAAQ,EAAK,IAAM,GAAO,EAAE,UAAY,GAAI,EAAE,CACjE,OAAO,OAAO,EAAW,CAAE,gBAAiB,EAAM,EAAO,OAAQ,CAAC,CAKpE,OAHI,GACF,OAAO,OAAO,EAAW,CAAE,iBAAkB,EAAW,CAAC,CAEpD,EChNT,MAAa,EAA0B,EAAa,QAAQ,IAAI,wBAA4B,GAAG,CAE/F,SAAgB,EAAc,EAA4B,CAKxD,OAJK,EAAW,EAAK,CAId,CAAE,QADM,KAAK,MAAM,EAAa,EAAM,OAAO,CAAC,CAC5B,SAAW,EAAE,CAAE,CAH/B,CAAE,QAAS,EAAE,CAAE,CAM1B,SAAgB,EAAc,EAAc,EAA0B,CACpE,EAAU,EAAQ,EAAK,CAAE,CAAE,UAAW,GAAM,CAAC,CAC7C,EAAc,EAAM,GAAG,KAAK,UAAU,EAAM,KAAM,EAAE,CAAC,IAAK,OAAO,CAInE,SAAgB,EACd,EACA,EACA,EAC2B,CAC3B,OAAO,EAAK,QAAQ,KAAM,GAAM,EAAE,UAAY,GAAW,EAAE,eAAiB,EAAa,CAI3F,SAAgB,EACd,EACA,EACA,EACc,CACd,IAAM,EAAoD,EAAE,CAC5D,IAAK,IAAM,KAAK,EAAU,MACxB,EAAc,EAAE,QAAU,EAAE,SAG9B,IAAM,EAAuB,CAC3B,QAAS,EAAU,QACnB,aAAc,EAAU,aACxB,WAAY,EAAU,UACtB,SACA,gBACD,CAQD,OAPI,EAAU,kBAAoB,IAAA,IAChC,OAAO,OAAO,EAAO,CAAE,gBAAiB,EAAU,gBAAiB,CAAC,CAM/D,CAAE,QAAS,CAAC,GAHN,EAAK,QAAQ,OACvB,GAAM,EAAE,EAAE,UAAY,EAAM,SAAW,EAAE,eAAiB,EAAM,cAClE,CAC2B,EAAM,CAAE,CAItC,SAAgB,EACd,EACA,EACmB,CACnB,GAAI,EAAU,QACZ,MAAO,CACL,OAAQ,cACR,SAAU,CACR,qBAAqB,EAAU,eAAiB,KAAK,QAAQ,EAAE,CAAC,0CACjE,CACD,aAAc,EAAE,CACjB,CAEH,GAAI,CAAC,EACH,MAAO,CACL,OAAQ,cACR,SAAU,CACR,0BAA0B,EAAU,QAAQ,UAAU,EAAU,aAAa,yCAC9E,CACD,aAAc,EAAE,CACjB,CAGH,IAAM,EAAqB,EAAE,CACvB,EAAoB,EAAE,CAE5B,IAAK,IAAM,KAAK,EAAU,MAAO,CAC/B,GAAI,CAAC,EAAE,cACL,SAEF,IAAM,EAAS,EAAS,cAAc,EAAE,QACxC,GAAI,IAAW,GAAK,EAAE,WAAa,EAAG,CACpC,EAAQ,KAAK,EAAE,OAAO,CACtB,EAAS,KAAK,eAAe,EAAE,OAAO,yBAAyB,CAC/D,SAKE,EAAE,WAAa,IAAA,IAAa,IAAW,IAAA,IACzC,EAAS,KACP,IAAI,EAAE,OAAO,oFACd,CAIL,IAAI,EACA,EAAU,kBAAoB,IAAA,IAAa,EAAS,kBAAoB,IAAA,KAC1E,GAAW,EAAU,gBAAkB,EAAS,iBAAmB,IAC/D,EAAU,CAAC,GACb,EAAS,KACP,2BAA2B,KAAK,IAAI,EAAQ,CAAC,QAAQ,EAAE,CAAC,OACjD,EAAS,gBAAkB,KAAK,QAAQ,EAAE,CAAC,OAAO,EAAU,gBAAkB,KAAK,QAAQ,EAAE,CAAC,IACtG,EAOL,IAAM,EAA6B,CACjC,OAHA,EAAQ,OAAS,GAAM,IAAY,IAAA,IAAa,EAAU,CAAC,EAGvC,YAAc,KAClC,SAAU,EAAS,OAAS,EAAI,EAAW,CAAC,yBAAyB,CACrE,aAAc,EACf,CAID,OAHI,IAAY,IAAA,IACd,OAAO,OAAO,EAAS,CAAE,gBAAiB,EAAS,CAAC,CAE/C,ECtIT,SAAS,EAAI,EAAmC,CAC9C,OAAO,IAAU,IAAA,GAAY,IAAM,IAAI,EAAQ,KAAK,QAAQ,EAAE,CAAC,GAGjE,SAAgBC,EAAa,EAAsB,EAAoC,CACrF,IAAM,EAAkB,EAAE,CAEpB,EACJ,EAAQ,SAAW,YACf,YACA,EAAQ,SAAW,KACjB,KACA,EAAQ,SAAW,cACjB,yBACA,cAEV,EAAM,KAAK,eAAe,IAAW,CACrC,EAAM,KACJ,SAAS,EAAU,QAAQ,UAAU,EAAU,aAAa,OAAO,EAAU,YAC9E,CACD,EAAM,KAAK,GAAG,CAEd,EAAM,KAAK,uEAAuE,CAClF,IAAK,IAAM,KAAK,EAAU,MAAO,CAC/B,IAAM,EAAK,EAAE,OAAO,OAAO,GAAG,CAAC,MAAM,EAAG,GAAG,CACrC,EAAQ,EAAE,MAAM,OAAO,EAAE,CACzB,GAAQ,EAAE,cAAgB,EAAI,EAAE,SAAS,CAAG,OAAO,SAAS,EAAE,CAC9D,EAAO,GAAG,EAAE,OAAO,GAAG,EAAE,YAAY,SAAS,EAAE,CAC/C,EAAQ,OAAO,EAAE,YAAY,CAAC,SAAS,EAAE,CACzC,EAAO,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,CAChD,EAAM,KAAK,GAAG,EAAG,GAAG,EAAM,GAAG,EAAK,GAAG,EAAK,GAAG,EAAM,GAAG,IAAO,CAO/D,GALA,EAAM,KAAK,GAAG,CAEd,EAAM,KACJ,oBAAoB,EAAI,EAAU,gBAAgB,CAAC,YAAY,EAAU,YAAY,kBAAkB,EAAU,cAAc,GAChI,CACG,EAAQ,kBAAoB,IAAA,GAAW,CACzC,IAAM,EAAO,EAAQ,iBAAmB,EAAI,IAAM,GAClD,EAAM,KAAK,oBAAoB,IAAO,EAAQ,gBAAgB,QAAQ,EAAE,CAAC,IAAI,CAE/E,EAAM,KAAK,qBAAqB,EAAU,eAAiB,KAAK,QAAQ,EAAE,CAAC,WAAW,CAClF,EAAU,mBAAqB,GAGjC,EAAM,KAAK,qBAAqB,EAAU,mBAAmB,4CAA4C,CAG3G,IAAM,EAAI,EAAU,iBAChB,GACF,EAAM,KACJ,oBAAoB,EAAE,MAAM,QAAQ,EAAE,CAAC,WAAW,EAAE,UAAU,QAAQ,EAAE,CAAC,UACnE,EAAE,WAAW,QAAQ,EAAE,CAAC,UAAU,KAAK,MAAM,EAAE,YAAc,EAAE,aAAa,CAAC,SACpF,CAGH,EAAM,KAAK,GAAG,CACd,IAAK,IAAM,KAAW,EAAQ,SAC5B,EAAM,KAAK,KAAK,IAAU,CAU5B,OAPI,EAAU,cAAgB,IAC5B,EAAM,KAAK,GAAG,CACd,EAAM,KACJ,SAAS,EAAU,cAAc,2FAClC,EAGI,EAAM,KAAK;EAAK,CCxCzB,MAAM,EAAqB,CAAC,OAAQ,WAAY,YAAY,CAG5D,SAAgB,EAAqB,EAAmB,EAAkC,CACxF,IAAI,EACE,EAAyB,EAAE,CAC3B,EAAe,IAAI,IACrB,EAAQ,EAEZ,IAAK,IAAM,KAAQ,EAAQ,MAAM;EAAK,CAAE,CACtC,GAAI,EAAK,MAAM,GAAK,GAClB,SAEF,IAAI,EACJ,GAAI,CACF,EAAQ,KAAK,MAAM,EAAK,MAClB,CACN,SAGF,GAAI,EAAM,OAAS,iBAAmB,IAAW,IAAA,GAAW,CAC1D,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAAY,EAAK,MAAM,GAAK,KAC9C,EAAS,GAEX,SAGF,GAAI,EAAM,OAAS,kBAAmB,CACpC,IAAM,EAAO,EAAM,SAAU,KACzB,OAAO,GAAS,UAClB,EAAa,KAAK,EAAK,CAEzB,IAAM,EAAO,EAAM,SAAU,UAC7B,GAAI,OAAO,GAAS,UAAY,EAC9B,IAAK,IAAM,KAAO,EAAoB,CACpC,IAAM,EAAS,EAAiC,GAC5C,OAAO,GAAU,UAAY,IAAU,IACzC,EAAa,IAAI,EAAM,CAI7B,SAGE,EAAM,OAAS,YACjB,IAIJ,MAAO,CACL,YACA,eACA,aAAc,CAAC,GAAG,EAAa,CAAC,MAAM,CACtC,QACA,GAAI,IAAW,IAAA,GAAyB,EAAE,CAAf,CAAE,SAAQ,CACtC,CAYH,SAAS,EAAe,EAAmC,CACzD,IAAM,EACJ,EAAS,SAAW,IAAA,GAKhB;0CAJA,cAAc,EAAS,OACpB,MAAM;EAAK,CACX,IAAK,GAAS,KAAK,IAAO,CAC1B,KAAK;EAAK,GAGb,EACJ,EAAS,aAAa,OAAS,EAC3B,gDAAgD,EAAS,aAAa,KAAK,MAAM,CAAC,IAClF,GAEN,MAAO,kCAAkC,EAAS,UAAU;;EAE5D,EAAY;;EAEZ,EAAS;;;EAMX,SAAS,EAAa,EAAmC,CACvD,IAAM,EACJ,EAAS,aAAa,OAAS,EAC3B,EAAS,aAAa,IAAK,GAAM,OAAO,EAAE,IAAI,CAAC,KAAK;EAAK,CACzD,uBAEN,MAAO,eAAe,EAAS,UAAU;;WAEhC,EAAS,MAAM;SACjB,EAAS,aAAa,OAAO;;;;EAIpC,EAAM;;;;;;;;;;;;;;;;EAkBR,SAAgB,EACd,EACA,EACA,EAA8D,GAC/C,CACf,IAAM,EAAU,EAAiB,EAAU,CAC3C,GAAI,IAAY,IAAA,GACd,MAAU,MACR,8BAA8B,EAAU,cAAc,EAAc,EAAU,CAAC,GAChF,CAGH,IAAM,EAAW,EAAqB,EAAW,EAAQ,CACnD,EAAU,EAAK,EAAW,aAAa,EAAU,MAAM,EAAG,EAAE,GAAG,CACrE,GAAI,EAAW,EAAQ,CACrB,MAAU,MACR,GAAG,EAAQ,uEACZ,CAOH,OAJA,EAAU,EAAK,EAAS,UAAU,CAAE,CAAE,UAAW,GAAM,CAAC,CACxD,EAAc,EAAK,EAAS,YAAY,CAAE,EAAe,EAAS,CAAE,OAAO,CAC3E,EAAc,EAAK,EAAS,gBAAgB,CAAE,EAAa,EAAS,CAAE,OAAO,CAEtE,CAAE,UAAS,WAAU,CAG9B,SAAS,GAAiB,EAAuC,CAC/D,IAAM,EAAO,EAAc,EAAU,CACrC,OAAO,EAAW,EAAK,CAAG,EAAa,EAAM,OAAO,CAAG,IAAA"}