@tangle-network/agent-bench 0.3.5 → 0.3.6

package/dist/chunk-LRRD7NAG.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/_harness.ts"],"sourcesContent":["/**\n * Shared code-bench harness. The \"stage the artifact → run the benchmark's own\n * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)\n * → read its JSON report → { resolved, score }\" spine that swe-bench,\n * terminal-bench, commit0, programbench and aec-bench all need. Factored out so\n * the Docker/venv/report-reading logic lives in ONE place instead of being\n * copy-pasted per adapter.\n *\n * It owns NO benchmark policy: each adapter passes the argv for its evaluator\n * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The\n * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config\n * and fail-loud diagnostics.\n *\n * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`\n * (the adapter passes the import line + the exact fix). A staged run that exits\n * nonzero throws with the captured stderr — never a fabricated score.\n */\n\nimport { execFile, spawn } from 'node:child_process'\nimport { createHash } from 'node:crypto'\nimport {\n  cp,\n  lstat,\n  mkdir,\n  mkdtemp,\n  readFile,\n  readlink,\n  readdir,\n  rename,\n  rm,\n  writeFile,\n} from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type {\n  BenchScore,\n  JudgeArtifactFileReceipt,\n  JudgeArtifactReceipt,\n} from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */\nexport const benchRoot = fileURLToPath(new URL('../..', import.meta.url))\n\n/** Resolve the shared interpreter without requiring an installed package to contain a venv. */\nexport function resolveBenchPython(\n  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,\n  root: string = benchRoot,\n): string {\n  const configured = env.AGENT_BENCH_PYTHON\n  if (configured === undefined) return join(root, '.venv', 'bin', 'python')\n  if (!isAbsolute(configured)) {\n    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')\n  }\n  return configured\n}\n\n/** The shared interpreter every Python-backed evaluator runs through. */\nexport const venvPython = resolveBenchPython()\n\n/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip\n *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)\n *  get their own venv and pass its python explicitly — keeping the shared one clean. */\nexport const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python')\n/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */\nexport const bigBuffer = 1024 * 1024 * 256\n\n/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */\nexport function venvBin(name: string): string {\n  return join(benchRoot, '.venv', 'bin', name)\n}\n\n/**\n * Run the bench venv python with an inline script (`-c`); return stdout. Throws\n * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather\n * than parse a partial dump.\n */\nexport async function runVenvPython(\n  script: string,\n  args: string[] = [],\n  timeoutMs = 0,\n  python: string = venvPython,\n): Promise<string> {\n  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {\n    maxBuffer: bigBuffer,\n    timeout: timeoutMs,\n  })\n  return stdout\n}\n\n/**\n * Preflight a python-backed harness: import the module(s) and (optionally) ping\n * Docker, all inside the bench venv. On failure THROWS the captured error joined\n * to the adapter's `fix` guidance — the contract every code-bench preflight wants.\n */\nexport async function preflightVenvImports(opts: {\n  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */\n  modules: string[]\n  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */\n  requireDocker?: boolean\n  /** Actionable remediation appended to the thrown message. */\n  fix: string\n  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */\n  python?: string\n}): Promise<void> {\n  const imports = opts.modules.filter((m) => m.length > 0)\n  const lines = [...imports.map((m) => `import ${m}`)]\n  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')\n  lines.push(\"print('ok')\")\n  try {\n    await runVenvPython(lines.join('\\n'), [], 0, opts.python ?? venvPython)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(`${msg}\\n${opts.fix}`)\n  }\n}\n\n/**\n * Run a bench-local python driver script (in the bench venv) while piping\n * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST\n * stdout line; callers parse that and inspect an `error` field (fail loud).\n *\n * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s\n * `input` option, because that option is NOT honored by async execFile — stdin\n * is left open and a driver that does `sys.stdin.read()` blocks forever. The\n * artifact-piping judges (commit0, appworld) MUST go through this.\n */\nexport function runVenvScriptStdin(\n  scriptPath: string,\n  args: string[],\n  input: string,\n  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},\n): Promise<string> {\n  return new Promise<string>((resolve, reject) => {\n    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {\n      cwd: opts.cwd ?? benchRoot,\n      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),\n    })\n    let stdout = ''\n    let stderr = ''\n    let bytes = 0\n    child.stdout.on('data', (c: Buffer) => {\n      bytes += c.length\n      if (bytes <= bigBuffer) stdout += c.toString('utf8')\n    })\n    child.stderr.on('data', (c: Buffer) => {\n      stderr += c.toString('utf8')\n    })\n    child.on('error', (err) => reject(err))\n    child.on('close', (code) => {\n      if (code === 0) resolve(stdout)\n      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))\n    })\n    child.stdin.end(input)\n  })\n}\n\nexport interface StagedRunSpec {\n  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */\n  tmpPrefix: string\n  /**\n   * Write the artifact + any harness inputs into the temp dir. Returns nothing;\n   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.\n   */\n  stage(dir: string): Promise<void>\n  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */\n  bin?: string\n  /** argv for the evaluator, computed from the temp `dir`. */\n  argv(dir: string): string[]\n  /** Working directory for the evaluator. Defaults to the temp `dir`. */\n  cwd?(dir: string): string\n  /** Hard timeout for the evaluator (ms); 0 = none. */\n  timeoutMs?: number\n  /**\n   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws\n   * if the expected report is absent/malformed (fail loud — no default score).\n   */\n  parseReport(dir: string): Promise<BenchScore>\n  /**\n   * Copy the complete evaluator directory plus raw process stdout/stderr to this\n   * caller-owned directory before cleanup. The destination must not exist.\n   */\n  capture?: StagedRunCaptureSpec\n  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */\n  keepTmp?: boolean\n}\n\nexport interface StagedRunCaptureSpec {\n  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */\n  destination: string\n}\n\n/** A staged run failed after any requested evidence was durably retained. */\nexport class StagedJudgeError extends Error {\n  readonly judgeArtifacts?: JudgeArtifactReceipt\n\n  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {\n    super(message, options)\n    this.name = 'StagedJudgeError'\n    this.judgeArtifacts = judgeArtifacts\n  }\n}\n\nfunction sha256(bytes: Uint8Array): `sha256:${string}` {\n  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`\n}\n\nfunction portablePath(path: string): string {\n  return path.split(sep).join('/')\n}\n\nfunction compareText(left: string, right: string): number {\n  return left < right ? -1 : left > right ? 1 : 0\n}\n\nasync function collectArtifactFiles(\n  root: string,\n  current: string,\n): Promise<JudgeArtifactFileReceipt[]> {\n  const absolute = join(root, current)\n  const entries = await readdir(absolute, { withFileTypes: true })\n  const files: JudgeArtifactFileReceipt[] = []\n  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {\n    const relativePath = join(current, entry.name)\n    const path = join(root, relativePath)\n    if (entry.isDirectory()) {\n      files.push(...await collectArtifactFiles(root, relativePath))\n      continue\n    }\n    if (entry.isFile()) {\n      const bytes = await readFile(path)\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: bytes.byteLength,\n        sha256: sha256(bytes),\n        kind: 'file',\n      })\n      continue\n    }\n    if (entry.isSymbolicLink()) {\n      const targetBytes = await readlink(path, { encoding: 'buffer' })\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: targetBytes.byteLength,\n        sha256: sha256(targetBytes),\n        kind: 'symlink',\n      })\n      continue\n    }\n    throw new Error(`staged judge capture does not support ${relativePath}`)\n  }\n  return files\n}\n\nfunction isWithin(parent: string, candidate: string): boolean {\n  const path = relative(parent, candidate)\n  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))\n}\n\nasync function assertDestinationAbsent(destination: string): Promise<void> {\n  try {\n    await lstat(destination)\n  } catch (error) {\n    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return\n    throw error\n  }\n  throw new Error(`staged judge capture destination already exists: ${destination}`)\n}\n\nasync function captureStagedRun(\n  sourceDirectory: string,\n  spec: StagedRunCaptureSpec,\n  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,\n  evaluatorSucceeded: boolean,\n): Promise<JudgeArtifactReceipt> {\n  const source = resolve(sourceDirectory)\n  const destination = resolve(spec.destination)\n  if (isWithin(source, destination)) {\n    throw new Error('staged judge capture destination must be outside the evaluator directory')\n  }\n  await mkdir(dirname(destination), { recursive: true })\n  await assertDestinationAbsent(destination)\n  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))\n  try {\n    await cp(source, join(staging, 'evaluator'), {\n      recursive: true,\n      errorOnExist: true,\n      force: false,\n      preserveTimestamps: true,\n      verbatimSymlinks: true,\n    })\n    await mkdir(join(staging, 'process'))\n    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)\n    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)\n\n    const files = [\n      ...await collectArtifactFiles(staging, 'evaluator'),\n      ...await collectArtifactFiles(staging, 'process'),\n    ].sort((left, right) => compareText(left.path, right.path))\n    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)\n    const treeBytes = Buffer.from(\n      files\n        .map((file) => `${file.path}\\0${file.kind}\\0${file.byteLength}\\0${file.sha256}\\n`)\n        .join(''),\n      'utf8',\n    )\n    const receipt: JudgeArtifactReceipt = {\n      schema: 'agent-bench/judge-artifacts/v1',\n      directory: destination,\n      evaluatorDirectory: join(destination, 'evaluator'),\n      manifestPath: join(destination, 'receipt.json'),\n      evaluatorSucceeded,\n      files,\n      fileCount: files.length,\n      byteLength,\n      treeSha256: sha256(treeBytes),\n    }\n    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\\n`)\n    await rename(staging, destination)\n    return receipt\n  } catch (error) {\n    await rm(staging, { recursive: true, force: true }).catch(() => {})\n    throw error\n  }\n}\n\nfunction processBytes(value: unknown): Buffer {\n  if (Buffer.isBuffer(value)) return value\n  if (value === undefined || value === null) return Buffer.alloc(0)\n  return Buffer.from(String(value), 'utf8')\n}\n\n/**\n * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →\n * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is\n * always removed in `finally` unless `keepTmp`.\n */\nexport async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {\n  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))\n  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let evaluatorSucceeded = false\n  let score: BenchScore | undefined\n  let failure: unknown\n  try {\n    try {\n      await spec.stage(dir)\n      const bin = spec.bin ?? venvPython\n      const argv = spec.argv(dir)\n      try {\n        const output = await execFileAsync(bin, argv, {\n          cwd: spec.cwd ? spec.cwd(dir) : dir,\n          encoding: 'buffer',\n          maxBuffer: bigBuffer,\n          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),\n        })\n        stdout = processBytes(output.stdout)\n        stderr = processBytes(output.stderr)\n        evaluatorSucceeded = true\n      } catch (err) {\n        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }\n        stdout = processBytes(e.stdout)\n        stderr = processBytes(e.stderr)\n        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))\n          .toString('utf8')\n          .slice(0, 2000)\n        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\\n${detail}`)\n      }\n      score = await spec.parseReport(dir)\n    } catch (err) {\n      failure = err\n    }\n  } finally {\n    let judgeArtifacts: JudgeArtifactReceipt | undefined\n    if (spec.capture) {\n      try {\n        judgeArtifacts = await captureStagedRun(\n          dir,\n          spec.capture,\n          { stdout, stderr },\n          evaluatorSucceeded,\n        )\n      } catch (captureError) {\n        failure = new Error(\n          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,\n          { cause: failure ?? captureError },\n        )\n      }\n    }\n    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})\n    if (failure) {\n      throw new StagedJudgeError(\n        failure instanceof Error ? failure.message : String(failure),\n        judgeArtifacts,\n        { cause: failure },\n      )\n    }\n    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)\n    return judgeArtifacts ? { ...score, judgeArtifacts } : score\n  }\n}\n\n/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */\nexport async function readJsonReport<T>(path: string): Promise<T> {\n  let raw: string\n  try {\n    raw = await readFile(path, 'utf8')\n  } catch (err) {\n    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)\n  }\n  try {\n    return JSON.parse(raw) as T\n  } catch (err) {\n    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)\n  }\n}\n\n/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */\nexport async function stageFile(path: string, content: string): Promise<void> {\n  await writeFile(path, content)\n}\n\n/** Sanitize an instance id into a filesystem/run-id-safe token. */\nexport function safeRunId(prefix: string, id: string): string {\n  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')\n}\n"],"mappings":";AAkBA,SAAS,UAAU,aAAa;AAChC,SAAS,kBAAkB;AAC3B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AACvB,SAAS,UAAU,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AAC5E,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAO1B,IAAM,gBAAgB,UAAU,QAAQ;AAGjC,IAAM,YAAY,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAGjE,SAAS,mBACd,MAAiD,QAAQ,KACzD,OAAe,WACP;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,OAAW,QAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;AACxE,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AACA,SAAO;AACT;AAGO,IAAM,aAAa,mBAAmB;AAKtC,IAAM,eAAe,CAAC,YAA4B,KAAK,WAAW,SAAS,OAAO,QAAQ;AAE1F,IAAM,YAAY,OAAO,OAAO;AAGhC,SAAS,QAAQ,MAAsB;AAC5C,SAAO,KAAK,WAAW,SAAS,OAAO,IAAI;AAC7C;AAOA,eAAsB,cACpB,QACA,OAAiB,CAAC,GAClB,YAAY,GACZ,SAAiB,YACA;AACjB,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IACtE,WAAW;AAAA,IACX,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AACT;AAOA,eAAsB,qBAAqB,MASzB;AAChB,QAAM,UAAU,KAAK,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACvD,QAAM,QAAQ,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,EAAE,CAAC;AACnD,MAAI,KAAK,cAAe,OAAM,KAAK,iBAAiB,0BAA0B;AAC9E,QAAM,KAAK,aAAa;AACxB,MAAI;AACF,UAAM,cAAc,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,GAAG,KAAK,UAAU,UAAU;AAAA,EACxE,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,GAAG,GAAG;AAAA,EAAK,KAAK,GAAG,EAAE;AAAA,EACvC;AACF;AAYO,SAAS,mBACd,YACA,MACA,OACA,OAA8D,CAAC,GAC9C;AACjB,SAAO,IAAI,QAAgB,CAACA,UAAS,WAAW;AAC9C,UAAM,QAAQ,MAAM,KAAK,UAAU,YAAY,CAAC,YAAY,GAAG,IAAI,GAAG;AAAA,MACpE,KAAK,KAAK,OAAO;AAAA,MACjB,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,IACtD,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,QAAQ;AACZ,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,eAAS,EAAE;AACX,UAAI,SAAS,UAAW,WAAU,EAAE,SAAS,MAAM;AAAA,IACrD,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,gBAAU,EAAE,SAAS,MAAM;AAAA,IAC7B,CAAC;AACD,UAAM,GAAG,SAAS,CAAC,QAAQ,OAAO,GAAG,CAAC;AACtC,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,SAAS,EAAG,CAAAA,SAAQ,MAAM;AAAA,UACzB,QAAO,IAAI,OAAO,UAAU,UAAU,QAAQ,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC;AAAA,IAC5E,CAAC;AACD,UAAM,MAAM,IAAI,KAAK;AAAA,EACvB,CAAC;AACH;AAsCO,IAAM,mBAAN,cAA+B,MAAM;AAAA,EACjC;AAAA,EAET,YAAY,SAAiB,gBAAuC,SAAwB;AAC1F,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,OAAO,OAAuC;AACrD,SAAO,UAAU,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;AACnE;AAEA,SAAS,aAAa,MAAsB;AAC1C,SAAO,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG;AACjC;AAEA,SAAS,YAAY,MAAc,OAAuB;AACxD,SAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI;AAChD;AAEA,eAAe,qBACb,MACA,SACqC;AACrC,QAAM,WAAW,KAAK,MAAM,OAAO;AACnC,QAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,eAAe,KAAK,CAAC;AAC/D,QAAM,QAAoC,CAAC;AAC3C,aAAW,SAAS,QAAQ,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG;AACrF,UAAM,eAAe,KAAK,SAAS,MAAM,IAAI;AAC7C,UAAM,OAAO,KAAK,MAAM,YAAY;AACpC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,KAAK,GAAG,MAAM,qBAAqB,MAAM,YAAY,CAAC;AAC5D;AAAA,IACF;AACA,QAAI,MAAM,OAAO,GAAG;AAClB,YAAM,QAAQ,MAAM,SAAS,IAAI;AACjC,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,MAAM;AAAA,QAClB,QAAQ,OAAO,KAAK;AAAA,QACpB,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,QAAI,MAAM,eAAe,GAAG;AAC1B,YAAM,cAAc,MAAM,SAAS,MAAM,EAAE,UAAU,SAAS,CAAC;AAC/D,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,YAAY;AAAA,QACxB,QAAQ,OAAO,WAAW;AAAA,QAC1B,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,yCAAyC,YAAY,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAgB,WAA4B;AAC5D,QAAM,OAAO,SAAS,QAAQ,SAAS;AACvC,SAAO,SAAS,MAAO,CAAC,KAAK,WAAW,KAAK,GAAG,EAAE,KAAK,SAAS,QAAQ,CAAC,WAAW,IAAI;AAC1F;AAEA,eAAe,wBAAwB,aAAoC;AACzE,MAAI;AACF,UAAM,MAAM,WAAW;AAAA,EACzB,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU;AACxD,UAAM;AAAA,EACR;AACA,QAAM,IAAI,MAAM,oDAAoD,WAAW,EAAE;AACnF;AAEA,eAAe,iBACb,iBACA,MACA,eACA,oBAC+B;AAC/B,QAAM,SAAS,QAAQ,eAAe;AACtC,QAAM,cAAc,QAAQ,KAAK,WAAW;AAC5C,MAAI,SAAS,QAAQ,WAAW,GAAG;AACjC,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AACA,QAAM,MAAM,QAAQ,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,QAAM,wBAAwB,WAAW;AACzC,QAAM,UAAU,MAAM,QAAQ,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,CAAC,WAAW,CAAC;AAC9F,MAAI;AACF,UAAM,GAAG,QAAQ,KAAK,SAAS,WAAW,GAAG;AAAA,MAC3C,WAAW;AAAA,MACX,cAAc;AAAA,MACd,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB,CAAC;AACD,UAAM,MAAM,KAAK,SAAS,SAAS,CAAC;AACpC,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAC5E,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAE5E,UAAM,QAAQ;AAAA,MACZ,GAAG,MAAM,qBAAqB,SAAS,WAAW;AAAA,MAClD,GAAG,MAAM,qBAAqB,SAAS,SAAS;AAAA,IAClD,EAAE,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC;AAC1D,UAAM,aAAa,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,CAAC;AAC3E,UAAM,YAAY,OAAO;AAAA,MACvB,MACG,IAAI,CAAC,SAAS,GAAG,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,KAAK,KAAK,MAAM;AAAA,CAAI,EAChF,KAAK,EAAE;AAAA,MACV;AAAA,IACF;AACA,UAAM,UAAgC;AAAA,MACpC,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,oBAAoB,KAAK,aAAa,WAAW;AAAA,MACjD,cAAc,KAAK,aAAa,cAAc;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,IAC9B;AACA,UAAM,UAAU,KAAK,SAAS,cAAc,GAAG,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,CAAI;AACtF,UAAM,OAAO,SAAS,WAAW;AACjC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAClE,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,MAAI,OAAO,SAAS,KAAK,EAAG,QAAO;AACnC,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO,OAAO,MAAM,CAAC;AAChE,SAAO,OAAO,KAAK,OAAO,KAAK,GAAG,MAAM;AAC1C;AAOA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,KAAK,SAAS,CAAC;AACxD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,qBAAqB;AACzB,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI;AACF,YAAM,KAAK,MAAM,GAAG;AACpB,YAAM,MAAM,KAAK,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,GAAG;AAC1B,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,KAAK,MAAM;AAAA,UAC5C,KAAK,KAAK,MAAM,KAAK,IAAI,GAAG,IAAI;AAAA,UAChC,UAAU;AAAA,UACV,WAAW;AAAA,UACX,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,iBAAS,aAAa,OAAO,MAAM;AACnC,iBAAS,aAAa,OAAO,MAAM;AACnC,6BAAqB;AAAA,MACvB,SAAS,KAAK;AACZ,cAAM,IAAI;AACV,iBAAS,aAAa,EAAE,MAAM;AAC9B,iBAAS,aAAa,EAAE,MAAM;AAC9B,cAAM,SAAS,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,CAAC,EACzE,SAAS,MAAM,EACf,MAAM,GAAG,GAAI;AAChB,cAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,EAAE,CAAC,sBAAsB,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;AAAA,EAAO,MAAM,EAAE;AAAA,MAC/G;AACA,cAAQ,MAAM,KAAK,YAAY,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,gBAAU;AAAA,IACZ;AAAA,EACF,UAAE;AACA,QAAI;AACJ,QAAI,KAAK,SAAS;AAChB,UAAI;AACF,yBAAiB,MAAM;AAAA,UACrB;AAAA,UACA,KAAK;AAAA,UACL,EAAE,QAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF,SAAS,cAAc;AACrB,kBAAU,IAAI;AAAA,UACZ,uDAAuD,wBAAwB,QAAQ,aAAa,UAAU,YAAY;AAAA,UAC1H,EAAE,OAAO,WAAW,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAS,OAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACjF,QAAI,SAAS;AACX,YAAM,IAAI;AAAA,QACR,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QAC3D;AAAA,QACA,EAAE,OAAO,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,iBAAiB,0CAA0C,cAAc;AAC/F,WAAO,iBAAiB,EAAE,GAAG,OAAO,eAAe,IAAI;AAAA,EACzD;AACF;AAGA,eAAsB,eAAkB,MAA0B;AAChE,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,SAAS,MAAM,MAAM;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,gCAAgC,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACA,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,4BAA4B,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,EACjG;AACF;AAGA,eAAsB,UAAU,MAAc,SAAgC;AAC5E,QAAM,UAAU,MAAM,OAAO;AAC/B;AAGO,SAAS,UAAU,QAAgB,IAAoB;AAC5D,SAAO,GAAG,MAAM,IAAI,EAAE,GAAG,QAAQ,oBAAoB,GAAG;AAC1D;","names":["resolve"]}

package/dist/chunk-ODT47UAY.js

@@ -0,0 +1,221 @@
+import {
+  benchRoot,
+  preflightVenvImports,
+  readJsonReport,
+  runStagedJudge,
+  stageFile,
+  venvPython
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/aec-bench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "aec-bench.json");
+var REPO = "TheodoreGalanos/aec-bench";
+var RAW = `https://raw.githubusercontent.com/${REPO}/main`;
+var TREE = `https://api.github.com/repos/${REPO}/git/trees/main?recursive=1`;
+var verifyPathPattern = /^tasks\/(.+)\/tests\/verify\.py$/;
+var DEFAULT_LIMIT = 10;
+function recordToTask(rec) {
+  const meta = {
+    taskId: rec.id,
+    discipline: rec.discipline,
+    taskToml: rec.task_toml,
+    verifyPy: rec.verify_py,
+    goldenPassMd: rec.golden_pass_md
+  };
+  return {
+    id: rec.id,
+    split: rec.discipline,
+    // instruction.md is fully self-contained and already specifies the exact JSON
+    // output schema + the "write to /workspace/output.md" contract the verifier
+    // keys off — we pass it through verbatim so the verify.py extractor matches.
+    prompt: rec.instruction,
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.verifyPy !== "string" || typeof md.taskId !== "string") {
+    throw new Error(`aec-bench task ${task.id} missing verifier metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function fetchText(url) {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`);
+  return res.text();
+}
+async function fetchTextOrNull(url) {
+  const res = await fetch(url);
+  if (res.status === 404) return null;
+  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`);
+  return res.text();
+}
+async function listAllInstances() {
+  const res = await fetch(TREE);
+  if (!res.ok) throw new Error(`aec-bench tree ${res.status}: ${TREE}`);
+  const { tree } = await res.json();
+  const ids = [];
+  for (const entry of tree) {
+    const m = verifyPathPattern.exec(entry.path);
+    if (m?.[1]) ids.push(m[1]);
+  }
+  return ids;
+}
+async function fetchInstance(id) {
+  const base = `${RAW}/tasks/${id}`;
+  const verify = await fetch(`${base}/tests/verify.py`);
+  if (verify.status === 404) return null;
+  if (!verify.ok) throw new Error(`aec-bench fetch ${verify.status}: ${id}/tests/verify.py`);
+  const [instruction, task_toml, golden_pass_md] = await Promise.all([
+    fetchText(`${base}/instruction.md`),
+    fetchText(`${base}/task.toml`),
+    fetchTextOrNull(`${base}/tests/fixtures/golden_pass.md`)
+  ]);
+  return {
+    id,
+    discipline: id.split("/")[0] ?? "",
+    instruction,
+    task_toml,
+    verify_py: await verify.text(),
+    golden_pass_md
+  };
+}
+function selectFixtures(records, opts) {
+  let tasks = records.map(recordToTask);
+  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  return tasks;
+}
+async function loadFixtures(opts) {
+  const records = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(
+    `[aec-bench] AEC_FIXTURES=1 \u2014 loading ${records.length} committed fixtures from ${FIXTURES} (no GitHub fetch)`
+  );
+  return selectFixtures(records, opts);
+}
+async function loadLive(opts) {
+  if (opts.ids) {
+    const records2 = [];
+    for (const id of opts.ids) {
+      const rec = await fetchInstance(id);
+      if (!rec) throw new Error(`aec-bench: ${id} has no tests/verify.py (seed-only or wrong id)`);
+      records2.push(rec);
+    }
+    return records2.map(recordToTask);
+  }
+  const limit = opts.limit ?? DEFAULT_LIMIT;
+  let ids = await listAllInstances();
+  if (opts.split) ids = ids.filter((id) => id.startsWith(`${opts.split}/`));
+  const records = [];
+  for (const id of ids) {
+    if (records.length >= limit) break;
+    try {
+      const rec = await fetchInstance(id);
+      if (rec) records.push(rec);
+    } catch (err) {
+      console.warn(`[aec-bench] skipping ${id}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  if (records.length === 0) {
+    throw new Error(
+      `aec-bench loadTasks found no runnable instances for ${JSON.stringify(opts)} (no tests/verify.py matched the requested split). Set AEC_FIXTURES=1 to run offline.`
+    );
+  }
+  return records.map(recordToTask);
+}
+async function runVerifier(meta, artifact) {
+  return runStagedJudge({
+    tmpPrefix: "aecbench-",
+    timeoutMs: 12e4,
+    async stage(dir) {
+      await Promise.all([
+        stageFile(join(dir, "output.md"), artifact),
+        stageFile(join(dir, "verify.py"), meta.verifyPy)
+      ]);
+    },
+    bin: venvPython,
+    argv: (dir) => [join(dir, "verify.py"), "--input", join(dir, "output.md"), "--output", join(dir, "reward.json")],
+    async parseReport(dir) {
+      const report = await readJsonReport(join(dir, "reward.json"));
+      const reward = report.reward;
+      if (typeof reward !== "number" || !Number.isFinite(reward)) {
+        throw new Error(`aec-bench verify.py wrote no numeric reward for ${meta.taskId}: ${JSON.stringify(report)}`);
+      }
+      const details = await readFile(join(dir, "details.json"), "utf8").then(
+        (s) => JSON.parse(s),
+        () => ({})
+      );
+      const score = Math.max(0, Math.min(1, reward));
+      return {
+        // resolved = full credit (all fields within tolerance), matching aec-bench's perfect_rate.
+        resolved: score >= 1,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, discipline: meta.discipline, reward, fields: details })
+      };
+    }
+  });
+}
+function createAecBenchAdapter() {
+  const fixturesMode = process.env.AEC_FIXTURES === "1";
+  return {
+    name: "aec-bench",
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["math", "json", "re"],
+        requireDocker: false,
+        fix: "Fix: python3 -m venv bench/.venv (verify.py only needs the stdlib \u2014 no pip install)."
+      });
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8").catch((err) => {
+          throw new Error(`AEC_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+        });
+        return;
+      }
+      const res = await fetch(`${RAW}/README.md`, { method: "HEAD" }).catch((err) => {
+        throw new Error(
+          `aec-bench preflight failed reaching ${RAW}: ${err instanceof Error ? err.message : err}
+Fix: ensure network access to raw.githubusercontent.com, or set AEC_FIXTURES=1 to run offline.`
+        );
+      });
+      if (!res.ok) {
+        throw new Error(
+          `aec-bench preflight: ${REPO} README HEAD ${res.status}. Set AEC_FIXTURES=1 to run against committed fixtures.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      try {
+        return await loadLive(opts);
+      } catch (err) {
+        if (err instanceof Error && /fetch \d|ENOTFOUND|getaddrinfo|network/i.test(err.message)) {
+          console.warn(
+            `[aec-bench] live fetch failed (${err.message.slice(0, 160)}); falling back to committed fixtures at ${FIXTURES}`
+          );
+          return loadFixtures(opts);
+        }
+        throw err;
+      }
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      return meta.goldenPassMd ?? void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      return runVerifier(meta, artifact);
+    }
+  };
+}
+
+export {
+  createAecBenchAdapter
+};
+//# sourceMappingURL=chunk-ODT47UAY.js.map

package/dist/chunk-ODT47UAY.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/aec-bench.ts"],"sourcesContent":["/**\n * AEC-Bench adapter (TheodoreGalanos/aec-bench, MIT) — closed-form\n * Architecture/Engineering/Construction calculation tasks. Worker artifact = a\n * markdown solution ending in a fenced ```json block with the required numeric\n * fields. Judge = the task's OWN `tests/verify.py`, run with python3: it\n * recomputes ground truth from the embedded engineering formulas, extracts the\n * last JSON block from the artifact, scores each field by math.isclose within a\n * per-field rel_tol, and writes {\"reward\": mean} + per-field details.json.\n * GRADED / partial-credit, FULLY DETERMINISTIC — no LLM judge.\n *\n * Distinct from the multimodal nomic-ai/aec-bench; this is the deterministic\n * calculation platform. The runnable-instance verify.py only needs python3 (no\n * Docker, no Harbor) for the pure-calc disciplines, so the local gate runs at\n * conc<=2 without a container backend.\n *\n * Requires for a live run: network to raw.githubusercontent.com /\n * api.github.com (the in-repo tasks tree) + a python3 interpreter (the bench\n * venv) to run verify.py. For offline/CI, set AEC_FIXTURES=1 to load the\n * committed fixtures (bench/fixtures/aec-bench.json) — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot, preflightVenvImports, readJsonReport, runStagedJudge, stageFile, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'aec-bench.json')\n\nconst REPO = 'TheodoreGalanos/aec-bench'\nconst RAW = `https://raw.githubusercontent.com/${REPO}/main`\nconst TREE = `https://api.github.com/repos/${REPO}/git/trees/main?recursive=1`\n\n/** Matches every runnable-instance task id at ANY depth under tasks/. */\nconst verifyPathPattern = /^tasks\\/(.+)\\/tests\\/verify\\.py$/\n\n/** Default cap on tasks enumerated before a `limit` is applied. */\nconst DEFAULT_LIMIT = 10\n\ninterface AecRecord {\n  /** Canonical task id `<discipline>/<task>`, e.g. 'electrical/catenary-sag'. */\n  id: string\n  discipline: string\n  /** instruction.md — the self-contained prompt (table + required outputs + JSON schema). */\n  instruction: string\n  /** task.toml — metadata/difficulty/timeouts (carried for trace context). */\n  task_toml: string\n  /** tests/verify.py — the deterministic verifier (recomputes GT, scores fields). */\n  verify_py: string\n  /** tests/fixtures/golden_pass.md — the oracle artifact that scores reward 1.0,\n   *  when the task ships one. Null when the task only ships a non-md ground truth\n   *  (e.g. tests/ground_truth.json); the judge never needs it, only goldArtifact does. */\n  golden_pass_md: string | null\n}\n\ninterface AecMeta {\n  taskId: string\n  discipline: string\n  taskToml: string\n  verifyPy: string\n  goldenPassMd: string | null\n}\n\nfunction recordToTask(rec: AecRecord): BenchTask {\n  const meta: AecMeta = {\n    taskId: rec.id,\n    discipline: rec.discipline,\n    taskToml: rec.task_toml,\n    verifyPy: rec.verify_py,\n    goldenPassMd: rec.golden_pass_md,\n  }\n  return {\n    id: rec.id,\n    split: rec.discipline,\n    // instruction.md is fully self-contained and already specifies the exact JSON\n    // output schema + the \"write to /workspace/output.md\" contract the verifier\n    // keys off — we pass it through verbatim so the verify.py extractor matches.\n    prompt: rec.instruction,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AecMeta {\n  const md = task.metadata\n  // Gold is optional (goldenPassMd may be null) — only the verifier + id are required.\n  if (!md || typeof md.verifyPy !== 'string' || typeof md.taskId !== 'string') {\n    throw new Error(`aec-bench task ${task.id} missing verifier metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AecMeta\n}\n\nasync function fetchText(url: string): Promise<string> {\n  const res = await fetch(url)\n  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`)\n  return res.text()\n}\n\n/** Like fetchText but returns null on a 404 (absent optional file); throws on any other non-OK. */\nasync function fetchTextOrNull(url: string): Promise<string | null> {\n  const res = await fetch(url)\n  if (res.status === 404) return null\n  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`)\n  return res.text()\n}\n\ninterface GitTree {\n  tree: Array<{ path: string; type: string }>\n}\n\n/**\n * One recursive git-tree call enumerates EVERY runnable-instance id at any depth:\n * a task is runnable iff it ships tests/verify.py. The captured group is the id\n * `tasks/<id>/tests/verify.py` → `<id>` (e.g. 'electrical/pf-droop', or a deeper\n * '<discipline>/<family>/<task>'). Throws loud on a non-OK tree response.\n */\nasync function listAllInstances(): Promise<string[]> {\n  const res = await fetch(TREE)\n  if (!res.ok) throw new Error(`aec-bench tree ${res.status}: ${TREE}`)\n  const { tree } = (await res.json()) as GitTree\n  const ids: string[] = []\n  for (const entry of tree) {\n    const m = verifyPathPattern.exec(entry.path)\n    if (m?.[1]) ids.push(m[1])\n  }\n  return ids\n}\n\n/**\n * Fetch one task's instruction.md + task.toml + tests/verify.py + golden_pass.md.\n * Returns null when the dir is a SEED (no runnable verify.py) so enumeration can\n * skip it without faking a task. Gold is OPTIONAL — a task that ships a non-md\n * ground truth (e.g. tests/ground_truth.json) yields golden_pass_md=null; the\n * deterministic judge needs only verify.py.\n */\nasync function fetchInstance(id: string): Promise<AecRecord | null> {\n  const base = `${RAW}/tasks/${id}`\n  const verify = await fetch(`${base}/tests/verify.py`)\n  if (verify.status === 404) return null\n  if (!verify.ok) throw new Error(`aec-bench fetch ${verify.status}: ${id}/tests/verify.py`)\n  const [instruction, task_toml, golden_pass_md] = await Promise.all([\n    fetchText(`${base}/instruction.md`),\n    fetchText(`${base}/task.toml`),\n    fetchTextOrNull(`${base}/tests/fixtures/golden_pass.md`),\n  ])\n  return {\n    id,\n    discipline: id.split('/')[0] ?? '',\n    instruction,\n    task_toml,\n    verify_py: await verify.text(),\n    golden_pass_md,\n  }\n}\n\nfunction selectFixtures(records: AecRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records.map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as AecRecord[]\n  console.warn(\n    `[aec-bench] AEC_FIXTURES=1 — loading ${records.length} committed fixtures from ${FIXTURES} (no GitHub fetch)`,\n  )\n  return selectFixtures(records, opts)\n}\n\n/** Enumerate live tasks: explicit ids (each required — throws on a bad id), or a\n *  capped slice of the recursive tree (optionally filtered to one split). Per-task\n *  resilient: a single fetch failure warns + SKIPS that task, never aborting the\n *  batch. Skips seed dirs (no verify.py) — never fabricates. */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  if (opts.ids) {\n    const records: AecRecord[] = []\n    for (const id of opts.ids) {\n      const rec = await fetchInstance(id)\n      if (!rec) throw new Error(`aec-bench: ${id} has no tests/verify.py (seed-only or wrong id)`)\n      records.push(rec)\n    }\n    return records.map(recordToTask)\n  }\n  const limit = opts.limit ?? DEFAULT_LIMIT\n  let ids = await listAllInstances()\n  if (opts.split) ids = ids.filter((id) => id.startsWith(`${opts.split}/`))\n  const records: AecRecord[] = []\n  for (const id of ids) {\n    if (records.length >= limit) break\n    try {\n      const rec = await fetchInstance(id)\n      if (rec) records.push(rec)\n    } catch (err) {\n      // One bad task must NEVER abort the batch — warn and skip it.\n      console.warn(`[aec-bench] skipping ${id}: ${err instanceof Error ? err.message : String(err)}`)\n    }\n  }\n  if (records.length === 0) {\n    throw new Error(\n      `aec-bench loadTasks found no runnable instances for ${JSON.stringify(opts)} ` +\n        `(no tests/verify.py matched the requested split). Set AEC_FIXTURES=1 to run offline.`,\n    )\n  }\n  return records.map(recordToTask)\n}\n\n/**\n * Run the task's own verify.py with python3 over the artifact via the shared\n * staged-judge spine (mkdtemp → stage → spawn → parseReport → cleanup). verify.py\n * writes {\"reward\": mean} to --output and per-field details.json as a sibling. We\n * read both: reward → graded score; details → sub-scores for the trace-analyst.\n * Fail loud if the verifier never wrote a numeric reward (a crashed verifier is\n * NOT a silent 0 — verify.py's own except-trap writes reward 0.0, so an absent /\n * non-numeric reward.json is a real bug, surfaced by readJsonReport).\n */\nasync function runVerifier(meta: AecMeta, artifact: string): Promise<BenchScore> {\n  return runStagedJudge({\n    tmpPrefix: 'aecbench-',\n    timeoutMs: 120_000,\n    async stage(dir) {\n      await Promise.all([\n        stageFile(join(dir, 'output.md'), artifact),\n        stageFile(join(dir, 'verify.py'), meta.verifyPy),\n      ])\n    },\n    bin: venvPython,\n    argv: (dir) => [join(dir, 'verify.py'), '--input', join(dir, 'output.md'), '--output', join(dir, 'reward.json')],\n    async parseReport(dir) {\n      const report = await readJsonReport<{ reward?: number }>(join(dir, 'reward.json'))\n      const reward = report.reward\n      if (typeof reward !== 'number' || !Number.isFinite(reward)) {\n        throw new Error(`aec-bench verify.py wrote no numeric reward for ${meta.taskId}: ${JSON.stringify(report)}`)\n      }\n      const details = await readFile(join(dir, 'details.json'), 'utf8').then(\n        (s) => JSON.parse(s) as Record<string, number>,\n        () => ({}),\n      )\n      const score = Math.max(0, Math.min(1, reward))\n      return {\n        // resolved = full credit (all fields within tolerance), matching aec-bench's perfect_rate.\n        resolved: score >= 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, discipline: meta.discipline, reward, fields: details }),\n      }\n    },\n  })\n}\n\nexport function createAecBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AEC_FIXTURES === '1'\n\n  return {\n    name: 'aec-bench',\n\n    async preflight() {\n      // The verifier is python3 over the stdlib (math/json/re) — no pip install.\n      // Reuse the shared import-probe so the bench venv interpreter is proven to\n      // exist + run before any judge spawns verify.py.\n      await preflightVenvImports({\n        modules: ['math', 'json', 're'],\n        requireDocker: false,\n        fix: 'Fix: python3 -m venv bench/.venv (verify.py only needs the stdlib — no pip install).',\n      })\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`AEC_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      const res = await fetch(`${RAW}/README.md`, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `aec-bench preflight failed reaching ${RAW}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set AEC_FIXTURES=1 to run offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `aec-bench preflight: ${REPO} README HEAD ${res.status}. Set AEC_FIXTURES=1 to run against committed fixtures.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      try {\n        return await loadLive(opts)\n      } catch (err) {\n        // A reachability failure falls back to fixtures with an explicit warn; a\n        // wrong-id / seed-only error (the loader's own throw) propagates.\n        if (err instanceof Error && /fetch \\d|ENOTFOUND|getaddrinfo|network/i.test(err.message)) {\n          console.warn(\n            `[aec-bench] live fetch failed (${err.message.slice(0, 160)}); falling back to committed fixtures at ${FIXTURES}`,\n          )\n          return loadFixtures(opts)\n        }\n        throw err\n      }\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's own golden_pass.md (scores reward 1.0 through the SAME\n      // verify.py the real artifact takes), proving the judge end-to-end. Tasks\n      // without a golden_pass.md (non-md ground truth) have no oracle artifact.\n      const meta = readMeta(task)\n      return meta.goldenPassMd ?? undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      // verify.py fail-closes an empty/unparseable artifact to reward 0.0 itself,\n      // so we pass it straight through (no pre-judging here).\n      return runVerifier(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;AAqBA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAIrB,IAAM,WAAW,KAAK,WAAW,YAAY,gBAAgB;AAE7D,IAAM,OAAO;AACb,IAAM,MAAM,qCAAqC,IAAI;AACrD,IAAM,OAAO,gCAAgC,IAAI;AAGjD,IAAM,oBAAoB;AAG1B,IAAM,gBAAgB;AA0BtB,SAAS,aAAa,KAA2B;AAC/C,QAAM,OAAgB;AAAA,IACpB,QAAQ,IAAI;AAAA,IACZ,YAAY,IAAI;AAAA,IAChB,UAAU,IAAI;AAAA,IACd,UAAU,IAAI;AAAA,IACd,cAAc,IAAI;AAAA,EACpB;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA;AAAA;AAAA;AAAA,IAIX,QAAQ,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA0B;AAC1C,QAAM,KAAK,KAAK;AAEhB,MAAI,CAAC,MAAM,OAAO,GAAG,aAAa,YAAY,OAAO,GAAG,WAAW,UAAU;AAC3E,UAAM,IAAI,MAAM,kBAAkB,KAAK,EAAE,iEAA4D;AAAA,EACvG;AACA,SAAO;AACT;AAEA,eAAe,UAAU,KAA8B;AACrD,QAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,mBAAmB,IAAI,MAAM,KAAK,GAAG,EAAE;AACpE,SAAO,IAAI,KAAK;AAClB;AAGA,eAAe,gBAAgB,KAAqC;AAClE,QAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,MAAI,IAAI,WAAW,IAAK,QAAO;AAC/B,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,mBAAmB,IAAI,MAAM,KAAK,GAAG,EAAE;AACpE,SAAO,IAAI,KAAK;AAClB;AAYA,eAAe,mBAAsC;AACnD,QAAM,MAAM,MAAM,MAAM,IAAI;AAC5B,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,kBAAkB,IAAI,MAAM,KAAK,IAAI,EAAE;AACpE,QAAM,EAAE,KAAK,IAAK,MAAM,IAAI,KAAK;AACjC,QAAM,MAAgB,CAAC;AACvB,aAAW,SAAS,MAAM;AACxB,UAAM,IAAI,kBAAkB,KAAK,MAAM,IAAI;AAC3C,QAAI,IAAI,CAAC,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAC3B;AACA,SAAO;AACT;AASA,eAAe,cAAc,IAAuC;AAClE,QAAM,OAAO,GAAG,GAAG,UAAU,EAAE;AAC/B,QAAM,SAAS,MAAM,MAAM,GAAG,IAAI,kBAAkB;AACpD,MAAI,OAAO,WAAW,IAAK,QAAO;AAClC,MAAI,CAAC,OAAO,GAAI,OAAM,IAAI,MAAM,mBAAmB,OAAO,MAAM,KAAK,EAAE,kBAAkB;AACzF,QAAM,CAAC,aAAa,WAAW,cAAc,IAAI,MAAM,QAAQ,IAAI;AAAA,IACjE,UAAU,GAAG,IAAI,iBAAiB;AAAA,IAClC,UAAU,GAAG,IAAI,YAAY;AAAA,IAC7B,gBAAgB,GAAG,IAAI,gCAAgC;AAAA,EACzD,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA,YAAY,GAAG,MAAM,GAAG,EAAE,CAAC,KAAK;AAAA,IAChC;AAAA,IACA;AAAA,IACA,WAAW,MAAM,OAAO,KAAK;AAAA,IAC7B;AAAA,EACF;AACF;AAEA,SAAS,eAAe,SAAsB,MAAgC;AAC5E,MAAI,QAAQ,QAAQ,IAAI,YAAY;AACpC,MAAI,KAAK,MAAO,SAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,KAAK;AAClE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC3D,UAAQ;AAAA,IACN,6CAAwC,QAAQ,MAAM,4BAA4B,QAAQ;AAAA,EAC5F;AACA,SAAO,eAAe,SAAS,IAAI;AACrC;AAMA,eAAe,SAAS,MAAyC;AAC/D,MAAI,KAAK,KAAK;AACZ,UAAMA,WAAuB,CAAC;AAC9B,eAAW,MAAM,KAAK,KAAK;AACzB,YAAM,MAAM,MAAM,cAAc,EAAE;AAClC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,cAAc,EAAE,iDAAiD;AAC3F,MAAAA,SAAQ,KAAK,GAAG;AAAA,IAClB;AACA,WAAOA,SAAQ,IAAI,YAAY;AAAA,EACjC;AACA,QAAM,QAAQ,KAAK,SAAS;AAC5B,MAAI,MAAM,MAAM,iBAAiB;AACjC,MAAI,KAAK,MAAO,OAAM,IAAI,OAAO,CAAC,OAAO,GAAG,WAAW,GAAG,KAAK,KAAK,GAAG,CAAC;AACxE,QAAM,UAAuB,CAAC;AAC9B,aAAW,MAAM,KAAK;AACpB,QAAI,QAAQ,UAAU,MAAO;AAC7B,QAAI;AACF,YAAM,MAAM,MAAM,cAAc,EAAE;AAClC,UAAI,IAAK,SAAQ,KAAK,GAAG;AAAA,IAC3B,SAAS,KAAK;AAEZ,cAAQ,KAAK,wBAAwB,EAAE,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,IAChG;AAAA,EACF;AACA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI;AAAA,MACR,uDAAuD,KAAK,UAAU,IAAI,CAAC;AAAA,IAE7E;AAAA,EACF;AACA,SAAO,QAAQ,IAAI,YAAY;AACjC;AAWA,eAAe,YAAY,MAAe,UAAuC;AAC/E,SAAO,eAAe;AAAA,IACpB,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,MAAM,KAAK;AACf,YAAM,QAAQ,IAAI;AAAA,QAChB,UAAU,KAAK,KAAK,WAAW,GAAG,QAAQ;AAAA,QAC1C,UAAU,KAAK,KAAK,WAAW,GAAG,KAAK,QAAQ;AAAA,MACjD,CAAC;AAAA,IACH;AAAA,IACA,KAAK;AAAA,IACL,MAAM,CAAC,QAAQ,CAAC,KAAK,KAAK,WAAW,GAAG,WAAW,KAAK,KAAK,WAAW,GAAG,YAAY,KAAK,KAAK,aAAa,CAAC;AAAA,IAC/G,MAAM,YAAY,KAAK;AACrB,YAAM,SAAS,MAAM,eAAoC,KAAK,KAAK,aAAa,CAAC;AACjF,YAAM,SAAS,OAAO;AACtB,UAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,MAAM,GAAG;AAC1D,cAAM,IAAI,MAAM,mDAAmD,KAAK,MAAM,KAAK,KAAK,UAAU,MAAM,CAAC,EAAE;AAAA,MAC7G;AACA,YAAM,UAAU,MAAM,SAAS,KAAK,KAAK,cAAc,GAAG,MAAM,EAAE;AAAA,QAChE,CAAC,MAAM,KAAK,MAAM,CAAC;AAAA,QACnB,OAAO,CAAC;AAAA,MACV;AACA,YAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAC7C,aAAO;AAAA;AAAA,QAEL,UAAU,SAAS;AAAA,QACnB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,YAAY,KAAK,YAAY,QAAQ,QAAQ,QAAQ,CAAC;AAAA,MACtG;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,iBAAiB;AAElD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAIhB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,QAAQ,QAAQ,IAAI;AAAA,QAC9B,eAAe;AAAA,QACf,KAAK;AAAA,MACP,CAAC;AACD,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI,MAAM,sBAAsB,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,QAC1G,CAAC;AACD;AAAA,MACF;AACA,YAAM,MAAM,MAAM,MAAM,GAAG,GAAG,cAAc,EAAE,QAAQ,OAAO,CAAC,EAAE,MAAM,CAAC,QAAQ;AAC7E,cAAM,IAAI;AAAA,UACR,uCAAuC,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA;AAAA,QAEzF;AAAA,MACF,CAAC;AACD,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,IAAI;AAAA,UACR,wBAAwB,IAAI,gBAAgB,IAAI,MAAM;AAAA,QACxD;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACF,eAAO,MAAM,SAAS,IAAI;AAAA,MAC5B,SAAS,KAAK;AAGZ,YAAI,eAAe,SAAS,0CAA0C,KAAK,IAAI,OAAO,GAAG;AACvF,kBAAQ;AAAA,YACN,kCAAkC,IAAI,QAAQ,MAAM,GAAG,GAAG,CAAC,4CAA4C,QAAQ;AAAA,UACjH;AACA,iBAAO,aAAa,IAAI;AAAA,QAC1B;AACA,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,gBAAgB;AAAA,IAC9B;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAG1B,aAAO,YAAY,MAAM,QAAQ;AAAA,IACnC;AAAA,EACF;AACF;","names":["records"]}

package/dist/chunk-PA2ZKHJC.js

@@ -0,0 +1,230 @@
+// src/benchmarks/hotpotqa.ts
+import { execFile } from "child_process";
+import { readFile } from "fs/promises";
+import { join } from "path";
+import { fileURLToPath } from "url";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+var PY = join(BENCH_ROOT, ".venv", "bin", "python");
+var FIXTURES = join(BENCH_ROOT, "fixtures", "hotpotqa.json");
+var DATASET = "hotpotqa/hotpot_qa";
+var DATASET_CONFIG = "distractor";
+var DATASET_SPLIT = "validation";
+var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+var DEFAULT_F1_PASS = 0.6;
+var WORKER_CONTRACT = [
+  "",
+  "Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.",
+  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+  "The answer after the sentinel must be the bare value only (no explanation on that line)."
+].join("\n");
+async function py(script, args = []) {
+  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
+    maxBuffer: 1024 * 1024 * 256
+  });
+  return stdout;
+}
+var ARTICLES = /* @__PURE__ */ new Set(["a", "an", "the"]);
+function normalizeAnswer(input) {
+  const lower = input.toLowerCase();
+  const noPunct = lower.replace(/[^\w\s]/g, " ");
+  const tokens = noPunct.split(/\s+/).filter((t) => t.length > 0).filter((t) => !ARTICLES.has(t));
+  return tokens.join(" ").trim();
+}
+function answerTokens(input) {
+  const n = normalizeAnswer(input);
+  return n.length === 0 ? [] : n.split(" ");
+}
+function exactMatch(prediction, gold) {
+  return normalizeAnswer(prediction) === normalizeAnswer(gold);
+}
+function tokenF1(prediction, gold) {
+  const predTokens = answerTokens(prediction);
+  const goldTokens = answerTokens(gold);
+  if (predTokens.length === 0 || goldTokens.length === 0) {
+    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0;
+  }
+  const goldCounts = /* @__PURE__ */ new Map();
+  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1);
+  let common = 0;
+  for (const t of predTokens) {
+    const left = goldCounts.get(t);
+    if (left !== void 0 && left > 0) {
+      common += 1;
+      goldCounts.set(t, left - 1);
+    }
+  }
+  if (common === 0) return 0;
+  const precision = common / predTokens.length;
+  const recall = common / goldTokens.length;
+  return 2 * precision * recall / (precision + recall);
+}
+function f1PassThreshold() {
+  const raw = process.env.HOTPOTQA_F1_PASS;
+  if (raw === void 0 || raw.length === 0) return DEFAULT_F1_PASS;
+  const v = Number(raw);
+  if (!Number.isFinite(v) || v < 0 || v > 1) {
+    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`);
+  }
+  return v;
+}
+function parseFinalAnswer(artifact) {
+  const lines = artifact.split(/\r?\n/);
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i] ?? "";
+    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
+  }
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const t = (lines[i] ?? "").trim();
+    if (t.length > 0) return t;
+  }
+  return "";
+}
+function rowToTask(row) {
+  const meta = {
+    gold: row.answer,
+    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },
+    type: row.type ?? "",
+    level: row.level ?? "",
+    rawQuestion: row.question
+  };
+  return {
+    id: `hotpotqa-${row.id}`,
+    split: DATASET_SPLIT,
+    prompt: row.question + WORKER_CONTRACT,
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.gold !== "string") {
+    throw new Error(`HotpotQA task ${task.id} missing metadata.gold \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(
+    `[hotpotqa] HOTPOTQA_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`
+  );
+  let tasks = rows.map(rowToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  return tasks;
+}
+function createHotpotqaAdapter() {
+  const fixturesMode = process.env.HOTPOTQA_FIXTURES === "1";
+  f1PassThreshold();
+  return {
+    name: "hotpotqa",
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8").catch((err) => {
+          throw new Error(
+            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
+          );
+        });
+        return;
+      }
+      try {
+        await py(
+          `from datasets import load_dataset
+load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+print('ok')`
+        );
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `hotpotqa preflight failed: ${msg}
+Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; (2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const limit = opts.limit ?? 10;
+      const script = `
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    rid = f"hotpotqa-{r['id']}"
+    if ids is not None and rid not in ids:
+        continue
+    sf = r.get('supporting_facts', {}) or {}
+    out.append({
+        "id": r["id"],
+        "question": r.get("question", ""),
+        "answer": r.get("answer", ""),
+        "type": str(r.get("type", "")),
+        "level": str(r.get("level", "")),
+        "supporting_facts": {
+            "title": list(sf.get("title", [])),
+            "sent_id": [int(x) for x in sf.get("sent_id", [])],
+        },
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`;
+      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
+      const rows = JSON.parse(stdout);
+      return rows.map(rowToTask);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const finalAnswer = parseFinalAnswer(artifact);
+      if (finalAnswer.length === 0) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({
+            reason: "no parseable answer",
+            em: false,
+            f1: 0,
+            normalizedGold: normalizeAnswer(meta.gold)
+          })
+        };
+      }
+      const em = exactMatch(finalAnswer, meta.gold);
+      const f1 = tokenF1(finalAnswer, meta.gold);
+      const threshold = f1PassThreshold();
+      const resolved = em || f1 >= threshold;
+      return {
+        resolved,
+        score: f1,
+        detail: JSON.stringify({
+          em,
+          f1,
+          threshold,
+          normalizedAnswer: normalizeAnswer(finalAnswer),
+          normalizedGold: normalizeAnswer(meta.gold),
+          type: meta.type,
+          level: meta.level,
+          supportingFacts: meta.supportingFacts
+        })
+      };
+    }
+  };
+}
+
+export {
+  normalizeAnswer,
+  exactMatch,
+  tokenF1,
+  parseFinalAnswer,
+  createHotpotqaAdapter
+};
+//# sourceMappingURL=chunk-PA2ZKHJC.js.map

package/dist/chunk-PA2ZKHJC.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/hotpotqa.ts"],"sourcesContent":["/**\n * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').\n * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.\n *\n * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no\n * LLM. Both the predicted final answer and the gold are normalized (lowercase,\n * strip articles a/an/the, strip punctuation, collapse whitespace), then scored\n * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS\n * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the\n * suite a judge that needs no model tokens at all.\n *\n * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of\n * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,\n * never part of the score.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed + network\n * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the\n * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'hotpotqa.json')\n\nconst DATASET = 'hotpotqa/hotpot_qa'\nconst DATASET_CONFIG = 'distractor'\nconst DATASET_SPLIT = 'validation'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Default F1 pass threshold; HotpotQA's leaderboard reports EM and F1 separately. */\nconst DEFAULT_F1_PASS = 0.6\n\n/** Worker contract appended to every task prompt. The judge keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\ninterface SupportingFacts {\n  title: string[]\n  sent_id: number[]\n}\n\ninterface HotpotRow {\n  id: string\n  question: string\n  answer: string\n  type: string\n  level: string\n  supporting_facts: SupportingFacts\n}\n\ninterface HotpotMeta {\n  gold: string\n  supportingFacts: SupportingFacts\n  type: string\n  level: string\n  rawQuestion: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\n\n/**\n * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,\n * drop articles (a/an/the), collapse whitespace. Token comparisons run on the\n * output of this exactly as the published evaluator does.\n */\nexport function normalizeAnswer(input: string): string {\n  const lower = input.toLowerCase()\n  // strip punctuation: keep word chars + whitespace only\n  const noPunct = lower.replace(/[^\\w\\s]/g, ' ')\n  const tokens = noPunct\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Normalized-token list (the F1 bag-of-tokens unit). */\nfunction answerTokens(input: string): string[] {\n  const n = normalizeAnswer(input)\n  return n.length === 0 ? [] : n.split(' ')\n}\n\n/** Exact match on the normalized strings. */\nexport function exactMatch(prediction: string, gold: string): boolean {\n  return normalizeAnswer(prediction) === normalizeAnswer(gold)\n}\n\n/**\n * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of\n * normalized tokens. Mirrors the published evaluator's special-case handling of\n * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both\n * are empty, else 0.\n */\nexport function tokenF1(prediction: string, gold: string): number {\n  const predTokens = answerTokens(prediction)\n  const goldTokens = answerTokens(gold)\n  if (predTokens.length === 0 || goldTokens.length === 0) {\n    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const goldCounts = new Map<string, number>()\n  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1)\n  let common = 0\n  for (const t of predTokens) {\n    const left = goldCounts.get(t)\n    if (left !== undefined && left > 0) {\n      common += 1\n      goldCounts.set(t, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / predTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\n/** Read the configured F1 pass threshold; fail loud on a malformed override. */\nfunction f1PassThreshold(): number {\n  const raw = process.env.HOTPOTQA_F1_PASS\n  if (raw === undefined || raw.length === 0) return DEFAULT_F1_PASS\n  const v = Number(raw)\n  if (!Number.isFinite(v) || v < 0 || v > 1) {\n    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`)\n  }\n  return v\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the trimmed last non-empty line. Returns '' when nothing is\n * parseable (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\nfunction rowToTask(row: HotpotRow): BenchTask {\n  const meta: HotpotMeta = {\n    gold: row.answer,\n    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },\n    type: row.type ?? '',\n    level: row.level ?? '',\n    rawQuestion: row.question,\n  }\n  return {\n    id: `hotpotqa-${row.id}`,\n    split: DATASET_SPLIT,\n    prompt: row.question + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): HotpotMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`HotpotQA task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as HotpotMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as HotpotRow[]\n  console.warn(\n    `[hotpotqa] HOTPOTQA_FIXTURES=1 — loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`,\n  )\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createHotpotqaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.HOTPOTQA_FIXTURES === '1'\n  // Validate the threshold at construction so a malformed env fails before any run.\n  f1PassThreshold()\n\n  return {\n    name: 'hotpotqa',\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `hotpotqa preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; ` +\n            `or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    rid = f\"hotpotqa-{r['id']}\"\n    if ids is not None and rid not in ids:\n        continue\n    sf = r.get('supporting_facts', {}) or {}\n    out.append({\n        \"id\": r[\"id\"],\n        \"question\": r.get(\"question\", \"\"),\n        \"answer\": r.get(\"answer\", \"\"),\n        \"type\": str(r.get(\"type\", \"\")),\n        \"level\": str(r.get(\"level\", \"\")),\n        \"supporting_facts\": {\n            \"title\": list(sf.get(\"title\", [])),\n            \"sent_id\": [int(x) for x in sf.get(\"sent_id\", [])],\n        },\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as HotpotRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            reason: 'no parseable answer',\n            em: false,\n            f1: 0,\n            normalizedGold: normalizeAnswer(meta.gold),\n          }),\n        }\n      }\n\n      const em = exactMatch(finalAnswer, meta.gold)\n      const f1 = tokenF1(finalAnswer, meta.gold)\n      const threshold = f1PassThreshold()\n      const resolved = em || f1 >= threshold\n      return {\n        resolved,\n        score: f1,\n        detail: JSON.stringify({\n          em,\n          f1,\n          threshold,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          type: meta.type,\n          level: meta.level,\n          supportingFacts: meta.supportingFacts,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AAoBA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,IAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,IAAM,UAAU;AAChB,IAAM,iBAAiB;AACvB,IAAM,gBAAgB;AACtB,IAAM,wBAAwB;AAG9B,IAAM,kBAAkB;AAGxB,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA,iDAAiD,qBAAqB;AAAA,EACtE;AACF,EAAE,KAAK,IAAI;AAyBX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IAClE,WAAW,OAAO,OAAO;AAAA,EAC3B,CAAC;AACD,SAAO;AACT;AAEA,IAAM,WAAW,oBAAI,IAAI,CAAC,KAAK,MAAM,KAAK,CAAC;AAOpC,SAAS,gBAAgB,OAAuB;AACrD,QAAM,QAAQ,MAAM,YAAY;AAEhC,QAAM,UAAU,MAAM,QAAQ,YAAY,GAAG;AAC7C,QAAM,SAAS,QACZ,MAAM,KAAK,EACX,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;AACjC,SAAO,OAAO,KAAK,GAAG,EAAE,KAAK;AAC/B;AAGA,SAAS,aAAa,OAAyB;AAC7C,QAAM,IAAI,gBAAgB,KAAK;AAC/B,SAAO,EAAE,WAAW,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;AAC1C;AAGO,SAAS,WAAW,YAAoB,MAAuB;AACpE,SAAO,gBAAgB,UAAU,MAAM,gBAAgB,IAAI;AAC7D;AAQO,SAAS,QAAQ,YAAoB,MAAsB;AAChE,QAAM,aAAa,aAAa,UAAU;AAC1C,QAAM,aAAa,aAAa,IAAI;AACpC,MAAI,WAAW,WAAW,KAAK,WAAW,WAAW,GAAG;AACtD,WAAO,WAAW,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;AAAA,EAClE;AACA,QAAM,aAAa,oBAAI,IAAoB;AAC3C,aAAW,KAAK,WAAY,YAAW,IAAI,IAAI,WAAW,IAAI,CAAC,KAAK,KAAK,CAAC;AAC1E,MAAI,SAAS;AACb,aAAW,KAAK,YAAY;AAC1B,UAAM,OAAO,WAAW,IAAI,CAAC;AAC7B,QAAI,SAAS,UAAa,OAAO,GAAG;AAClC,gBAAU;AACV,iBAAW,IAAI,GAAG,OAAO,CAAC;AAAA,IAC5B;AAAA,EACF;AACA,MAAI,WAAW,EAAG,QAAO;AACzB,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,SAAS,SAAS,WAAW;AACnC,SAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAGA,SAAS,kBAA0B;AACjC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,QAAQ,UAAa,IAAI,WAAW,EAAG,QAAO;AAClD,QAAM,IAAI,OAAO,GAAG;AACpB,MAAI,CAAC,OAAO,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AACzC,UAAM,IAAI,MAAM,mDAAmD,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EAC1F;AACA,SAAO;AACT;AAQO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,KAAK,MAAM,CAAC,KAAK,IAAI,KAAK;AAChC,QAAI,EAAE,SAAS,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAEA,SAAS,UAAU,KAA2B;AAC5C,QAAM,OAAmB;AAAA,IACvB,MAAM,IAAI;AAAA,IACV,iBAAiB,IAAI,oBAAoB,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClE,MAAM,IAAI,QAAQ;AAAA,IAClB,OAAO,IAAI,SAAS;AAAA,IACpB,aAAa,IAAI;AAAA,EACnB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,IAAI,EAAE;AAAA,IACtB,OAAO;AAAA,IACP,QAAQ,IAAI,WAAW;AAAA,IACvB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA6B;AAC7C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAAU;AACtC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,6DAAwD;AAAA,EAClG;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,iDAA4C,KAAK,MAAM,4BAA4B,QAAQ;AAAA,EAC7F;AACA,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,kBAAgB;AAEhB,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,2BAA2B,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAC7F;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,eACK,KAAK,UAAU,OAAO,CAAC,KAAK,KAAK,UAAU,cAAc,CAAC,WAAW,KAAK,UAAU,aAAa,CAAC;AAAA;AAAA,QAEzG;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,+HAEkB,OAAO,KAAK,cAAc;AAAA,QAE/E;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,SAAS;AAAA;AAAA;AAAA,oBAGD,KAAK,UAAU,OAAO,CAAC,KAAK,KAAK,UAAU,cAAc,CAAC,WAAW,KAAK,UAAU,aAAa,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAmBjF,KAAK;AAAA;AAAA;AAAA;AAIpC,YAAM,SAAS,MAAM,GAAG,QAAQ,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;AAC1E,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK,IAAI,SAAS;AAAA,IAC3B;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,GAAG,qBAAqB,IAAI,KAAK,IAAI;AAAA,IAC9C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,cAAc,iBAAiB,QAAQ;AAE7C,UAAI,YAAY,WAAW,GAAG;AAE5B,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU;AAAA,YACrB,QAAQ;AAAA,YACR,IAAI;AAAA,YACJ,IAAI;AAAA,YACJ,gBAAgB,gBAAgB,KAAK,IAAI;AAAA,UAC3C,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,KAAK,WAAW,aAAa,KAAK,IAAI;AAC5C,YAAM,KAAK,QAAQ,aAAa,KAAK,IAAI;AACzC,YAAM,YAAY,gBAAgB;AAClC,YAAM,WAAW,MAAM,MAAM;AAC7B,aAAO;AAAA,QACL;AAAA,QACA,OAAO;AAAA,QACP,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA;AAAA,UACA,kBAAkB,gBAAgB,WAAW;AAAA,UAC7C,gBAAgB,gBAAgB,KAAK,IAAI;AAAA,UACzC,MAAM,KAAK;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,iBAAiB,KAAK;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}