@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/chunk-5H5XV76F.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/humaneval.ts"],"sourcesContent":["/**\n * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the\n * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout\n * through the `Supervisor` that self-corrects across rounds, vs blind\n * random@k. This is the experiment `humaneval-gate.mts` names as \"the next one\" —\n * the gate measures SELECTION over stateless single completions; this measures\n * whether observe→steer (self-correction) beats blind compute at equal k.\n *\n * Worker artifact = the model's reply (a Python function, fenced or raw). The\n * DETERMINISTIC judge runs the candidate against the task's own `test` in an\n * isolated `--network=none` python container — exit 0 = pass. No gold\n * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only\n * to self-verify the judge before spending tokens.\n *\n * The primitives (loader / extractor / Docker checker) live here and are reused by\n * `humaneval-gate.mts` — one home, no duplication.\n */\n\nimport { execFile } from 'node:child_process'\nimport { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { gunzipSync } from 'node:zlib'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'\nconst dockerImage = 'python:3.12-slim'\nconst dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)\n\nexport interface HumanEvalTask {\n  taskId: string\n  prompt: string\n  test: string\n  entryPoint: string\n  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */\n  canonicalSolution?: string\n}\n\n/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch\n *  or a malformed line — a silently-short task set would poison the gate. `offset`\n *  selects a deeper slice (the later tasks are harder) so the worker has a\n *  correctable middle band rather than a saturated easy prefix. */\nexport async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {\n  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits\n  // (429) under repeated runs. Fall back to the network fetch when unset.\n  const localGz = process.env.HUMANEVAL_GZ\n  let gz: Buffer\n  if (localGz) {\n    gz = readFileSync(localGz)\n  } else {\n    const res = await fetch(humanevalUrl)\n    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)\n    gz = Buffer.from(await res.arrayBuffer())\n  }\n  const text = gunzipSync(gz).toString('utf8')\n  const tasks: HumanEvalTask[] = []\n  for (const line of text.split('\\n')) {\n    if (line.trim() === '') continue\n    const d = JSON.parse(line) as {\n      task_id?: string\n      prompt?: string\n      test?: string\n      entry_point?: string\n      canonical_solution?: string\n    }\n    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {\n      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)\n    }\n    tasks.push({\n      taskId: d.task_id,\n      prompt: d.prompt,\n      test: d.test,\n      entryPoint: d.entry_point,\n      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),\n    })\n  }\n  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')\n  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)\n  return tasks.slice(offset, offset + limit)\n}\n\nconst solveInstruction =\n  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'\n\nexport function basePrompt(task: HumanEvalTask): string {\n  return `${solveInstruction}\\n\\n\\`\\`\\`python\\n${task.prompt}\\`\\`\\``\n}\n\n/** Extract the function source from a model reply: prefer a fenced ```python (or\n *  bare ```) block, else fall back to the raw text. The deployable program adds the\n *  prompt header (imports + signature context), so a candidate that returns only a\n *  body still runs; a candidate that re-defines the function shadows the header. */\nexport function extractCode(reply: string): string {\n  const fenced = reply.match(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i)\n  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()\n  return reply.trim()\n}\n\n/** The deployable test program: the prompt header (imports + signature/docstring the\n *  model was given), then the candidate (its def shadows the header's stub), then the\n *  task's own check() suite and the call. No gold solution anywhere. */\nfunction buildProgram(task: HumanEvalTask, candidate: string): string {\n  return `${task.prompt}\\n${candidate}\\n\\n${task.test}\\n\\ncheck(${task.entryPoint})\\n`\n}\n\nexport interface CheckResult {\n  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */\n  pass: number\n  /** On failure: the interpreter stderr tail (traceback / failing assertion). The\n   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */\n  detail?: string\n}\n\nexport interface PythonProgramResult {\n  exitCode: number\n  stdout: string\n  stderr: string\n}\n\n/** Run one candidate's deployable test program in an isolated container:\n *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.\n *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image\n *  unavailable) is NOT a test failure — it throws so the harness fails loud rather\n *  than scoring every candidate 0 from a broken checker. */\nlet dockerRunSeq = 0\n\n/** Run arbitrary Python in the same isolated container used by the HumanEval\n * checker. This is also the execution-tool primitive for experiments that let\n * a model test its own snippets; model-written code must never run on the host. */\nexport function runPythonProgram(\n  program: string,\n  timeoutMs = dockerTimeoutMs,\n): Promise<PythonProgramResult> {\n  const dir = mkdtempSync(join(tmpdir(), 'hev-'))\n  // Rootless Docker maps container root to a subordinate host uid that cannot\n  // traverse mkdtemp's default 0700 directory. The directory contains only the\n  // disposable candidate script and is mounted read-only into the container.\n  chmodSync(dir, 0o755)\n  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })\n  // Unique container name so we can force-reap it regardless of the docker client's state.\n  const name = `hev-${process.pid}-${dockerRunSeq++}`\n  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`\n  return new Promise<PythonProgramResult>((resolvePromise, reject) => {\n    let settled = false\n    const cleanup = () => {\n      rmSync(dir, { recursive: true, force: true })\n      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung\n      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;\n      // the name is unique, so no reuse race).\n      execFile('docker', ['rm', '-f', name], () => {})\n    }\n    const finish = (res: PythonProgramResult) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      resolvePromise(res)\n    }\n    const fail = (e: Error) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      reject(e)\n    }\n    // Candidate timeouts are enforced inside the container. If the outer Docker\n    // client or daemon misses this larger deadline, that is infrastructure failure,\n    // not a wrong answer.\n    const outerTimeoutMs = timeoutMs + 3_000\n    const backstop = setTimeout(\n      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),\n      outerTimeoutMs + 3_000,\n    )\n    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))\n    execFile(\n      'docker',\n      [\n        'run',\n        '--rm',\n        '--pull=never',\n        '--name',\n        name,\n        '--network=none',\n        '--cpus=1',\n        '--memory=512m',\n        '--pids-limit=64',\n        '--cap-drop=ALL',\n        '--security-opt=no-new-privileges',\n        '--read-only',\n        '--tmpfs',\n        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',\n        '--user',\n        '65534:65534',\n        '-v',\n        `${dir}:/w:ro`,\n        '-w',\n        '/w',\n        dockerImage,\n        'sh',\n        '-c',\n        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf \"%s\\\\n\" \"$1\"; exec timeout -s KILL \"$2\" python /w/p.py',\n        'agent-runtime-checker',\n        startMarker,\n        `${inContainerSeconds}s`,\n      ],\n      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },\n      (err, stdout, stderr) => {\n        const markerLine = `${startMarker}\\n`\n        const candidateStarted = stdout?.startsWith(markerLine) === true\n        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')\n        if (err) {\n          const e = err as Error & { killed?: boolean; code?: number | string }\n          if (e.code === 'ENOENT') {\n            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))\n            return\n          }\n          if (e.killed) {\n            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))\n            return\n          }\n          if (!candidateStarted) {\n            if (\n              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(\n                stderr,\n              )\n            ) {\n              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {\n              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            fail(\n              new Error(\n                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,\n              ),\n            )\n            return\n          }\n          const exitCode = typeof e.code === 'number' ? e.code : 1\n          finish({\n            exitCode,\n            stdout: candidateStdout,\n            stderr: stderr ?? '',\n          })\n          return\n        }\n        if (!candidateStarted) {\n          fail(new Error('docker checker exited without starting the candidate'))\n          return\n        }\n        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })\n      },\n    )\n  })\n}\n\nexport async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {\n  const result = await runPythonProgram(buildProgram(task, candidate))\n  return result.exitCode === 0\n    ? { pass: 1 }\n    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }\n}\n\n/** A HumanEval task carries its checker inputs in metadata so the deterministic\n *  judge can rebuild the deployable program from a `BenchTask` alone. */\ninterface HumanEvalMeta extends Record<string, unknown> {\n  promptHeader: string\n  test: string\n  entryPoint: string\n  canonicalSolution?: string\n}\n\nfunction toBenchTask(t: HumanEvalTask): BenchTask {\n  const metadata: HumanEvalMeta = {\n    promptHeader: t.prompt,\n    test: t.test,\n    entryPoint: t.entryPoint,\n    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),\n  }\n  return { id: t.taskId, prompt: basePrompt(t), metadata }\n}\n\nfunction taskFromMeta(task: BenchTask): HumanEvalTask {\n  const m = task.metadata as HumanEvalMeta | undefined\n  if (!m?.promptHeader || !m.test || !m.entryPoint) {\n    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)\n  }\n  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }\n}\n\n/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle\n *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */\nexport function createHumanEvalAdapter(): BenchmarkAdapter {\n  return {\n    name: 'humaneval',\n    async preflight() {\n      // Prove the daemon and exact local image before any model call. Scoring uses\n      // --pull=never so a registry or cold pull cannot become a candidate outcome.\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {\n          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))\n          else resolve()\n        })\n      })\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['image', 'inspect', dockerImage], (err) => {\n          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))\n          else resolve()\n        })\n      })\n    },\n    async loadTasks(opts?: LoadOptions) {\n      const offset = Number(process.env.OFFSET ?? 0)\n      // Pull a generous window when filtering by id, else exactly `limit` from offset.\n      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)\n      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all\n      return picked.map(toBenchTask)\n    },\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))\n      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }\n    },\n    async goldArtifact(task: BenchTask) {\n      const m = task.metadata as HumanEvalMeta | undefined\n      const sol = m?.canonicalSolution\n      // Return the COMPLETE function (signature header + canonical body), i.e. what a\n      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose\n      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its\n      // leading indent and breaks it, so a body-only gold fails its own judge. A full\n      // def starts at column 0, trims safely, and self-verifies.\n      return sol ? `${m!.promptHeader}${sol}` : undefined\n    },\n  }\n}\n"],"mappings":";AAkBA,SAAS,gBAAgB;AACzB,SAAS,WAAW,aAAa,cAAc,QAAQ,qBAAqB;AAC5E,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAG3B,IAAM,eAAe;AACrB,IAAM,cAAc;AACpB,IAAM,kBAAkB,OAAO,QAAQ,IAAI,qBAAqB,GAAK;AAerE,eAAsB,cAAc,OAAe,SAAS,GAA6B;AAGvF,QAAM,UAAU,QAAQ,IAAI;AAC5B,MAAI;AACJ,MAAI,SAAS;AACX,SAAK,aAAa,OAAO;AAAA,EAC3B,OAAO;AACL,UAAM,MAAM,MAAM,MAAM,YAAY;AACpC,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,KAAK,YAAY,EAAE;AAClF,SAAK,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC;AAAA,EAC1C;AACA,QAAM,OAAO,WAAW,EAAE,EAAE,SAAS,MAAM;AAC3C,QAAM,QAAyB,CAAC;AAChC,aAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,QAAI,KAAK,KAAK,MAAM,GAAI;AACxB,UAAM,IAAI,KAAK,MAAM,IAAI;AAOzB,QAAI,CAAC,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,aAAa;AACxD,YAAM,IAAI,MAAM,+BAA+B,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrE;AACA,UAAM,KAAK;AAAA,MACT,QAAQ,EAAE;AAAA,MACV,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,YAAY,EAAE;AAAA,MACd,GAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,IAAI,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B;AACrE,MAAI,UAAU,MAAM,OAAQ,OAAM,IAAI,MAAM,UAAU,MAAM,oBAAoB,MAAM,MAAM,EAAE;AAC9F,SAAO,MAAM,MAAM,QAAQ,SAAS,KAAK;AAC3C;AAEA,IAAM,mBACJ;AAEK,SAAS,WAAW,MAA6B;AACtD,SAAO,GAAG,gBAAgB;AAAA;AAAA;AAAA,EAAqB,KAAK,MAAM;AAC5D;AAMO,SAAS,YAAY,OAAuB;AACjD,QAAM,SAAS,MAAM,MAAM,sCAAsC;AACjE,MAAI,UAAU,OAAO,OAAO,CAAC,MAAM,SAAU,QAAO,OAAO,CAAC,EAAE,KAAK;AACnE,SAAO,MAAM,KAAK;AACpB;AAKA,SAAS,aAAa,MAAqB,WAA2B;AACpE,SAAO,GAAG,KAAK,MAAM;AAAA,EAAK,SAAS;AAAA;AAAA,EAAO,KAAK,IAAI;AAAA;AAAA,QAAa,KAAK,UAAU;AAAA;AACjF;AAqBA,IAAI,eAAe;AAKZ,SAAS,iBACd,SACA,YAAY,iBACkB;AAC9B,QAAM,MAAM,YAAY,KAAK,OAAO,GAAG,MAAM,CAAC;AAI9C,YAAU,KAAK,GAAK;AACpB,gBAAc,KAAK,KAAK,MAAM,GAAG,SAAS,EAAE,MAAM,IAAM,CAAC;AAEzD,QAAM,OAAO,OAAO,QAAQ,GAAG,IAAI,cAAc;AACjD,QAAM,cAAc,qCAAqC,IAAI;AAC7D,SAAO,IAAI,QAA6B,CAAC,gBAAgB,WAAW;AAClE,QAAI,UAAU;AACd,UAAM,UAAU,MAAM;AACpB,aAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAI5C,eAAS,UAAU,CAAC,MAAM,MAAM,IAAI,GAAG,MAAM;AAAA,MAAC,CAAC;AAAA,IACjD;AACA,UAAM,SAAS,CAAC,QAA6B;AAC3C,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,qBAAe,GAAG;AAAA,IACpB;AACA,UAAM,OAAO,CAAC,MAAa;AACzB,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,aAAO,CAAC;AAAA,IACV;AAIA,UAAM,iBAAiB,YAAY;AACnC,UAAM,WAAW;AAAA,MACf,MAAM,KAAK,IAAI,MAAM,wCAAwC,iBAAiB,GAAK,IAAI,CAAC;AAAA,MACxF,iBAAiB;AAAA,IACnB;AACA,UAAM,qBAAqB,KAAK,IAAI,GAAG,KAAK,KAAK,YAAY,GAAK,CAAC;AACnE;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,GAAG;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,kBAAkB;AAAA,MACvB;AAAA,MACA,EAAE,SAAS,gBAAgB,YAAY,WAAW,WAAW,IAAI,OAAO,KAAK;AAAA,MAC7E,CAAC,KAAK,QAAQ,WAAW;AACvB,cAAM,aAAa,GAAG,WAAW;AAAA;AACjC,cAAM,mBAAmB,QAAQ,WAAW,UAAU,MAAM;AAC5D,cAAM,kBAAkB,mBAAmB,OAAO,MAAM,WAAW,MAAM,IAAK,UAAU;AACxF,YAAI,KAAK;AACP,gBAAM,IAAI;AACV,cAAI,EAAE,SAAS,UAAU;AACvB,iBAAK,IAAI,MAAM,0EAAqE,CAAC;AACrF;AAAA,UACF;AACA,cAAI,EAAE,QAAQ;AACZ,iBAAK,IAAI,MAAM,+BAA+B,cAAc,kBAAkB,CAAC;AAC/E;AAAA,UACF;AACA,cAAI,CAAC,kBAAkB;AACrB,gBACE,8GAA8G;AAAA,cAC5G;AAAA,YACF,GACA;AACA,mBAAK,IAAI,MAAM,8BAA8B,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AACpE;AAAA,YACF;AACA,gBAAI,0EAA0E,KAAK,MAAM,GAAG;AAC1F,mBAAK,IAAI,MAAM,gBAAgB,WAAW,iBAAiB,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AAClF;AAAA,YACF;AACA;AAAA,cACE,IAAI;AAAA,gBACF,+CAA+C,OAAO,MAAM,GAAG,GAAG,KAAK,EAAE,OAAO;AAAA,cAClF;AAAA,YACF;AACA;AAAA,UACF;AACA,gBAAM,WAAW,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;AACvD,iBAAO;AAAA,YACL;AAAA,YACA,QAAQ;AAAA,YACR,QAAQ,UAAU;AAAA,UACpB,CAAC;AACD;AAAA,QACF;AACA,YAAI,CAAC,kBAAkB;AACrB,eAAK,IAAI,MAAM,sDAAsD,CAAC;AACtE;AAAA,QACF;AACA,eAAO,EAAE,UAAU,GAAG,QAAQ,iBAAiB,QAAQ,UAAU,GAAG,CAAC;AAAA,MACvE;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,eAAsB,WAAW,MAAqB,WAAyC;AAC7F,QAAM,SAAS,MAAM,iBAAiB,aAAa,MAAM,SAAS,CAAC;AACnE,SAAO,OAAO,aAAa,IACvB,EAAE,MAAM,EAAE,IACV,EAAE,MAAM,GAAG,QAAQ,OAAO,OAAO,MAAM,IAAI,KAAK,wBAAwB;AAC9E;AAWA,SAAS,YAAY,GAA6B;AAChD,QAAM,WAA0B;AAAA,IAC9B,cAAc,EAAE;AAAA,IAChB,MAAM,EAAE;AAAA,IACR,YAAY,EAAE;AAAA,IACd,GAAI,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC;AAAA,EAC1E;AACA,SAAO,EAAE,IAAI,EAAE,QAAQ,QAAQ,WAAW,CAAC,GAAG,SAAS;AACzD;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,IAAI,KAAK;AACf,MAAI,CAAC,GAAG,gBAAgB,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY;AAChD,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,2BAA2B;AAAA,EAC7E;AACA,SAAO,EAAE,QAAQ,KAAK,IAAI,QAAQ,EAAE,cAAc,MAAM,EAAE,MAAM,YAAY,EAAE,WAAW;AAC3F;AAIO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,YAAY;AAGhB,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,WAAW,YAAY,qBAAqB,GAAG,CAAC,QAAQ;AAC1E,cAAI,IAAK,QAAO,IAAI,MAAM,kFAAkF,CAAC;AAAA,cACxG,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AACD,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,SAAS,WAAW,WAAW,GAAG,CAAC,QAAQ;AAC7D,cAAI,IAAK,QAAO,IAAI,MAAM,iDAAiD,WAAW,EAAE,CAAC;AAAA,cACpF,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,IACA,MAAM,UAAU,MAAoB;AAClC,YAAM,SAAS,OAAO,QAAQ,IAAI,UAAU,CAAC;AAE7C,YAAM,MAAM,MAAM,cAAc,MAAM,MAAM,MAAO,MAAM,SAAS,GAAI,MAAM;AAC5E,YAAM,SAAS,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,SAAS,EAAE,MAAM,CAAC,IAAI;AAC7E,aAAO,OAAO,IAAI,WAAW;AAAA,IAC/B;AAAA,IACA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,KAAK,IAAI,MAAM,WAAW,aAAa,IAAI,GAAG,YAAY,QAAQ,CAAC;AAC3E,aAAO,EAAE,UAAU,SAAS,GAAG,OAAO,MAAM,QAAQ,SAAS,IAAI,iBAAiB,eAAe;AAAA,IACnG;AAAA,IACA,MAAM,aAAa,MAAiB;AAClC,YAAM,IAAI,KAAK;AACf,YAAM,MAAM,GAAG;AAMf,aAAO,MAAM,GAAG,EAAG,YAAY,GAAG,GAAG,KAAK;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-7WSD27QQ.js

@@ -0,0 +1,118 @@
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/agentbench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "agentbench-dbbench.json");
+var DEFAULT_SPLIT = "dev";
+var agentbenchDir = () => process.env.AGENTBENCH_DIR;
+var agentbenchAnswerOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+function normalizeAnswer(value) {
+  return value.toLowerCase().replace(/\s+/g, " ").replace(/^["'`]+|["'`]+$/g, "").trim();
+}
+function rowToTask(row, index, split) {
+  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ""}`).join(", ");
+  const sampleRows = row.table?.table_info?.rows?.slice(0, 40);
+  const meta = {
+    labels: row.label,
+    split,
+    subset: "dbbench",
+    table: row.table
+  };
+  return {
+    id: `dbbench-${split}-${index}`,
+    split,
+    prompt: [
+      "Answer this AgentBench DBBench question using the table below.",
+      "Return only the answer value.",
+      "",
+      `Question: ${row.description}`,
+      row.table?.table_name ? `Table: ${row.table.table_name}` : void 0,
+      columns ? `Columns: ${columns}` : void 0,
+      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.labels)) {
+    throw new Error(`agentbench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts, split) {
+  let tasks = rows.map((row, index) => rowToTask(row, index, split));
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+async function loadJsonl(path) {
+  const raw = await readFile(path, "utf8");
+  return raw.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
+}
+async function loadFixtures(opts, split) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 \u2014 loading ${rows.length} DBBench adapter fixtures`);
+  return selectRows(rows, opts, split);
+}
+function createAgentBenchAdapter() {
+  const fixturesMode = process.env.AGENTBENCH_FIXTURES === "1";
+  return {
+    name: "agentbench",
+    output: agentbenchAnswerOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = agentbenchDir();
+      if (!dir) {
+        throw new Error("AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.");
+      }
+      await loadJsonl(join(dir, "data", "dbbench", `${DEFAULT_SPLIT}.jsonl`));
+    },
+    async loadTasks(opts = {}) {
+      const split = opts.split ?? DEFAULT_SPLIT;
+      if (fixturesMode) return loadFixtures(opts, split);
+      const dir = agentbenchDir();
+      if (!dir) throw new Error("AGENTBENCH_DIR is required to load AgentBench DBBench tasks");
+      return selectRows(await loadJsonl(join(dir, "data", "dbbench", `${split}.jsonl`)), opts, split);
+    },
+    async goldArtifact(task) {
+      return readMeta(task).labels[0];
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const got = normalizeAnswer(artifact);
+      const expected = meta.labels.map(normalizeAnswer);
+      const resolved = expected.includes(got);
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact })
+      };
+    }
+  };
+}
+
+export {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+};
+//# sourceMappingURL=chunk-7WSD27QQ.js.map

package/dist/chunk-7WSD27QQ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/agentbench.ts"],"sourcesContent":["/**\n * AgentBench deterministic subset adapter.\n *\n * This targets AgentBench DBBench rows only: question + table + published label.\n * It does not wrap AgentBench's controller protocol or the non-deterministic game\n * environments. Worker artifact = final answer text. Judge = exact match against\n * the official DBBench label list after light whitespace/case normalization.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'agentbench-dbbench.json')\nconst DEFAULT_SPLIT = 'dev'\n\ninterface AgentBenchDbRow {\n  description: string\n  label: string[]\n  table?: {\n    table_name?: string\n    table_info?: {\n      columns?: Array<{ name: string; type?: string }>\n      rows?: unknown[][]\n    }\n  }\n}\n\ninterface AgentBenchMeta {\n  labels: string[]\n  split: string\n  subset: 'dbbench'\n  table?: AgentBenchDbRow['table']\n}\n\nconst agentbenchDir = (): string | undefined => process.env.AGENTBENCH_DIR\n\nexport const agentbenchAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction normalizeAnswer(value: string): string {\n  return value\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/^[\"'`]+|[\"'`]+$/g, '')\n    .trim()\n}\n\nfunction rowToTask(row: AgentBenchDbRow, index: number, split: string): BenchTask {\n  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ''}`).join(', ')\n  const sampleRows = row.table?.table_info?.rows?.slice(0, 40)\n  const meta: AgentBenchMeta = {\n    labels: row.label,\n    split,\n    subset: 'dbbench',\n    table: row.table,\n  }\n  return {\n    id: `dbbench-${split}-${index}`,\n    split,\n    prompt: [\n      'Answer this AgentBench DBBench question using the table below.',\n      'Return only the answer value.',\n      '',\n      `Question: ${row.description}`,\n      row.table?.table_name ? `Table: ${row.table.table_name}` : undefined,\n      columns ? `Columns: ${columns}` : undefined,\n      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AgentBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.labels)) {\n    throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AgentBenchMeta\n}\n\nfunction selectRows(rows: AgentBenchDbRow[], opts: LoadOptions, split: string): BenchTask[] {\n  let tasks = rows.map((row, index) => rowToTask(row, index, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadJsonl(path: string): Promise<AgentBenchDbRow[]> {\n  const raw = await readFile(path, 'utf8')\n  return raw\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter(Boolean)\n    .map((line) => JSON.parse(line) as AgentBenchDbRow)\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as AgentBenchDbRow[]\n  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`)\n  return selectRows(rows, opts, split)\n}\n\nexport function createAgentBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AGENTBENCH_FIXTURES === '1'\n\n  return {\n    name: 'agentbench',\n    output: agentbenchAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = agentbenchDir()\n      if (!dir) {\n        throw new Error('AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.')\n      }\n      await loadJsonl(join(dir, 'data', 'dbbench', `${DEFAULT_SPLIT}.jsonl`))\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = agentbenchDir()\n      if (!dir) throw new Error('AGENTBENCH_DIR is required to load AgentBench DBBench tasks')\n      return selectRows(await loadJsonl(join(dir, 'data', 'dbbench', `${split}.jsonl`)), opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).labels[0]\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = normalizeAnswer(artifact)\n      const expected = meta.labels.map(normalizeAnswer)\n      const resolved = expected.includes(got)\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,yBAAyB;AACtE,IAAM,gBAAgB;AAqBtB,IAAM,gBAAgB,MAA0B,QAAQ,IAAI;AAErD,IAAM,yBAAgD;AAAA,EAC3D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,wCAAwC,CAAC;AAC1E,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,gBAAgB,OAAuB;AAC9C,SAAO,MACJ,YAAY,EACZ,QAAQ,QAAQ,GAAG,EACnB,QAAQ,oBAAoB,EAAE,EAC9B,KAAK;AACV;AAEA,SAAS,UAAU,KAAsB,OAAe,OAA0B;AAChF,QAAM,UAAU,IAAI,OAAO,YAAY,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,GAAG,EAAE,OAAO,KAAK,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,KAAK,IAAI;AAChH,QAAM,aAAa,IAAI,OAAO,YAAY,MAAM,MAAM,GAAG,EAAE;AAC3D,QAAM,OAAuB;AAAA,IAC3B,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,QAAQ;AAAA,IACR,OAAO,IAAI;AAAA,EACb;AACA,SAAO;AAAA,IACL,IAAI,WAAW,KAAK,IAAI,KAAK;AAAA,IAC7B;AAAA,IACA,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,IAAI,WAAW;AAAA,MAC5B,IAAI,OAAO,aAAa,UAAU,IAAI,MAAM,UAAU,KAAK;AAAA,MAC3D,UAAU,YAAY,OAAO,KAAK;AAAA,MAClC,aAAa,cAAc,KAAK,UAAU,UAAU,CAAC,KAAK;AAAA,IAC5D,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiC;AACjD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,GAAG;AACpC,UAAM,IAAI,MAAM,mBAAmB,KAAK,EAAE,wDAAmD;AAAA,EAC/F;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAyB,MAAmB,OAA4B;AAC1F,MAAI,QAAQ,KAAK,IAAI,CAAC,KAAK,UAAU,UAAU,KAAK,OAAO,KAAK,CAAC;AACjE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,wCAAwC,KAAK,UAAU,IAAI,CAAC,EAAE;AACtG,SAAO;AACT;AAEA,eAAe,UAAU,MAA0C;AACjE,QAAM,MAAM,MAAM,SAAS,MAAM,MAAM;AACvC,SAAO,IACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAoB;AACtD;AAEA,eAAe,aAAa,MAAmB,OAAqC;AAClF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,qDAAgD,KAAK,MAAM,2BAA2B;AACnG,SAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEO,SAAS,0BAA4C;AAC1D,QAAM,eAAe,QAAQ,IAAI,wBAAwB;AAEzD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,cAAc;AAC1B,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,wHAAwH;AAAA,MAC1I;AACA,YAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,aAAa,QAAQ,CAAC;AAAA,IACxE;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,UAAI,aAAc,QAAO,aAAa,MAAM,KAAK;AACjD,YAAM,MAAM,cAAc;AAC1B,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6DAA6D;AACvF,aAAO,WAAW,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,KAAK,QAAQ,CAAC,GAAG,MAAM,KAAK;AAAA,IAChG;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,SAAS,IAAI,EAAE,OAAO,CAAC;AAAA,IAChC;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAM,gBAAgB,QAAQ;AACpC,YAAM,WAAW,KAAK,OAAO,IAAI,eAAe;AAChD,YAAM,WAAW,SAAS,SAAS,GAAG;AACtC,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,OAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,KAAK,SAAS,CAAC;AAAA,MACzG;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-C7T7WEK2.js

@@ -0,0 +1,103 @@
+// src/benchmarks/cadgenbench.ts
+import { execFile as execFile2 } from "child_process";
+import { mkdtemp as mkdtemp2, rm as rm2, writeFile as writeFile2 } from "fs/promises";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+import { promisify as promisify2 } from "util";
+
+// src/worker-build123d.ts
+import { execFile } from "child_process";
+import { existsSync, readFileSync } from "fs";
+import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+import { routerChatWithUsage } from "@tangle-network/agent-runtime/loops";
+var execFileAsync = promisify(execFile);
+var CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? "/tmp/cgb-venv/bin/python";
+var CGB_DIR = process.env.CADGENBENCH_DIR ?? "/tmp/cadgenbench";
+
+// src/benchmarks/cadgenbench.ts
+var execFileAsync2 = promisify2(execFile2);
+var SCORE_PY = `
+import sys, json, tempfile, shutil
+from pathlib import Path
+from cadgenbench.eval.evaluate import evaluate_result
+cand, gt = Path(sys.argv[1]), Path(sys.argv[2])
+with tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:
+    rd, gd = Path(rd), Path(gd)
+    (rd / 'result.json').write_text('{}')
+    shutil.copy(gt, gd / 'ground_truth.step')
+    try:
+        evaluate_result(rd, gd, candidate_step=cand)
+        d = json.loads((rd / 'result.json').read_text())
+        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))
+    except Exception as e:
+        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))
+`.trim();
+function fixtureTasks() {
+  const g = join2(CGB_DIR, "tests/fixtures/geometry");
+  return [
+    { id: "box-10x20x30", prompt: "A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).", gtStep: join2(g, "box_10_20_30.step") },
+    { id: "cube-10", prompt: "A cube, 10 units on every side.", gtStep: join2(g, "box_10_10_10.step") },
+    { id: "sphere-10", prompt: "A sphere of radius 10 units, centered at the origin.", gtStep: join2(g, "sphere_10.step") }
+  ];
+}
+function createCadGenBenchAdapter() {
+  return {
+    name: "cadgenbench",
+    async preflight() {
+      const r = await execFileAsync2(CGB_VENV_PY, ["-c", 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print("ok")'], { timeout: 6e4 }).catch(
+        (e) => ({ stdout: "", stderr: e instanceof Error ? e.message : String(e) })
+      );
+      if (!/ok/.test(r.stdout)) {
+        throw new Error(
+          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}
+Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      let tasks = fixtureTasks();
+      const hard = process.env.CGB_HARD_DIR;
+      if (hard) {
+        const { readFile: readFile2 } = await import("fs/promises");
+        tasks = JSON.parse(await readFile2(join2(hard, "tasks.json"), "utf8"));
+      }
+      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
+      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
+      const meta = (gtStep) => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) });
+      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) }));
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const { gtStep, resolveThreshold } = task.metadata;
+      if (!artifact.includes("ISO-10303-21")) return { resolved: false, score: 0, detail: "artifact is not a STEP file" };
+      const dir = await mkdtemp2(join2(tmpdir2(), "cgb-judge-"));
+      const cand = join2(dir, "candidate.step");
+      const scorer = join2(dir, "score.py");
+      try {
+        await writeFile2(cand, artifact);
+        await writeFile2(scorer, SCORE_PY);
+        const r = await execFileAsync2("xvfb-run", ["-a", CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 18e4 }).catch(
+          (e) => ({ stdout: e.stdout ?? "", stderr: e instanceof Error ? e.message : String(e) })
+        );
+        const m = /CGB_SCORE (\{.*\})/.exec(r.stdout);
+        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` };
+        const v = JSON.parse(m[1]);
+        const score = typeof v.cad_score === "number" ? v.cad_score : 0;
+        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ""}` };
+      } finally {
+        await rm2(dir, { recursive: true, force: true }).catch(() => {
+        });
+      }
+    }
+  };
+}
+
+export {
+  createCadGenBenchAdapter
+};
+//# sourceMappingURL=chunk-C7T7WEK2.js.map

package/dist/chunk-C7T7WEK2.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/cadgenbench.ts","../src/worker-build123d.ts"],"sourcesContent":["/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n","/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const { content, usage: u } = await routerChatWithUsage(cfg, [\n        { role: 'system', content: sys },\n        { role: 'user', content: user },\n      ])\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n"],"mappings":";AAgBA,SAAS,YAAAA,iBAAgB;AACzB,SAAS,WAAAC,UAAS,MAAAC,KAAI,aAAAC,kBAAiB;AACvC,SAAS,UAAAC,eAAc;AACvB,SAAS,QAAAC,aAAY;AACrB,SAAS,aAAAC,kBAAiB;;;ACP1B,SAAS,gBAAgB;AACzB,SAAS,YAAY,oBAAoB;AACzC,SAAS,SAAS,UAAU,IAAI,iBAAiB;AACjD,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAK1B,SAAS,2BAA2B;AAIpC,IAAM,gBAAgB,UAAU,QAAQ;AAEjC,IAAM,cAAc,QAAQ,IAAI,oBAAoB;AACpD,IAAM,UAAU,QAAQ,IAAI,mBAAmB;;;ADNtD,IAAMC,iBAAgBC,WAAUC,SAAQ;AAKxC,IAAM,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAef,KAAK;AASP,SAAS,eAAsE;AAC7E,QAAM,IAAIC,MAAK,SAAS,yBAAyB;AACjD,SAAO;AAAA,IACL,EAAE,IAAI,gBAAgB,QAAQ,yFAAyF,QAAQA,MAAK,GAAG,mBAAmB,EAAE;AAAA,IAC5J,EAAE,IAAI,WAAW,QAAQ,mCAAmC,QAAQA,MAAK,GAAG,mBAAmB,EAAE;AAAA,IACjG,EAAE,IAAI,aAAa,QAAQ,wDAAwD,QAAQA,MAAK,GAAG,gBAAgB,EAAE;AAAA,EACvH;AACF;AAEO,SAAS,2BAA6C;AAC3D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,IAAI,MAAMH,eAAc,aAAa,CAAC,MAAM,+EAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,EAAE;AAAA,QACvJ,CAAC,OAAO,EAAE,QAAQ,IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,EAAE;AAAA,MAC3E;AACA,UAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GAAG;AACxB,cAAM,IAAI;AAAA,UACR,sCAAsC,WAAW,MAAM,EAAE,OAAO,MAAM,GAAG,GAAG,CAAC;AAAA,4DACd,OAAO,6EAA6E,OAAO;AAAA,QAC5J;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AAGtC,UAAI,QAAQ,aAAa;AACzB,YAAM,OAAO,QAAQ,IAAI;AACzB,UAAI,MAAM;AACR,cAAM,EAAE,UAAAI,UAAS,IAAI,MAAM,OAAO,aAAkB;AACpD,gBAAQ,KAAK,MAAM,MAAMA,UAASD,MAAK,MAAM,YAAY,GAAG,MAAM,CAAC;AAAA,MACrE;AACA,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,YAAM,OAAO,CAAC,YAA6B,EAAE,QAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,GAAG,EAAE;AACxH,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,KAAK,EAAE,MAAM,EAAwC,EAAE;AAAA,IACrI;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,QAAQ,iBAAiB,IAAI,KAAK;AAC1C,UAAI,CAAC,SAAS,SAAS,cAAc,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,8BAA8B;AAClH,YAAM,MAAM,MAAME,SAAQF,MAAKG,QAAO,GAAG,YAAY,CAAC;AACtD,YAAM,OAAOH,MAAK,KAAK,gBAAgB;AACvC,YAAM,SAASA,MAAK,KAAK,UAAU;AACnC,UAAI;AACF,cAAMI,WAAU,MAAM,QAAQ;AAC9B,cAAMA,WAAU,QAAQ,QAAQ;AAEhC,cAAM,IAAI,MAAMP,eAAc,YAAY,CAAC,MAAM,aAAa,QAAQ,MAAM,MAAM,GAAG,EAAE,WAAW,KAAK,IAAI,SAAS,KAAQ,CAAC,EAAE;AAAA,UAC7H,CAAC,OAAO,EAAE,QAAS,EAA0B,UAAU,IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,EAAE;AAAA,QAChH;AACA,cAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;AAC5C,YAAI,CAAC,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,QAAQ,MAAM,GAAG,GAAG,CAAC,GAAG;AAC1H,cAAM,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;AACzB,cAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;AAC9D,eAAO,EAAE,UAAU,SAAS,kBAAkB,OAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,CAAC,WAAW,EAAE,MAAM,GAAG,EAAE,QAAQ,KAAK,EAAE,KAAK,MAAM,EAAE,GAAG;AAAA,MACnJ,UAAE;AACA,cAAMQ,IAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AACF;","names":["execFile","mkdtemp","rm","writeFile","tmpdir","join","promisify","execFileAsync","promisify","execFile","join","readFile","mkdtemp","tmpdir","writeFile","rm"]}

package/dist/chunk-CKUVRZ2T.js

@@ -0,0 +1,251 @@
+import {
+  preflightVenvImports,
+  readJsonReport,
+  runStagedJudge,
+  runVenvPython,
+  safeRunId,
+  stageFile
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/swe-bench.ts
+import { join } from "path";
+var SWE_REPO_DIR = "/work";
+var swePatchOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)];
+    const last = fences.at(-1)?.[1];
+    return (last ?? text).trim();
+  }
+};
+var DATASET = "princeton-nlp/SWE-bench_Verified";
+var SWE_CACHE_LEVELS = /* @__PURE__ */ new Set(["none", "base", "env", "instance"]);
+function scorerNamespace() {
+  const namespace = process.env.SWEBENCH_NAMESPACE ?? "swebench";
+  if (namespace !== "swebench" && namespace !== "none") {
+    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`);
+  }
+  return namespace;
+}
+var TEST_FILE_EXCLUDES = [
+  "':(exclude,glob)**/tests/**'",
+  "':(exclude,glob)**/test/**'",
+  "':(exclude,glob)test_*.py'",
+  "':(exclude,glob)**/test_*.py'",
+  "':(exclude,glob)*_test.py'",
+  "':(exclude,glob)**/*_test.py'",
+  "':(exclude,glob)conftest.py'",
+  "':(exclude,glob)**/conftest.py'"
+].join(" ");
+function stringIds(report, key) {
+  const value = report[key];
+  if (value === void 0) return [];
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
+    throw new Error(`swe-bench: malformed ${key}`);
+  }
+  return value;
+}
+function scoreSweReport(taskId, value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("swe-bench: report must be an object");
+  }
+  const report = value;
+  const statusIds = {
+    resolved: stringIds(report, "resolved_ids"),
+    unresolved: stringIds(report, "unresolved_ids"),
+    emptyPatch: stringIds(report, "empty_patch_ids"),
+    completed: stringIds(report, "completed_ids"),
+    incomplete: stringIds(report, "incomplete_ids"),
+    error: stringIds(report, "error_ids")
+  };
+  const submitted = stringIds(report, "submitted_ids");
+  const mentioned = Object.values(statusIds).flat();
+  if (mentioned.some((id) => id !== taskId) || submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId)) {
+    throw new Error(`swe-bench: report identity mismatch for ${taskId}`);
+  }
+  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {
+    throw new Error(`swe-bench: evaluator failed for ${taskId}`);
+  }
+  const outcomes = [
+    statusIds.resolved.includes(taskId),
+    statusIds.unresolved.includes(taskId),
+    statusIds.emptyPatch.includes(taskId)
+  ];
+  if (outcomes.filter(Boolean).length !== 1) {
+    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`);
+  }
+  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {
+    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`);
+  }
+  const resolved = outcomes[0];
+  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) };
+}
+function sweEvaluationArgv(args) {
+  return [
+    "-m",
+    "swebench.harness.run_evaluation",
+    "--dataset_name",
+    DATASET,
+    "--predictions_path",
+    args.predictionsPath,
+    "--run_id",
+    args.runId,
+    "--instance_ids",
+    args.instanceId,
+    "--max_workers",
+    "1",
+    "--namespace",
+    args.namespace ?? scorerNamespace(),
+    "--cache_level",
+    args.cacheLevel
+  ];
+}
+function shellQuote(value) {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+function sweMetadata(task) {
+  const repo = String(task.metadata?.repo ?? "");
+  const base = String(task.metadata?.base_commit ?? "");
+  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
+    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`);
+  }
+  if (!/^[0-9a-f]{7,40}$/i.test(base)) {
+    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`);
+  }
+  return { repo, base };
+}
+function createSweBenchAdapter(options = {}) {
+  if (options.timeoutMs !== void 0 && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)) throw new Error("swe-bench: timeoutMs must be a positive integer");
+  const cacheLevel = options.cacheLevel ?? "env";
+  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error("swe-bench: invalid cacheLevel");
+  if (options.captureEvaluatorArtifacts !== void 0 && typeof options.captureEvaluatorArtifacts !== "function") throw new Error("swe-bench: captureEvaluatorArtifacts must be a function");
+  let attemptSequence = 0;
+  return {
+    name: "swe-bench-verified",
+    output: swePatchOutput,
+    // Extract the patch from repo STATE, not printed text: stage every edit the
+    // agent made in the cloned repo and diff it against the checked-out
+    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold
+    // `test_patch` itself, so an agent edit to a test would collide on apply. The
+    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold
+    // patch format the swebench judge's `git apply` expects.
+    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the
+    // agent only edits (the harness owns the checkout — a stochastic model can't be
+    // trusted to clone to an exact path). `--quiet` keeps the exec output small.
+    boxSetup(task) {
+      const { repo, base } = sweMetadata(task);
+      return {
+        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`
+      };
+    },
+    boxExtract() {
+      return {
+        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`
+      };
+    },
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["swebench"],
+        requireDocker: true,
+        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; (2) ensure the Docker daemon is running (the judge builds per-instance images).`
+      });
+    },
+    async loadTasks(opts = {}) {
+      const limit = opts.limit ?? 10;
+      const split = opts.split ?? "test";
+      const script = `
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    if ids is not None and r["instance_id"] not in ids:
+        continue
+    out.append({
+        "instance_id": r["instance_id"], "repo": r["repo"], "base_commit": r["base_commit"],
+        "problem_statement": r["problem_statement"], "patch": r["patch"], "test_patch": r["test_patch"],
+        "FAIL_TO_PASS": r["FAIL_TO_PASS"], "PASS_TO_PASS": r["PASS_TO_PASS"],
+        "version": r.get("version"), "environment_setup_commit": r.get("environment_setup_commit"),
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`;
+      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
+      const rows = JSON.parse(stdout);
+      return rows.map(
+        (r) => ({
+          id: String(r.instance_id),
+          split,
+          prompt: [
+            `Repository: ${r.repo} @ ${r.base_commit}`,
+            "",
+            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
+            "",
+            "Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files \u2014 the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.",
+            "Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff \u2014 the harness reads your committed edits directly from the repo.",
+            "",
+            "--- Issue ---",
+            String(r.problem_statement ?? "")
+          ].join("\n"),
+          metadata: r
+        })
+      );
+    },
+    async goldArtifact(task) {
+      const gold = task.metadata?.patch;
+      return typeof gold === "string" ? gold : void 0;
+    },
+    async judge(task, artifact) {
+      const runId = safeRunId("bench", task.id);
+      const capture = options.captureEvaluatorArtifacts?.({
+        taskId: task.id,
+        runId,
+        attemptSequence: ++attemptSequence
+      });
+      return runStagedJudge({
+        tmpPrefix: "swebench-",
+        ...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
+        ...capture === void 0 ? {} : { capture },
+        // Debug: retain the staged dir (holds swebench's per-instance apply/run
+        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.
+        ...process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {},
+        async stage(dir) {
+          await stageFile(
+            join(dir, "preds.json"),
+            JSON.stringify([
+              { instance_id: task.id, model_name_or_path: "agent-runtime-bench", model_patch: artifact }
+            ])
+          );
+        },
+        // The official evaluation harness. Pulls/builds the instance image, applies
+        // the patch, runs the test spec, writes a per-run report JSON in cwd.
+        argv: (dir) => sweEvaluationArgv({
+          predictionsPath: join(dir, "preds.json"),
+          runId,
+          instanceId: task.id,
+          cacheLevel,
+          namespace: scorerNamespace()
+        }),
+        async parseReport(dir) {
+          const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`));
+          return scoreSweReport(task.id, report);
+        }
+      });
+    }
+  };
+}
+
+export {
+  swePatchOutput,
+  scoreSweReport,
+  sweEvaluationArgv,
+  createSweBenchAdapter
+};
+//# sourceMappingURL=chunk-CKUVRZ2T.js.map

package/dist/chunk-CKUVRZ2T.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;AAcA,SAAS,YAAY;AAmBrB,IAAM,eAAe;AAWd,IAAM,iBAAwC;AAAA,EACnD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AAGA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,uCAAuC,CAAC;AACzE,UAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC;AAC9B,YAAQ,QAAQ,MAAM,KAAK;AAAA,EAC7B;AACF;AAEA,IAAM,UAAU;AAsBhB,IAAM,mBAAmB,oBAAI,IAAwB,CAAC,QAAQ,QAAQ,OAAO,UAAU,CAAC;AAExF,SAAS,kBAAuC;AAC9C,QAAM,YAAY,QAAQ,IAAI,sBAAsB;AACpD,MAAI,cAAc,cAAc,cAAc,QAAQ;AACpD,UAAM,IAAI,MAAM,kDAAkD,SAAS,GAAG;AAAA,EAChF;AACA,SAAO;AACT;AACA,IAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;AAClF,QAAM,QAAQ,OAAO,GAAG;AACxB,MAAI,UAAU,OAAW,QAAO,CAAC;AACjC,MAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAC,UAAU,OAAO,UAAU,QAAQ,GAAG;AAC7E,UAAM,IAAI,MAAM,wBAAwB,GAAG,EAAE;AAAA,EAC/C;AACA,SAAO;AACT;AAGO,SAAS,eAAe,QAAgB,OAA4B;AACzE,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,UAAM,IAAI,MAAM,qCAAqC;AAAA,EACvD;AACA,QAAM,SAAS;AACf,QAAM,YAAY;AAAA,IAChB,UAAU,UAAU,QAAQ,cAAc;AAAA,IAC1C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,YAAY,UAAU,QAAQ,iBAAiB;AAAA,IAC/C,WAAW,UAAU,QAAQ,eAAe;AAAA,IAC5C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,OAAO,UAAU,QAAQ,WAAW;AAAA,EACtC;AACA,QAAM,YAAY,UAAU,QAAQ,eAAe;AACnD,QAAM,YAAY,OAAO,OAAO,SAAS,EAAE,KAAK;AAChD,MACE,UAAU,KAAK,CAAC,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,CAAC,MAAM,SACxE;AACA,UAAM,IAAI,MAAM,2CAA2C,MAAM,EAAE;AAAA,EACrE;AACA,MAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAAG;AAC7E,UAAM,IAAI,MAAM,mCAAmC,MAAM,EAAE;AAAA,EAC7D;AACA,QAAM,WAAW;AAAA,IACf,UAAU,SAAS,SAAS,MAAM;AAAA,IAClC,UAAU,WAAW,SAAS,MAAM;AAAA,IACpC,UAAU,WAAW,SAAS,MAAM;AAAA,EACtC;AACA,MAAI,SAAS,OAAO,OAAO,EAAE,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,+CAA+C,MAAM,EAAE;AAAA,EACzE;AACA,OAAK,SAAS,CAAC,KAAK,SAAS,CAAC,MAAM,CAAC,UAAU,UAAU,SAAS,MAAM,GAAG;AACzE,UAAM,IAAI,MAAM,sDAAsD,MAAM,EAAE;AAAA,EAChF;AACA,QAAM,WAAW,SAAS,CAAC;AAC3B,SAAO,EAAE,UAAU,OAAO,WAAW,IAAI,GAAG,QAAQ,KAAK,UAAU,MAAM,EAAE;AAC7E;AAEO,SAAS,kBAAkB,MAMrB;AACX,SAAO;AAAA,IACL;AAAA,IAAM;AAAA,IACN;AAAA,IAAkB;AAAA,IAClB;AAAA,IAAsB,KAAK;AAAA,IAC3B;AAAA,IAAY,KAAK;AAAA,IACjB;AAAA,IAAkB,KAAK;AAAA,IACvB;AAAA,IAAiB;AAAA,IACjB;AAAA,IAAe,KAAK,aAAa,gBAAgB;AAAA,IACjD;AAAA,IAAiB,KAAK;AAAA,EACxB;AACF;AAEA,SAAS,WAAW,OAAuB;AACzC,SAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,CAAC;AACzC;AAEA,SAAS,YAAY,MAAiD;AACpE,QAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;AAC7C,QAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;AACpD,MAAI,CAAC,qCAAqC,KAAK,IAAI,GAAG;AACpD,UAAM,IAAI,MAAM,wCAAwC,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EAC5E;AACA,MAAI,CAAC,oBAAoB,KAAK,IAAI,GAAG;AACnC,UAAM,IAAI,MAAM,+CAA+C,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EACnF;AACA,SAAO,EAAE,MAAM,KAAK;AACtB;AAEO,SAAS,sBAAsB,UAAkC,CAAC,GAAqB;AAC5F,MACE,QAAQ,cAAc,WAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,GACrE,OAAM,IAAI,MAAM,iDAAiD;AACnE,QAAM,aAAa,QAAQ,cAAc;AACzC,MAAI,CAAC,iBAAiB,IAAI,UAAU,EAAG,OAAM,IAAI,MAAM,+BAA+B;AACtF,MACE,QAAQ,8BAA8B,UACnC,OAAO,QAAQ,8BAA8B,WAChD,OAAM,IAAI,MAAM,yDAAyD;AAC3E,MAAI,kBAAkB;AACtB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWR,SAAS,MAAM;AACb,YAAM,EAAE,MAAM,KAAK,IAAI,YAAY,IAAI;AACvC,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,yBAAyB,WAAW,sBAAsB,IAAI,EAAE,CAAC,IAAI,WAAW,YAAY,CAAC,cAAc,WAAW,YAAY,CAAC,qBAAqB,WAAW,IAAI,CAAC;AAAA,MACrN;AAAA,IACF;AAAA,IACA,aAAa;AACX,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,qBAAqB,WAAW,YAAY,CAAC,uBAAuB,kBAAkB;AAAA,MACnI;AAAA,IACF;AAAA,IAEA,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAEJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,QAAQ,KAAK,SAAS;AAE5B,YAAM,SAAS;AAAA;AAAA;AAAA,oBAGD,KAAK,UAAU,OAAO,CAAC,WAAW,KAAK,UAAU,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAYtC,KAAK;AAAA;AAAA;AAAA;AAIpC,YAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;AACrF,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,OAAO,EAAE,WAAW;AAAA,UACxB;AAAA,UACA,QAAQ;AAAA,YACN,eAAe,EAAE,IAAI,MAAM,EAAE,WAAW;AAAA,YACxC;AAAA,YACA,uCAAuC,YAAY,2BAA2B,EAAE,WAAW,+BAA+B,YAAY;AAAA,YACtI;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,qBAAqB,EAAE;AAAA,UAClC,EAAE,KAAK,IAAI;AAAA,UACX,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK,UAAU;AAC5B,aAAO,OAAO,SAAS,WAAW,OAAO;AAAA,IAC3C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;AACxC,YAAM,UAAU,QAAQ,4BAA4B;AAAA,QAClD,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,iBAAiB,EAAE;AAAA,MACrB,CAAC;AACD,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,GAAI,QAAQ,cAAc,SAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;AAAA,QAC1E,GAAI,YAAY,SAAY,CAAC,IAAI,EAAE,QAAQ;AAAA;AAAA;AAAA,QAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;AAAA,QACzD,MAAM,MAAM,KAAK;AACf,gBAAM;AAAA,YACJ,KAAK,KAAK,YAAY;AAAA,YACtB,KAAK,UAAU;AAAA,cACb,EAAE,aAAa,KAAK,IAAI,oBAAoB,uBAAuB,aAAa,SAAS;AAAA,YAC3F,CAAC;AAAA,UACH;AAAA,QACF;AAAA;AAAA;AAAA,QAGA,MAAM,CAAC,QAAQ,kBAAkB;AAAA,UAC/B,iBAAiB,KAAK,KAAK,YAAY;AAAA,UACvC;AAAA,UACA,YAAY,KAAK;AAAA,UACjB;AAAA,UACA,WAAW,gBAAgB;AAAA,QAC7B,CAAC;AAAA,QACD,MAAM,YAAY,KAAK;AAErB,gBAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,KAAK,OAAO,CAAC;AAC7F,iBAAO,eAAe,KAAK,IAAI,MAAM;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}