@tangle-network/agent-bench 0.4.1 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/adapters.d.ts +4 -11
- package/dist/adapters.js +78 -41
- package/dist/adapters.js.map +1 -1
- package/dist/benchmarks/_harness.d.ts +48 -66
- package/dist/benchmarks/_harness.js +329 -33
- package/dist/benchmarks/_harness.js.map +1 -1
- package/dist/benchmarks/aec-bench.d.ts +4 -25
- package/dist/benchmarks/aec-bench.js +242 -7
- package/dist/benchmarks/aec-bench.js.map +1 -1
- package/dist/benchmarks/agentbench.d.ts +5 -13
- package/dist/benchmarks/agentbench.js +114 -9
- package/dist/benchmarks/agentbench.js.map +1 -1
- package/dist/benchmarks/appworld.d.ts +9 -29
- package/dist/benchmarks/appworld.js +317 -12
- package/dist/benchmarks/appworld.js.map +1 -1
- package/dist/benchmarks/bfcl.d.ts +5 -15
- package/dist/benchmarks/bfcl.js +264 -9
- package/dist/benchmarks/bfcl.js.map +1 -1
- package/dist/benchmarks/cad-design.d.ts +16 -41
- package/dist/benchmarks/cad-design.js +512 -6
- package/dist/benchmarks/cad-design.js.map +1 -1
- package/dist/benchmarks/cadbench.d.ts +4 -17
- package/dist/benchmarks/cadbench.js +2 -8
- package/dist/benchmarks/cadgenbench.d.ts +4 -20
- package/dist/benchmarks/cadgenbench.js +2 -8
- package/dist/benchmarks/commit0.d.ts +5 -27
- package/dist/benchmarks/commit0.js +187 -9
- package/dist/benchmarks/commit0.js.map +1 -1
- package/dist/benchmarks/crag.d.ts +4 -12
- package/dist/benchmarks/crag.js +110 -8
- package/dist/benchmarks/crag.js.map +1 -1
- package/dist/benchmarks/dabstep.d.ts +5 -15
- package/dist/benchmarks/dabstep.js +177 -9
- package/dist/benchmarks/dabstep.js.map +1 -1
- package/dist/benchmarks/enterpriseops-gym.d.ts +5 -34
- package/dist/benchmarks/enterpriseops-gym.js +236 -9
- package/dist/benchmarks/enterpriseops-gym.js.map +1 -1
- package/dist/benchmarks/finresearchbench.d.ts +4 -13
- package/dist/benchmarks/finresearchbench.js +218 -7
- package/dist/benchmarks/finresearchbench.js.map +1 -1
- package/dist/benchmarks/finsearchcomp.d.ts +8 -39
- package/dist/benchmarks/finsearchcomp.js +267 -6
- package/dist/benchmarks/finsearchcomp.js.map +1 -1
- package/dist/benchmarks/frames.d.ts +15 -37
- package/dist/benchmarks/frames.js +408 -11
- package/dist/benchmarks/frames.js.map +1 -1
- package/dist/benchmarks/hotpotqa.d.ts +4 -24
- package/dist/benchmarks/hotpotqa.js +250 -14
- package/dist/benchmarks/hotpotqa.js.map +1 -1
- package/dist/benchmarks/humaneval.d.ts +19 -37
- package/dist/benchmarks/humaneval.js +279 -16
- package/dist/benchmarks/humaneval.js.map +1 -1
- package/dist/benchmarks/mind2web.d.ts +7 -34
- package/dist/benchmarks/mind2web.js +257 -8
- package/dist/benchmarks/mind2web.js.map +1 -1
- package/dist/benchmarks/nomiracl.d.ts +4 -13
- package/dist/benchmarks/nomiracl.js +146 -8
- package/dist/benchmarks/nomiracl.js.map +1 -1
- package/dist/benchmarks/open-rag-bench.d.ts +4 -12
- package/dist/benchmarks/open-rag-bench.js +110 -8
- package/dist/benchmarks/open-rag-bench.js.map +1 -1
- package/dist/benchmarks/programbench.d.ts +5 -29
- package/dist/benchmarks/programbench.js +161 -9
- package/dist/benchmarks/programbench.js.map +1 -1
- package/dist/benchmarks/rag-shared.d.ts +21 -20
- package/dist/benchmarks/rag-shared.js +245 -38
- package/dist/benchmarks/rag-shared.js.map +1 -1
- package/dist/benchmarks/ragbench.d.ts +4 -14
- package/dist/benchmarks/ragbench.js +127 -8
- package/dist/benchmarks/ragbench.js.map +1 -1
- package/dist/benchmarks/simpleqa.d.ts +17 -44
- package/dist/benchmarks/simpleqa.js +293 -10
- package/dist/benchmarks/simpleqa.js.map +1 -1
- package/dist/benchmarks/swe-bench.d.ts +23 -36
- package/dist/benchmarks/swe-bench.js +234 -13
- package/dist/benchmarks/swe-bench.js.map +1 -1
- package/dist/benchmarks/t2-ragbench.d.ts +4 -12
- package/dist/benchmarks/t2-ragbench.js +118 -8
- package/dist/benchmarks/t2-ragbench.js.map +1 -1
- package/dist/benchmarks/tau-bench-shared.d.ts +14 -22
- package/dist/benchmarks/tau-bench-shared.js +169 -9
- package/dist/benchmarks/tau-bench-shared.js.map +1 -1
- package/dist/benchmarks/tau2-bench.d.ts +5 -5
- package/dist/benchmarks/tau2-bench.js +28 -10
- package/dist/benchmarks/tau2-bench.js.map +1 -1
- package/dist/benchmarks/tau3-banking.d.ts +4 -13
- package/dist/benchmarks/tau3-banking.js +28 -8
- package/dist/benchmarks/tau3-banking.js.map +1 -1
- package/dist/benchmarks/terminal-bench.d.ts +4 -22
- package/dist/benchmarks/terminal-bench.js +140 -7
- package/dist/benchmarks/terminal-bench.js.map +1 -1
- package/dist/benchmarks/toollm.d.ts +5 -13
- package/dist/benchmarks/toollm.js +184 -9
- package/dist/benchmarks/toollm.js.map +1 -1
- package/dist/benchmarks/trata-hedge.d.ts +4 -30
- package/dist/benchmarks/trata-hedge.js +336 -6
- package/dist/benchmarks/trata-hedge.js.map +1 -1
- package/dist/benchmarks/types.d.ts +85 -94
- package/dist/benchmarks/types.js +1 -1
- package/dist/benchmarks/webarena-verified.d.ts +5 -13
- package/dist/benchmarks/webarena-verified.js +152 -9
- package/dist/benchmarks/webarena-verified.js.map +1 -1
- package/dist/cadbench-DpQWZHp4.js +285 -0
- package/dist/cadbench-DpQWZHp4.js.map +1 -0
- package/dist/cadgenbench-DRhczfsG.js +151 -0
- package/dist/cadgenbench-DRhczfsG.js.map +1 -0
- package/dist/index.d.ts +245 -293
- package/dist/index.js +1669 -1791
- package/dist/index.js.map +1 -1
- package/package.json +18 -15
- package/scripts/verify-packed-consumer.mjs +20 -17
- package/src/corpus.test.mts +13 -0
- package/src/corpus.ts +4 -0
- package/src/pier-trial-controller.test.mts +15 -7
- package/src/profile-coordinates.ts +2 -2
- package/src/rollout-ledger/backfill-swe-arena.mts +5 -1
- package/src/rollout-ledger/settle-capture.mts +7 -1
- package/src/search-bench/profiles.ts +1 -1
- package/src/skill-sandbox-smoke.mts +2 -1
- package/src/swe-arena/gepa-seat.mts +1 -1
- package/src/swe-arena/ledger-orphans.test.mts +36 -34
- package/dist/benchmarks/cadbench.js.map +0 -1
- package/dist/benchmarks/cadgenbench.js.map +0 -1
- package/dist/benchmarks/types.js.map +0 -1
- package/dist/chunk-53UPUNBZ.js +0 -325
- package/dist/chunk-53UPUNBZ.js.map +0 -1
- package/dist/chunk-5FEQDSCT.js +0 -211
- package/dist/chunk-5FEQDSCT.js.map +0 -1
- package/dist/chunk-5H5XV76F.js +0 -240
- package/dist/chunk-5H5XV76F.js.map +0 -1
- package/dist/chunk-67ACKDCX.js +0 -118
- package/dist/chunk-67ACKDCX.js.map +0 -1
- package/dist/chunk-7FKBWOQT.js +0 -147
- package/dist/chunk-7FKBWOQT.js.map +0 -1
- package/dist/chunk-BEN6IF2X.js +0 -221
- package/dist/chunk-BEN6IF2X.js.map +0 -1
- package/dist/chunk-BZY5QARD.js +0 -120
- package/dist/chunk-BZY5QARD.js.map +0 -1
- package/dist/chunk-C7T7WEK2.js +0 -103
- package/dist/chunk-C7T7WEK2.js.map +0 -1
- package/dist/chunk-CLIKAXKH.js +0 -276
- package/dist/chunk-CLIKAXKH.js.map +0 -1
- package/dist/chunk-CWIOBFSP.js +0 -197
- package/dist/chunk-CWIOBFSP.js.map +0 -1
- package/dist/chunk-CXDUTWQE.js +0 -318
- package/dist/chunk-CXDUTWQE.js.map +0 -1
- package/dist/chunk-DWALFME7.js +0 -182
- package/dist/chunk-DWALFME7.js.map +0 -1
- package/dist/chunk-EEOC6QPJ.js +0 -144
- package/dist/chunk-EEOC6QPJ.js.map +0 -1
- package/dist/chunk-EIETHPD5.js +0 -321
- package/dist/chunk-EIETHPD5.js.map +0 -1
- package/dist/chunk-GC2EPS6L.js +0 -130
- package/dist/chunk-GC2EPS6L.js.map +0 -1
- package/dist/chunk-GCHL6XPM.js +0 -169
- package/dist/chunk-GCHL6XPM.js.map +0 -1
- package/dist/chunk-HQ5HCCKF.js +0 -142
- package/dist/chunk-HQ5HCCKF.js.map +0 -1
- package/dist/chunk-HVW25KSX.js +0 -208
- package/dist/chunk-HVW25KSX.js.map +0 -1
- package/dist/chunk-INNOYXCP.js +0 -387
- package/dist/chunk-INNOYXCP.js.map +0 -1
- package/dist/chunk-J6BU3NTM.js +0 -251
- package/dist/chunk-J6BU3NTM.js.map +0 -1
- package/dist/chunk-JSQOUKXS.js +0 -149
- package/dist/chunk-JSQOUKXS.js.map +0 -1
- package/dist/chunk-JTHWEDEW.js +0 -32
- package/dist/chunk-JTHWEDEW.js.map +0 -1
- package/dist/chunk-NRMGT25X.js +0 -116
- package/dist/chunk-NRMGT25X.js.map +0 -1
- package/dist/chunk-PA2ZKHJC.js +0 -230
- package/dist/chunk-PA2ZKHJC.js.map +0 -1
- package/dist/chunk-PUIRNYI7.js +0 -189
- package/dist/chunk-PUIRNYI7.js.map +0 -1
- package/dist/chunk-QZZEAHWJ.js +0 -136
- package/dist/chunk-QZZEAHWJ.js.map +0 -1
- package/dist/chunk-SEVJPLZC.js +0 -260
- package/dist/chunk-SEVJPLZC.js.map +0 -1
- package/dist/chunk-TBKU5XQI.js +0 -228
- package/dist/chunk-TBKU5XQI.js.map +0 -1
- package/dist/chunk-UPAMRDX4.js +0 -233
- package/dist/chunk-UPAMRDX4.js.map +0 -1
- package/dist/chunk-VQRS7VUC.js +0 -342
- package/dist/chunk-VQRS7VUC.js.map +0 -1
- package/dist/chunk-WG7TM7UV.js +0 -30
- package/dist/chunk-WG7TM7UV.js.map +0 -1
- package/dist/chunk-X3BTXCJ4.js +0 -262
- package/dist/chunk-X3BTXCJ4.js.map +0 -1
- package/dist/chunk-XXFF3RRD.js +0 -162
- package/dist/chunk-XXFF3RRD.js.map +0 -1
- package/dist/chunk-ZFNOM7WR.js +0 -27
- package/dist/chunk-ZFNOM7WR.js.map +0 -1
- package/dist/chunk-ZNCCYTFG.js +0 -170
- package/dist/chunk-ZNCCYTFG.js.map +0 -1
package/dist/chunk-ZNCCYTFG.js
DELETED
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
benchRoot,
|
|
3
|
-
preflightVenvImports,
|
|
4
|
-
runVenvScriptStdin,
|
|
5
|
-
venvPythonAt
|
|
6
|
-
} from "./chunk-EIETHPD5.js";
|
|
7
|
-
|
|
8
|
-
// src/benchmarks/commit0.ts
|
|
9
|
-
import { join } from "path";
|
|
10
|
-
import { readFile } from "fs/promises";
|
|
11
|
-
var FIXTURES = join(benchRoot, "fixtures", "commit0.json");
|
|
12
|
-
var commit0VenvDir = () => process.env.COMMIT0_VENV ?? ".venv-commit0";
|
|
13
|
-
var commit0Python = () => venvPythonAt(commit0VenvDir());
|
|
14
|
-
var DATASET = "wentingzhao/commit0_combined";
|
|
15
|
-
var DATASET_SPLIT = "test";
|
|
16
|
-
var ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`;
|
|
17
|
-
var commit0DiffOutput = {
|
|
18
|
-
parse(events) {
|
|
19
|
-
let text = "";
|
|
20
|
-
for (const ev of events) {
|
|
21
|
-
const d = ev?.data;
|
|
22
|
-
const t = d?.finalText ?? d?.text ?? d?.result;
|
|
23
|
-
if (typeof t === "string" && t.length > 0) text = t;
|
|
24
|
-
}
|
|
25
|
-
const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)];
|
|
26
|
-
const body = (fences.at(-1)?.[1] ?? text).trim();
|
|
27
|
-
return body.length > 0 ? `${body}
|
|
28
|
-
` : body;
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
function rowToTask(row) {
|
|
32
|
-
const meta = {
|
|
33
|
-
instanceId: row.instance_id,
|
|
34
|
-
repo: row.repo,
|
|
35
|
-
originalRepo: row.original_repo,
|
|
36
|
-
baseCommit: row.base_commit,
|
|
37
|
-
referenceCommit: row.reference_commit,
|
|
38
|
-
srcDir: row.src_dir,
|
|
39
|
-
testDir: row.test.test_dir,
|
|
40
|
-
testCmd: row.test.test_cmd,
|
|
41
|
-
specification: row.setup.specification
|
|
42
|
-
};
|
|
43
|
-
return {
|
|
44
|
-
id: row.instance_id,
|
|
45
|
-
split: DATASET_SPLIT,
|
|
46
|
-
prompt: [
|
|
47
|
-
`Clone https://github.com/${row.repo} into /work, then \`cd /work && git checkout ${row.base_commit}\`.`,
|
|
48
|
-
`This is the STUBBED library: the public functions/classes under \`${row.src_dir}\` have empty bodies (\`pass\`/\`...\`).`,
|
|
49
|
-
`Fill in COMPLETE implementations under \`${row.src_dir}\` so the existing test suite under \`${row.test.test_dir}\` passes. Read those tests and the spec to learn the required behavior.`,
|
|
50
|
-
`Specification / docs: ${row.setup.specification}`,
|
|
51
|
-
"Do NOT edit the test files \u2014 the evaluation re-runs the existing tests on a fresh clone. Implement only the source.",
|
|
52
|
-
`When done, from /work run EXACTLY: \`git add -A && git diff --cached -- ${row.src_dir}\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \`\`\`diff \u2026 \`\`\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`
|
|
53
|
-
].join("\n"),
|
|
54
|
-
metadata: meta
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
function readMeta(task) {
|
|
58
|
-
const md = task.metadata;
|
|
59
|
-
if (!md || typeof md.instanceId !== "string" || typeof md.referenceCommit !== "string" || typeof md.srcDir !== "string") {
|
|
60
|
-
throw new Error(`commit0 task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
|
|
61
|
-
}
|
|
62
|
-
return md;
|
|
63
|
-
}
|
|
64
|
-
function selectRows(rows, opts) {
|
|
65
|
-
let tasks = rows.map(rowToTask);
|
|
66
|
-
if (opts.ids) {
|
|
67
|
-
const want = new Set(opts.ids);
|
|
68
|
-
tasks = tasks.filter((t) => want.has(t.id));
|
|
69
|
-
} else if (opts.limit !== void 0) {
|
|
70
|
-
tasks = tasks.slice(0, opts.limit);
|
|
71
|
-
}
|
|
72
|
-
return tasks;
|
|
73
|
-
}
|
|
74
|
-
async function loadFixtures(opts) {
|
|
75
|
-
const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
|
|
76
|
-
console.warn(
|
|
77
|
-
`[commit0] COMMIT0_FIXTURES=1 \u2014 loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`
|
|
78
|
-
);
|
|
79
|
-
return selectRows(rows, opts);
|
|
80
|
-
}
|
|
81
|
-
async function fetchRows(opts) {
|
|
82
|
-
const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
|
|
83
|
-
const rows = [];
|
|
84
|
-
const want = opts.ids ? new Set(opts.ids) : null;
|
|
85
|
-
const page = 100;
|
|
86
|
-
for (let offset = 0; offset < 64 && rows.length < target; offset += page) {
|
|
87
|
-
const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`);
|
|
88
|
-
if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
|
|
89
|
-
const body = await res.json();
|
|
90
|
-
const got = body.rows ?? [];
|
|
91
|
-
if (got.length === 0) break;
|
|
92
|
-
for (const r of got) {
|
|
93
|
-
if (want && !want.has(r.row.instance_id)) continue;
|
|
94
|
-
rows.push(r.row);
|
|
95
|
-
}
|
|
96
|
-
if (got.length < page) break;
|
|
97
|
-
}
|
|
98
|
-
if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`);
|
|
99
|
-
return rows;
|
|
100
|
-
}
|
|
101
|
-
async function runHarness(meta, artifact) {
|
|
102
|
-
const judge = join(benchRoot, "scripts", "commit0_judge.py");
|
|
103
|
-
let stdout;
|
|
104
|
-
try {
|
|
105
|
-
stdout = await runVenvScriptStdin(
|
|
106
|
-
judge,
|
|
107
|
-
["--dataset", DATASET, "--split", DATASET_SPLIT, "--instance", meta.instanceId, "--src-dir", meta.srcDir],
|
|
108
|
-
artifact,
|
|
109
|
-
{ cwd: benchRoot, python: commit0Python() }
|
|
110
|
-
);
|
|
111
|
-
} catch (err) {
|
|
112
|
-
const e = err;
|
|
113
|
-
throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
|
|
114
|
-
}
|
|
115
|
-
const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
|
|
116
|
-
if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`);
|
|
117
|
-
if (typeof report.passed !== "number" || typeof report.total !== "number") {
|
|
118
|
-
throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
|
|
119
|
-
}
|
|
120
|
-
if (report.total <= 0) {
|
|
121
|
-
throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`);
|
|
122
|
-
}
|
|
123
|
-
const score = report.passed / report.total;
|
|
124
|
-
return {
|
|
125
|
-
resolved: report.passed === report.total,
|
|
126
|
-
score,
|
|
127
|
-
detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total })
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
function createCommit0Adapter() {
|
|
131
|
-
const fixturesMode = process.env.COMMIT0_FIXTURES === "1";
|
|
132
|
-
return {
|
|
133
|
-
name: "commit0",
|
|
134
|
-
output: commit0DiffOutput,
|
|
135
|
-
async preflight() {
|
|
136
|
-
await preflightVenvImports({
|
|
137
|
-
modules: ["commit0"],
|
|
138
|
-
requireDocker: true,
|
|
139
|
-
python: commit0Python(),
|
|
140
|
-
fix: `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets (an ISOLATED venv \u2014 commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; (2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`
|
|
141
|
-
});
|
|
142
|
-
},
|
|
143
|
-
async loadTasks(opts = {}) {
|
|
144
|
-
if (fixturesMode) return loadFixtures(opts);
|
|
145
|
-
let rows;
|
|
146
|
-
try {
|
|
147
|
-
rows = await fetchRows(opts);
|
|
148
|
-
} catch (err) {
|
|
149
|
-
console.warn(
|
|
150
|
-
`[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
|
|
151
|
-
);
|
|
152
|
-
return loadFixtures(opts);
|
|
153
|
-
}
|
|
154
|
-
return selectRows(rows, opts);
|
|
155
|
-
},
|
|
156
|
-
async goldArtifact() {
|
|
157
|
-
return void 0;
|
|
158
|
-
},
|
|
159
|
-
async judge(task, artifact) {
|
|
160
|
-
const meta = readMeta(task);
|
|
161
|
-
return runHarness(meta, artifact);
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
export {
|
|
167
|
-
commit0DiffOutput,
|
|
168
|
-
createCommit0Adapter
|
|
169
|
-
};
|
|
170
|
-
//# sourceMappingURL=chunk-ZNCCYTFG.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/benchmarks/commit0.ts"],"sourcesContent":["/**\n * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.\n * Each record is a Python repo stubbed at `base_commit` (public function bodies\n * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker\n * artifact = a unified diff that fills in `src_dir`. Judge = the official\n * `commit0` test harness: it stages the starter repo, applies the worker's diff,\n * builds the library's deps and runs `pytest`, writing a per-repo pytest-json\n * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the\n * macro-averaged unit-test pass-rate the leaderboard reports). Fully\n * deterministic — no LLM judge.\n *\n * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its\n * implementation as a fenced ```diff against the stubbed repo — same deliverable\n * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the\n * real `commit0` harness on a local Docker backend (NOT reimplemented here).\n *\n * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed\n * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +\n * a Docker daemon (`--backend local`). For offline/CI dataset listing set\n * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)\n * — judging still needs the harness + Docker and fails loud, never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'commit0.json')\n\n// commit0's pip deps (pydantic/sqlalchemy v1, modal, …) conflict with the shared\n// bench .venv, so its harness runs in an ISOLATED venv. Override with COMMIT0_VENV.\n// Resolved at call-time so the env is honored at run-time (not frozen at import).\nconst commit0VenvDir = (): string => process.env.COMMIT0_VENV ?? '.venv-commit0'\nconst commit0Python = (): string => venvPythonAt(commit0VenvDir())\n\nconst DATASET = 'wentingzhao/commit0_combined'\nconst DATASET_SPLIT = 'test'\n// HF rows server — columnar pull with no `datasets` install required for listing.\nconst ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`\n\n/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */\nexport const commit0DiffOutput: OutputAdapter<string> = {\n parse(events) {\n let text = ''\n for (const ev of events) {\n const d = (ev as { data?: Record<string, unknown> })?.data\n const t = d?.finalText ?? d?.text ?? d?.result\n if (typeof t === 'string' && t.length > 0) text = t\n }\n const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n const body = (fences.at(-1)?.[1] ?? text).trim()\n // `git apply` rejects a patch that is not newline-terminated (\"corrupt patch at\n // line N+1\"); the .trim() above strips the final newline, so restore exactly one.\n return body.length > 0 ? `${body}\\n` : body\n },\n}\n\ninterface Commit0Setup {\n install: string\n packages: string[] | null\n pip_packages: string[] | null\n pre_install: string[] | null\n python: string\n specification: string\n}\n\ninterface Commit0Row {\n instance_id: string\n repo: string\n original_repo: string\n base_commit: string\n reference_commit: string\n setup: Commit0Setup\n test: { test_cmd: string; test_dir: string }\n src_dir: string\n}\n\ninterface Commit0Meta {\n instanceId: string\n repo: string\n originalRepo: string\n baseCommit: string\n referenceCommit: string\n srcDir: string\n testDir: string\n testCmd: string\n specification: string\n}\n\nfunction rowToTask(row: Commit0Row): BenchTask {\n const meta: Commit0Meta = {\n instanceId: row.instance_id,\n repo: row.repo,\n originalRepo: row.original_repo,\n baseCommit: row.base_commit,\n referenceCommit: row.reference_commit,\n srcDir: row.src_dir,\n testDir: row.test.test_dir,\n testCmd: row.test.test_cmd,\n specification: row.setup.specification,\n }\n return {\n id: row.instance_id,\n split: DATASET_SPLIT,\n prompt: [\n `Clone https://github.com/${row.repo} into /work, then \\`cd /work && git checkout ${row.base_commit}\\`.`,\n `This is the STUBBED library: the public functions/classes under \\`${row.src_dir}\\` have empty bodies (\\`pass\\`/\\`...\\`).`,\n `Fill in COMPLETE implementations under \\`${row.src_dir}\\` so the existing test suite under \\`${row.test.test_dir}\\` passes. Read those tests and the spec to learn the required behavior.`,\n `Specification / docs: ${row.setup.specification}`,\n 'Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.',\n `When done, from /work run EXACTLY: \\`git add -A && git diff --cached -- ${row.src_dir}\\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \\`\\`\\`diff … \\`\\`\\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`,\n ].join('\\n'),\n metadata: meta as unknown as Record<string, unknown>,\n }\n}\n\nfunction readMeta(task: BenchTask): Commit0Meta {\n const md = task.metadata\n if (!md || typeof md.instanceId !== 'string' || typeof md.referenceCommit !== 'string' || typeof md.srcDir !== 'string') {\n throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`)\n }\n return md as unknown as Commit0Meta\n}\n\nfunction selectRows(rows: Commit0Row[], opts: LoadOptions): BenchTask[] {\n let tasks = rows.map(rowToTask)\n if (opts.ids) {\n const want = new Set(opts.ids)\n tasks = tasks.filter((t) => want.has(t.id))\n } else if (opts.limit !== undefined) {\n tasks = tasks.slice(0, opts.limit)\n }\n return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as Commit0Row[]\n console.warn(\n `[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`,\n )\n return selectRows(rows, opts)\n}\n\n/** Pull real rows from the HF rows server (paged). limit caps the pull; ids\n * filtered client-side. Throws on a non-OK response (fail loud). */\nasync function fetchRows(opts: LoadOptions): Promise<Commit0Row[]> {\n const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n const rows: Commit0Row[] = []\n const want = opts.ids ? new Set(opts.ids) : null\n const page = 100\n for (let offset = 0; offset < 64 && rows.length < target; offset += page) {\n const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`)\n if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n const body = (await res.json()) as { rows?: Array<{ row: Commit0Row }>; num_rows_total?: number }\n const got = body.rows ?? []\n if (got.length === 0) break\n for (const r of got) {\n if (want && !want.has(r.row.instance_id)) continue\n rows.push(r.row)\n }\n if (got.length < page) break\n }\n if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`)\n return rows\n}\n\n/**\n * Run the official commit0 harness for one repo over the worker's diff. The\n * harness clones base_commit, applies the diff into src_dir, installs deps and\n * runs pytest, then writes a per-repo pytest-json report. We read that report and\n * compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.\n *\n * This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,\n * not reimplemented. The driver script lives in scripts/commit0_judge.py so the\n * harness call + report parse are one auditable python entrypoint.\n */\nasync function runHarness(meta: Commit0Meta, artifact: string): Promise<BenchScore> {\n const judge = join(benchRoot, 'scripts', 'commit0_judge.py')\n let stdout: string\n try {\n // The worker's diff is piped to the driver's stdin (shared stdin-aware runner —\n // execFile's `input` option is not honored async and hangs the reader).\n stdout = await runVenvScriptStdin(\n judge,\n ['--dataset', DATASET, '--split', DATASET_SPLIT, '--instance', meta.instanceId, '--src-dir', meta.srcDir],\n artifact,\n { cwd: benchRoot, python: commit0Python() },\n )\n } catch (err) {\n const e = err as { message?: string }\n throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n }\n const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n passed?: number\n total?: number\n error?: string\n }\n if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`)\n if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n }\n // total=0 means the harness MEASURED NOTHING (collection error with no declared\n // test ids) — an unmeasured attempt, not a 0% one. Throw so the caller excludes\n // it as infra instead of recording a fabricated zero.\n if (report.total <= 0) {\n throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`)\n }\n const score = report.passed / report.total\n return {\n resolved: report.passed === report.total,\n score,\n detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n }\n}\n\nexport function createCommit0Adapter(): BenchmarkAdapter {\n const fixturesMode = process.env.COMMIT0_FIXTURES === '1'\n\n return {\n name: 'commit0',\n output: commit0DiffOutput,\n\n async preflight() {\n await preflightVenvImports({\n modules: ['commit0'],\n requireDocker: true,\n python: commit0Python(),\n fix:\n `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets ` +\n `(an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; ` +\n `(2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). ` +\n `Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`,\n })\n },\n\n async loadTasks(opts: LoadOptions = {}) {\n if (fixturesMode) return loadFixtures(opts)\n let rows: Commit0Row[]\n try {\n rows = await fetchRows(opts)\n } catch (err) {\n console.warn(\n `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n )\n return loadFixtures(opts)\n }\n return selectRows(rows, opts)\n },\n\n async goldArtifact() {\n // The oracle is the reference_commit's src_dir, which the commit0 harness\n // checks out by ref — not expressible as a portable diff string without\n // cloning. verify-judge against this adapter requires the live harness, so\n // we return undefined (no offline gold-diff). Judge correctness is proven by\n // running the harness on a real solve, not by a synthetic gold patch.\n return undefined\n },\n\n async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n const meta = readMeta(task)\n return runHarness(meta, artifact)\n },\n }\n}\n"],"mappings":";;;;;;;;AAuBA,SAAS,YAAY;AACrB,SAAS,gBAAgB;AAKzB,IAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAK3D,IAAM,iBAAiB,MAAc,QAAQ,IAAI,gBAAgB;AACjE,IAAM,gBAAgB,MAAc,aAAa,eAAe,CAAC;AAEjE,IAAM,UAAU;AAChB,IAAM,gBAAgB;AAEtB,IAAM,WAAW,uDAAuD,mBAAmB,OAAO,CAAC,yBAAyB,aAAa;AAGlI,IAAM,oBAA2C;AAAA,EACtD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,uCAAuC,CAAC;AACzE,UAAM,QAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAG/C,WAAO,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO;AAAA,EACzC;AACF;AAkCA,SAAS,UAAU,KAA4B;AAC7C,QAAM,OAAoB;AAAA,IACxB,YAAY,IAAI;AAAA,IAChB,MAAM,IAAI;AAAA,IACV,cAAc,IAAI;AAAA,IAClB,YAAY,IAAI;AAAA,IAChB,iBAAiB,IAAI;AAAA,IACrB,QAAQ,IAAI;AAAA,IACZ,SAAS,IAAI,KAAK;AAAA,IAClB,SAAS,IAAI,KAAK;AAAA,IAClB,eAAe,IAAI,MAAM;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO;AAAA,IACP,QAAQ;AAAA,MACN,4BAA4B,IAAI,IAAI,gDAAgD,IAAI,WAAW;AAAA,MACnG,qEAAqE,IAAI,OAAO;AAAA,MAChF,4CAA4C,IAAI,OAAO,yCAAyC,IAAI,KAAK,QAAQ;AAAA,MACjH,yBAAyB,IAAI,MAAM,aAAa;AAAA,MAChD;AAAA,MACA,2EAA2E,IAAI,OAAO,6KAAwK,IAAI,WAAW;AAAA,IAC/Q,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA8B;AAC9C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,YAAY,OAAO,GAAG,oBAAoB,YAAY,OAAO,GAAG,WAAW,UAAU;AACvH,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wDAAmD;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAoB,MAAgC;AACtE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,+CAA0C,KAAK,MAAM,6BAA6B,QAAQ;AAAA,EAC5F;AACA,SAAO,WAAW,MAAM,IAAI;AAC9B;AAIA,eAAe,UAAU,MAA0C;AACjE,QAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;AAC/D,QAAM,OAAqB,CAAC;AAC5B,QAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;AAC5C,QAAM,OAAO;AACb,WAAS,SAAS,GAAG,SAAS,MAAM,KAAK,SAAS,QAAQ,UAAU,MAAM;AACxE,UAAM,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,MAAM,WAAW,IAAI,EAAE;AACrE,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,qBAAqB,IAAI,MAAM,YAAY,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AACtH,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,MAAM,KAAK,QAAQ,CAAC;AAC1B,QAAI,IAAI,WAAW,EAAG;AACtB,eAAW,KAAK,KAAK;AACnB,UAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,WAAW,EAAG;AAC1C,WAAK,KAAK,EAAE,GAAG;AAAA,IACjB;AACA,QAAI,IAAI,SAAS,KAAM;AAAA,EACzB;AACA,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B,KAAK,UAAU,IAAI,CAAC,SAAS,OAAO,EAAE;AACzG,SAAO;AACT;AAYA,eAAe,WAAW,MAAmB,UAAuC;AAClF,QAAM,QAAQ,KAAK,WAAW,WAAW,kBAAkB;AAC3D,MAAI;AACJ,MAAI;AAGF,aAAS,MAAM;AAAA,MACb;AAAA,MACA,CAAC,aAAa,SAAS,WAAW,eAAe,cAAc,KAAK,YAAY,aAAa,KAAK,MAAM;AAAA,MACxG;AAAA,MACA,EAAE,KAAK,WAAW,QAAQ,cAAc,EAAE;AAAA,IAC5C;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,8BAA8B,KAAK,UAAU,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EAC/G;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAKlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,KAAK,OAAO,KAAK,EAAE;AACjG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,6CAA6C,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACrF;AAIA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,eAAe,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC7G;AACA,QAAM,QAAQ,OAAO,SAAS,OAAO;AACrC,SAAO;AAAA,IACL,UAAU,OAAO,WAAW,OAAO;AAAA,IACnC;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,YAAY,KAAK,YAAY,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACpG;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,qBAAqB;AAEtD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,SAAS;AAAA,QACnB,eAAe;AAAA,QACf,QAAQ,cAAc;AAAA,QACtB,KACE,kCAAkC,eAAe,CAAC,aAAa,eAAe,CAAC;AAAA,MAInF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,UAAU,IAAI;AAAA,MAC7B,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,qCAAqC,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAA4C,QAAQ;AAAA,QACnI;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,WAAW,MAAM,IAAI;AAAA,IAC9B;AAAA,IAEA,MAAM,eAAe;AAMnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}
|