@zhijiewang/openharness 2.40.0 → 2.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/dist/evals/orchestrator.js +24 -9
- package/dist/evals/pack-loader.js +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -892,7 +892,7 @@ Output lives at `~/.oh/evals/runs/<run-id>/`:
|
|
|
892
892
|
|
|
893
893
|
A pluggable pack contract (`pack.json` + `instances.jsonl` + `fixtures/<id>/`) lets you author packs against any test suite. The `scripts/build-evals-pack.mjs` helper bakes a SWE-bench-Lite-compatible repo at a given base_commit into a fixture; see [CONTRIBUTING.md](CONTRIBUTING.md#authoring-eval-packs).
|
|
894
894
|
|
|
895
|
-
A bundled `swe-bench-lite-mini` pack (10 cherry-picked instances, ready to run out-of-the-box) is shipping in v2.40.
|
|
895
|
+
A bundled `swe-bench-lite-mini` pack (10 cherry-picked instances, ready to run out-of-the-box) is shipping in v2.40.2.
|
|
896
896
|
|
|
897
897
|
## How It Works
|
|
898
898
|
|
package/README.zh-CN.md
CHANGED
|
@@ -891,7 +891,7 @@ oh evals show 2026-05-05T14-30-00
|
|
|
891
891
|
|
|
892
892
|
可插拔的 pack 协议(`pack.json` + `instances.jsonl` + `fixtures/<id>/`)允许你针对任意测试套件编写 pack。`scripts/build-evals-pack.mjs` 工具可将 SWE-bench-Lite 兼容仓库在指定 base_commit 处烘焙为 fixture,详见 [CONTRIBUTING.md](CONTRIBUTING.md#authoring-eval-packs)。
|
|
893
893
|
|
|
894
|
-
内置的 `swe-bench-lite-mini` pack(10 个精选 instance,开箱即跑)将在 v2.40.
|
|
894
|
+
内置的 `swe-bench-lite-mini` pack(10 个精选 instance,开箱即跑)将在 v2.40.2 版本发布。
|
|
895
895
|
|
|
896
896
|
## 工作原理
|
|
897
897
|
|
|
@@ -325,17 +325,32 @@ function captureGitDiff(worktreeDir) {
|
|
|
325
325
|
}
|
|
326
326
|
}
|
|
327
327
|
async function extractFixture(packDir, instanceId, dest) {
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
328
|
+
const fxDir = join(packDir, "fixtures", instanceId);
|
|
329
|
+
// Prefer .tar.gz (bundled by gzip — universally available); fall back to
|
|
330
|
+
// .tar.zst for older packs that were built before v2.40.1.
|
|
331
|
+
const candidates = [
|
|
332
|
+
{ path: join(fxDir, "repo.tar.gz"), flag: "-xzf" },
|
|
333
|
+
{ path: join(fxDir, "repo.tar.zst"), flag: "" },
|
|
334
|
+
];
|
|
335
|
+
for (const c of candidates) {
|
|
336
|
+
if (!existsSync(c.path))
|
|
337
|
+
continue;
|
|
338
|
+
if (readFileSync(c.path).length === 0) {
|
|
339
|
+
// Empty tarball = test mode (synthetic pack). Caller's setup.sh
|
|
340
|
+
// handles initialization; we just ensure the dest dir exists.
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
if (c.flag === "-xzf") {
|
|
344
|
+
execFileSync("tar", ["-xzf", c.path, "-C", dest], { stdio: ["ignore", "pipe", "pipe"] });
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
// Legacy .tar.zst path: requires the system `zstd` binary on PATH.
|
|
348
|
+
execFileSync("tar", ["--use-compress-program=zstd -d", "-xf", c.path, "-C", dest], {
|
|
349
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
350
|
+
});
|
|
351
|
+
}
|
|
332
352
|
return;
|
|
333
353
|
}
|
|
334
|
-
// Extract via system `tar` (Linux/macOS/Win10+ all ship one).
|
|
335
|
-
// -I/--use-compress-program zstd is available on tar 1.31+.
|
|
336
|
-
execFileSync("tar", ["--use-compress-program=zstd -d", "-xf", tarPath, "-C", dest], {
|
|
337
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
338
|
-
});
|
|
339
354
|
}
|
|
340
355
|
async function runSetupScript(packDir, instanceId, worktreeDir) {
|
|
341
356
|
const setupPath = join(packDir, "fixtures", instanceId, "setup.sh");
|
|
@@ -85,8 +85,8 @@ export function validatePack(packDir) {
|
|
|
85
85
|
errors.push(`fixture dir missing for ${instanceId} at ${fixtureDir}`);
|
|
86
86
|
}
|
|
87
87
|
else {
|
|
88
|
-
if (!existsSync(join(fixtureDir, "repo.tar.zst"))) {
|
|
89
|
-
errors.push(`fixture missing repo.tar.zst for ${instanceId}`);
|
|
88
|
+
if (!existsSync(join(fixtureDir, "repo.tar.gz")) && !existsSync(join(fixtureDir, "repo.tar.zst"))) {
|
|
89
|
+
errors.push(`fixture missing repo.tar.gz (or legacy repo.tar.zst) for ${instanceId}`);
|
|
90
90
|
}
|
|
91
91
|
if (!existsSync(join(fixtureDir, "setup.sh"))) {
|
|
92
92
|
errors.push(`fixture missing setup.sh for ${instanceId}`);
|