independent-tests 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -1
- package/README.md +112 -54
- package/SKILL.md +13 -10
- package/bin/backends.mjs +55 -25
- package/bin/generate.mjs +3 -3
- package/bin/handout.mjs +3 -3
- package/bin/lib.mjs +176 -79
- package/bin/platform.mjs +145 -0
- package/bin/run.mjs +6 -4
- package/bin/runners/boundary.mjs +27 -8
- package/bin/runners/guard.mjs +8 -6
- package/bin/runners/pytest.mjs +3 -2
- package/bin/runners/shared.mjs +10 -7
- package/package.json +8 -7
- package/prompts/agent.md +1 -1
- package/prompts/exec/agent.md +1 -1
- package/prompts/http/agent.md +1 -1
- package/template/package-lock.json +1 -1
- package/template/package.json +2 -2
- package/template/sandbox/run.mjs +3 -2
- package/template/sandbox/test.mjs +30 -0
- package/template-python/README.md +2 -1
- package/template-python/sandbox/run.py +5 -3
package/bin/lib.mjs
CHANGED
|
@@ -34,9 +34,23 @@ import {
|
|
|
34
34
|
rmSync,
|
|
35
35
|
writeFileSync,
|
|
36
36
|
} from "node:fs";
|
|
37
|
-
import { homedir, tmpdir } from "node:os";
|
|
38
|
-
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
37
|
+
import { freemem, homedir, tmpdir, totalmem } from "node:os";
|
|
38
|
+
import { basename, dirname, join, posix, relative, resolve, sep } from "node:path";
|
|
39
39
|
import { fileURLToPath } from "node:url";
|
|
40
|
+
import {
|
|
41
|
+
DEFAULT_PYTHON,
|
|
42
|
+
GROUP,
|
|
43
|
+
OS_SANDBOX,
|
|
44
|
+
VENV_BIN_DIR,
|
|
45
|
+
WINDOWS,
|
|
46
|
+
comparable,
|
|
47
|
+
inside,
|
|
48
|
+
killTree,
|
|
49
|
+
slash,
|
|
50
|
+
spawnCommand,
|
|
51
|
+
venvBin,
|
|
52
|
+
which,
|
|
53
|
+
} from "./platform.mjs";
|
|
40
54
|
import { withBins } from "./runners/shared.mjs";
|
|
41
55
|
|
|
42
56
|
/**
|
|
@@ -76,12 +90,12 @@ export const TRANSCRIPTS = join(homedir(), ".claude", "projects");
|
|
|
76
90
|
/** The bundled specs (ROOT/specs/: duration-parse, the README's example). */
|
|
77
91
|
export const BUNDLED_SPECS = join(ROOT, "specs");
|
|
78
92
|
|
|
93
|
+
// realpathSync.native: on Windows it also expands 8.3 short names (RUNNER~1) in TEMP.
|
|
94
|
+
const realPath = WINDOWS ? realpathSync.native : realpathSync;
|
|
79
95
|
const workBase =
|
|
80
|
-
process.env.INDEPENDENT_TESTS_WORK ?? join(
|
|
96
|
+
process.env.INDEPENDENT_TESTS_WORK ?? join(realPath(tmpdir()), "independent-tests-work");
|
|
81
97
|
mkdirSync(workBase, { recursive: true });
|
|
82
|
-
export const WORK_ROOT =
|
|
83
|
-
|
|
84
|
-
const inside = (path, folder) => path === folder || path.startsWith(`${folder}/`);
|
|
98
|
+
export const WORK_ROOT = realPath(workBase);
|
|
85
99
|
/**
|
|
86
100
|
* What no session may read: the package, the person's data and wherever
|
|
87
101
|
* SPECS and RUNS were moved, folded so none is listed under another.
|
|
@@ -159,7 +173,7 @@ export function ensureTemplate() {
|
|
|
159
173
|
filter: (from) => !relative(TEMPLATE_SOURCE, from).split(sep).includes("node_modules"),
|
|
160
174
|
});
|
|
161
175
|
log(`installing the toolchain into ${TEMPLATE} (npm@10.9.2 ci, once)`);
|
|
162
|
-
const out =
|
|
176
|
+
const out = spawnCommand("npx", ["-y", "npm@10.9.2", "ci", "--no-audit", "--no-fund"], {
|
|
163
177
|
cwd: partial,
|
|
164
178
|
encoding: "utf8",
|
|
165
179
|
maxBuffer: 64 * 1024 * 1024,
|
|
@@ -178,12 +192,12 @@ export function ensureTemplate() {
|
|
|
178
192
|
}
|
|
179
193
|
/**
|
|
180
194
|
* The interpreter the `python` toolchain and Python repositories run on:
|
|
181
|
-
* INDEPENDENT_TESTS_PYTHON, default `python3` on PATH.
|
|
195
|
+
* INDEPENDENT_TESTS_PYTHON, default `python3` on PATH (`python` on Windows).
|
|
182
196
|
*/
|
|
183
|
-
export const PYTHON = process.env.INDEPENDENT_TESTS_PYTHON ??
|
|
197
|
+
export const PYTHON = process.env.INDEPENDENT_TESTS_PYTHON ?? DEFAULT_PYTHON;
|
|
184
198
|
const PYTHON_SOURCE = join(ROOT, "template-python");
|
|
185
199
|
/** The tree's own interpreter, which holds the pinned pytest: relative to a staged Python tree. */
|
|
186
|
-
export const VENV_PYTHON =
|
|
200
|
+
export const VENV_PYTHON = venvBin("python");
|
|
187
201
|
const pythonTrees = new Map();
|
|
188
202
|
/**
|
|
189
203
|
* A toolchain tree with what its suites need installed, which `stage`
|
|
@@ -195,7 +209,7 @@ const pythonTrees = new Map();
|
|
|
195
209
|
*/
|
|
196
210
|
function toolchainTree(name, source, version, install) {
|
|
197
211
|
const files = listFiles(source, [".venv", "target", "node_modules"]).filter(
|
|
198
|
-
(path) => !path.split(
|
|
212
|
+
(path) => !path.split("/").includes("__pycache__"),
|
|
199
213
|
);
|
|
200
214
|
const hash = sha256(
|
|
201
215
|
JSON.stringify([version, ...files.map((path) => [path, sha256(readFileSync(join(source, path)))])]),
|
|
@@ -243,8 +257,8 @@ export function ensurePython() {
|
|
|
243
257
|
step(VENV_PYTHON, [...pip, "install", "--no-deps", "--require-hashes", "-r", "requirements.txt"]);
|
|
244
258
|
step(VENV_PYTHON, [...pip, "uninstall", "-y", "pip"]);
|
|
245
259
|
// The activate scripts name the partial folder; nothing here sources them.
|
|
246
|
-
for (const name of readdirSync(join(partial, ".venv",
|
|
247
|
-
if (/^(activate|pip)/i.test(name)) rmSync(join(partial, ".venv",
|
|
260
|
+
for (const name of readdirSync(join(partial, ".venv", VENV_BIN_DIR)))
|
|
261
|
+
if (/^(activate|pip)/i.test(name)) rmSync(join(partial, ".venv", VENV_BIN_DIR, name), { force: true });
|
|
248
262
|
});
|
|
249
263
|
pythonTrees.set(PYTHON, dest);
|
|
250
264
|
return dest;
|
|
@@ -315,7 +329,7 @@ export const TOOLCHAINS = {
|
|
|
315
329
|
suite,
|
|
316
330
|
files: listFiles(join(dir, "tests", suite))
|
|
317
331
|
.filter((path) => /\.test\.tsx?$/.test(path))
|
|
318
|
-
.map((path) => join("tests", suite, path)),
|
|
332
|
+
.map((path) => posix.join("tests", suite, path)),
|
|
319
333
|
build: [process.execPath, "node_modules/typescript/bin/tsc", "-p", "tsconfig.json"],
|
|
320
334
|
envFile: "sandbox.env",
|
|
321
335
|
reporter: "./sandbox/reporter.mjs",
|
|
@@ -334,12 +348,12 @@ export const TOOLCHAINS = {
|
|
|
334
348
|
generated: (path) =>
|
|
335
349
|
path.split("/").some((part) => part === "__pycache__" || part === ".pytest_cache") || path.endsWith(".pyc"),
|
|
336
350
|
marker: "sandbox/run.py",
|
|
337
|
-
checkCommand:
|
|
351
|
+
checkCommand: `${DEFAULT_PYTHON} sandbox/compile_check.py`,
|
|
338
352
|
check: (dir) => [join(dir, VENV_PYTHON), ["-I", "sandbox/compile_check.py"]],
|
|
339
353
|
suite: (dir, suite) => {
|
|
340
354
|
const files = listFiles(join(dir, "tests", suite))
|
|
341
355
|
.filter((path) => /(^|\/)test_[^/]*\.py$/.test(path))
|
|
342
|
-
.map((path) => join("tests", suite, path));
|
|
356
|
+
.map((path) => posix.join("tests", suite, path));
|
|
343
357
|
return [
|
|
344
358
|
process.execPath,
|
|
345
359
|
[
|
|
@@ -376,7 +390,7 @@ export const TOOLCHAINS = {
|
|
|
376
390
|
suite,
|
|
377
391
|
files: listFiles(join(dir, "tests", suite))
|
|
378
392
|
.filter((path) => path.endsWith("_test.go"))
|
|
379
|
-
.map((path) => join("tests", suite, path)),
|
|
393
|
+
.map((path) => posix.join("tests", suite, path)),
|
|
380
394
|
}),
|
|
381
395
|
],
|
|
382
396
|
],
|
|
@@ -402,7 +416,7 @@ export const TOOLCHAINS = {
|
|
|
402
416
|
suite,
|
|
403
417
|
files: listFiles(join(dir, "tests", suite))
|
|
404
418
|
.filter((path) => path.endsWith(".rs"))
|
|
405
|
-
.map((path) => join("tests", suite, path)),
|
|
419
|
+
.map((path) => posix.join("tests", suite, path)),
|
|
406
420
|
}),
|
|
407
421
|
],
|
|
408
422
|
],
|
|
@@ -419,18 +433,18 @@ export const TOOLCHAINS = {
|
|
|
419
433
|
check: (dir, chain) => {
|
|
420
434
|
if (chain.build === undefined) return [process.execPath, ["-e", ""]];
|
|
421
435
|
const cwd = join(dir, chain.package);
|
|
422
|
-
return [
|
|
436
|
+
return [chain.build, [], { cwd, env: withBins(cwd), shell: true, windowsHide: true }];
|
|
423
437
|
},
|
|
424
438
|
suite: (dir, suite, chain) => {
|
|
425
439
|
const cwd = join(dir, chain.package);
|
|
426
|
-
const files = suiteFiles(dir, suite, chain).map((path) => relative(cwd, join(dir, path)));
|
|
440
|
+
const files = suiteFiles(dir, suite, chain).map((path) => slash(relative(cwd, join(dir, path))));
|
|
427
441
|
const { build, rootDir, outDir, testTimeoutMs } = chain;
|
|
428
442
|
// pytest runs on the base tree's .venv, every file in one process, with
|
|
429
443
|
// the package's import root first on the path, so the tree's own source
|
|
430
444
|
// shadows the installed copy of the package.
|
|
431
445
|
const python =
|
|
432
446
|
chain.runner === "pytest"
|
|
433
|
-
? { python: relative(cwd, join(dir, VENV_PYTHON)), pythonPath: chain.pythonPath, perFile: false }
|
|
447
|
+
? { python: slash(relative(cwd, join(dir, VENV_PYTHON))), pythonPath: chain.pythonPath, perFile: false }
|
|
434
448
|
: {};
|
|
435
449
|
return [
|
|
436
450
|
process.execPath,
|
|
@@ -551,7 +565,7 @@ export function repoChain(config, slug) {
|
|
|
551
565
|
chain.pythonPath ??= ["."];
|
|
552
566
|
}
|
|
553
567
|
// A build's output, where the repository's .gitignore does not hide it.
|
|
554
|
-
const outDir = chain.outDir === undefined ? null : `${join(chain.package, chain.outDir)}/`;
|
|
568
|
+
const outDir = chain.outDir === undefined ? null : `${posix.join(chain.package, chain.outDir)}/`;
|
|
555
569
|
chain.generated = (path) =>
|
|
556
570
|
(outDir !== null && path.startsWith(outDir)) ||
|
|
557
571
|
(python && (path.split("/").some((part) => part === "__pycache__" || part === ".pytest_cache") || path.endsWith(".pyc")));
|
|
@@ -602,10 +616,10 @@ export function commandOf(chain, suite) {
|
|
|
602
616
|
// Sessions run the suite under the tree's copy of boundary.mjs (`stage` writes it into .git/).
|
|
603
617
|
const plain = commandOf({ ...chain, boundary: null }, suite);
|
|
604
618
|
const cd = /^cd [^&]+ && /.exec(plain)?.[0] ?? "";
|
|
605
|
-
const up = chain.package === "." ? "" : `${relative(chain.package, ".")}/`;
|
|
619
|
+
const up = chain.package === "." ? "" : `${posix.relative(chain.package, ".")}/`;
|
|
606
620
|
return `${cd}node ${up}.git/independent-tests/boundary.mjs -- sh -c '${plain.slice(cd.length).replace(/'/g, "'\\''")}'`;
|
|
607
621
|
}
|
|
608
|
-
const inPackage = (path) => relative(chain.package, path) || ".";
|
|
622
|
+
const inPackage = (path) => posix.relative(chain.package, path) || ".";
|
|
609
623
|
const folder = { public: chain.publicDir, own: chain.publicDir, private: chain.hiddenDir, existing: chain.testDir }[suite];
|
|
610
624
|
const build = chain.build === undefined ? "" : `npx ${chain.build} && `;
|
|
611
625
|
// From the tree's root: nothing to change into for a package at the root.
|
|
@@ -617,8 +631,10 @@ export function commandOf(chain, suite) {
|
|
|
617
631
|
private: `${inPackage(folder)}/`,
|
|
618
632
|
existing: inPackage(folder),
|
|
619
633
|
}[suite];
|
|
620
|
-
const python = relative(chain.package, VENV_PYTHON) || VENV_PYTHON;
|
|
621
|
-
|
|
634
|
+
const python = posix.relative(chain.package, VENV_PYTHON) || VENV_PYTHON;
|
|
635
|
+
// Windows' Python splits PYTHONPATH on ";", quoted for Git Bash, which sessions run there.
|
|
636
|
+
const pythonPath = WINDOWS ? `'${chain.pythonPath.join(";")}'` : chain.pythonPath.join(":");
|
|
637
|
+
return `${cd}PYTHONPATH=${pythonPath} ${python} -m pytest -p no:cacheprovider ${target}`;
|
|
622
638
|
}
|
|
623
639
|
if (chain.runner === "jest") {
|
|
624
640
|
// Jest takes regular expressions over test paths.
|
|
@@ -639,9 +655,10 @@ export function commandOf(chain, suite) {
|
|
|
639
655
|
}[suite];
|
|
640
656
|
return `${cd}npx vitest run ${filter}`.trimEnd();
|
|
641
657
|
}
|
|
642
|
-
const compiled = join(chain.outDir ?? ".", relative(chain.rootDir ?? ".", inPackage(folder)));
|
|
643
|
-
|
|
644
|
-
|
|
658
|
+
const compiled = posix.join(chain.outDir ?? ".", posix.relative(chain.rootDir ?? ".", inPackage(folder)));
|
|
659
|
+
// The files listed by the session's shell, not globbed by node --test, which Node 20 cannot do.
|
|
660
|
+
const files = { public: `${compiled}/${chain.slug}.*.js`, own: `${compiled}/${chain.slug}-own.*.js` }[suite] ?? `$(find ${compiled} -name '*.test.js')`;
|
|
661
|
+
return `${cd}${build}node --test ${files}`;
|
|
645
662
|
}
|
|
646
663
|
/** The command a writer is told checks its tree: the toolchain's own, or a repository's build. */
|
|
647
664
|
export const checkCommand = (chain) =>
|
|
@@ -674,10 +691,10 @@ function suiteFiles(dir, suite, chain) {
|
|
|
674
691
|
const under = (folder) =>
|
|
675
692
|
listFiles(join(dir, folder))
|
|
676
693
|
.filter((path) => matches.test(path.split("/").at(-1)))
|
|
677
|
-
.map((path) => join(folder, path));
|
|
694
|
+
.map((path) => posix.join(folder, path));
|
|
678
695
|
// Directly in publicDir: a hidden file named after the spec is not public.
|
|
679
696
|
const ownPublic = (path) =>
|
|
680
|
-
chain.slug !== null && dirname(path) === chain.publicDir && chain.isPublicName(path.split("/").at(-1));
|
|
697
|
+
chain.slug !== null && posix.dirname(path) === chain.publicDir && chain.isPublicName(path.split("/").at(-1));
|
|
681
698
|
const hidden = (path) => chain.hiddenDir !== undefined && path.startsWith(`${chain.hiddenDir}/`);
|
|
682
699
|
if (suite === "public") return under(chain.publicDir).filter(ownPublic);
|
|
683
700
|
if (suite === "private") return under(chain.hiddenDir);
|
|
@@ -765,10 +782,10 @@ const codeList = (items) => items.map((item) => `\`${item}\``).join(", ");
|
|
|
765
782
|
*/
|
|
766
783
|
function importOf(chain, from) {
|
|
767
784
|
if (chain.runner === "pytest") {
|
|
768
|
-
const root = join(chain.package, chain.pythonPath[0]);
|
|
769
|
-
return relative(root, chain.seam[0]).replace(/(\/__init__)?\.py$/, "").split("/").join(".");
|
|
785
|
+
const root = posix.join(chain.package, chain.pythonPath[0]);
|
|
786
|
+
return posix.relative(root, chain.seam[0]).replace(/(\/__init__)?\.py$/, "").split("/").join(".");
|
|
770
787
|
}
|
|
771
|
-
const path = relative(from, chain.seam[0]).replace(/\.[cm]?[jt]sx?$/, chain.runner === "node-test" ? ".js" : "");
|
|
788
|
+
const path = posix.relative(from, chain.seam[0]).replace(/\.[cm]?[jt]sx?$/, chain.runner === "node-test" ? ".js" : "");
|
|
772
789
|
return path.startsWith(".") ? path : `./${path}`;
|
|
773
790
|
}
|
|
774
791
|
function render(text, chain, name) {
|
|
@@ -846,13 +863,13 @@ export async function pool(items, limit, fn) {
|
|
|
846
863
|
return results;
|
|
847
864
|
}
|
|
848
865
|
|
|
849
|
-
/** Every file under `dir`, relative and sorted, skipping the named top-level entries. */
|
|
866
|
+
/** Every file under `dir`, relative with "/" on every OS and sorted, skipping the named top-level entries. */
|
|
850
867
|
export function listFiles(dir, skip = ["node_modules", ".git"]) {
|
|
851
868
|
const out = [];
|
|
852
869
|
const walk = (at) => {
|
|
853
870
|
for (const entry of readdirSync(at, { withFileTypes: true })) {
|
|
854
871
|
const path = join(at, entry.name);
|
|
855
|
-
const rel = relative(dir, path);
|
|
872
|
+
const rel = slash(relative(dir, path));
|
|
856
873
|
if (skip.includes(rel)) continue;
|
|
857
874
|
if (entry.isDirectory()) walk(path);
|
|
858
875
|
else out.push(rel);
|
|
@@ -933,6 +950,7 @@ export function copy(from, to) {
|
|
|
933
950
|
*/
|
|
934
951
|
function clone(from, to) {
|
|
935
952
|
mkdirSync(to, { recursive: true });
|
|
953
|
+
// Windows has neither: a plain copy.
|
|
936
954
|
const flags = { darwin: ["-Rc"], linux: ["-R", "--reflink=auto"] }[process.platform];
|
|
937
955
|
if (flags !== undefined && spawnSync("cp", [...flags, `${from}/.`, to]).status === 0) return;
|
|
938
956
|
copy(from, to);
|
|
@@ -967,16 +985,16 @@ const INSTALL = {
|
|
|
967
985
|
*/
|
|
968
986
|
function pythonTool(name, version) {
|
|
969
987
|
const dir = join(WORK_ROOT, "tools", `${name}-${version}`);
|
|
970
|
-
const bin = join(dir, "
|
|
988
|
+
const bin = join(dir, venvBin(name, "."));
|
|
971
989
|
if (existsSync(bin)) return bin;
|
|
972
990
|
const partial = `${dir}.partial-${process.pid}`;
|
|
973
991
|
rmSync(partial, { recursive: true, force: true });
|
|
974
992
|
mkdirSync(dirname(partial), { recursive: true });
|
|
975
993
|
for (const argv of [
|
|
976
994
|
[PYTHON, "-m", "venv", partial],
|
|
977
|
-
[join(partial, "
|
|
995
|
+
[join(partial, venvBin("python", ".")), "-m", "pip", "install", "-q", "--disable-pip-version-check", `${name}==${version}`],
|
|
978
996
|
]) {
|
|
979
|
-
const out = spawnSync(argv[0], argv.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
|
997
|
+
const out = spawnSync(argv[0], argv.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, windowsHide: true });
|
|
980
998
|
if (out.status !== 0) {
|
|
981
999
|
rmSync(partial, { recursive: true, force: true });
|
|
982
1000
|
throw new Error(`installing ${name} ${version} failed:\n${`${out.error ?? ""}${out.stdout}${out.stderr}`.slice(-4000)}`);
|
|
@@ -1037,7 +1055,7 @@ export function prepareBase(config) {
|
|
|
1037
1055
|
const partial = mkdtempSync(`${dest}.partial-`);
|
|
1038
1056
|
const started = Date.now();
|
|
1039
1057
|
const step = (label, bin, argv, options = {}) => {
|
|
1040
|
-
const out =
|
|
1058
|
+
const out = spawnCommand(bin, argv, { cwd: partial, encoding: "utf8", maxBuffer: 256 * 1024 * 1024, ...options });
|
|
1041
1059
|
if (out.status === 0) return out;
|
|
1042
1060
|
rmSync(partial, { recursive: true, force: true });
|
|
1043
1061
|
throw new Error(`${label} failed:\n${`${out.error ?? ""}${out.stdout}${out.stderr}`.slice(-4000)}`);
|
|
@@ -1070,7 +1088,9 @@ export function prepareBase(config) {
|
|
|
1070
1088
|
if (script === undefined) continue;
|
|
1071
1089
|
log(`${basename(config.repo)} ${config.base.slice(0, 12)}: build ${folder}`);
|
|
1072
1090
|
const cwd = join(partial, folder);
|
|
1073
|
-
|
|
1091
|
+
// Through the OS shell, as npm runs the package's own build script.
|
|
1092
|
+
step(`build of ${folder}`, script, [], { cwd, env: withBins(cwd), shell: true, windowsHide: true });
|
|
1093
|
+
|
|
1074
1094
|
}
|
|
1075
1095
|
// Build output is ignored by the repository's .gitignore, so a built base
|
|
1076
1096
|
// keeps the plain base's commit.
|
|
@@ -1272,25 +1292,50 @@ export function verdict({ refused = [], publicCount, hiddenCount, regressions =
|
|
|
1272
1292
|
return hiddenCount.total > 0 && hiddenCount.passed === hiddenCount.total ? "passed" : "overfit";
|
|
1273
1293
|
}
|
|
1274
1294
|
|
|
1275
|
-
|
|
1295
|
+
/**
|
|
1296
|
+
* The claude CLI, or null: CLAUDE_BIN, claude on PATH, the newest VS Code
|
|
1297
|
+
* extension's bundled binary. On Windows npm's `claude.cmd` shim is
|
|
1298
|
+
* replaced by the package it starts (its native claude.exe, or cli.js on
|
|
1299
|
+
* this Node), since a shim runs only through cmd.exe and a prompt does not
|
|
1300
|
+
* survive its quoting.
|
|
1301
|
+
*/
|
|
1302
|
+
export function findClaude() {
|
|
1276
1303
|
if (process.env.CLAUDE_BIN) return process.env.CLAUDE_BIN;
|
|
1277
|
-
const onPath =
|
|
1278
|
-
if (onPath
|
|
1304
|
+
const onPath = which("claude");
|
|
1305
|
+
if (onPath !== null && !/\.(cmd|bat|ps1)$/i.test(onPath)) return onPath;
|
|
1306
|
+
if (onPath !== null) {
|
|
1307
|
+
const pkg = join(dirname(onPath), "node_modules", "@anthropic-ai", "claude-code");
|
|
1308
|
+
const shipped = [join(pkg, "bin", "claude.exe"), join(pkg, "cli.js")].find((path) => existsSync(path));
|
|
1309
|
+
if (shipped !== undefined) return shipped;
|
|
1310
|
+
}
|
|
1279
1311
|
const extensions = join(homedir(), ".vscode", "extensions");
|
|
1280
1312
|
const bundled = existsSync(extensions)
|
|
1281
1313
|
? readdirSync(extensions)
|
|
1282
1314
|
.filter((name) => name.startsWith("anthropic.claude-code-"))
|
|
1283
|
-
.map((name) => join(extensions, name, "resources", "native-binary", "claude"))
|
|
1315
|
+
.map((name) => join(extensions, name, "resources", "native-binary", WINDOWS ? "claude.exe" : "claude"))
|
|
1284
1316
|
.filter((path) => existsSync(path))
|
|
1285
1317
|
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
|
|
1286
1318
|
: [];
|
|
1287
|
-
|
|
1288
|
-
throw new Error("No claude CLI: set CLAUDE_BIN, put claude on PATH, or install the VS Code extension.");
|
|
1289
|
-
return bundled.at(-1);
|
|
1319
|
+
return bundled.at(-1) ?? null;
|
|
1290
1320
|
}
|
|
1321
|
+
export function claudeBin() {
|
|
1322
|
+
const found = findClaude();
|
|
1323
|
+
if (found === null)
|
|
1324
|
+
throw new Error("No claude CLI: set CLAUDE_BIN, put claude on PATH, or install the VS Code extension. Or pick another backend: INDEPENDENT_TESTS_BACKEND=codex-cli or anthropic-api.");
|
|
1325
|
+
return found;
|
|
1326
|
+
}
|
|
1327
|
+
/** How to start `claude` with `argv`: [bin, argv]; a cli.js runs on this Node. */
|
|
1328
|
+
const claudeCommand = (argv) => {
|
|
1329
|
+
const bin = claudeBin();
|
|
1330
|
+
return /\.[cm]?js$/.test(bin) ? [process.execPath, [bin, ...argv]] : [bin, argv];
|
|
1331
|
+
};
|
|
1291
1332
|
|
|
1292
|
-
/**
|
|
1293
|
-
|
|
1333
|
+
/**
|
|
1334
|
+
* Absolute path as a permission rule takes it: `//` and the path, which on
|
|
1335
|
+
* Windows Claude Code matches in POSIX form (C:\Users\me as //c/Users/me).
|
|
1336
|
+
*/
|
|
1337
|
+
const posixForm = (path) => (WINDOWS ? `/${path[0].toLowerCase()}${slash(path.slice(2))}` : path);
|
|
1338
|
+
const rulePath = (path) => `/${posixForm(path)}/**`;
|
|
1294
1339
|
|
|
1295
1340
|
/** The workdirs that exist now, other than `own` and the judge root. */
|
|
1296
1341
|
const siblings = (own) =>
|
|
@@ -1305,6 +1350,11 @@ const resultsFile = () => (PRIVATE.some((folder) => inside(RESULTS, folder)) ? [
|
|
|
1305
1350
|
* A spec whose boundary is a service (`boundaryOf`) needs its sessions to
|
|
1306
1351
|
* listen on and connect to 127.0.0.1, so its sandbox allows local binding;
|
|
1307
1352
|
* every other session's settings are unchanged, still with no network.
|
|
1353
|
+
*
|
|
1354
|
+
* On Windows Claude Code has no OS sandbox to require: the sandbox is asked
|
|
1355
|
+
* for where it exists and not required, so Bash there is held by nothing
|
|
1356
|
+
* but the transcript audit (`peeks`, `audit`), and `claudeCli` records
|
|
1357
|
+
* `sandboxed: false` and says so once.
|
|
1308
1358
|
*/
|
|
1309
1359
|
export function sessionSettings(cwd, chain) {
|
|
1310
1360
|
const denied = [...PRIVATE, TRANSCRIPTS, JUDGE_ROOT, ...siblings(cwd)];
|
|
@@ -1312,7 +1362,7 @@ export function sessionSettings(cwd, chain) {
|
|
|
1312
1362
|
return {
|
|
1313
1363
|
sandbox: {
|
|
1314
1364
|
enabled: true,
|
|
1315
|
-
failIfUnavailable:
|
|
1365
|
+
failIfUnavailable: OS_SANDBOX,
|
|
1316
1366
|
allowUnsandboxedCommands: false,
|
|
1317
1367
|
filesystem: {
|
|
1318
1368
|
denyRead: [...PRIVATE, ...resultsFile(), TRANSCRIPTS, WORK_ROOT],
|
|
@@ -1325,7 +1375,7 @@ export function sessionSettings(cwd, chain) {
|
|
|
1325
1375
|
"WebFetch",
|
|
1326
1376
|
"WebSearch",
|
|
1327
1377
|
...denied.flatMap((path) => fileTools.map((tool) => `${tool}(${rulePath(path)})`)),
|
|
1328
|
-
...resultsFile().flatMap((path) => fileTools.map((tool) => `${tool}(/${path})`)),
|
|
1378
|
+
...resultsFile().flatMap((path) => fileTools.map((tool) => `${tool}(/${posixForm(path)})`)),
|
|
1329
1379
|
],
|
|
1330
1380
|
},
|
|
1331
1381
|
};
|
|
@@ -1334,14 +1384,16 @@ export function sessionSettings(cwd, chain) {
|
|
|
1334
1384
|
/**
|
|
1335
1385
|
* Every session of every script here takes a slot first: at most
|
|
1336
1386
|
* INDEPENDENT_TESTS_MAX_SESSIONS at once (default 3), and none starts while
|
|
1337
|
-
*
|
|
1338
|
-
*
|
|
1339
|
-
*
|
|
1387
|
+
* less than INDEPENDENT_TESTS_MIN_FREE percent of memory is free (default
|
|
1388
|
+
* 25): macOS's `memory_pressure`, elsewhere the OS's available memory. A
|
|
1389
|
+
* slot is a file under WORK_ROOT/slots holding its owner's pid; a dead
|
|
1390
|
+
* owner's slot is taken back.
|
|
1340
1391
|
*/
|
|
1341
1392
|
const MAX_SESSIONS = Number(process.env.INDEPENDENT_TESTS_MAX_SESSIONS ?? 3);
|
|
1342
1393
|
const MIN_FREE = Number(process.env.INDEPENDENT_TESTS_MIN_FREE ?? 25);
|
|
1343
1394
|
const SLOTS = join(WORK_ROOT, "slots");
|
|
1344
1395
|
export function freeMemoryPercent() {
|
|
1396
|
+
if (process.platform !== "darwin") return Math.round((freemem() / totalmem()) * 100);
|
|
1345
1397
|
const out = spawnSync("memory_pressure", [], { encoding: "utf8" }).stdout ?? "";
|
|
1346
1398
|
const match = /free percentage:\s*(\d+)%/.exec(out);
|
|
1347
1399
|
return match === null ? 100 : Number(match[1]);
|
|
@@ -1383,16 +1435,46 @@ export const BACKENDS = ["claude-cli", "anthropic-api", "codex-cli", "exec"];
|
|
|
1383
1435
|
/**
|
|
1384
1436
|
* A session's backend: `backend` when the caller names one (a run's
|
|
1385
1437
|
* --backend, the referee's), else for the writers and the reference
|
|
1386
|
-
* INDEPENDENT_TESTS_BACKEND, then toolchain.json `backend
|
|
1387
|
-
* implementing agent (`kind: "agent"`)
|
|
1388
|
-
* caller names
|
|
1438
|
+
* INDEPENDENT_TESTS_BACKEND, then toolchain.json `backend`, then
|
|
1439
|
+
* `defaultBackend`. The implementing agent (`kind: "agent"`) skips the
|
|
1440
|
+
* writers' choice unless its caller names one: a spec's writer backend is
|
|
1441
|
+
* not its implementer.
|
|
1389
1442
|
*/
|
|
1390
1443
|
export function backendOf({ backend, kind, chain } = {}) {
|
|
1391
1444
|
const chosen =
|
|
1392
|
-
backend ?? (kind === "agent" ? undefined : (process.env.INDEPENDENT_TESTS_BACKEND || chain?.backend)) ??
|
|
1445
|
+
backend ?? (kind === "agent" ? undefined : (process.env.INDEPENDENT_TESTS_BACKEND || chain?.backend)) ?? defaultBackend();
|
|
1393
1446
|
if (!BACKENDS.includes(chosen)) throw new Error(`No backend ${chosen} (${BACKENDS.join(", ")})`);
|
|
1394
1447
|
return chosen;
|
|
1395
1448
|
}
|
|
1449
|
+
/** Whether the person has a Codex login (CODEX_HOME or ~/.codex) or an OpenAI key for it. */
|
|
1450
|
+
const codexSignedIn = () =>
|
|
1451
|
+
existsSync(join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json")) ||
|
|
1452
|
+
Boolean(process.env.OPENAI_API_KEY || process.env.CODEX_API_KEY);
|
|
1453
|
+
let defaultChosen;
|
|
1454
|
+
/**
|
|
1455
|
+
* The backend a session runs on when nothing names one: whichever of the
|
|
1456
|
+
* person's accounts is there, in this order. claude-cli when a claude CLI
|
|
1457
|
+
* is found (`findClaude`), else codex-cli on their Codex login, else
|
|
1458
|
+
* anthropic-api on ANTHROPIC_API_KEY. None of them is an error that names
|
|
1459
|
+
* all three. Said once when it is not claude-cli.
|
|
1460
|
+
*/
|
|
1461
|
+
export function defaultBackend() {
|
|
1462
|
+
if (defaultChosen !== undefined) return defaultChosen;
|
|
1463
|
+
defaultChosen =
|
|
1464
|
+
findClaude() !== null
|
|
1465
|
+
? "claude-cli"
|
|
1466
|
+
: codexSignedIn()
|
|
1467
|
+
? "codex-cli"
|
|
1468
|
+
: process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN
|
|
1469
|
+
? "anthropic-api"
|
|
1470
|
+
: null;
|
|
1471
|
+
if (defaultChosen === null)
|
|
1472
|
+
throw new Error(
|
|
1473
|
+
"No agent to run sessions on. Any one of: Claude Code signed in (`claude`, or CLAUDE_BIN), a Codex login (`codex login`, or OPENAI_API_KEY), or ANTHROPIC_API_KEY. INDEPENDENT_TESTS_BACKEND picks one when you have several.",
|
|
1474
|
+
);
|
|
1475
|
+
if (defaultChosen !== "claude-cli") log(`no claude CLI found: sessions run on ${defaultChosen} (INDEPENDENT_TESTS_BACKEND picks another)`);
|
|
1476
|
+
return defaultChosen;
|
|
1477
|
+
}
|
|
1396
1478
|
/**
|
|
1397
1479
|
* One writer, reference, referee or agent session in `cwd` on its backend
|
|
1398
1480
|
* (`backendOf`). Every backend writes the same transcript shape, Claude
|
|
@@ -1420,9 +1502,12 @@ export async function session(options) {
|
|
|
1420
1502
|
* directories from `lsof -d cwd` on macOS, /proc on Linux; command lines
|
|
1421
1503
|
* from `ps`. Never this process or its own group. The session's own
|
|
1422
1504
|
* process group goes first (`claudeCli` and bin/backends.mjs start each
|
|
1423
|
-
* session as a group leader and kill that group when it closes).
|
|
1505
|
+
* session as a group leader and kill that group when it closes). Windows
|
|
1506
|
+
* has no process groups or readable working directories: there the
|
|
1507
|
+
* session's process tree is all that is killed (`killTree`).
|
|
1424
1508
|
*/
|
|
1425
1509
|
export function killStrays(cwd) {
|
|
1510
|
+
if (WINDOWS) return [];
|
|
1426
1511
|
let root;
|
|
1427
1512
|
try {
|
|
1428
1513
|
root = realpathSync(cwd);
|
|
@@ -1468,12 +1553,16 @@ export function killStrays(cwd) {
|
|
|
1468
1553
|
if (strays.length > 0) log(`killed ${strays.length} process(es) the session left running in ${root}`);
|
|
1469
1554
|
return strays;
|
|
1470
1555
|
}
|
|
1471
|
-
/**
|
|
1472
|
-
export
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1556
|
+
/** Kills a session's process group, or its tree on Windows (its leader `pid` was spawned with `GROUP`); quiet when it is gone. */
|
|
1557
|
+
export const killGroup = killTree;
|
|
1558
|
+
/** Said once per process when a session runs without an OS sandbox. */
|
|
1559
|
+
let warnedUnsandboxed = false;
|
|
1560
|
+
export function warnUnsandboxed() {
|
|
1561
|
+
if (warnedUnsandboxed) return;
|
|
1562
|
+
warnedUnsandboxed = true;
|
|
1563
|
+
log(
|
|
1564
|
+
`${process.platform}: no OS sandbox for sessions here. File tools are held by deny rules; Bash is not, so a read of the hidden suite is caught by the transcript audit, not prevented. For an enforced sandbox, run under WSL2, macOS or Linux.`,
|
|
1565
|
+
);
|
|
1477
1566
|
}
|
|
1478
1567
|
|
|
1479
1568
|
/**
|
|
@@ -1515,7 +1604,9 @@ export async function claudeCli({ cwd, text, transcript, kind, resume, model, ma
|
|
|
1515
1604
|
const { ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ...env } = process.env;
|
|
1516
1605
|
// Its own process group, killed when it closes (and if this process exits first): a
|
|
1517
1606
|
// server a session left in the background goes with it (`killStrays` catches the rest).
|
|
1518
|
-
|
|
1607
|
+
if (!OS_SANDBOX) warnUnsandboxed();
|
|
1608
|
+
const [bin, args] = claudeCommand(argv);
|
|
1609
|
+
const child = spawn(bin, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"], ...GROUP });
|
|
1519
1610
|
const reap = () => killGroup(child.pid);
|
|
1520
1611
|
process.once("exit", reap);
|
|
1521
1612
|
// Once the CLI itself exits, anything left in its group is a leftover that may hold its pipes open.
|
|
@@ -1567,6 +1658,8 @@ export async function claudeCli({ cwd, text, transcript, kind, resume, model, ma
|
|
|
1567
1658
|
apiError: result?.api_error ?? null,
|
|
1568
1659
|
finalText: typeof result?.result === "string" ? result.result : "",
|
|
1569
1660
|
stderr: stderr.slice(-2000),
|
|
1661
|
+
/** False where no OS sandbox held the session's Bash (Windows). */
|
|
1662
|
+
sandboxed: OS_SANDBOX,
|
|
1570
1663
|
});
|
|
1571
1664
|
});
|
|
1572
1665
|
});
|
|
@@ -1603,7 +1696,7 @@ export function scopeOf(chain, changed, read) {
|
|
|
1603
1696
|
: path.startsWith(`${chain.hiddenDir}/`)
|
|
1604
1697
|
? "the hidden suite's folder"
|
|
1605
1698
|
: // Added is never an edit: a run without the public suite may name its own tests so.
|
|
1606
|
-
dirname(path) === chain.publicDir && chain.isPublicName(name) && item.change !== "added"
|
|
1699
|
+
posix.dirname(path) === chain.publicDir && chain.isPublicName(name) && item.change !== "added"
|
|
1607
1700
|
? "the public suite"
|
|
1608
1701
|
: chain.isTest(path) && item.change !== "added"
|
|
1609
1702
|
? "an existing test"
|
|
@@ -1634,7 +1727,7 @@ function startOf(chain) {
|
|
|
1634
1727
|
const stubbed = (path) =>
|
|
1635
1728
|
chain.seam.includes(path) ? join(spec, "skeleton", path) : null;
|
|
1636
1729
|
const publicFile = (path) =>
|
|
1637
|
-
dirname(path) === chain.publicDir ? join(spec, "tests", "public", basename(path)) : null;
|
|
1730
|
+
posix.dirname(path) === chain.publicDir ? join(spec, "tests", "public", posix.basename(path)) : null;
|
|
1638
1731
|
return (path) => {
|
|
1639
1732
|
for (const file of [stubbed(path), publicFile(path)])
|
|
1640
1733
|
if (file !== null && existsSync(file)) return readFileSync(file);
|
|
@@ -1667,7 +1760,7 @@ export function checkoutChanges(chain, checkout, ref) {
|
|
|
1667
1760
|
...listed("ls-files", "-z", "--others", "--exclude-standard"),
|
|
1668
1761
|
]
|
|
1669
1762
|
: listed("diff", "--name-only", "-z", "--no-renames", chain.base, ref);
|
|
1670
|
-
const publicPaths = new Set(listFiles(join(specDir(chain.slug), "tests", "public")).map((name) => join(chain.publicDir, name)));
|
|
1763
|
+
const publicPaths = new Set(listFiles(join(specDir(chain.slug), "tests", "public")).map((name) => posix.join(chain.publicDir, name)));
|
|
1671
1764
|
const paths = [...new Set([...named, ...chain.seam, ...publicPaths])].sort();
|
|
1672
1765
|
const start = startOf(chain);
|
|
1673
1766
|
const now = (path) => {
|
|
@@ -1770,7 +1863,7 @@ export function audit(transcript, chain, since) {
|
|
|
1770
1863
|
/(^|[\s/])(specs|runs)(\/|\s|$)/.test(reach({ name: "Bash", input }));
|
|
1771
1864
|
const searchedOver = (input) =>
|
|
1772
1865
|
typeof input.path === "string" &&
|
|
1773
|
-
[ROOT, HOME, SPECS, join(SPECS, slug), RUNS].includes(input.path.replace(/\/$/, ""));
|
|
1866
|
+
[ROOT, HOME, SPECS, join(SPECS, slug), RUNS].map(comparable).includes(comparable(input.path).replace(/\/$/, ""));
|
|
1774
1867
|
const reach = (block) => {
|
|
1775
1868
|
const input = block.input ?? {};
|
|
1776
1869
|
if (block.name === "Bash") return (input.command ?? "").replace(/<<-?\s*(['"]?)(\w+)\1[\s\S]*?\n\2(?=\s|$)/g, "");
|
|
@@ -1779,8 +1872,8 @@ export function audit(transcript, chain, since) {
|
|
|
1779
1872
|
};
|
|
1780
1873
|
return scanTranscript(transcript, since, (block) => {
|
|
1781
1874
|
const input = block.input ?? {};
|
|
1782
|
-
const text = reach(block);
|
|
1783
|
-
const hit = names.find((name) => text.includes(name));
|
|
1875
|
+
const text = comparable(reach(block));
|
|
1876
|
+
const hit = names.find((name) => text.includes(comparable(name)));
|
|
1784
1877
|
if (hit !== undefined) return hit;
|
|
1785
1878
|
if (block.name === "Bash" && sweep(input)) return "a recursive search over specs/ or runs/";
|
|
1786
1879
|
if ((block.name === "Grep" || block.name === "Glob") && searchedOver(input)) return `a search over ${input.path}`;
|
|
@@ -1819,10 +1912,13 @@ function scanTranscript(transcript, since, match) {
|
|
|
1819
1912
|
*/
|
|
1820
1913
|
export function peeks(transcript, own, names = []) {
|
|
1821
1914
|
if (!existsSync(transcript)) return [];
|
|
1822
|
-
|
|
1915
|
+
// Compared as `comparable` has them: on Windows a path may come as C:\, C:/ or /c/.
|
|
1916
|
+
const patterns = [...PRIVATE, TRANSCRIPTS, ".claude/projects", ...names].map(comparable);
|
|
1917
|
+
const work = comparable(WORK_ROOT);
|
|
1918
|
+
const mine = comparable(own);
|
|
1823
1919
|
// The session's own project folder, where the CLI keeps a long tool
|
|
1824
1920
|
// output and tells the session to read it back: its own, not a peek.
|
|
1825
|
-
const ownProject = join(TRANSCRIPTS, own.replace(/[^a-zA-Z0-9]/g, "-"));
|
|
1921
|
+
const ownProject = comparable(join(TRANSCRIPTS, own.replace(/[^a-zA-Z0-9]/g, "-")));
|
|
1826
1922
|
const found = [];
|
|
1827
1923
|
for (const line of readFileSync(transcript, "utf8").split("\n")) {
|
|
1828
1924
|
if (!line.includes('"tool_use"')) continue;
|
|
@@ -1834,13 +1930,13 @@ export function peeks(transcript, own, names = []) {
|
|
|
1834
1930
|
}
|
|
1835
1931
|
for (const block of event.message?.content ?? []) {
|
|
1836
1932
|
if (block.type !== "tool_use") continue;
|
|
1837
|
-
const input = JSON.stringify(block.input ?? {}).replaceAll(`${ownProject}/`, "");
|
|
1933
|
+
const input = comparable(JSON.stringify(block.input ?? {})).replaceAll(`${ownProject}/`, "");
|
|
1838
1934
|
// A tail that does not start with "/" is a path beside WORK_ROOT, not
|
|
1839
1935
|
// under it.
|
|
1840
1936
|
const outside = input
|
|
1841
|
-
.split(
|
|
1937
|
+
.split(work)
|
|
1842
1938
|
.slice(1)
|
|
1843
|
-
.some((tail) => tail.startsWith("/") && !`${
|
|
1939
|
+
.some((tail) => tail.startsWith("/") && !`${work}${tail}`.startsWith(mine));
|
|
1844
1940
|
const hit = patterns.find((pattern) => input.includes(pattern));
|
|
1845
1941
|
if (hit !== undefined || outside)
|
|
1846
1942
|
found.push({ tool: block.name, match: hit ?? "another workdir", input: input.slice(0, 300) });
|
|
@@ -1851,6 +1947,7 @@ export function peeks(transcript, own, names = []) {
|
|
|
1851
1947
|
|
|
1852
1948
|
/** Removes a workdir, which lives under WORK_ROOT. */
|
|
1853
1949
|
export function discard(dir) {
|
|
1854
|
-
if (!dir.startsWith(`${WORK_ROOT}
|
|
1950
|
+
if (!dir.startsWith(`${WORK_ROOT}${sep}`)) throw new Error(`Not a workdir: ${dir}`);
|
|
1951
|
+
|
|
1855
1952
|
rmSync(dir, { recursive: true, force: true });
|
|
1856
1953
|
}
|