autor3search-typescript 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +210 -0
- package/dist/benchproto/types.d.ts +32 -0
- package/dist/benchproto/types.js +28 -0
- package/dist/benchproto/types.js.map +1 -0
- package/dist/benchrun/child.d.ts +1 -0
- package/dist/benchrun/child.js +222 -0
- package/dist/benchrun/child.js.map +1 -0
- package/dist/benchrun/invoke.d.ts +61 -0
- package/dist/benchrun/invoke.js +152 -0
- package/dist/benchrun/invoke.js.map +1 -0
- package/dist/cli/cmd-baseline.d.ts +15 -0
- package/dist/cli/cmd-baseline.js +292 -0
- package/dist/cli/cmd-baseline.js.map +1 -0
- package/dist/cli/cmd-doctor.d.ts +11 -0
- package/dist/cli/cmd-doctor.js +26 -0
- package/dist/cli/cmd-doctor.js.map +1 -0
- package/dist/cli/cmd-eval.d.ts +30 -0
- package/dist/cli/cmd-eval.js +173 -0
- package/dist/cli/cmd-eval.js.map +1 -0
- package/dist/cli/cmd-init.d.ts +16 -0
- package/dist/cli/cmd-init.js +343 -0
- package/dist/cli/cmd-init.js.map +1 -0
- package/dist/cli/cmd-profile.d.ts +11 -0
- package/dist/cli/cmd-profile.js +140 -0
- package/dist/cli/cmd-profile.js.map +1 -0
- package/dist/cli/cmd-report.d.ts +15 -0
- package/dist/cli/cmd-report.js +70 -0
- package/dist/cli/cmd-report.js.map +1 -0
- package/dist/cli/cmd-status.d.ts +11 -0
- package/dist/cli/cmd-status.js +143 -0
- package/dist/cli/cmd-status.js.map +1 -0
- package/dist/cli/cmd-stop.d.ts +15 -0
- package/dist/cli/cmd-stop.js +143 -0
- package/dist/cli/cmd-stop.js.map +1 -0
- package/dist/cli/main.d.ts +31 -0
- package/dist/cli/main.js +133 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/runctx.d.ts +29 -0
- package/dist/cli/runctx.js +49 -0
- package/dist/cli/runctx.js.map +1 -0
- package/dist/cli/speedup.d.ts +14 -0
- package/dist/cli/speedup.js +25 -0
- package/dist/cli/speedup.js.map +1 -0
- package/dist/config/load.d.ts +6 -0
- package/dist/config/load.js +172 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/schema.d.ts +42 -0
- package/dist/config/schema.js +25 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/discover/benchmarks.d.ts +26 -0
- package/dist/discover/benchmarks.js +88 -0
- package/dist/discover/benchmarks.js.map +1 -0
- package/dist/discover/files.d.ts +23 -0
- package/dist/discover/files.js +82 -0
- package/dist/discover/files.js.map +1 -0
- package/dist/doctor/doctor.d.ts +56 -0
- package/dist/doctor/doctor.js +255 -0
- package/dist/doctor/doctor.js.map +1 -0
- package/dist/freeze/freeze.d.ts +23 -0
- package/dist/freeze/freeze.js +107 -0
- package/dist/freeze/freeze.js.map +1 -0
- package/dist/freeze/manifest.d.ts +6 -0
- package/dist/freeze/manifest.js +9 -0
- package/dist/freeze/manifest.js.map +1 -0
- package/dist/freeze/safepath.d.ts +8 -0
- package/dist/freeze/safepath.js +22 -0
- package/dist/freeze/safepath.js.map +1 -0
- package/dist/gitx/git.d.ts +87 -0
- package/dist/gitx/git.js +235 -0
- package/dist/gitx/git.js.map +1 -0
- package/dist/measure/interleave.d.ts +32 -0
- package/dist/measure/interleave.js +42 -0
- package/dist/measure/interleave.js.map +1 -0
- package/dist/pipeline/eval.d.ts +67 -0
- package/dist/pipeline/eval.js +495 -0
- package/dist/pipeline/eval.js.map +1 -0
- package/dist/pm/detect.d.ts +17 -0
- package/dist/pm/detect.js +131 -0
- package/dist/pm/detect.js.map +1 -0
- package/dist/profile/profile.d.ts +53 -0
- package/dist/profile/profile.js +105 -0
- package/dist/profile/profile.js.map +1 -0
- package/dist/results/results.d.ts +65 -0
- package/dist/results/results.js +214 -0
- package/dist/results/results.js.map +1 -0
- package/dist/runner/exec.d.ts +69 -0
- package/dist/runner/exec.js +185 -0
- package/dist/runner/exec.js.map +1 -0
- package/dist/scope/scope.d.ts +20 -0
- package/dist/scope/scope.js +70 -0
- package/dist/scope/scope.js.map +1 -0
- package/dist/state/baseline.d.ts +35 -0
- package/dist/state/baseline.js +38 -0
- package/dist/state/baseline.js.map +1 -0
- package/dist/state/home.d.ts +9 -0
- package/dist/state/home.js +72 -0
- package/dist/state/home.js.map +1 -0
- package/dist/state/lock.d.ts +11 -0
- package/dist/state/lock.js +89 -0
- package/dist/state/lock.js.map +1 -0
- package/dist/state/runnaming.d.ts +27 -0
- package/dist/state/runnaming.js +47 -0
- package/dist/state/runnaming.js.map +1 -0
- package/dist/state/stop.d.ts +28 -0
- package/dist/state/stop.js +59 -0
- package/dist/state/stop.js.map +1 -0
- package/dist/stats/delta.d.ts +29 -0
- package/dist/stats/delta.js +41 -0
- package/dist/stats/delta.js.map +1 -0
- package/dist/stats/geomean.d.ts +4 -0
- package/dist/stats/geomean.js +21 -0
- package/dist/stats/geomean.js.map +1 -0
- package/dist/stats/mannwhitney.d.ts +27 -0
- package/dist/stats/mannwhitney.js +158 -0
- package/dist/stats/mannwhitney.js.map +1 -0
- package/dist/verdict/verdict.d.ts +46 -0
- package/dist/verdict/verdict.js +102 -0
- package/dist/verdict/verdict.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/package.json +35 -0
- package/templates/program.md +131 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spawns the measurement child (child.ts / child.js) for one benchmark in
|
|
3
|
+
* one directory, and turns its outcome into a `BenchResult` that never
|
|
4
|
+
* throws for a benchmark-level failure -- a timeout, a crash, or a child
|
|
5
|
+
* that writes nothing is reported as `ok: false` with a diagnosis, because
|
|
6
|
+
* the pipeline treats that as a CRASH verdict for the benchmark rather than
|
|
7
|
+
* a harness bug.
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync } from 'node:fs';
|
|
10
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
11
|
+
import { tmpdir } from 'node:os';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { createRequire } from 'node:module';
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
15
|
+
import { parseBenchResult } from '../benchproto/types.js';
|
|
16
|
+
import { run, tail } from '../runner/exec.js';
|
|
17
|
+
const require = createRequire(import.meta.url);
|
|
18
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
/**
|
|
20
|
+
* Picks the measurement child from a directory, preferring the built
|
|
21
|
+
* `child.js` and falling back to the source `child.ts`.
|
|
22
|
+
*
|
|
23
|
+
* Under a real install (`dist/`), only `child.js` exists next to this
|
|
24
|
+
* module -- that is the branch that actually runs for an installed user.
|
|
25
|
+
* Under Vitest, `import.meta.url` resolves to `src/benchrun/invoke.ts`,
|
|
26
|
+
* where only `child.ts` exists -- there is no build step before the test
|
|
27
|
+
* run. Either way the spawn below runs through the tsx loader, so a `.ts`
|
|
28
|
+
* path works identically to a `.js` one; this just picks whichever file is
|
|
29
|
+
* actually on disk.
|
|
30
|
+
*
|
|
31
|
+
* Pulled apart from `resolveChild` (a thin wrapper below) so both branches
|
|
32
|
+
* can be exercised directly with an injected existence check, since only
|
|
33
|
+
* one of the two ever exists in any single checkout.
|
|
34
|
+
*/
|
|
35
|
+
export function pickChildPath(dir, exists) {
|
|
36
|
+
const asJs = path.join(dir, 'child.js');
|
|
37
|
+
if (exists(asJs))
|
|
38
|
+
return asJs;
|
|
39
|
+
return path.join(dir, 'child.ts');
|
|
40
|
+
}
|
|
41
|
+
function resolveChild() {
|
|
42
|
+
return pickChildPath(HERE, existsSync);
|
|
43
|
+
}
|
|
44
|
+
export const CHILD = resolveChild();
|
|
45
|
+
/**
|
|
46
|
+
* The tsx ESM loader, resolved from OUR dependencies.
|
|
47
|
+
*
|
|
48
|
+
* The measured repository does not depend on us, so resolving this relative
|
|
49
|
+
* to the measured directory would work in our own fixtures and fail on
|
|
50
|
+
* every real user's repository. Resolving it here, from this module's own
|
|
51
|
+
* `import.meta.url`, always finds it in *our* node_modules regardless of
|
|
52
|
+
* what (if anything) the measured directory contains.
|
|
53
|
+
*/
|
|
54
|
+
function tsxLoaderUrl() {
|
|
55
|
+
return pathToFileURL(require.resolve('tsx/esm')).href;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Arbitrary throw values (a non-`Error`, or a `JSON.parse` `SyntaxError`)
|
|
59
|
+
* both need a printable message; this mirrors the same helper in child.ts.
|
|
60
|
+
*/
|
|
61
|
+
function messageOf(e) {
|
|
62
|
+
return e instanceof Error ? e.message : String(e);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Reads and parses the child's `--out` file, turning every failure mode
|
|
66
|
+
* into `{ ok: false }` instead of a rejection: a missing file (the child
|
|
67
|
+
* died before writing at all) and a present-but-malformed one (the file
|
|
68
|
+
* exists but doesn't parse, or doesn't have the right shape).
|
|
69
|
+
*
|
|
70
|
+
* The malformed case is not exotic: our own timeout path kills the child's
|
|
71
|
+
* process group with SIGKILL, which can land mid-`writeFile`, leaving a
|
|
72
|
+
* truncated-but-readable JSON document on disk. `readFile` succeeds on
|
|
73
|
+
* that; only `JSON.parse`/`parseBenchResult` would have caught it, and
|
|
74
|
+
* uncaught, that throw would have propagated out of `runChild` as a
|
|
75
|
+
* rejected promise -- exactly the "harness bug" failure mode `runChild`
|
|
76
|
+
* exists to avoid for a benchmark-level failure.
|
|
77
|
+
*
|
|
78
|
+
* Exported for testing: it lets a test write deliberately truncated JSON
|
|
79
|
+
* to a real, controlled `--out` path and assert `ok: false` directly,
|
|
80
|
+
* without needing to race a real subprocess's SIGKILL against its own
|
|
81
|
+
* `writeFile` to reproduce the truncation.
|
|
82
|
+
*/
|
|
83
|
+
export async function readChildResult(outFile, id, r, timeoutMs) {
|
|
84
|
+
let text;
|
|
85
|
+
try {
|
|
86
|
+
text = await readFile(outFile, 'utf8');
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// The child died before writing. Its own stderr is the useful diagnosis.
|
|
90
|
+
const why = r.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${r.exitCode}`;
|
|
91
|
+
return { ok: false, id, error: `benchmark child ${why}: ${tail(r.stderr, 20)}` };
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
return parseBenchResult(text);
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
return { ok: false, id, error: `malformed benchmark result: ${messageOf(e)}` };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Runs one benchmark, once, in a fresh child process.
|
|
102
|
+
*
|
|
103
|
+
* Never rejects for a benchmark-level failure: a timeout, a crash, or a
|
|
104
|
+
* child that exits without writing --out all come back as `{ ok: false }`
|
|
105
|
+
* with a diagnosis built from the child's own stderr, so a caller can
|
|
106
|
+
* record a CRASH verdict instead of aborting the whole run.
|
|
107
|
+
*/
|
|
108
|
+
export async function runChild(o) {
|
|
109
|
+
const dir = await mkdtemp(path.join(tmpdir(), 'ars-out-'));
|
|
110
|
+
const outFile = path.join(dir, 'result.json');
|
|
111
|
+
try {
|
|
112
|
+
const args = [
|
|
113
|
+
...o.nodeArgs,
|
|
114
|
+
'--import',
|
|
115
|
+
tsxLoaderUrl(),
|
|
116
|
+
CHILD,
|
|
117
|
+
'--file',
|
|
118
|
+
o.benchFileAbs,
|
|
119
|
+
'--fn',
|
|
120
|
+
o.fn,
|
|
121
|
+
'--id',
|
|
122
|
+
o.id,
|
|
123
|
+
'--benchtime-ms',
|
|
124
|
+
String(o.benchtimeMs),
|
|
125
|
+
'--warmup-ms',
|
|
126
|
+
String(o.warmupMs),
|
|
127
|
+
'--out',
|
|
128
|
+
outFile,
|
|
129
|
+
];
|
|
130
|
+
const r = await run(process.execPath, args, {
|
|
131
|
+
cwd: o.cwd,
|
|
132
|
+
timeoutMs: o.timeoutMs,
|
|
133
|
+
...(o.log ? { log: o.log } : {}),
|
|
134
|
+
});
|
|
135
|
+
return await readChildResult(outFile, o.id, r, o.timeoutMs);
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
// A failure here (permissions, an EBUSY-like transient on some
|
|
139
|
+
// platform) must never replace a result the try block already
|
|
140
|
+
// produced -- losing a computed measurement is strictly worse than
|
|
141
|
+
// leaking a temp directory, so the cleanup failure is swallowed rather
|
|
142
|
+
// than left to propagate out of `finally` and clobber the return value
|
|
143
|
+
// (or the rejection) above.
|
|
144
|
+
try {
|
|
145
|
+
await rm(dir, { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
/* leak the directory rather than discard the result */
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=invoke.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"invoke.js","sourceRoot":"","sources":["../../src/benchrun/invoke.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAoB,MAAM,wBAAwB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,IAAI,EAAmB,MAAM,mBAAmB,CAAA;AAE9D,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAEzD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAA8B;IACvE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;IACvC,IAAI,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;AACnC,CAAC;AAED,SAAS,YAAY;IACnB,OAAO,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,EAAE,CAAA;AAEnC;;;;;;;;GAQG;AACH,SAAS,YAAY;IACnB,OAAO,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;AACvD,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACnD,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,EAAU,EACV,CAAuD,EACvD,SAAiB;IAEjB,IAAI,IAAY,CAAA;IAChB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,yEAAyE;QACzE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB,SAAS,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;QAChF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,mBAAmB,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAA;IAClF,CAAC;IACD,IAAI,CAAC;QACH,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,+BAA+B,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IAChF,CAAC;AACH,CAAC;AAeD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,CAAkB;IAC/C,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;IAC7C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG;YACX,GAAG,CAAC,CAAC,QAAQ;YACb,UAAU;YACV,YAAY,EAAE;YACd,KAAK;YACL,QAAQ;YACR,CAAC,CAAC,YAAY;YACd,MAAM;YACN,CAAC,CAAC,EAAE;YACJ,MAAM;YACN,CAAC,CAAC,EAAE;YACJ,gBAAgB;YAChB,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;YACrB,aAAa;YACb,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,OAAO;YACP,OAAO;SACR,CAAA;QACD,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjC,CAAC,CAAA;QACF,OAAO,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAA;IAC7D,CAAC;YAAS,CAAC;QACT,+DAA+D;QAC/D,8DAA8D;QAC9D,mEAAmE;QACnE,uEAAuE;QACvE,uEAAuE;QACvE,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { RunCtx } from './runctx.js';
|
|
2
|
+
/**
|
|
3
|
+
* `baseline`: freezes the tests and benchmarks, creates the run branch,
|
|
4
|
+
* pins a detached worktree at HEAD, installs into it, and PROVES the
|
|
5
|
+
* declared benchmarks actually run there before declaring success.
|
|
6
|
+
*
|
|
7
|
+
* Every refusal below (bad flags, an unclean tree, a reused tag, a
|
|
8
|
+
* benchmark declared in config that no longer exists) happens before the
|
|
9
|
+
* run branch is created and before anything is written to the state
|
|
10
|
+
* directory. Everything after `createBranch` is a mutation, and any
|
|
11
|
+
* failure there is unwound: the worktree is removed, the run directory is
|
|
12
|
+
* deleted, and -- per Ruling 3 -- so is the branch, so a retry with the
|
|
13
|
+
* same tag never needs `-force`.
|
|
14
|
+
*/
|
|
15
|
+
export declare function cmdBaseline(ctx: RunCtx, argv: readonly string[]): Promise<number>;
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
import { access, readFile, rm } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { parseArgs } from 'node:util';
|
|
4
|
+
import { runChild } from '../benchrun/invoke.js';
|
|
5
|
+
import { loadConfig, parseDuration } from '../config/load.js';
|
|
6
|
+
import { CONFIG_PATH } from '../config/schema.js';
|
|
7
|
+
import { discoverBenchmarks } from '../discover/benchmarks.js';
|
|
8
|
+
import { freezableFiles } from '../discover/files.js';
|
|
9
|
+
import { snapshot } from '../freeze/freeze.js';
|
|
10
|
+
import { hashString } from '../freeze/manifest.js';
|
|
11
|
+
import { addWorktree, branchExists, changedFiles, createBranch, currentBranch, deleteBranch, headCommit, removeWorktree, } from '../gitx/git.js';
|
|
12
|
+
import { detect } from '../pm/detect.js';
|
|
13
|
+
import { ok, run, runShell, tail } from '../runner/exec.js';
|
|
14
|
+
import { writeBaseline } from '../state/baseline.js';
|
|
15
|
+
import { runDir } from '../state/home.js';
|
|
16
|
+
import { BRANCH_PREFIX, FROZEN_DIRNAME, WORKTREE_DIRNAME } from '../state/runnaming.js';
|
|
17
|
+
const CHECKOUT_TIMEOUT_MS = 60_000;
|
|
18
|
+
/**
|
|
19
|
+
* Duration of the smoke measurement run in the worktree. This is
|
|
20
|
+
* deliberately tiny: the point of this run is to prove each benchmark
|
|
21
|
+
* imports and executes without throwing, not to produce a usable
|
|
22
|
+
* measurement -- that is what `eval` (task 19) actually measures, against
|
|
23
|
+
* the config's own `benchtime`.
|
|
24
|
+
*/
|
|
25
|
+
const SMOKE_BENCHTIME_MS = 20;
|
|
26
|
+
const SMOKE_WARMUP_MS = 0;
|
|
27
|
+
function fail(message) {
|
|
28
|
+
process.stderr.write(`error: ${message}\n`);
|
|
29
|
+
return 2;
|
|
30
|
+
}
|
|
31
|
+
function messageOf(e) {
|
|
32
|
+
return e instanceof Error ? e.message : String(e);
|
|
33
|
+
}
|
|
34
|
+
async function exists(p) {
|
|
35
|
+
try {
|
|
36
|
+
await access(p);
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Checks out `ref` in the main repository (not a worktree). None of the
|
|
45
|
+
* already-built `gitx` helpers do this directly: `createBranch` always
|
|
46
|
+
* creates a new branch, and `repointWorktree` operates on a worktree's own
|
|
47
|
+
* checkout, not the main repo's. This is only ever used to move the main
|
|
48
|
+
* repo's checkout OFF the run branch before that branch is deleted --
|
|
49
|
+
* `git branch -D` refuses to delete the branch currently checked out.
|
|
50
|
+
*/
|
|
51
|
+
async function checkoutRef(root, ref) {
|
|
52
|
+
const r = await run('git', ['checkout', '-q', ref], { cwd: root, timeoutMs: CHECKOUT_TIMEOUT_MS });
|
|
53
|
+
if (!ok(r)) {
|
|
54
|
+
throw new Error(`git checkout ${ref} failed in ${root} (exit ${r.exitCode}): ${r.stderr.trim()}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** Declared benchmark ids that are not among the discovered benchmarks. */
|
|
58
|
+
function missingDeclared(declared, discovered) {
|
|
59
|
+
const found = new Set(discovered.map((b) => b.id));
|
|
60
|
+
return declared.filter((id) => !found.has(id));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* `baseline`: freezes the tests and benchmarks, creates the run branch,
|
|
64
|
+
* pins a detached worktree at HEAD, installs into it, and PROVES the
|
|
65
|
+
* declared benchmarks actually run there before declaring success.
|
|
66
|
+
*
|
|
67
|
+
* Every refusal below (bad flags, an unclean tree, a reused tag, a
|
|
68
|
+
* benchmark declared in config that no longer exists) happens before the
|
|
69
|
+
* run branch is created and before anything is written to the state
|
|
70
|
+
* directory. Everything after `createBranch` is a mutation, and any
|
|
71
|
+
* failure there is unwound: the worktree is removed, the run directory is
|
|
72
|
+
* deleted, and -- per Ruling 3 -- so is the branch, so a retry with the
|
|
73
|
+
* same tag never needs `-force`.
|
|
74
|
+
*/
|
|
75
|
+
export async function cmdBaseline(ctx, argv) {
|
|
76
|
+
let tag;
|
|
77
|
+
let force;
|
|
78
|
+
try {
|
|
79
|
+
const normalized = argv.map((a) => (/^-[A-Za-z][A-Za-z-]+$/.test(a) ? `-${a}` : a));
|
|
80
|
+
const { values } = parseArgs({
|
|
81
|
+
args: normalized,
|
|
82
|
+
options: {
|
|
83
|
+
tag: { type: 'string' },
|
|
84
|
+
force: { type: 'boolean', default: false },
|
|
85
|
+
},
|
|
86
|
+
strict: true,
|
|
87
|
+
allowPositionals: false,
|
|
88
|
+
});
|
|
89
|
+
if (typeof values.tag !== 'string' || values.tag === '') {
|
|
90
|
+
return fail('-tag <tag> is required, e.g. -tag sep6');
|
|
91
|
+
}
|
|
92
|
+
tag = values.tag;
|
|
93
|
+
force = values.force === true;
|
|
94
|
+
}
|
|
95
|
+
catch (e) {
|
|
96
|
+
return fail(messageOf(e));
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
// Validates the tag (it becomes a directory name) before anything else
|
|
100
|
+
// runs -- an invalid tag must be refused before any git or filesystem
|
|
101
|
+
// mutation, not discovered midway through one.
|
|
102
|
+
const dir = runDir(ctx.repoRoot, tag);
|
|
103
|
+
const branchName = `${BRANCH_PREFIX}${tag}`;
|
|
104
|
+
const detected = await detect(ctx.repoRoot);
|
|
105
|
+
const config = await loadConfig(ctx.configPath);
|
|
106
|
+
// Deliberately `changedFiles(repoRoot, HEAD)` rather than `isClean`
|
|
107
|
+
// (`git status --porcelain`): `isClean` trusts whatever `.gitignore` is
|
|
108
|
+
// on disk, including one the agent just wrote -- the same blind spot
|
|
109
|
+
// fixed in `pipeline/eval.ts` gate 8, and for the identical reason. A
|
|
110
|
+
// baseline pinned while an agent-hidden, uncommitted file sits on disk
|
|
111
|
+
// is exactly as unreproducible as one pinned with any other uncommitted
|
|
112
|
+
// change; `changedFiles` (Priority 2's ignore-immune enumeration) is
|
|
113
|
+
// what actually proves nothing on disk differs from HEAD.
|
|
114
|
+
const headBeforeBaseline = await headCommit(ctx.repoRoot);
|
|
115
|
+
if ((await changedFiles(ctx.repoRoot, headBeforeBaseline)).length > 0) {
|
|
116
|
+
return fail('the working tree is not clean (uncommitted changes or untracked files). A baseline ' +
|
|
117
|
+
'pinned to what is on disk rather than what is committed could not be reproduced by ' +
|
|
118
|
+
'anyone else, including a retry on this same machine after those files are gone. ' +
|
|
119
|
+
'Commit or stash your changes and try again.');
|
|
120
|
+
}
|
|
121
|
+
if (await exists(dir)) {
|
|
122
|
+
if (!force) {
|
|
123
|
+
return fail(`a baseline for tag ${JSON.stringify(tag)} already exists at ${dir}; pass -force to replace it`);
|
|
124
|
+
}
|
|
125
|
+
// With -force, tear down whatever the earlier run left behind before
|
|
126
|
+
// anything new is created: removeWorktree first, or git keeps a
|
|
127
|
+
// stale worktree registration and addWorktree below fails.
|
|
128
|
+
const worktreeDir = path.join(dir, WORKTREE_DIRNAME);
|
|
129
|
+
if (await exists(worktreeDir)) {
|
|
130
|
+
try {
|
|
131
|
+
await removeWorktree(ctx.repoRoot, worktreeDir);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Not a registered worktree (e.g. a previous attempt failed
|
|
135
|
+
// before addWorktree ran) -- the rm below still removes it from
|
|
136
|
+
// disk either way.
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
await rm(dir, { recursive: true, force: true });
|
|
140
|
+
}
|
|
141
|
+
const discovered = await discoverBenchmarks(ctx.repoRoot);
|
|
142
|
+
const declared = config.benchmarks;
|
|
143
|
+
let benchmarks;
|
|
144
|
+
if (declared.length > 0) {
|
|
145
|
+
const missing = missingDeclared(declared, discovered);
|
|
146
|
+
if (missing.length > 0) {
|
|
147
|
+
return fail(`config declares benchmark(s) that no longer exist: ${missing.join(', ')}. Update ` +
|
|
148
|
+
`${CONFIG_PATH} or restore the missing benchmark(s).`);
|
|
149
|
+
}
|
|
150
|
+
const byId = new Map(discovered.map((b) => [b.id, b]));
|
|
151
|
+
benchmarks = declared.map((id) => byId.get(id));
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
benchmarks = discovered;
|
|
155
|
+
}
|
|
156
|
+
if (benchmarks.length === 0) {
|
|
157
|
+
return fail('no benchmarks found: there is nothing for this baseline to measure');
|
|
158
|
+
}
|
|
159
|
+
// Everything above is a pure refusal: nothing has been created or
|
|
160
|
+
// written, in git or on disk, past this point.
|
|
161
|
+
const originalBranch = await currentBranch(ctx.repoRoot);
|
|
162
|
+
const originalCommit = await headCommit(ctx.repoRoot);
|
|
163
|
+
const fallbackRef = originalBranch === 'HEAD' ? originalCommit : originalBranch;
|
|
164
|
+
// A branch of this name can already exist here for two different
|
|
165
|
+
// reasons, which need two different responses:
|
|
166
|
+
//
|
|
167
|
+
// - The repo is ALREADY checked out on it: this is a `-force` re-run
|
|
168
|
+
// of a tag whose previous baseline succeeded (a successful run never
|
|
169
|
+
// switches away from the branch it creates). There is nothing to
|
|
170
|
+
// recreate -- proceed with the existing checkout, at whatever commit
|
|
171
|
+
// it is already on.
|
|
172
|
+
// - It exists but is checked out nowhere here (a stale branch left
|
|
173
|
+
// over some other way): safe to delete outright, with no need to
|
|
174
|
+
// check out away from it first, and recreate fresh below.
|
|
175
|
+
//
|
|
176
|
+
// `createdBranch` records whether THIS invocation is the one that
|
|
177
|
+
// created the branch -- only then does the unwind below delete it
|
|
178
|
+
// (Ruling 3): a branch this invocation merely reused on `-force` was
|
|
179
|
+
// not this run's to destroy on failure.
|
|
180
|
+
let createdBranch = false;
|
|
181
|
+
if (await branchExists(ctx.repoRoot, branchName)) {
|
|
182
|
+
if (originalBranch !== branchName) {
|
|
183
|
+
await deleteBranch(ctx.repoRoot, branchName);
|
|
184
|
+
await createBranch(ctx.repoRoot, branchName);
|
|
185
|
+
createdBranch = true;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
await createBranch(ctx.repoRoot, branchName);
|
|
190
|
+
createdBranch = true;
|
|
191
|
+
}
|
|
192
|
+
const worktreeDir = path.join(dir, WORKTREE_DIRNAME);
|
|
193
|
+
const frozenDir = path.join(dir, FROZEN_DIRNAME);
|
|
194
|
+
let worktreeAdded = false;
|
|
195
|
+
try {
|
|
196
|
+
const head = await headCommit(ctx.repoRoot);
|
|
197
|
+
// `config.unfreeze` names test/bench files deliberately exempted from
|
|
198
|
+
// the freeze (see `renderConfigYaml`'s own comment on the key, and
|
|
199
|
+
// spec section 6) -- they must never be snapshotted or hashed into the
|
|
200
|
+
// manifest in the first place, or gate 3's restore would overwrite the
|
|
201
|
+
// agent's edits to them on every eval regardless of this config.
|
|
202
|
+
const unfreeze = new Set(config.unfreeze);
|
|
203
|
+
const freezable = (await freezableFiles(ctx.repoRoot)).filter((f) => !unfreeze.has(f));
|
|
204
|
+
const manifest = await snapshot(ctx.repoRoot, freezable, frozenDir);
|
|
205
|
+
await addWorktree(ctx.repoRoot, worktreeDir, head);
|
|
206
|
+
worktreeAdded = true;
|
|
207
|
+
const timeoutMs = parseDuration(config.timeout);
|
|
208
|
+
const install = await runShell(detected.installCommand, { cwd: worktreeDir, timeoutMs });
|
|
209
|
+
if (!ok(install)) {
|
|
210
|
+
throw new Error(`installing dependencies in the worktree failed (${detected.installCommand}, exit ` +
|
|
211
|
+
`${install.exitCode}): ${tail(install.stderr || install.stdout, 40)}`);
|
|
212
|
+
}
|
|
213
|
+
// The smoke run: prove every declared benchmark actually runs in the
|
|
214
|
+
// worktree before this baseline is declared successful. Without
|
|
215
|
+
// this, a benchmark that cannot execute would produce a "successful"
|
|
216
|
+
// baseline and fail mysteriously at the user's first `eval`.
|
|
217
|
+
for (const b of benchmarks) {
|
|
218
|
+
const result = await runChild({
|
|
219
|
+
cwd: worktreeDir,
|
|
220
|
+
benchFileAbs: path.join(worktreeDir, b.file),
|
|
221
|
+
fn: b.fn,
|
|
222
|
+
id: b.id,
|
|
223
|
+
benchtimeMs: SMOKE_BENCHTIME_MS,
|
|
224
|
+
warmupMs: SMOKE_WARMUP_MS,
|
|
225
|
+
timeoutMs,
|
|
226
|
+
nodeArgs: config.nodeArgs,
|
|
227
|
+
});
|
|
228
|
+
if (!result.ok) {
|
|
229
|
+
throw new Error(`smoke run of ${b.id} failed: ${result.error}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const configText = await readFile(ctx.configPath, 'utf8');
|
|
233
|
+
const lockfileText = await readFile(path.join(ctx.repoRoot, detected.lockfile), 'utf8');
|
|
234
|
+
const record = {
|
|
235
|
+
tag,
|
|
236
|
+
// frozenCommit and measureCommit both start at HEAD, identically --
|
|
237
|
+
// measureCommit is the only one of the two that ever advances
|
|
238
|
+
// (after a KEEP), and frozenCommit must never move once set.
|
|
239
|
+
frozenCommit: head,
|
|
240
|
+
measureCommit: head,
|
|
241
|
+
configHash: hashString(configText),
|
|
242
|
+
lockfileHash: hashString(lockfileText),
|
|
243
|
+
lockfileName: detected.lockfile,
|
|
244
|
+
benchmarks: benchmarks.map((b) => b.id),
|
|
245
|
+
manifest,
|
|
246
|
+
createdAt: new Date().toISOString(),
|
|
247
|
+
};
|
|
248
|
+
await writeBaseline(dir, record);
|
|
249
|
+
process.stdout.write(`baseline ${JSON.stringify(tag)} created at ${head.slice(0, 7)}\n`);
|
|
250
|
+
process.stdout.write(` branch: ${branchName}\n`);
|
|
251
|
+
process.stdout.write(` worktree: ${worktreeDir}\n`);
|
|
252
|
+
process.stdout.write(` state: ${dir}\n`);
|
|
253
|
+
process.stdout.write(' benchmarks smoke-tested:\n');
|
|
254
|
+
for (const b of benchmarks)
|
|
255
|
+
process.stdout.write(` - ${b.id}\n`);
|
|
256
|
+
return 0;
|
|
257
|
+
}
|
|
258
|
+
catch (e) {
|
|
259
|
+
const reason = messageOf(e);
|
|
260
|
+
if (worktreeAdded) {
|
|
261
|
+
try {
|
|
262
|
+
await removeWorktree(ctx.repoRoot, worktreeDir);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
// Best effort -- the directory removal below still gets rid of
|
|
266
|
+
// it on disk even if git's own bookkeeping is left stale.
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
await rm(dir, { recursive: true, force: true });
|
|
270
|
+
try {
|
|
271
|
+
// Only ever delete a branch THIS invocation created (Ruling 3) --
|
|
272
|
+
// one it merely reused under `-force` was not this run's to
|
|
273
|
+
// destroy on failure.
|
|
274
|
+
if (createdBranch) {
|
|
275
|
+
if ((await currentBranch(ctx.repoRoot)) === branchName) {
|
|
276
|
+
await checkoutRef(ctx.repoRoot, fallbackRef);
|
|
277
|
+
}
|
|
278
|
+
await deleteBranch(ctx.repoRoot, branchName);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
catch (unwindErr) {
|
|
282
|
+
return fail(`${reason}\n(cleanup also failed to delete branch ${branchName}: ` +
|
|
283
|
+
`${messageOf(unwindErr)}; remove it manually with "git branch -D ${branchName}" before retrying)`);
|
|
284
|
+
}
|
|
285
|
+
return fail(reason);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
catch (e) {
|
|
289
|
+
return fail(messageOf(e));
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
//# sourceMappingURL=cmd-baseline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cmd-baseline.js","sourceRoot":"","sources":["../../src/cli/cmd-baseline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,kBAAkB,EAAkB,MAAM,2BAA2B,CAAA;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EACL,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,UAAU,EACV,cAAc,GACf,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAA;AAC3D,OAAO,EAAE,aAAa,EAAuB,MAAM,sBAAsB,CAAA;AACzE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAGvF,MAAM,mBAAmB,GAAG,MAAM,CAAA;AAElC;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAC7B,MAAM,eAAe,GAAG,CAAC,CAAA;AAEzB,SAAS,IAAI,CAAC,OAAe;IAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,IAAI,CAAC,CAAA;IAC3C,OAAO,CAAC,CAAA;AACV,CAAC;AAED,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACnD,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAA;QACf,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,GAAW;IAClD,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAA;IAClG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,cAAc,IAAI,UAAU,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACnG,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,SAAS,eAAe,CAAC,QAA2B,EAAE,UAAgC;IACpF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAClD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AAChD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAW,EAAE,IAAuB;IACpE,IAAI,GAAW,CAAA;IACf,IAAI,KAAc,CAAA;IAClB,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACnF,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;YAC3B,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE;gBACP,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;aAC3C;YACD,MAAM,EAAE,IAAI;YACZ,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAA;QACF,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC,wCAAwC,CAAC,CAAA;QACvD,CAAC;QACD,GAAG,GAAG,MAAM,CAAC,GAAG,CAAA;QAChB,KAAK,GAAG,MAAM,CAAC,KAAK,KAAK,IAAI,CAAA;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,CAAC;QACH,uEAAuE;QACvE,sEAAsE;QACtE,+CAA+C;QAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QACrC,MAAM,UAAU,GAAG,GAAG,aAAa,GAAG,GAAG,EAAE,CAAA;QAE3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAE/C,oEAAoE;QACpE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,uEAAuE;QACvE,wEAAwE;QACxE,qEAAqE;QACrE,0DAA0D;QAC1D,MAAM,kBAAkB,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACzD,IAAI,CAAC,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,OAAO,IAAI,CACT,qFAAqF;gBACnF,qFAAqF;gBACrF,kFAAkF;gBAClF,6CAA6C,CAChD,CAAA;QACH,CAAC;QAED,IAAI,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,IAAI,CACT,sBAAsB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,sBAAsB,GAAG,6BAA6B,CAChG,CAAA;YACH,CAAC;YACD,qEAAqE;YACrE,gEAAgE;YAChE,2DAA2D;YAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAA;YACpD,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,4DAA4D;oBAC5D,gEAAgE;oBAChE,mBAAmB;gBACrB,CAAC;YACH,CAAC;YACD,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAA;QAClC,IAAI,UAAuB,CAAA;QAC3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YACrD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,IAAI,CACT,sDAAsD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW;oBACjF,GAAG,WAAW,uCAAuC,CACxD,CAAA;YACH,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACtD,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAA;QAClD,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,UAAU,CAAA;QACzB,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,oEAAoE,CAAC,CAAA;QACnF,CAAC;QAED,kEAAkE;QAClE,+CAA+C;QAE/C,MAAM,cAAc,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACxD,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACrD,MAAM,WAAW,GAAG,cAAc,KAAK,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAA;QAE/E,iEAAiE;QACjE,+CAA+C;QAC/C,EAAE;QACF,qEAAqE;QACrE,uEAAuE;QACvE,mEAAmE;QACnE,uEAAuE;QACvE,sBAAsB;QACtB,mEAAmE;QACnE,mEAAmE;QACnE,4DAA4D;QAC5D,EAAE;QACF,kEAAkE;QAClE,kEAAkE;QAClE,qEAAqE;QACrE,wCAAwC;QACxC,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,IAAI,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;YACjD,IAAI,cAAc,KAAK,UAAU,EAAE,CAAC;gBAClC,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;gBAC5C,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;gBAC5C,aAAa,GAAG,IAAI,CAAA;YACtB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAC5C,aAAa,GAAG,IAAI,CAAA;QACtB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAA;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAA;QAChD,IAAI,aAAa,GAAG,KAAK,CAAA;QAEzB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAE3C,sEAAsE;YACtE,mEAAmE;YACnE,uEAAuE;YACvE,uEAAuE;YACvE,iEAAiE;YACjE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YACzC,MAAM,SAAS,GAAG,CAAC,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YACtF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YAEnE,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAA;YAClD,aAAa,GAAG,IAAI,CAAA;YAEpB,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAE/C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;YACxF,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CACb,mDAAmD,QAAQ,CAAC,cAAc,SAAS;oBACjF,GAAG,OAAO,CAAC,QAAQ,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CACxE,CAAA;YACH,CAAC;YAED,qEAAqE;YACrE,gEAAgE;YAChE,qEAAqE;YACrE,6DAA6D;YAC7D,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;oBAC5B,GAAG,EAAE,WAAW;oBAChB,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC;oBAC5C,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,WAAW,EAAE,kBAAkB;oBAC/B,QAAQ,EAAE,eAAe;oBACzB,SAAS;oBACT,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC,CAAA;gBACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,EAAE,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBACjE,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;YACzD,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAA;YAEvF,MAAM,MAAM,GAAmB;gBAC7B,GAAG;gBACH,oEAAoE;gBACpE,8DAA8D;gBAC9D,6DAA6D;gBAC7D,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,IAAI;gBACnB,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC;gBAClC,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC;gBACtC,YAAY,EAAE,QAAQ,CAAC,QAAQ;gBAC/B,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,QAAQ;gBACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAA;YACD,MAAM,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAEhC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;YACxF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,UAAU,IAAI,CAAC,CAAA;YACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,WAAW,IAAI,CAAC,CAAA;YACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,CAAA;YAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;YACpD,KAAK,MAAM,CAAC,IAAI,UAAU;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;YACnE,OAAO,CAAC,CAAA;QACV,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC3B,IAAI,aAAa,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,+DAA+D;oBAC/D,0DAA0D;gBAC5D,CAAC;YACH,CAAC;YACD,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/C,IAAI,CAAC;gBACH,kEAAkE;gBAClE,4DAA4D;gBAC5D,sBAAsB;gBACtB,IAAI,aAAa,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;wBACvD,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;oBAC9C,CAAC;oBACD,MAAM,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;gBAC9C,CAAC;YACH,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,OAAO,IAAI,CACT,GAAG,MAAM,2CAA2C,UAAU,IAAI;oBAChE,GAAG,SAAS,CAAC,SAAS,CAAC,4CAA4C,UAAU,oBAAoB,CACpG,CAAA;YACH,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAA;QACrB,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RunCtx } from './runctx.js';
|
|
2
|
+
/**
|
|
3
|
+
* Reports whether this machine can measure reliably: informational only.
|
|
4
|
+
*
|
|
5
|
+
* `doctor` deliberately always returns 0, regardless of how many checks
|
|
6
|
+
* fail. Every other command's exit code carries a verdict the harness or
|
|
7
|
+
* a script may act on; this one does not -- it exists to be run before
|
|
8
|
+
* trusting an overnight run's numbers, and a diagnostic that blocks the
|
|
9
|
+
* next step is a diagnostic people stop running.
|
|
10
|
+
*/
|
|
11
|
+
export declare function cmdDoctor(_ctx: RunCtx, _argv: string[]): Promise<number>;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { runChecks } from '../doctor/doctor.js';
|
|
2
|
+
/**
|
|
3
|
+
* Reports whether this machine can measure reliably: informational only.
|
|
4
|
+
*
|
|
5
|
+
* `doctor` deliberately always returns 0, regardless of how many checks
|
|
6
|
+
* fail. Every other command's exit code carries a verdict the harness or
|
|
7
|
+
* a script may act on; this one does not -- it exists to be run before
|
|
8
|
+
* trusting an overnight run's numbers, and a diagnostic that blocks the
|
|
9
|
+
* next step is a diagnostic people stop running.
|
|
10
|
+
*/
|
|
11
|
+
export async function cmdDoctor(_ctx, _argv) {
|
|
12
|
+
const checks = await runChecks();
|
|
13
|
+
process.stdout.write('autor3search-typescript doctor -- can this machine measure reliably?\n\n');
|
|
14
|
+
for (const c of checks) {
|
|
15
|
+
const mark = c.ok ? 'OK ' : 'WARN';
|
|
16
|
+
process.stdout.write(`[${mark}] ${c.name}: ${c.detail}\n`);
|
|
17
|
+
}
|
|
18
|
+
const warnings = checks.filter((c) => !c.ok).length;
|
|
19
|
+
process.stdout.write('\n');
|
|
20
|
+
process.stdout.write(warnings === 0
|
|
21
|
+
? 'No issues found.\n'
|
|
22
|
+
: `${warnings} check(s) flagged above -- address what you can before an overnight run.\n`);
|
|
23
|
+
process.stdout.write('This command is informational only and always exits 0.\n');
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=cmd-doctor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cmd-doctor.js","sourceRoot":"","sources":["../../src/cli/cmd-doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAG/C;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,KAAe;IAC3D,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAA;IAEhC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0EAA0E,CAAC,CAAA;IAChG,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAA;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAA;IACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,QAAQ,KAAK,CAAC;QACZ,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,GAAG,QAAQ,4EAA4E,CAC5F,CAAA;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAEhF,OAAO,CAAC,CAAA;AACV,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { RunCtx } from './runctx.js';
|
|
2
|
+
/**
|
|
3
|
+
* SIGTERM's default disposition kills node immediately -- once ANY listener
|
|
4
|
+
* is added, Node stops doing that for us, so this handler must actually
|
|
5
|
+
* terminate the process itself once it's done cleaning up.
|
|
6
|
+
*
|
|
7
|
+
* Both steps below are what `stop -force` (spec section 2 item 4) needs and
|
|
8
|
+
* does not otherwise get for free: `eval.ts` spawns every measurement child
|
|
9
|
+
* `detached: true` (its own process-group leader), so it is not reaped
|
|
10
|
+
* merely by `eval` itself dying; and the eval lock is a file this process
|
|
11
|
+
* would otherwise never release, since `acquireEvalLock`'s `finally` in
|
|
12
|
+
* `runEval` never runs if the process exits out from under it instead.
|
|
13
|
+
*/
|
|
14
|
+
/** Exported for testing: see `cmd-eval.sigterm.test.ts`. */
|
|
15
|
+
export declare function installSigtermHandler(dir: string): () => void;
|
|
16
|
+
/**
|
|
17
|
+
* `eval`: runs one experiment through the gate chain (`pipeline/eval.ts`)
|
|
18
|
+
* and reports the verdict. The exit code IS the signal an unattended agent
|
|
19
|
+
* loop acts on -- 0 KEEP, 1 DISCARD, 2 FAIL, 3 CRASH -- so every path here
|
|
20
|
+
* that can be reached after a real attempt returns whatever `EXIT_CODES`
|
|
21
|
+
* says for that outcome; only a usage error (bad flags, no resolvable tag)
|
|
22
|
+
* returns 2 without ever having attempted an experiment.
|
|
23
|
+
*
|
|
24
|
+
* Under `--json`, exactly one JSON object is printed to stdout and nothing
|
|
25
|
+
* else: subprocess transcripts go to `run.log`, and every warning is folded
|
|
26
|
+
* into the object's own `warnings` array rather than printed as a separate
|
|
27
|
+
* line. Human output prints those same warnings as `WARNING:` lines above
|
|
28
|
+
* the verdict instead.
|
|
29
|
+
*/
|
|
30
|
+
export declare function cmdEval(ctx: RunCtx, argv: readonly string[]): Promise<number>;
|