autor3search-typescript 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +210 -0
- package/dist/benchproto/types.d.ts +32 -0
- package/dist/benchproto/types.js +28 -0
- package/dist/benchproto/types.js.map +1 -0
- package/dist/benchrun/child.d.ts +1 -0
- package/dist/benchrun/child.js +222 -0
- package/dist/benchrun/child.js.map +1 -0
- package/dist/benchrun/invoke.d.ts +61 -0
- package/dist/benchrun/invoke.js +152 -0
- package/dist/benchrun/invoke.js.map +1 -0
- package/dist/cli/cmd-baseline.d.ts +15 -0
- package/dist/cli/cmd-baseline.js +292 -0
- package/dist/cli/cmd-baseline.js.map +1 -0
- package/dist/cli/cmd-doctor.d.ts +11 -0
- package/dist/cli/cmd-doctor.js +26 -0
- package/dist/cli/cmd-doctor.js.map +1 -0
- package/dist/cli/cmd-eval.d.ts +30 -0
- package/dist/cli/cmd-eval.js +173 -0
- package/dist/cli/cmd-eval.js.map +1 -0
- package/dist/cli/cmd-init.d.ts +16 -0
- package/dist/cli/cmd-init.js +343 -0
- package/dist/cli/cmd-init.js.map +1 -0
- package/dist/cli/cmd-profile.d.ts +11 -0
- package/dist/cli/cmd-profile.js +140 -0
- package/dist/cli/cmd-profile.js.map +1 -0
- package/dist/cli/cmd-report.d.ts +15 -0
- package/dist/cli/cmd-report.js +70 -0
- package/dist/cli/cmd-report.js.map +1 -0
- package/dist/cli/cmd-status.d.ts +11 -0
- package/dist/cli/cmd-status.js +143 -0
- package/dist/cli/cmd-status.js.map +1 -0
- package/dist/cli/cmd-stop.d.ts +15 -0
- package/dist/cli/cmd-stop.js +143 -0
- package/dist/cli/cmd-stop.js.map +1 -0
- package/dist/cli/main.d.ts +31 -0
- package/dist/cli/main.js +133 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/runctx.d.ts +29 -0
- package/dist/cli/runctx.js +49 -0
- package/dist/cli/runctx.js.map +1 -0
- package/dist/cli/speedup.d.ts +14 -0
- package/dist/cli/speedup.js +25 -0
- package/dist/cli/speedup.js.map +1 -0
- package/dist/config/load.d.ts +6 -0
- package/dist/config/load.js +172 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/schema.d.ts +42 -0
- package/dist/config/schema.js +25 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/discover/benchmarks.d.ts +26 -0
- package/dist/discover/benchmarks.js +88 -0
- package/dist/discover/benchmarks.js.map +1 -0
- package/dist/discover/files.d.ts +23 -0
- package/dist/discover/files.js +82 -0
- package/dist/discover/files.js.map +1 -0
- package/dist/doctor/doctor.d.ts +56 -0
- package/dist/doctor/doctor.js +255 -0
- package/dist/doctor/doctor.js.map +1 -0
- package/dist/freeze/freeze.d.ts +23 -0
- package/dist/freeze/freeze.js +107 -0
- package/dist/freeze/freeze.js.map +1 -0
- package/dist/freeze/manifest.d.ts +6 -0
- package/dist/freeze/manifest.js +9 -0
- package/dist/freeze/manifest.js.map +1 -0
- package/dist/freeze/safepath.d.ts +8 -0
- package/dist/freeze/safepath.js +22 -0
- package/dist/freeze/safepath.js.map +1 -0
- package/dist/gitx/git.d.ts +87 -0
- package/dist/gitx/git.js +235 -0
- package/dist/gitx/git.js.map +1 -0
- package/dist/measure/interleave.d.ts +32 -0
- package/dist/measure/interleave.js +42 -0
- package/dist/measure/interleave.js.map +1 -0
- package/dist/pipeline/eval.d.ts +67 -0
- package/dist/pipeline/eval.js +495 -0
- package/dist/pipeline/eval.js.map +1 -0
- package/dist/pm/detect.d.ts +17 -0
- package/dist/pm/detect.js +131 -0
- package/dist/pm/detect.js.map +1 -0
- package/dist/profile/profile.d.ts +53 -0
- package/dist/profile/profile.js +105 -0
- package/dist/profile/profile.js.map +1 -0
- package/dist/results/results.d.ts +65 -0
- package/dist/results/results.js +214 -0
- package/dist/results/results.js.map +1 -0
- package/dist/runner/exec.d.ts +69 -0
- package/dist/runner/exec.js +185 -0
- package/dist/runner/exec.js.map +1 -0
- package/dist/scope/scope.d.ts +20 -0
- package/dist/scope/scope.js +70 -0
- package/dist/scope/scope.js.map +1 -0
- package/dist/state/baseline.d.ts +35 -0
- package/dist/state/baseline.js +38 -0
- package/dist/state/baseline.js.map +1 -0
- package/dist/state/home.d.ts +9 -0
- package/dist/state/home.js +72 -0
- package/dist/state/home.js.map +1 -0
- package/dist/state/lock.d.ts +11 -0
- package/dist/state/lock.js +89 -0
- package/dist/state/lock.js.map +1 -0
- package/dist/state/runnaming.d.ts +27 -0
- package/dist/state/runnaming.js +47 -0
- package/dist/state/runnaming.js.map +1 -0
- package/dist/state/stop.d.ts +28 -0
- package/dist/state/stop.js +59 -0
- package/dist/state/stop.js.map +1 -0
- package/dist/stats/delta.d.ts +29 -0
- package/dist/stats/delta.js +41 -0
- package/dist/stats/delta.js.map +1 -0
- package/dist/stats/geomean.d.ts +4 -0
- package/dist/stats/geomean.js +21 -0
- package/dist/stats/geomean.js.map +1 -0
- package/dist/stats/mannwhitney.d.ts +27 -0
- package/dist/stats/mannwhitney.js +158 -0
- package/dist/stats/mannwhitney.js.map +1 -0
- package/dist/verdict/verdict.d.ts +46 -0
- package/dist/verdict/verdict.js +102 -0
- package/dist/verdict/verdict.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/package.json +35 -0
- package/templates/program.md +131 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { access, readFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { LOCKFILES } from '../config/schema.js';
|
|
4
|
+
/**
|
|
5
|
+
* The manager and install command for each recognised lockfile.
|
|
6
|
+
*
|
|
7
|
+
* The filenames themselves come from `LOCKFILES` in `src/config/schema.ts`
|
|
8
|
+
* (also the base of `IMMUTABLE_FILES`) rather than being re-declared here, so
|
|
9
|
+
* the two lists cannot drift apart. The `switch` is exhaustive over
|
|
10
|
+
* `LockfileName`: if schema.ts ever adds a lockfile without a case being
|
|
11
|
+
* added here, this fails to typecheck rather than silently detecting nothing.
|
|
12
|
+
*/
|
|
13
|
+
function managerFor(lockfile) {
|
|
14
|
+
switch (lockfile) {
|
|
15
|
+
case 'package-lock.json':
|
|
16
|
+
return { pm: 'npm', installCommand: 'npm ci' };
|
|
17
|
+
case 'pnpm-lock.yaml':
|
|
18
|
+
return { pm: 'pnpm', installCommand: 'pnpm install --frozen-lockfile' };
|
|
19
|
+
case 'yarn.lock':
|
|
20
|
+
return { pm: 'yarn', installCommand: 'yarn install --immutable' };
|
|
21
|
+
case 'bun.lockb':
|
|
22
|
+
case 'bun.lock':
|
|
23
|
+
return { pm: 'bun', installCommand: 'bun install --frozen-lockfile' };
|
|
24
|
+
default: {
|
|
25
|
+
const exhaustive = lockfile;
|
|
26
|
+
throw new Error(`unhandled lockfile: ${String(exhaustive)}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function exists(p) {
|
|
31
|
+
try {
|
|
32
|
+
await access(p);
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* True if a parsed `workspaces` field expresses a real workspace.
|
|
41
|
+
*
|
|
42
|
+
* `"workspaces": []` and `"workspaces": {}` declare no members, but declaring
|
|
43
|
+
* the key at all signals workspace intent (npm and yarn both treat its mere
|
|
44
|
+
* presence as opting a repository into workspace mode). Since a subtly wrong
|
|
45
|
+
* guess here would weaken a guarantee the user believes they have, an empty
|
|
46
|
+
* declaration is refused in the same way a populated one is.
|
|
47
|
+
*
|
|
48
|
+
* `"workspaces": null` is different: neither npm's nor yarn's schema accepts
|
|
49
|
+
* `null` for this field, and `null` is the idiomatic JSON spelling of "not
|
|
50
|
+
* set" — so, unlike `[]`/`{}`, it does not signal intent and is treated as
|
|
51
|
+
* absent.
|
|
52
|
+
*/
|
|
53
|
+
function isWorkspaceDeclaration(workspaces) {
|
|
54
|
+
return workspaces !== undefined && workspaces !== null;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Refuses a workspace root.
|
|
58
|
+
*
|
|
59
|
+
* v0.1 supports single-package repositories only. Saying so plainly is much
|
|
60
|
+
* better than half-working: in a workspace the scope gate, the freeze set and
|
|
61
|
+
* the install layout all mean something different, and getting any of them
|
|
62
|
+
* subtly wrong weakens a guarantee the user believes they have.
|
|
63
|
+
*/
|
|
64
|
+
export async function assertSinglePackage(pkgJsonText, root) {
|
|
65
|
+
let pkg;
|
|
66
|
+
try {
|
|
67
|
+
pkg = JSON.parse(pkgJsonText);
|
|
68
|
+
}
|
|
69
|
+
catch (e) {
|
|
70
|
+
throw new Error(`package.json is not valid JSON: ${e.message}. ` +
|
|
71
|
+
'Fix the JSON syntax and try again.');
|
|
72
|
+
}
|
|
73
|
+
const workspaces = pkg.workspaces;
|
|
74
|
+
if (isWorkspaceDeclaration(workspaces)) {
|
|
75
|
+
throw new Error('this repository declares npm/yarn workspaces (a "workspaces" key in package.json). ' +
|
|
76
|
+
'autor3search-typescript v0.1 supports single-package repositories only: in a ' +
|
|
77
|
+
'workspace the scope gate, the freeze set and the install layout all mean something ' +
|
|
78
|
+
'different, and getting any of them wrong would weaken the guarantee this tool gives. ' +
|
|
79
|
+
'Run autor3search-typescript from inside one of the workspace packages instead.');
|
|
80
|
+
}
|
|
81
|
+
if (await exists(path.join(root, 'pnpm-workspace.yaml'))) {
|
|
82
|
+
throw new Error('this repository is a pnpm workspace (a pnpm-workspace.yaml is present). ' +
|
|
83
|
+
'autor3search-typescript v0.1 supports single-package repositories only: in a ' +
|
|
84
|
+
'workspace the scope gate, the freeze set and the install layout all mean something ' +
|
|
85
|
+
'different, and getting any of them wrong would weaken the guarantee this tool gives. ' +
|
|
86
|
+
'Run autor3search-typescript from inside one of the workspace packages instead.');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
export async function detect(root) {
|
|
90
|
+
const pkgPath = path.join(root, 'package.json');
|
|
91
|
+
let pkgText;
|
|
92
|
+
try {
|
|
93
|
+
pkgText = await readFile(pkgPath, 'utf8');
|
|
94
|
+
}
|
|
95
|
+
catch (e) {
|
|
96
|
+
if (e.code === 'ENOENT') {
|
|
97
|
+
throw new Error(`no package.json in ${root}: this does not look like a Node package. ` +
|
|
98
|
+
'Run autor3search-typescript from the package root (the directory that contains ' +
|
|
99
|
+
'package.json).');
|
|
100
|
+
}
|
|
101
|
+
// package.json exists but could not be read (permissions, it's a directory,
|
|
102
|
+
// etc.) — that is a different problem than a missing package.json, and
|
|
103
|
+
// telling the user to create a file that already exists would be
|
|
104
|
+
// actively misleading.
|
|
105
|
+
throw new Error(`could not read package.json in ${root}: ${e.message}. ` +
|
|
106
|
+
'Check that it is a regular, readable file and try again.');
|
|
107
|
+
}
|
|
108
|
+
await assertSinglePackage(pkgText, root);
|
|
109
|
+
const present = [];
|
|
110
|
+
for (const lockfile of LOCKFILES) {
|
|
111
|
+
if (await exists(path.join(root, lockfile))) {
|
|
112
|
+
present.push({ lockfile, ...managerFor(lockfile) });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (present.length === 0) {
|
|
116
|
+
throw new Error(`no lockfile found (looked for ${LOCKFILES.join(', ')}). ` +
|
|
117
|
+
'A baseline pinned without a lockfile could not be reproduced, so this is refused. ' +
|
|
118
|
+
'Commit a lockfile and try again.');
|
|
119
|
+
}
|
|
120
|
+
// Two bun lockfile names (bun.lockb, bun.lock) map to the same manager; that
|
|
121
|
+
// is not ambiguity, so we compare distinct managers, not lockfile count.
|
|
122
|
+
const managers = new Set(present.map((p) => p.pm));
|
|
123
|
+
if (managers.size > 1) {
|
|
124
|
+
throw new Error(`more than one lockfile is present (${present.map((p) => p.lockfile).join(', ')}), ` +
|
|
125
|
+
'implying more than one package manager. Which one owns this repository would then ' +
|
|
126
|
+
'be a guess, and guessing wrong installs the baseline differently from the ' +
|
|
127
|
+
'candidate. Remove the stale lockfile and try again.');
|
|
128
|
+
}
|
|
129
|
+
return present[0];
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=detect.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/pm/detect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAa/C;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,QAAsB;IACxC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,mBAAmB;YACtB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAA;QAChD,KAAK,gBAAgB;YACnB,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,gCAAgC,EAAE,CAAA;QACzE,KAAK,WAAW;YACd,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,0BAA0B,EAAE,CAAA;QACnE,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,CAAA;QACvE,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,UAAU,GAAU,QAAQ,CAAA;YAClC,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAA;QACf,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,sBAAsB,CAAC,UAAmB;IACjD,OAAO,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI,CAAA;AACxD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE,IAAY;IACzE,IAAI,GAAY,CAAA;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,mCAAoC,CAAW,CAAC,OAAO,IAAI;YACzD,oCAAoC,CACvC,CAAA;IACH,CAAC;IACD,MAAM,UAAU,GAAI,GAAgC,CAAC,UAAU,CAAA;IAC/D,IAAI,sBAAsB,CAAC,UAAU,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,qFAAqF;YACnF,+EAA+E;YAC/E,qFAAqF;YACrF,uFAAuF;YACvF,gFAAgF,CACnF,CAAA;IACH,CAAC;IACD,IAAI,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,+EAA+E;YAC/E,qFAAqF;YACrF,uFAAuF;YACvF,gFAAgF,CACnF,CAAA;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAA;IAC/C,IAAI,OAAe,CAAA;IACnB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC3C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACb,sBAAsB,IAAI,4CAA4C;gBACpE,iFAAiF;gBACjF,gBAAgB,CACnB,CAAA;QACH,CAAC;QACD,4EAA4E;QAC5E,uEAAuE;QACvE,iEAAiE;QACjE,uBAAuB;QACvB,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,KAAM,CAAW,CAAC,OAAO,IAAI;YACjE,0DAA0D,CAC7D,CAAA;IACH,CAAC;IACD,MAAM,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAExC,MAAM,OAAO,GAAe,EAAE,CAAA;IAC9B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QACrD,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;YACxD,oFAAoF;YACpF,kCAAkC,CACrC,CAAA;IACH,CAAC;IACD,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAClD,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,sCAAsC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;YAClF,oFAAoF;YACpF,4EAA4E;YAC5E,qDAAqD,CACxD,CAAA;IACH,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAE,CAAA;AACpB,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export interface HotFrame {
|
|
2
|
+
functionName: string;
|
|
3
|
+
url: string;
|
|
4
|
+
lineNumber: number;
|
|
5
|
+
/** Sum of the microsecond timeDeltas attributed to this function+url, across every sample. */
|
|
6
|
+
selfTimeUs: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Aggregates a raw `.cpuprofile` JSON document into self time per
|
|
10
|
+
* `functionName` + `url`, dropping Node-internal and anonymous-root frames,
|
|
11
|
+
* sorted with the hottest function first.
|
|
12
|
+
*
|
|
13
|
+
* Self time per node id is the sum of `timeDeltas[i]` for every index `i`
|
|
14
|
+
* where `samples[i]` is that node -- `samples` and `timeDeltas` are parallel
|
|
15
|
+
* arrays, not independent ones. A profile that samples nothing but Node
|
|
16
|
+
* internals (a benchmark too fast to catch any user code in a sample) is a
|
|
17
|
+
* real input, not a bug: this returns an empty array rather than throwing,
|
|
18
|
+
* so a caller can tell "genuinely nothing to report" from a parse failure.
|
|
19
|
+
*/
|
|
20
|
+
export declare function parseCpuProfile(text: string): HotFrame[];
|
|
21
|
+
export interface ProfileOneOptions {
|
|
22
|
+
/** The worktree/repo to measure in -- sets module resolution and tsconfig context. */
|
|
23
|
+
cwd: string;
|
|
24
|
+
benchFileAbs: string;
|
|
25
|
+
fn: string;
|
|
26
|
+
id: string;
|
|
27
|
+
benchtimeMs: number;
|
|
28
|
+
warmupMs: number;
|
|
29
|
+
timeoutMs: number;
|
|
30
|
+
nodeArgs: string[];
|
|
31
|
+
/** Directory the `.cpuprofile` file is written into; created if missing. */
|
|
32
|
+
profileDir: string;
|
|
33
|
+
log?: ((s: string) => void) | undefined;
|
|
34
|
+
}
|
|
35
|
+
export interface ProfileResult {
|
|
36
|
+
id: string;
|
|
37
|
+
/** Absolute path to the written `.cpuprofile` file. */
|
|
38
|
+
profilePath: string;
|
|
39
|
+
/** Aggregated, sorted (hottest first) user-code hot frames. */
|
|
40
|
+
hotFrames: HotFrame[];
|
|
41
|
+
/** Sum of every kept hot frame's self time -- the denominator for percentages. */
|
|
42
|
+
totalSelfUs: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Profiles one benchmark once, by spawning the same measurement child
|
|
46
|
+
* `runChild` uses elsewhere, with `--cpu-prof` added to its node args.
|
|
47
|
+
*
|
|
48
|
+
* Never requires a baseline: this only ever spawns a fresh child in `cwd`
|
|
49
|
+
* and reads back what Node's own profiler wrote, so it works before a run
|
|
50
|
+
* has ever started -- it is reconnaissance for deciding whether the
|
|
51
|
+
* benchmarks even point at the right code.
|
|
52
|
+
*/
|
|
53
|
+
export declare function profileBenchmark(o: ProfileOneOptions): Promise<ProfileResult>;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs one benchmark under Node's built-in `--cpu-prof` sampling profiler
|
|
3
|
+
* and turns the resulting `.cpuprofile` document into an aggregated,
|
|
4
|
+
* sorted list of hot functions.
|
|
5
|
+
*
|
|
6
|
+
* This exists so the optimizing agent works from real profiler data instead
|
|
7
|
+
* of guessing at hot spots from reading source -- a profiler that only ever
|
|
8
|
+
* reports Node internals gives it nothing to act on.
|
|
9
|
+
*/
|
|
10
|
+
import { mkdir, readFile, rm } from 'node:fs/promises';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { runChild } from '../benchrun/invoke.js';
|
|
13
|
+
/** True for a frame this report should never surface: it is not the user's own code. */
|
|
14
|
+
function isNodeInternal(url) {
|
|
15
|
+
return url === '' || url.startsWith('node:');
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Aggregates a raw `.cpuprofile` JSON document into self time per
|
|
19
|
+
* `functionName` + `url`, dropping Node-internal and anonymous-root frames,
|
|
20
|
+
* sorted with the hottest function first.
|
|
21
|
+
*
|
|
22
|
+
* Self time per node id is the sum of `timeDeltas[i]` for every index `i`
|
|
23
|
+
* where `samples[i]` is that node -- `samples` and `timeDeltas` are parallel
|
|
24
|
+
* arrays, not independent ones. A profile that samples nothing but Node
|
|
25
|
+
* internals (a benchmark too fast to catch any user code in a sample) is a
|
|
26
|
+
* real input, not a bug: this returns an empty array rather than throwing,
|
|
27
|
+
* so a caller can tell "genuinely nothing to report" from a parse failure.
|
|
28
|
+
*/
|
|
29
|
+
export function parseCpuProfile(text) {
|
|
30
|
+
const doc = JSON.parse(text);
|
|
31
|
+
const nodeById = new Map(doc.nodes.map((n) => [n.id, n]));
|
|
32
|
+
const selfUsByNodeId = new Map();
|
|
33
|
+
for (let i = 0; i < doc.samples.length; i++) {
|
|
34
|
+
const nodeId = doc.samples[i];
|
|
35
|
+
const delta = doc.timeDeltas[i];
|
|
36
|
+
if (nodeId === undefined || delta === undefined)
|
|
37
|
+
continue;
|
|
38
|
+
selfUsByNodeId.set(nodeId, (selfUsByNodeId.get(nodeId) ?? 0) + delta);
|
|
39
|
+
}
|
|
40
|
+
const byKey = new Map();
|
|
41
|
+
for (const [nodeId, selfTimeUs] of selfUsByNodeId) {
|
|
42
|
+
const node = nodeById.get(nodeId);
|
|
43
|
+
if (!node)
|
|
44
|
+
continue;
|
|
45
|
+
const { functionName, url, lineNumber } = node.callFrame;
|
|
46
|
+
if (isNodeInternal(url))
|
|
47
|
+
continue;
|
|
48
|
+
// A "|" separator is enough here: functionName and url both come from
|
|
49
|
+
// source code the profiler itself observed, and even a collision would
|
|
50
|
+
// only ever merge two frames' self times together (never crash or
|
|
51
|
+
// misattribute to a third, unrelated frame) -- an acceptable, extremely
|
|
52
|
+
// unlikely edge case for a reporting tool, not a security boundary.
|
|
53
|
+
const key = functionName + '|' + url;
|
|
54
|
+
const existing = byKey.get(key);
|
|
55
|
+
if (existing) {
|
|
56
|
+
existing.selfTimeUs += selfTimeUs;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
byKey.set(key, { functionName, url, lineNumber, selfTimeUs });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return [...byKey.values()].sort((a, b) => b.selfTimeUs - a.selfTimeUs);
|
|
63
|
+
}
|
|
64
|
+
/** Turns a benchmark id into a filesystem-safe `.cpuprofile` basename. */
|
|
65
|
+
function profileFileName(id) {
|
|
66
|
+
return id.replace(/[^A-Za-z0-9._-]/g, '_') + '.cpuprofile';
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Profiles one benchmark once, by spawning the same measurement child
|
|
70
|
+
* `runChild` uses elsewhere, with `--cpu-prof` added to its node args.
|
|
71
|
+
*
|
|
72
|
+
* Never requires a baseline: this only ever spawns a fresh child in `cwd`
|
|
73
|
+
* and reads back what Node's own profiler wrote, so it works before a run
|
|
74
|
+
* has ever started -- it is reconnaissance for deciding whether the
|
|
75
|
+
* benchmarks even point at the right code.
|
|
76
|
+
*/
|
|
77
|
+
export async function profileBenchmark(o) {
|
|
78
|
+
await mkdir(o.profileDir, { recursive: true });
|
|
79
|
+
const name = profileFileName(o.id);
|
|
80
|
+
const profilePath = path.join(o.profileDir, name);
|
|
81
|
+
// Remove any stale file at this exact path first, so a previous run's
|
|
82
|
+
// leftover (e.g. this benchmark crashing before writing one) is never
|
|
83
|
+
// mistaken for the profile this call is about to produce.
|
|
84
|
+
await rm(profilePath, { force: true });
|
|
85
|
+
const nodeArgs = [...o.nodeArgs, '--cpu-prof', '--cpu-prof-dir', o.profileDir, '--cpu-prof-name', name];
|
|
86
|
+
const result = await runChild({
|
|
87
|
+
cwd: o.cwd,
|
|
88
|
+
benchFileAbs: o.benchFileAbs,
|
|
89
|
+
fn: o.fn,
|
|
90
|
+
id: o.id,
|
|
91
|
+
benchtimeMs: o.benchtimeMs,
|
|
92
|
+
warmupMs: o.warmupMs,
|
|
93
|
+
timeoutMs: o.timeoutMs,
|
|
94
|
+
nodeArgs,
|
|
95
|
+
...(o.log ? { log: o.log } : {}),
|
|
96
|
+
});
|
|
97
|
+
if (!result.ok) {
|
|
98
|
+
throw new Error(`profiling ${o.id} failed: ${result.error}`);
|
|
99
|
+
}
|
|
100
|
+
const text = await readFile(profilePath, 'utf8');
|
|
101
|
+
const hotFrames = parseCpuProfile(text);
|
|
102
|
+
const totalSelfUs = hotFrames.reduce((sum, f) => sum + f.selfTimeUs, 0);
|
|
103
|
+
return { id: o.id, profilePath, hotFrames, totalSelfUs };
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=profile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile.js","sourceRoot":"","sources":["../../src/profile/profile.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAA;AA8BhD,wFAAwF;AACxF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,KAAK,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;AAC9C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAA;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAC/B,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QACzD,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;IACzC,KAAK,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,cAAc,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QACjC,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAA;QACxD,IAAI,cAAc,CAAC,GAAG,CAAC;YAAE,SAAQ;QACjC,sEAAsE;QACtE,uEAAuE;QACvE,kEAAkE;QAClE,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,GAAG,GAAG,YAAY,GAAG,GAAG,GAAG,GAAG,CAAA;QACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,UAAU,IAAI,UAAU,CAAA;QACnC,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;QAC/D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;AACxE,CAAC;AA2BD,0EAA0E;AAC1E,SAAS,eAAe,CAAC,EAAU;IACjC,OAAO,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,GAAG,aAAa,CAAA;AAC5D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,CAAoB;IACzD,MAAM,KAAK,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC9C,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;IACjD,sEAAsE;IACtE,sEAAsE;IACtE,0DAA0D;IAC1D,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC,UAAU,EAAE,iBAAiB,EAAE,IAAI,CAAC,CAAA;IAEvG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ;QACR,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACjC,CAAC,CAAA;IAEF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IAChD,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;IAEvE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,CAAA;AAC1D,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { DiscardReason, Status } from '../verdict/verdict.js';
|
|
2
|
+
/** Repo-root-relative default location of the log. */
|
|
3
|
+
export declare const RESULTS_PATH = "results.tsv";
|
|
4
|
+
export declare const HEADER = "commit\tscore\tbest_bench_delta\tp_min\tstatus\treason\tdescription";
|
|
5
|
+
export interface Row {
|
|
6
|
+
commit: string;
|
|
7
|
+
score: number;
|
|
8
|
+
bestBenchDelta: number;
|
|
9
|
+
pMin: number;
|
|
10
|
+
/**
|
|
11
|
+
* Typed against verdict's own `Status` rather than a plain string: every
|
|
12
|
+
* producer of a row already holds one of these four values, so a stray
|
|
13
|
+
* status string would be a bug at the call site, not something this log
|
|
14
|
+
* should quietly tolerate.
|
|
15
|
+
*/
|
|
16
|
+
status: Status;
|
|
17
|
+
/**
|
|
18
|
+
* A `DiscardReason` on a discard row; the empty string on every other
|
|
19
|
+
* row (keep, fail, crash). Never `undefined` -- a field that sometimes
|
|
20
|
+
* exists and sometimes doesn't cannot round-trip through a fixed
|
|
21
|
+
* 7-column TSV, whose row shape does not vary with status.
|
|
22
|
+
*/
|
|
23
|
+
reason: DiscardReason | '';
|
|
24
|
+
description: string;
|
|
25
|
+
}
|
|
26
|
+
export interface Summary {
|
|
27
|
+
counts: Record<string, number>;
|
|
28
|
+
/** Product of every kept row's score; 1 (no-op) when there are none. */
|
|
29
|
+
cumulativeSpeedup: number;
|
|
30
|
+
topWins: Row[];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Appends one row, writing the header first if the file is new or empty.
|
|
34
|
+
* Not safe for concurrent writers -- callers are expected to hold the eval
|
|
35
|
+
* lock across the whole experiment, so only one process ever appends at a
|
|
36
|
+
* time.
|
|
37
|
+
*/
|
|
38
|
+
export declare function appendRow(path: string, r: Row): Promise<void>;
|
|
39
|
+
/**
|
|
40
|
+
* Loads every row. A missing file is an empty log, not an error -- this is
|
|
41
|
+
* the state of a repository that has never run an experiment.
|
|
42
|
+
*
|
|
43
|
+
* Anything else is strict: a line that does not split into exactly 7
|
|
44
|
+
* tab-separated fields, or whose numeric or enum fields don't parse,
|
|
45
|
+
* fails the WHOLE load, naming the file and the 1-based line number.
|
|
46
|
+
* Sanitizing on write (see `sanitize`) already makes a malformed row
|
|
47
|
+
* nearly impossible to produce honestly, so encountering one is a real
|
|
48
|
+
* signal -- a torn write, a hand edit -- not noise. Silently dropping the
|
|
49
|
+
* bad row instead would let a corrupted log masquerade as a short one,
|
|
50
|
+
* which is the worse failure for a file that is the sole record of an
|
|
51
|
+
* unattended overnight run. The same reasoning extends past field count
|
|
52
|
+
* to field *content*: a torn write is at least as likely to garble a
|
|
53
|
+
* value as to drop a whole field, and a garbled numeric field that
|
|
54
|
+
* parses "successfully" to 0 or NaN is a worse failure than a visible
|
|
55
|
+
* parse error, since it distorts every summary silently.
|
|
56
|
+
*/
|
|
57
|
+
export declare function loadRows(path: string): Promise<Row[]>;
|
|
58
|
+
/**
|
|
59
|
+
* Counts per status, the cumulative speedup, and the biggest individual
|
|
60
|
+
* wins. Never throws and never produces NaN, even for an empty log: an
|
|
61
|
+
* empty log is a real, valid state (no experiments have run yet), not an
|
|
62
|
+
* error, and a report reading it should say "1x, nothing tried yet"
|
|
63
|
+
* rather than crash.
|
|
64
|
+
*/
|
|
65
|
+
export declare function summarize(rows: Row[]): Summary;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { appendFile, readFile, stat } from 'node:fs/promises';
|
|
2
|
+
/** Repo-root-relative default location of the log. */
|
|
3
|
+
export const RESULTS_PATH = 'results.tsv';
|
|
4
|
+
export const HEADER = 'commit\tscore\tbest_bench_delta\tp_min\tstatus\treason\tdescription';
|
|
5
|
+
const FIELD_COUNT = 7;
|
|
6
|
+
/** Character (not byte) cap on `description`, ellipsis included. */
|
|
7
|
+
const DESCRIPTION_LIMIT = 256;
|
|
8
|
+
const VALID_STATUSES = ['keep', 'discard', 'fail', 'crash'];
|
|
9
|
+
const VALID_REASONS = [
|
|
10
|
+
'no_significant_improvement',
|
|
11
|
+
'improvement_below_min_effect',
|
|
12
|
+
'significant_regression',
|
|
13
|
+
];
|
|
14
|
+
/**
|
|
15
|
+
* Tabs, CRs and LFs are the TSV's own field and row delimiters. Left
|
|
16
|
+
* untouched, a single embedded newline in e.g. `description` would split
|
|
17
|
+
* one experiment into two lines on disk -- and the second half, missing
|
|
18
|
+
* its leading columns, would fail `loadRows`'s field-count check on every
|
|
19
|
+
* future read. Flattening them to spaces before a value ever reaches disk
|
|
20
|
+
* makes that failure mode structurally impossible rather than merely
|
|
21
|
+
* unlikely, whatever the field.
|
|
22
|
+
*/
|
|
23
|
+
function sanitize(s) {
|
|
24
|
+
return s.replace(/[\t\r\n]/g, ' ');
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Truncates by Unicode code point, never by byte or by UTF-16 code unit.
|
|
28
|
+
* `Array.from` iterates a string by code point, so a surrogate pair (an
|
|
29
|
+
* astral character, e.g. an emoji) is kept or dropped whole. A byte-count
|
|
30
|
+
* cut (e.g. slicing a UTF-8 `Buffer`) could stop mid-sequence and decode
|
|
31
|
+
* back as a replacement character; a naive `.slice` on UTF-16 units could
|
|
32
|
+
* split a surrogate pair into two lone, invalid halves. Either would let an
|
|
33
|
+
* agent's pasted stack trace or diff -- pasted verbatim into `-desc` --
|
|
34
|
+
* produce a row that doesn't even round-trip as valid text.
|
|
35
|
+
*/
|
|
36
|
+
function truncateDescription(s) {
|
|
37
|
+
const chars = Array.from(s);
|
|
38
|
+
if (chars.length <= DESCRIPTION_LIMIT)
|
|
39
|
+
return s;
|
|
40
|
+
return `${chars.slice(0, DESCRIPTION_LIMIT - 3).join('')}...`;
|
|
41
|
+
}
|
|
42
|
+
function formatRow(r) {
|
|
43
|
+
const fields = [
|
|
44
|
+
sanitize(r.commit),
|
|
45
|
+
// 4 decimals: score is a ratio hovering around 1.0, where a 0.01%
|
|
46
|
+
// difference is noise; 4 places is enough to distinguish real
|
|
47
|
+
// improvements without implying false precision.
|
|
48
|
+
r.score.toFixed(4),
|
|
49
|
+
// best_bench_delta is a percentage; 2 decimal places (0.01 pp) is
|
|
50
|
+
// already finer than the run-to-run noise it's measuring.
|
|
51
|
+
r.bestBenchDelta.toFixed(2),
|
|
52
|
+
// p_min is a probability, not a percentage, and can be extremely
|
|
53
|
+
// small (e.g. 1.08e-5). Formatting it to a fixed 2 decimals would
|
|
54
|
+
// print "0.00" for every significant result, making them
|
|
55
|
+
// indistinguishable from each other and from a genuine p=0. JS's
|
|
56
|
+
// own number-to-string conversion is lossless (it prints the
|
|
57
|
+
// shortest decimal that reads back to the exact same double), and
|
|
58
|
+
// switches to exponential notation on its own for very small
|
|
59
|
+
// magnitudes, so it is used as-is instead of a fixed decimal count.
|
|
60
|
+
String(r.pMin),
|
|
61
|
+
sanitize(r.status),
|
|
62
|
+
sanitize(r.reason),
|
|
63
|
+
truncateDescription(sanitize(r.description)),
|
|
64
|
+
];
|
|
65
|
+
return fields.join('\t');
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Appends one row, writing the header first if the file is new or empty.
|
|
69
|
+
* Not safe for concurrent writers -- callers are expected to hold the eval
|
|
70
|
+
* lock across the whole experiment, so only one process ever appends at a
|
|
71
|
+
* time.
|
|
72
|
+
*/
|
|
73
|
+
export async function appendRow(path, r) {
|
|
74
|
+
let needsHeader = true;
|
|
75
|
+
try {
|
|
76
|
+
const info = await stat(path);
|
|
77
|
+
needsHeader = info.size === 0;
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
if (err.code !== 'ENOENT')
|
|
81
|
+
throw err;
|
|
82
|
+
needsHeader = true;
|
|
83
|
+
}
|
|
84
|
+
const prefix = needsHeader ? `${HEADER}\n` : '';
|
|
85
|
+
await appendFile(path, `${prefix}${formatRow(r)}\n`, 'utf8');
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Parses one numeric column strictly. `Number('')` is `0` in JavaScript --
|
|
89
|
+
* left unchecked, a torn write that leaves an empty field (still 7 fields
|
|
90
|
+
* total, so the field-count check alone would never catch it) would
|
|
91
|
+
* silently produce a legitimate-looking score of 0 rather than an obvious
|
|
92
|
+
* parse failure. A blank `score` on a `keep` row is the worst case: it
|
|
93
|
+
* collapses `summarize`'s cumulative-speedup PRODUCT for the ENTIRE log to
|
|
94
|
+
* 0, with no visible anomaly. So blank is checked and rejected explicitly,
|
|
95
|
+
* before ever calling `Number` on it; `Number.isFinite` then also catches
|
|
96
|
+
* genuinely non-numeric text (which parses to `NaN`) and `Infinity`.
|
|
97
|
+
*/
|
|
98
|
+
function parseNumberField(path, lineNo, field, raw) {
|
|
99
|
+
if (raw.trim() === '') {
|
|
100
|
+
throw new Error(`${path}:${lineNo}: ${field}: empty field, want a number`);
|
|
101
|
+
}
|
|
102
|
+
const n = Number(raw);
|
|
103
|
+
if (!Number.isFinite(n)) {
|
|
104
|
+
throw new Error(`${path}:${lineNo}: ${field}: invalid number "${raw}"`);
|
|
105
|
+
}
|
|
106
|
+
return n;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* `status` must be one of the four values `Status` allows. An unrecognized
|
|
110
|
+
* status has a milder blast radius than a bad number -- it simply never
|
|
111
|
+
* matches the `'keep'` filter in `summarize`, so it can't poison the
|
|
112
|
+
* cumulative-speedup product -- but it would still let `counts` silently
|
|
113
|
+
* accumulate an unexpected key, which is the same "corrupted log
|
|
114
|
+
* masquerading as valid" failure the field-count check exists to prevent.
|
|
115
|
+
*/
|
|
116
|
+
function parseStatus(path, lineNo, raw) {
|
|
117
|
+
if (!VALID_STATUSES.includes(raw)) {
|
|
118
|
+
throw new Error(`${path}:${lineNo}: status: unknown status "${raw}"`);
|
|
119
|
+
}
|
|
120
|
+
return raw;
|
|
121
|
+
}
|
|
122
|
+
/** `reason` must be empty, or one of the three `DiscardReason` values. */
|
|
123
|
+
function parseReason(path, lineNo, raw) {
|
|
124
|
+
if (raw === '' || VALID_REASONS.includes(raw)) {
|
|
125
|
+
return raw;
|
|
126
|
+
}
|
|
127
|
+
throw new Error(`${path}:${lineNo}: reason: unknown reason "${raw}"`);
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Loads every row. A missing file is an empty log, not an error -- this is
|
|
131
|
+
* the state of a repository that has never run an experiment.
|
|
132
|
+
*
|
|
133
|
+
* Anything else is strict: a line that does not split into exactly 7
|
|
134
|
+
* tab-separated fields, or whose numeric or enum fields don't parse,
|
|
135
|
+
* fails the WHOLE load, naming the file and the 1-based line number.
|
|
136
|
+
* Sanitizing on write (see `sanitize`) already makes a malformed row
|
|
137
|
+
* nearly impossible to produce honestly, so encountering one is a real
|
|
138
|
+
* signal -- a torn write, a hand edit -- not noise. Silently dropping the
|
|
139
|
+
* bad row instead would let a corrupted log masquerade as a short one,
|
|
140
|
+
* which is the worse failure for a file that is the sole record of an
|
|
141
|
+
* unattended overnight run. The same reasoning extends past field count
|
|
142
|
+
* to field *content*: a torn write is at least as likely to garble a
|
|
143
|
+
* value as to drop a whole field, and a garbled numeric field that
|
|
144
|
+
* parses "successfully" to 0 or NaN is a worse failure than a visible
|
|
145
|
+
* parse error, since it distorts every summary silently.
|
|
146
|
+
*/
|
|
147
|
+
export async function loadRows(path) {
|
|
148
|
+
let text;
|
|
149
|
+
try {
|
|
150
|
+
text = await readFile(path, 'utf8');
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
if (err.code === 'ENOENT')
|
|
154
|
+
return [];
|
|
155
|
+
throw err;
|
|
156
|
+
}
|
|
157
|
+
const lines = text.split('\n');
|
|
158
|
+
// A well-formed file ends with a trailing newline, which turns into one
|
|
159
|
+
// trailing empty element after split; drop it rather than counting it as
|
|
160
|
+
// a (blank, malformed) final line.
|
|
161
|
+
if (lines.length > 0 && lines[lines.length - 1] === '')
|
|
162
|
+
lines.pop();
|
|
163
|
+
const rows = [];
|
|
164
|
+
for (let i = 0; i < lines.length; i++) {
|
|
165
|
+
const lineNo = i + 1;
|
|
166
|
+
const rawLine = lines[i] ?? '';
|
|
167
|
+
if (lineNo === 1 && rawLine === HEADER)
|
|
168
|
+
continue;
|
|
169
|
+
const fields = rawLine.split('\t');
|
|
170
|
+
if (fields.length !== FIELD_COUNT) {
|
|
171
|
+
throw new Error(`${path}:${lineNo}: got ${fields.length} fields, want ${FIELD_COUNT}`);
|
|
172
|
+
}
|
|
173
|
+
const [commit, scoreStr, deltaStr, pMinStr, status, reason, description] = fields;
|
|
174
|
+
rows.push({
|
|
175
|
+
commit,
|
|
176
|
+
score: parseNumberField(path, lineNo, 'score', scoreStr),
|
|
177
|
+
bestBenchDelta: parseNumberField(path, lineNo, 'best_bench_delta', deltaStr),
|
|
178
|
+
pMin: parseNumberField(path, lineNo, 'p_min', pMinStr),
|
|
179
|
+
status: parseStatus(path, lineNo, status),
|
|
180
|
+
reason: parseReason(path, lineNo, reason),
|
|
181
|
+
description,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
return rows;
|
|
185
|
+
}
|
|
186
|
+
/** Number of "top wins" to report -- the most impactful kept experiments. */
|
|
187
|
+
const TOP_WINS = 3;
|
|
188
|
+
/**
|
|
189
|
+
* Counts per status, the cumulative speedup, and the biggest individual
|
|
190
|
+
* wins. Never throws and never produces NaN, even for an empty log: an
|
|
191
|
+
* empty log is a real, valid state (no experiments have run yet), not an
|
|
192
|
+
* error, and a report reading it should say "1x, nothing tried yet"
|
|
193
|
+
* rather than crash.
|
|
194
|
+
*/
|
|
195
|
+
export function summarize(rows) {
|
|
196
|
+
const counts = {};
|
|
197
|
+
for (const r of rows) {
|
|
198
|
+
counts[r.status] = (counts[r.status] ?? 0) + 1;
|
|
199
|
+
}
|
|
200
|
+
const kept = rows.filter((r) => r.status === 'keep');
|
|
201
|
+
// The PRODUCT of every kept score, not the latest and not a mean. Each
|
|
202
|
+
// KEEP advances the measurement baseline to the just-kept commit (see
|
|
203
|
+
// baseline.measureCommit), so every subsequent score is only that
|
|
204
|
+
// experiment's own incremental contribution on top of the last win, not
|
|
205
|
+
// a re-measurement against the original baseline. Successive real
|
|
206
|
+
// improvements compound the way successive percentage changes do, so
|
|
207
|
+
// only their product reflects the true end-to-end speedup. `reduce`'s
|
|
208
|
+
// seed of 1 is what makes an empty (or keep-less) log report "no
|
|
209
|
+
// speedup yet" instead of dividing by, or starting from, zero.
|
|
210
|
+
const cumulativeSpeedup = kept.reduce((acc, r) => acc * r.score, 1);
|
|
211
|
+
const topWins = [...kept].sort((a, b) => a.score - b.score).slice(0, TOP_WINS);
|
|
212
|
+
return { counts, cumulativeSpeedup, topWins };
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=results.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/results/results.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAG7D,sDAAsD;AACtD,MAAM,CAAC,MAAM,YAAY,GAAG,aAAa,CAAA;AAEzC,MAAM,CAAC,MAAM,MAAM,GAAG,qEAAqE,CAAA;AAE3F,MAAM,WAAW,GAAG,CAAC,CAAA;AACrB,oEAAoE;AACpE,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAE7B,MAAM,cAAc,GAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AAC9E,MAAM,aAAa,GAA6B;IAC9C,4BAA4B;IAC5B,8BAA8B;IAC9B,wBAAwB;CACzB,CAAA;AA+BD;;;;;;;;GAQG;AACH,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;AACpC,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,mBAAmB,CAAC,CAAS;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3B,IAAI,KAAK,CAAC,MAAM,IAAI,iBAAiB;QAAE,OAAO,CAAC,CAAA;IAC/C,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED,SAAS,SAAS,CAAC,CAAM;IACvB,MAAM,MAAM,GAAG;QACb,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;QAClB,kEAAkE;QAClE,8DAA8D;QAC9D,iDAAiD;QACjD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAClB,kEAAkE;QAClE,0DAA0D;QAC1D,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3B,iEAAiE;QACjE,kEAAkE;QAClE,yDAAyD;QACzD,iEAAiE;QACjE,6DAA6D;QAC7D,kEAAkE;QAClE,6DAA6D;QAC7D,oEAAoE;QACpE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QACd,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;QAClB,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;QAClB,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;KAC7C,CAAA;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,CAAM;IAClD,IAAI,WAAW,GAAG,IAAI,CAAA;IACtB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7B,WAAW,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC,CAAA;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,GAAG,CAAA;QAC/D,WAAW,GAAG,IAAI,CAAA;IACpB,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/C,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAC9D,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,gBAAgB,CAAC,IAAY,EAAE,MAAc,EAAE,KAAa,EAAE,GAAW;IAChF,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,KAAK,8BAA8B,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;IACrB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,KAAK,qBAAqB,GAAG,GAAG,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,IAAY,EAAE,MAAc,EAAE,GAAW;IAC5D,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAa,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,6BAA6B,GAAG,GAAG,CAAC,CAAA;IACvE,CAAC;IACD,OAAO,GAAa,CAAA;AACtB,CAAC;AAED,0EAA0E;AAC1E,SAAS,WAAW,CAAC,IAAY,EAAE,MAAc,EAAE,GAAW;IAC5D,IAAI,GAAG,KAAK,EAAE,IAAI,aAAa,CAAC,QAAQ,CAAC,GAAoB,CAAC,EAAE,CAAC;QAC/D,OAAO,GAAyB,CAAA;IAClC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,6BAA6B,GAAG,GAAG,CAAC,CAAA;AACvE,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,IAAI,IAAY,CAAA;IAChB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAA;QAC/D,MAAM,GAAG,CAAA;IACX,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC9B,wEAAwE;IACxE,yEAAyE;IACzE,mCAAmC;IACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,CAAC,GAAG,EAAE,CAAA;IAEnE,MAAM,IAAI,GAAU,EAAE,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAC9B,IAAI,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,MAAM;YAAE,SAAQ;QAChD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,SAAS,MAAM,CAAC,MAAM,iBAAiB,WAAW,EAAE,CAAC,CAAA;QACxF,CAAC;QACD,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,GAAG,MAQ1E,CAAA;QACD,IAAI,CAAC,IAAI,CAAC;YACR,MAAM;YACN,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC;YACxD,cAAc,EAAE,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,CAAC;YAC5E,IAAI,EAAE,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;YACtD,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC;YACzC,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC;YACzC,WAAW;SACZ,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,6EAA6E;AAC7E,MAAM,QAAQ,GAAG,CAAC,CAAA;AAElB;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,IAAW;IACnC,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;IAEpD,uEAAuE;IACvE,sEAAsE;IACtE,kEAAkE;IAClE,wEAAwE;IACxE,kEAAkE;IAClE,qEAAqE;IACrE,sEAAsE;IACtE,iEAAiE;IACjE,+DAA+D;IAC/D,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAEnE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE9E,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAA;AAC/C,CAAC"}
|