@zgeoff/rig 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bench/bench-report-entry-schema.d.ts +10 -0
- package/dist/bench/build-base-side-root.d.ts +1 -0
- package/dist/bench/collect-bench-comparison.d.ts +3 -0
- package/dist/bench/define-bench.d.ts +2 -0
- package/dist/bench/find-base-setup-verdict.d.ts +3 -0
- package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
- package/dist/bench/format-bench-comparison.d.ts +2 -0
- package/dist/bench/format-bench-report.d.ts +3 -0
- package/dist/bench/format-benchmark-result.d.ts +1 -1
- package/dist/bench/is-concurrency-list.d.ts +1 -0
- package/dist/bench/parse-bench-report.d.ts +9 -0
- package/dist/bench/pick-bench-exit-code.d.ts +2 -0
- package/dist/bench/pick-bench-plan.d.ts +13 -0
- package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
- package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
- package/dist/bench/pick-comparison-verdict.d.ts +7 -0
- package/dist/bench/pick-head-verdict.d.ts +3 -0
- package/dist/bench/pick-named-bench-files.d.ts +1 -0
- package/dist/bench/pick-quantile.d.ts +1 -0
- package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
- package/dist/bench/read-bench-args.d.ts +9 -0
- package/dist/bench/read-bench-file-args.d.ts +9 -0
- package/dist/bench/run-benchmark.d.ts +3 -3
- package/dist/bench/to-bench-name.d.ts +1 -0
- package/dist/bench/types.d.ts +20 -0
- package/dist/build-bench-folder-path.d.ts +1 -0
- package/dist/create-bench-work-folder.d.ts +6 -0
- package/dist/format-parse-error.d.ts +1 -0
- package/dist/read-command-output.d.ts +8 -0
- package/dist/read-git-common-dir.d.ts +6 -0
- package/dist/read-git-prefix.d.ts +6 -0
- package/dist/run-bench.d.ts +7 -0
- package/dist/run-cli.d.ts +1 -2
- package/package.json +4 -4
- package/src/bench/bench-report-entry-schema.ts +17 -0
- package/src/bench/build-base-side-root.ts +7 -0
- package/src/bench/collect-bench-comparison.ts +43 -0
- package/src/bench/define-bench.ts +15 -0
- package/src/bench/find-base-setup-verdict.ts +21 -0
- package/src/bench/find-missing-bench-verdict.ts +33 -0
- package/src/bench/format-bench-comparison.ts +72 -0
- package/src/bench/format-bench-report.ts +24 -0
- package/src/bench/format-benchmark-result.ts +7 -3
- package/src/bench/is-concurrency-list.ts +10 -0
- package/src/bench/parse-bench-definition.ts +21 -3
- package/src/bench/parse-bench-report.ts +42 -0
- package/src/bench/pick-bench-exit-code.ts +15 -0
- package/src/bench/pick-bench-plan.ts +25 -0
- package/src/bench/pick-checkout-cleanup.ts +29 -0
- package/src/bench/pick-cleaned-up-result.ts +30 -0
- package/src/bench/pick-comparison-verdict.ts +36 -0
- package/src/bench/pick-head-verdict.ts +11 -0
- package/src/bench/pick-named-bench-files.ts +8 -0
- package/src/bench/pick-quantile.ts +9 -0
- package/src/bench/pick-unremoved-checkout-state.ts +30 -0
- package/src/bench/read-bench-args.ts +45 -0
- package/src/bench/read-bench-file-args.ts +38 -0
- package/src/bench/run-benchmark.ts +88 -21
- package/src/bench/to-bench-name.ts +5 -0
- package/src/bench/types.ts +37 -0
- package/src/build-bench-folder-path.ts +7 -0
- package/src/cli.ts +31 -0
- package/src/create-bench-work-folder.ts +26 -0
- package/src/format-parse-error.ts +9 -0
- package/src/read-cli-args.ts +1 -10
- package/src/read-command-output.ts +30 -0
- package/src/read-git-common-dir.ts +27 -0
- package/src/read-git-prefix.ts +27 -0
- package/src/run-bench-file.ts +23 -13
- package/src/run-bench.ts +348 -39
- package/src/run-cli.ts +1 -3
- package/dist/bench/pick-median.d.ts +0 -1
- package/src/bench/pick-median.ts +0 -10
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SpawnOptions } from './browser/process-runtime.ts';
|
|
2
|
+
import type { ReadCommandOutput } from './read-command-output.ts';
|
|
3
|
+
|
|
4
|
+
export interface GitCommonDirDeps {
|
|
5
|
+
readonly readCommandOutput: ReadCommandOutput;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
// Returns git's own answer: in the main checkout it is relative to the folder git runs in, such as
|
|
9
|
+
// ../../.git from a package folder, and in a linked worktree it is absolute.
|
|
10
|
+
export async function readGitCommonDir(
|
|
11
|
+
repoRoot: string,
|
|
12
|
+
environment: SpawnOptions['environment'],
|
|
13
|
+
deps: GitCommonDirDeps,
|
|
14
|
+
): Promise<string> {
|
|
15
|
+
const output = await deps.readCommandOutput(['git', 'rev-parse', '--git-common-dir'], {
|
|
16
|
+
cwd: repoRoot,
|
|
17
|
+
environment,
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
if (output.exitCode !== 0) {
|
|
21
|
+
throw new Error(
|
|
22
|
+
`git rev-parse --git-common-dir failed in ${repoRoot} with exit code ${output.exitCode}`,
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
return output.stdout.trim();
|
|
27
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SpawnOptions } from './browser/process-runtime.ts';
|
|
2
|
+
import type { ReadCommandOutput } from './read-command-output.ts';
|
|
3
|
+
|
|
4
|
+
export interface GitPrefixDeps {
|
|
5
|
+
readonly readCommandOutput: ReadCommandOutput;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
// Returns the folder's path inside its repo as git prints it, with a trailing slash, or an empty
|
|
9
|
+
// string at the repo's top level.
|
|
10
|
+
export async function readGitPrefix(
|
|
11
|
+
folder: string,
|
|
12
|
+
environment: SpawnOptions['environment'],
|
|
13
|
+
deps: GitPrefixDeps,
|
|
14
|
+
): Promise<string> {
|
|
15
|
+
const output = await deps.readCommandOutput(['git', 'rev-parse', '--show-prefix'], {
|
|
16
|
+
cwd: folder,
|
|
17
|
+
environment,
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
if (output.exitCode !== 0) {
|
|
21
|
+
throw new Error(
|
|
22
|
+
`git rev-parse --show-prefix failed in ${folder} with exit code ${output.exitCode}`,
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
return output.stdout.trim();
|
|
27
|
+
}
|
package/src/run-bench-file.ts
CHANGED
|
@@ -1,25 +1,35 @@
|
|
|
1
|
+
import { appendFile } from 'node:fs/promises';
|
|
2
|
+
import { formatBenchReport } from './bench/format-bench-report.ts';
|
|
1
3
|
import { formatBenchmarkResult } from './bench/format-benchmark-result.ts';
|
|
2
4
|
import { parseBenchDefinition } from './bench/parse-bench-definition.ts';
|
|
5
|
+
import { readBenchFileArgs } from './bench/read-bench-file-args.ts';
|
|
3
6
|
import { runBenchmark } from './bench/run-benchmark.ts';
|
|
4
7
|
|
|
5
|
-
// `rig bench` runs this once per bench file, in the environment the benchmarks get
|
|
6
|
-
|
|
8
|
+
// `rig bench` runs this once per bench file, in the environment the benchmarks get, and judges the
|
|
9
|
+
// budget from the report this appends, so a bench that ran exits 0 whatever its median.
|
|
10
|
+
const args = await readBenchFileArgs(process.argv.slice(2));
|
|
7
11
|
|
|
8
|
-
|
|
12
|
+
if (args.kind === 'refused') {
|
|
13
|
+
process.stderr.write(`rig bench: ${args.reason}\n`);
|
|
9
14
|
|
|
10
|
-
|
|
11
|
-
|
|
15
|
+
process.exitCode = 2;
|
|
16
|
+
} else {
|
|
17
|
+
const loaded: unknown = await import(args.file);
|
|
12
18
|
|
|
13
|
-
const
|
|
19
|
+
const defaultExport: unknown =
|
|
20
|
+
typeof loaded === 'object' && loaded !== null ? Reflect.get(loaded, 'default') : null;
|
|
14
21
|
|
|
15
|
-
|
|
16
|
-
process.stderr.write(`rig bench: ${file}: ${parsed.reason}\n`);
|
|
22
|
+
const parsed = parseBenchDefinition(defaultExport);
|
|
17
23
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
24
|
+
if (parsed.kind === 'refused') {
|
|
25
|
+
process.stderr.write(`rig bench: ${args.file}: ${parsed.reason}\n`);
|
|
26
|
+
|
|
27
|
+
process.exitCode = 2;
|
|
28
|
+
} else {
|
|
29
|
+
const results = await runBenchmark(parsed.definition, () => performance.now());
|
|
21
30
|
|
|
22
|
-
|
|
31
|
+
process.stdout.write(formatBenchmarkResult(parsed.definition, results));
|
|
23
32
|
|
|
24
|
-
|
|
33
|
+
await appendFile(args.reportPath, formatBenchReport(args.file, parsed.definition, results));
|
|
34
|
+
}
|
|
25
35
|
}
|
package/src/run-bench.ts
CHANGED
|
@@ -1,10 +1,28 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { join, relative } from 'node:path';
|
|
2
|
+
import { match } from 'ts-pattern';
|
|
3
|
+
import { buildBaseSideRoot } from './bench/build-base-side-root.ts';
|
|
4
|
+
import { findBaseSetupVerdict } from './bench/find-base-setup-verdict.ts';
|
|
5
|
+
import { findMissingBenchVerdict } from './bench/find-missing-bench-verdict.ts';
|
|
6
|
+
import { parseBenchReport } from './bench/parse-bench-report.ts';
|
|
7
|
+
import type { ParsedBenchReport } from './bench/parse-bench-report.ts';
|
|
8
|
+
import { pickBenchPlan } from './bench/pick-bench-plan.ts';
|
|
9
|
+
import { pickCheckoutCleanup } from './bench/pick-checkout-cleanup.ts';
|
|
10
|
+
import { pickCleanedUpResult } from './bench/pick-cleaned-up-result.ts';
|
|
11
|
+
import { pickComparisonVerdict } from './bench/pick-comparison-verdict.ts';
|
|
12
|
+
import { pickHeadVerdict } from './bench/pick-head-verdict.ts';
|
|
13
|
+
import { pickNamedBenchFiles } from './bench/pick-named-bench-files.ts';
|
|
14
|
+
import { pickUnremovedCheckoutState } from './bench/pick-unremoved-checkout-state.ts';
|
|
15
|
+
import { readBenchArgs } from './bench/read-bench-args.ts';
|
|
16
|
+
import type { BenchOutput, BenchVerdict, CheckoutState } from './bench/types.ts';
|
|
2
17
|
import { pickExitCodeAfterTests } from './browser/pick-exit-code-after-tests.ts';
|
|
3
18
|
import { buildDietCommand } from './build-diet-command.ts';
|
|
4
19
|
import { buildProgramLaneHooks } from './build-program-lane-hooks.ts';
|
|
5
20
|
import { collectBenchFiles } from './collect-bench-files.ts';
|
|
21
|
+
import { defineRig } from './define-rig.ts';
|
|
6
22
|
import type { LaneHooks } from './lane-hooks.ts';
|
|
7
23
|
import { makeRepoLaneHooks } from './make-repo-lane-hooks.ts';
|
|
24
|
+
import type { ReadCommandOutput } from './read-command-output.ts';
|
|
25
|
+
import { readGitPrefix } from './read-git-prefix.ts';
|
|
8
26
|
import { runLaneHooks } from './run-lane-hooks.ts';
|
|
9
27
|
import type { ProgramBuildDeps } from './run-program-builds.ts';
|
|
10
28
|
import { runProgramBuilds } from './run-program-builds.ts';
|
|
@@ -16,82 +34,154 @@ export interface BenchDeps extends ProgramBuildDeps {
|
|
|
16
34
|
// The script that runs one bench file.
|
|
17
35
|
readonly benchScript: string;
|
|
18
36
|
|
|
37
|
+
readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
|
|
38
|
+
|
|
19
39
|
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
20
40
|
|
|
21
41
|
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
22
42
|
|
|
23
43
|
readonly importModule: (path: string) => Promise<unknown>;
|
|
44
|
+
|
|
45
|
+
// Makes a new folder for one run's reports and base checkout.
|
|
46
|
+
readonly createWorkFolder: (repoRoot: string) => Promise<string>;
|
|
47
|
+
|
|
48
|
+
readonly readText: (path: string) => Promise<string | null>;
|
|
49
|
+
|
|
50
|
+
readonly probeFolder: (path: string) => Promise<boolean>;
|
|
51
|
+
|
|
52
|
+
readonly removeFolder: (path: string) => Promise<void>;
|
|
53
|
+
|
|
54
|
+
readonly readCommandOutput: ReadCommandOutput;
|
|
24
55
|
}
|
|
25
56
|
|
|
26
57
|
// Builds each program once, as `rig e2e` does, then runs each bench file in a process of its own,
|
|
27
|
-
// between the start and stop hooks of the bench lane when the config names one.
|
|
28
|
-
//
|
|
58
|
+
// between the start and stop hooks of the bench lane when the config names one. A base ref runs
|
|
59
|
+
// the same way from the same folder of a checkout of that ref, with that folder's config.
|
|
29
60
|
export async function runBench(
|
|
30
61
|
args: readonly string[],
|
|
31
62
|
config: RigConfig,
|
|
32
63
|
run: RigRun,
|
|
33
64
|
deps: BenchDeps,
|
|
34
65
|
): Promise<number> {
|
|
35
|
-
|
|
36
|
-
deps.output.err('rig bench: takes at most one benchmark name\n');
|
|
66
|
+
const parsed = await readBenchArgs(args);
|
|
37
67
|
|
|
38
|
-
|
|
68
|
+
if (parsed.kind === 'refused') {
|
|
69
|
+
return writeBenchVerdict(
|
|
70
|
+
{ output: [{ stream: 'err', text: `rig bench: ${parsed.reason}\n` }], exitCode: 2 },
|
|
71
|
+
deps,
|
|
72
|
+
);
|
|
39
73
|
}
|
|
40
74
|
|
|
41
|
-
const
|
|
42
|
-
|
|
75
|
+
const headBenchFiles = await readBenchFiles(run.repoRoot, deps);
|
|
76
|
+
|
|
77
|
+
return match(pickBenchPlan(parsed.name, parsed.baseRef, headBenchFiles))
|
|
78
|
+
.with({ kind: 'finished' }, (plan) => writeBenchVerdict(plan.verdict, deps))
|
|
79
|
+
.with({ kind: 'head' }, (plan) => runHeadBench(plan.files, config, run, deps))
|
|
80
|
+
.with({ kind: 'comparison' }, (plan) =>
|
|
81
|
+
runBenchComparison(
|
|
82
|
+
{
|
|
83
|
+
baseRef: plan.baseRef,
|
|
84
|
+
name: parsed.name,
|
|
85
|
+
headConfig: config,
|
|
86
|
+
headFiles: plan.files,
|
|
87
|
+
headBenchFiles,
|
|
88
|
+
},
|
|
89
|
+
run,
|
|
90
|
+
deps,
|
|
91
|
+
),
|
|
92
|
+
)
|
|
93
|
+
.exhaustive();
|
|
94
|
+
}
|
|
43
95
|
|
|
44
|
-
|
|
45
|
-
|
|
96
|
+
function writeBenchVerdict(verdict: BenchVerdict, deps: BenchDeps): number {
|
|
97
|
+
writeBenchOutput(verdict.output, deps);
|
|
46
98
|
|
|
47
|
-
|
|
48
|
-
|
|
99
|
+
return verdict.exitCode;
|
|
100
|
+
}
|
|
49
101
|
|
|
50
|
-
|
|
51
|
-
|
|
102
|
+
function writeBenchOutput(output: readonly BenchOutput[], deps: BenchDeps): void {
|
|
103
|
+
for (const entry of output) {
|
|
104
|
+
deps.output[entry.stream](entry.text);
|
|
52
105
|
}
|
|
106
|
+
}
|
|
53
107
|
|
|
54
|
-
|
|
108
|
+
async function readBenchFiles(root: string, deps: BenchDeps): Promise<readonly string[]> {
|
|
109
|
+
const isWorkspaceRepo = await deps.probeWorkspaceRepo(root);
|
|
110
|
+
const trackedFiles = await deps.readTrackedFiles(root);
|
|
55
111
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
112
|
+
return collectBenchFiles(trackedFiles, isWorkspaceRepo);
|
|
113
|
+
}
|
|
59
114
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
run.repoRoot,
|
|
63
|
-
config.programs,
|
|
64
|
-
);
|
|
115
|
+
interface BenchSide {
|
|
116
|
+
readonly root: string;
|
|
65
117
|
|
|
66
|
-
|
|
118
|
+
readonly config: RigConfig;
|
|
67
119
|
|
|
68
|
-
|
|
69
|
-
}
|
|
120
|
+
readonly files: readonly string[];
|
|
70
121
|
|
|
71
|
-
|
|
72
|
-
return basename(file, '.bench.ts');
|
|
122
|
+
readonly reportPath: string;
|
|
73
123
|
}
|
|
74
124
|
|
|
75
|
-
function
|
|
76
|
-
|
|
77
|
-
|
|
125
|
+
async function runHeadBench(
|
|
126
|
+
files: readonly string[],
|
|
127
|
+
config: RigConfig,
|
|
128
|
+
run: RigRun,
|
|
78
129
|
deps: BenchDeps,
|
|
79
|
-
): number {
|
|
80
|
-
|
|
81
|
-
|
|
130
|
+
): Promise<number> {
|
|
131
|
+
const workFolder = await deps.createWorkFolder(run.repoRoot);
|
|
132
|
+
|
|
133
|
+
const head: BenchSide = {
|
|
134
|
+
root: run.repoRoot,
|
|
135
|
+
config,
|
|
136
|
+
files,
|
|
137
|
+
reportPath: join(workFolder, 'head.jsonl'),
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
const [ran] = await Promise.allSettled([runHeadSide(head, run, deps)]);
|
|
141
|
+
const [cleaned] = await Promise.allSettled([deps.removeFolder(workFolder)]);
|
|
142
|
+
|
|
143
|
+
const result = pickCleanedUpResult(ran, cleaned);
|
|
82
144
|
|
|
145
|
+
if (result.kind === 'throw') {
|
|
146
|
+
throw result.error;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return pickExitCodeAfterTests(result.value, run.signal.aborted);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function runHeadSide(head: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
|
|
153
|
+
const exitCode = await runBenchSide(head, run, deps);
|
|
154
|
+
const report = await readBenchReport(head, deps);
|
|
155
|
+
|
|
156
|
+
return writeBenchVerdict(pickHeadVerdict(report, exitCode), deps);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Runs one side's bench files against that side's root, so a base checkout builds its own programs
|
|
160
|
+
// and starts its own lane.
|
|
161
|
+
async function runBenchSide(side: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
|
|
162
|
+
if (side.files.length === 0) {
|
|
83
163
|
return 0;
|
|
84
164
|
}
|
|
85
165
|
|
|
86
|
-
const
|
|
166
|
+
const sideRun: RigRun = { ...run, repoRoot: side.root };
|
|
167
|
+
|
|
168
|
+
const buildExitCode = await runProgramBuilds(side.config.programs, sideRun, deps);
|
|
87
169
|
|
|
88
|
-
|
|
170
|
+
if (buildExitCode !== 0) {
|
|
171
|
+
return buildExitCode;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const hooks = buildProgramLaneHooks(
|
|
175
|
+
makeBenchHooks(side, sideRun, deps),
|
|
176
|
+
side.root,
|
|
177
|
+
side.config.programs,
|
|
178
|
+
);
|
|
89
179
|
|
|
90
|
-
return
|
|
180
|
+
return runLaneHooks(side.files, hooks);
|
|
91
181
|
}
|
|
92
182
|
|
|
93
|
-
function makeBenchHooks(
|
|
94
|
-
const lane = config.lanes.find((candidate) => candidate.name === config.benchLane);
|
|
183
|
+
function makeBenchHooks(side: BenchSide, run: RigRun, deps: BenchDeps): LaneHooks {
|
|
184
|
+
const lane = side.config.lanes.find((candidate) => candidate.name === side.config.benchLane);
|
|
95
185
|
|
|
96
186
|
const laneHooks: Pick<LaneHooks, 'start' | 'stop'> =
|
|
97
187
|
lane === undefined
|
|
@@ -119,6 +209,8 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
|
|
|
119
209
|
'bun',
|
|
120
210
|
deps.benchScript,
|
|
121
211
|
join(run.repoRoot, file),
|
|
212
|
+
'--report',
|
|
213
|
+
side.reportPath,
|
|
122
214
|
]),
|
|
123
215
|
{ cwd: run.repoRoot, environment, signal: run.signal },
|
|
124
216
|
);
|
|
@@ -130,3 +222,220 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
|
|
|
130
222
|
},
|
|
131
223
|
};
|
|
132
224
|
}
|
|
225
|
+
|
|
226
|
+
// A bench file that failed before it measured anything appends no report. Each entry's file
|
|
227
|
+
// becomes relative to the side's root, so the same file matches on the base and on the head.
|
|
228
|
+
async function readBenchReport(side: BenchSide, deps: BenchDeps): Promise<ParsedBenchReport> {
|
|
229
|
+
const text = await deps.readText(side.reportPath);
|
|
230
|
+
|
|
231
|
+
const report = parseBenchReport(text ?? '');
|
|
232
|
+
|
|
233
|
+
if (report.kind === 'refused') {
|
|
234
|
+
return report;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return {
|
|
238
|
+
kind: 'parsed',
|
|
239
|
+
entries: report.entries.map((entry) => ({
|
|
240
|
+
file: relative(side.root, entry.file),
|
|
241
|
+
name: entry.name,
|
|
242
|
+
concurrency: entry.concurrency,
|
|
243
|
+
medianMs: entry.medianMs,
|
|
244
|
+
spreadMs: entry.spreadMs,
|
|
245
|
+
budgetMs: entry.budgetMs,
|
|
246
|
+
})),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
interface BenchComparison {
|
|
251
|
+
readonly baseRef: string;
|
|
252
|
+
|
|
253
|
+
readonly name: string | null;
|
|
254
|
+
|
|
255
|
+
readonly headConfig: RigConfig;
|
|
256
|
+
|
|
257
|
+
readonly headFiles: readonly string[];
|
|
258
|
+
|
|
259
|
+
// Every bench file of the head, for the names a missing benchmark lists.
|
|
260
|
+
readonly headBenchFiles: readonly string[];
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
interface BaseCheckout {
|
|
264
|
+
readonly workFolder: string;
|
|
265
|
+
|
|
266
|
+
// The top folder of the checkout, which git adds and removes.
|
|
267
|
+
readonly checkoutRoot: string;
|
|
268
|
+
|
|
269
|
+
// The folder inside the checkout that matches the folder rig started in.
|
|
270
|
+
readonly sideRoot: string;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function runBenchComparison(
|
|
274
|
+
comparison: BenchComparison,
|
|
275
|
+
run: RigRun,
|
|
276
|
+
deps: BenchDeps,
|
|
277
|
+
): Promise<number> {
|
|
278
|
+
const gitPrefix = await readGitPrefix(run.repoRoot, run.environment, deps);
|
|
279
|
+
const workFolder = await deps.createWorkFolder(run.repoRoot);
|
|
280
|
+
|
|
281
|
+
const checkoutRoot = join(workFolder, 'base');
|
|
282
|
+
|
|
283
|
+
const base: BaseCheckout = {
|
|
284
|
+
workFolder,
|
|
285
|
+
checkoutRoot,
|
|
286
|
+
sideRoot: buildBaseSideRoot(checkoutRoot, gitPrefix),
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
// An abort makes the add report failure even when git finished the checkout, so every path,
|
|
290
|
+
// a failed add included, runs the same removal.
|
|
291
|
+
const [compared] = await Promise.allSettled([runCheckoutComparison(comparison, base, run, deps)]);
|
|
292
|
+
const [cleaned] = await Promise.allSettled([removeComparisonCheckout(base, run, deps)]);
|
|
293
|
+
|
|
294
|
+
const result = pickCleanedUpResult(compared, cleaned);
|
|
295
|
+
|
|
296
|
+
if (result.kind === 'throw') {
|
|
297
|
+
throw result.error;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return pickExitCodeAfterTests(Math.max(result.value, result.cleanup), run.signal.aborted);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function runCheckoutComparison(
|
|
304
|
+
comparison: BenchComparison,
|
|
305
|
+
checkout: BaseCheckout,
|
|
306
|
+
run: RigRun,
|
|
307
|
+
deps: BenchDeps,
|
|
308
|
+
): Promise<number> {
|
|
309
|
+
const addExitCode = await deps.runCommand(
|
|
310
|
+
['git', 'worktree', 'add', '--detach', checkout.checkoutRoot, comparison.baseRef],
|
|
311
|
+
{ cwd: run.repoRoot, environment: run.environment, signal: run.signal },
|
|
312
|
+
);
|
|
313
|
+
|
|
314
|
+
const addFailure = findBaseSetupVerdict('checkout', comparison.baseRef, addExitCode);
|
|
315
|
+
|
|
316
|
+
if (addFailure !== null) {
|
|
317
|
+
return writeBenchVerdict(addFailure, deps);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return runCheckedOutComparison(comparison, checkout, run, deps);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function runCheckedOutComparison(
|
|
324
|
+
comparison: BenchComparison,
|
|
325
|
+
checkout: BaseCheckout,
|
|
326
|
+
run: RigRun,
|
|
327
|
+
deps: BenchDeps,
|
|
328
|
+
): Promise<number> {
|
|
329
|
+
const installExitCode = await deps.runCommand(['bun', 'install', '--frozen-lockfile'], {
|
|
330
|
+
cwd: checkout.checkoutRoot,
|
|
331
|
+
environment: run.environment,
|
|
332
|
+
signal: run.signal,
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
const installFailure = findBaseSetupVerdict('install', comparison.baseRef, installExitCode);
|
|
336
|
+
|
|
337
|
+
if (installFailure !== null) {
|
|
338
|
+
return writeBenchVerdict(installFailure, deps);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const baseSide = await readBaseSide(checkout.sideRoot, deps);
|
|
342
|
+
|
|
343
|
+
const missing = findMissingBenchVerdict(
|
|
344
|
+
comparison.name,
|
|
345
|
+
comparison.headBenchFiles,
|
|
346
|
+
baseSide.benchFiles,
|
|
347
|
+
);
|
|
348
|
+
|
|
349
|
+
if (missing !== null) {
|
|
350
|
+
return writeBenchVerdict(missing, deps);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const base: BenchSide = {
|
|
354
|
+
root: checkout.sideRoot,
|
|
355
|
+
config: baseSide.config,
|
|
356
|
+
files: pickNamedBenchFiles(baseSide.benchFiles, comparison.name),
|
|
357
|
+
reportPath: join(checkout.workFolder, 'base.jsonl'),
|
|
358
|
+
};
|
|
359
|
+
|
|
360
|
+
const head: BenchSide = {
|
|
361
|
+
root: run.repoRoot,
|
|
362
|
+
config: comparison.headConfig,
|
|
363
|
+
files: comparison.headFiles,
|
|
364
|
+
reportPath: join(checkout.workFolder, 'head.jsonl'),
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
const baseExitCode = await runBenchSide(base, run, deps);
|
|
368
|
+
const headExitCode = await runBenchSide(head, run, deps);
|
|
369
|
+
const baseReport = await readBenchReport(base, deps);
|
|
370
|
+
const headReport = await readBenchReport(head, deps);
|
|
371
|
+
|
|
372
|
+
return writeBenchVerdict(
|
|
373
|
+
pickComparisonVerdict(
|
|
374
|
+
{ exitCode: baseExitCode, report: baseReport },
|
|
375
|
+
{ exitCode: headExitCode, report: headReport },
|
|
376
|
+
),
|
|
377
|
+
deps,
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
interface BaseSide {
|
|
382
|
+
readonly config: RigConfig;
|
|
383
|
+
|
|
384
|
+
readonly benchFiles: readonly string[];
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// A package that only the head has gives an empty base side, which runs nothing, so the table
|
|
388
|
+
// shows `-` for the base.
|
|
389
|
+
async function readBaseSide(root: string, deps: BenchDeps): Promise<BaseSide> {
|
|
390
|
+
const isPresent = await deps.probeFolder(root);
|
|
391
|
+
|
|
392
|
+
if (!isPresent) {
|
|
393
|
+
return { config: defineRig({}), benchFiles: [] };
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const config = await deps.readConfig(root);
|
|
397
|
+
const benchFiles = await readBenchFiles(root, deps);
|
|
398
|
+
|
|
399
|
+
return { config, benchFiles };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function removeComparisonCheckout(
|
|
403
|
+
base: BaseCheckout,
|
|
404
|
+
run: RigRun,
|
|
405
|
+
deps: BenchDeps,
|
|
406
|
+
): Promise<number> {
|
|
407
|
+
const checkout = await removeBaseCheckout(base.checkoutRoot, run, deps);
|
|
408
|
+
|
|
409
|
+
const cleanup = pickCheckoutCleanup(base.checkoutRoot, checkout);
|
|
410
|
+
|
|
411
|
+
writeBenchOutput(cleanup.output, deps);
|
|
412
|
+
|
|
413
|
+
if (!cleanup.isWorkFolderKept) {
|
|
414
|
+
await deps.removeFolder(base.workFolder);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
return cleanup.exitCode;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// An aborted run still removes the checkout, so the removal gets a signal of its own.
|
|
421
|
+
async function removeBaseCheckout(
|
|
422
|
+
checkoutRoot: string,
|
|
423
|
+
run: RigRun,
|
|
424
|
+
deps: BenchDeps,
|
|
425
|
+
): Promise<CheckoutState> {
|
|
426
|
+
const removeExitCode = await deps.runCommand(
|
|
427
|
+
['git', 'worktree', 'remove', '--force', checkoutRoot],
|
|
428
|
+
{ cwd: run.repoRoot, environment: run.environment, signal: new AbortController().signal },
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
if (removeExitCode === 0) {
|
|
432
|
+
return 'removed';
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const listing = await deps.readCommandOutput(['git', 'worktree', 'list', '--porcelain', '-z'], {
|
|
436
|
+
cwd: run.repoRoot,
|
|
437
|
+
environment: run.environment,
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
return pickUnremovedCheckoutState(listing, checkoutRoot);
|
|
441
|
+
}
|
package/src/run-cli.ts
CHANGED
|
@@ -17,9 +17,7 @@ import type { UnitDeps } from './run-unit.ts';
|
|
|
17
17
|
import type { RigConfig, RigRun } from './types.ts';
|
|
18
18
|
|
|
19
19
|
export interface CLIDeps
|
|
20
|
-
extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
|
|
21
|
-
readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
|
|
22
|
-
}
|
|
20
|
+
extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {}
|
|
23
21
|
|
|
24
22
|
export async function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number> {
|
|
25
23
|
const args = await readCLIArgs(argv);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function pickMedian(samples: readonly number[]): number;
|
package/src/bench/pick-median.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export function pickMedian(samples: readonly number[]): number {
|
|
2
|
-
const sorted = samples.toSorted((left, right) => left - right);
|
|
3
|
-
const middle = Math.floor(sorted.length / 2);
|
|
4
|
-
|
|
5
|
-
if (sorted.length % 2 === 1) {
|
|
6
|
-
return sorted[middle] ?? 0;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
|
|
10
|
-
}
|