@zgeoff/rig 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/dist/bench/bench-report-entry-schema.d.ts +10 -0
- package/dist/bench/build-base-side-root.d.ts +1 -0
- package/dist/bench/collect-bench-comparison.d.ts +3 -0
- package/dist/bench/define-bench.d.ts +2 -0
- package/dist/bench/find-base-setup-verdict.d.ts +3 -0
- package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
- package/dist/bench/format-bench-comparison.d.ts +2 -0
- package/dist/bench/format-bench-report.d.ts +3 -0
- package/dist/bench/format-benchmark-result.d.ts +1 -1
- package/dist/bench/is-concurrency-list.d.ts +1 -0
- package/dist/bench/parse-bench-report.d.ts +9 -0
- package/dist/bench/pick-bench-exit-code.d.ts +2 -0
- package/dist/bench/pick-bench-plan.d.ts +13 -0
- package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
- package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
- package/dist/bench/pick-comparison-verdict.d.ts +7 -0
- package/dist/bench/pick-head-verdict.d.ts +3 -0
- package/dist/bench/pick-named-bench-files.d.ts +1 -0
- package/dist/bench/pick-quantile.d.ts +1 -0
- package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
- package/dist/bench/read-bench-args.d.ts +9 -0
- package/dist/bench/read-bench-file-args.d.ts +9 -0
- package/dist/bench/run-benchmark.d.ts +3 -3
- package/dist/bench/to-bench-name.d.ts +1 -0
- package/dist/bench/types.d.ts +20 -0
- package/dist/build-bench-folder-path.d.ts +1 -0
- package/dist/create-bench-work-folder.d.ts +6 -0
- package/dist/format-parse-error.d.ts +1 -0
- package/dist/has-lane-suffix.d.ts +1 -0
- package/dist/index.js +33 -2
- package/dist/is-test-file.d.ts +1 -0
- package/dist/make-ignore-check.d.ts +1 -0
- package/dist/read-command-output.d.ts +8 -0
- package/dist/read-git-common-dir.d.ts +6 -0
- package/dist/read-git-prefix.d.ts +6 -0
- package/dist/run-bench.d.ts +7 -0
- package/dist/run-cli.d.ts +1 -2
- package/dist/types.d.ts +2 -0
- package/package.json +4 -4
- package/src/bench/bench-report-entry-schema.ts +17 -0
- package/src/bench/build-base-side-root.ts +7 -0
- package/src/bench/collect-bench-comparison.ts +43 -0
- package/src/bench/define-bench.ts +15 -0
- package/src/bench/find-base-setup-verdict.ts +21 -0
- package/src/bench/find-missing-bench-verdict.ts +33 -0
- package/src/bench/format-bench-comparison.ts +72 -0
- package/src/bench/format-bench-report.ts +24 -0
- package/src/bench/format-benchmark-result.ts +7 -3
- package/src/bench/is-concurrency-list.ts +10 -0
- package/src/bench/parse-bench-definition.ts +21 -3
- package/src/bench/parse-bench-report.ts +42 -0
- package/src/bench/pick-bench-exit-code.ts +15 -0
- package/src/bench/pick-bench-plan.ts +25 -0
- package/src/bench/pick-checkout-cleanup.ts +29 -0
- package/src/bench/pick-cleaned-up-result.ts +30 -0
- package/src/bench/pick-comparison-verdict.ts +36 -0
- package/src/bench/pick-head-verdict.ts +11 -0
- package/src/bench/pick-named-bench-files.ts +8 -0
- package/src/bench/pick-quantile.ts +9 -0
- package/src/bench/pick-unremoved-checkout-state.ts +30 -0
- package/src/bench/read-bench-args.ts +45 -0
- package/src/bench/read-bench-file-args.ts +38 -0
- package/src/bench/run-benchmark.ts +88 -21
- package/src/bench/to-bench-name.ts +5 -0
- package/src/bench/types.ts +37 -0
- package/src/build-bench-folder-path.ts +7 -0
- package/src/build-test-selection.ts +5 -9
- package/src/build-unit-command.ts +3 -2
- package/src/cli.ts +31 -0
- package/src/create-bench-work-folder.ts +26 -0
- package/src/define-rig.ts +23 -1
- package/src/format-parse-error.ts +9 -0
- package/src/has-lane-suffix.ts +3 -0
- package/src/is-test-file.ts +5 -0
- package/src/make-ignore-check.ts +15 -0
- package/src/parse-rig-config.ts +29 -1
- package/src/read-cli-args.ts +1 -10
- package/src/read-command-output.ts +30 -0
- package/src/read-git-common-dir.ts +27 -0
- package/src/read-git-prefix.ts +27 -0
- package/src/resolve-related-tests.ts +4 -4
- package/src/run-bench-file.ts +23 -13
- package/src/run-bench.ts +348 -39
- package/src/run-cli.ts +1 -3
- package/src/run-lane.ts +41 -14
- package/src/types.ts +4 -0
- package/dist/bench/pick-median.d.ts +0 -1
- package/src/bench/pick-median.ts +0 -10
package/src/run-bench.ts
CHANGED
|
@@ -1,10 +1,28 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { join, relative } from 'node:path';
|
|
2
|
+
import { match } from 'ts-pattern';
|
|
3
|
+
import { buildBaseSideRoot } from './bench/build-base-side-root.ts';
|
|
4
|
+
import { findBaseSetupVerdict } from './bench/find-base-setup-verdict.ts';
|
|
5
|
+
import { findMissingBenchVerdict } from './bench/find-missing-bench-verdict.ts';
|
|
6
|
+
import { parseBenchReport } from './bench/parse-bench-report.ts';
|
|
7
|
+
import type { ParsedBenchReport } from './bench/parse-bench-report.ts';
|
|
8
|
+
import { pickBenchPlan } from './bench/pick-bench-plan.ts';
|
|
9
|
+
import { pickCheckoutCleanup } from './bench/pick-checkout-cleanup.ts';
|
|
10
|
+
import { pickCleanedUpResult } from './bench/pick-cleaned-up-result.ts';
|
|
11
|
+
import { pickComparisonVerdict } from './bench/pick-comparison-verdict.ts';
|
|
12
|
+
import { pickHeadVerdict } from './bench/pick-head-verdict.ts';
|
|
13
|
+
import { pickNamedBenchFiles } from './bench/pick-named-bench-files.ts';
|
|
14
|
+
import { pickUnremovedCheckoutState } from './bench/pick-unremoved-checkout-state.ts';
|
|
15
|
+
import { readBenchArgs } from './bench/read-bench-args.ts';
|
|
16
|
+
import type { BenchOutput, BenchVerdict, CheckoutState } from './bench/types.ts';
|
|
2
17
|
import { pickExitCodeAfterTests } from './browser/pick-exit-code-after-tests.ts';
|
|
3
18
|
import { buildDietCommand } from './build-diet-command.ts';
|
|
4
19
|
import { buildProgramLaneHooks } from './build-program-lane-hooks.ts';
|
|
5
20
|
import { collectBenchFiles } from './collect-bench-files.ts';
|
|
21
|
+
import { defineRig } from './define-rig.ts';
|
|
6
22
|
import type { LaneHooks } from './lane-hooks.ts';
|
|
7
23
|
import { makeRepoLaneHooks } from './make-repo-lane-hooks.ts';
|
|
24
|
+
import type { ReadCommandOutput } from './read-command-output.ts';
|
|
25
|
+
import { readGitPrefix } from './read-git-prefix.ts';
|
|
8
26
|
import { runLaneHooks } from './run-lane-hooks.ts';
|
|
9
27
|
import type { ProgramBuildDeps } from './run-program-builds.ts';
|
|
10
28
|
import { runProgramBuilds } from './run-program-builds.ts';
|
|
@@ -16,82 +34,154 @@ export interface BenchDeps extends ProgramBuildDeps {
|
|
|
16
34
|
// The script that runs one bench file.
|
|
17
35
|
readonly benchScript: string;
|
|
18
36
|
|
|
37
|
+
readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
|
|
38
|
+
|
|
19
39
|
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
20
40
|
|
|
21
41
|
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
22
42
|
|
|
23
43
|
readonly importModule: (path: string) => Promise<unknown>;
|
|
44
|
+
|
|
45
|
+
// Makes a new folder for one run's reports and base checkout.
|
|
46
|
+
readonly createWorkFolder: (repoRoot: string) => Promise<string>;
|
|
47
|
+
|
|
48
|
+
readonly readText: (path: string) => Promise<string | null>;
|
|
49
|
+
|
|
50
|
+
readonly probeFolder: (path: string) => Promise<boolean>;
|
|
51
|
+
|
|
52
|
+
readonly removeFolder: (path: string) => Promise<void>;
|
|
53
|
+
|
|
54
|
+
readonly readCommandOutput: ReadCommandOutput;
|
|
24
55
|
}
|
|
25
56
|
|
|
26
57
|
// Builds each program once, as `rig e2e` does, then runs each bench file in a process of its own,
|
|
27
|
-
// between the start and stop hooks of the bench lane when the config names one.
|
|
28
|
-
//
|
|
58
|
+
// between the start and stop hooks of the bench lane when the config names one. A base ref runs
|
|
59
|
+
// the same way from the same folder of a checkout of that ref, with that folder's config.
|
|
29
60
|
export async function runBench(
|
|
30
61
|
args: readonly string[],
|
|
31
62
|
config: RigConfig,
|
|
32
63
|
run: RigRun,
|
|
33
64
|
deps: BenchDeps,
|
|
34
65
|
): Promise<number> {
|
|
35
|
-
|
|
36
|
-
deps.output.err('rig bench: takes at most one benchmark name\n');
|
|
66
|
+
const parsed = await readBenchArgs(args);
|
|
37
67
|
|
|
38
|
-
|
|
68
|
+
if (parsed.kind === 'refused') {
|
|
69
|
+
return writeBenchVerdict(
|
|
70
|
+
{ output: [{ stream: 'err', text: `rig bench: ${parsed.reason}\n` }], exitCode: 2 },
|
|
71
|
+
deps,
|
|
72
|
+
);
|
|
39
73
|
}
|
|
40
74
|
|
|
41
|
-
const
|
|
42
|
-
|
|
75
|
+
const headBenchFiles = await readBenchFiles(run.repoRoot, deps);
|
|
76
|
+
|
|
77
|
+
return match(pickBenchPlan(parsed.name, parsed.baseRef, headBenchFiles))
|
|
78
|
+
.with({ kind: 'finished' }, (plan) => writeBenchVerdict(plan.verdict, deps))
|
|
79
|
+
.with({ kind: 'head' }, (plan) => runHeadBench(plan.files, config, run, deps))
|
|
80
|
+
.with({ kind: 'comparison' }, (plan) =>
|
|
81
|
+
runBenchComparison(
|
|
82
|
+
{
|
|
83
|
+
baseRef: plan.baseRef,
|
|
84
|
+
name: parsed.name,
|
|
85
|
+
headConfig: config,
|
|
86
|
+
headFiles: plan.files,
|
|
87
|
+
headBenchFiles,
|
|
88
|
+
},
|
|
89
|
+
run,
|
|
90
|
+
deps,
|
|
91
|
+
),
|
|
92
|
+
)
|
|
93
|
+
.exhaustive();
|
|
94
|
+
}
|
|
43
95
|
|
|
44
|
-
|
|
45
|
-
|
|
96
|
+
function writeBenchVerdict(verdict: BenchVerdict, deps: BenchDeps): number {
|
|
97
|
+
writeBenchOutput(verdict.output, deps);
|
|
46
98
|
|
|
47
|
-
|
|
48
|
-
|
|
99
|
+
return verdict.exitCode;
|
|
100
|
+
}
|
|
49
101
|
|
|
50
|
-
|
|
51
|
-
|
|
102
|
+
function writeBenchOutput(output: readonly BenchOutput[], deps: BenchDeps): void {
|
|
103
|
+
for (const entry of output) {
|
|
104
|
+
deps.output[entry.stream](entry.text);
|
|
52
105
|
}
|
|
106
|
+
}
|
|
53
107
|
|
|
54
|
-
|
|
108
|
+
async function readBenchFiles(root: string, deps: BenchDeps): Promise<readonly string[]> {
|
|
109
|
+
const isWorkspaceRepo = await deps.probeWorkspaceRepo(root);
|
|
110
|
+
const trackedFiles = await deps.readTrackedFiles(root);
|
|
55
111
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
112
|
+
return collectBenchFiles(trackedFiles, isWorkspaceRepo);
|
|
113
|
+
}
|
|
59
114
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
run.repoRoot,
|
|
63
|
-
config.programs,
|
|
64
|
-
);
|
|
115
|
+
interface BenchSide {
|
|
116
|
+
readonly root: string;
|
|
65
117
|
|
|
66
|
-
|
|
118
|
+
readonly config: RigConfig;
|
|
67
119
|
|
|
68
|
-
|
|
69
|
-
}
|
|
120
|
+
readonly files: readonly string[];
|
|
70
121
|
|
|
71
|
-
|
|
72
|
-
return basename(file, '.bench.ts');
|
|
122
|
+
readonly reportPath: string;
|
|
73
123
|
}
|
|
74
124
|
|
|
75
|
-
function
|
|
76
|
-
|
|
77
|
-
|
|
125
|
+
async function runHeadBench(
|
|
126
|
+
files: readonly string[],
|
|
127
|
+
config: RigConfig,
|
|
128
|
+
run: RigRun,
|
|
78
129
|
deps: BenchDeps,
|
|
79
|
-
): number {
|
|
80
|
-
|
|
81
|
-
|
|
130
|
+
): Promise<number> {
|
|
131
|
+
const workFolder = await deps.createWorkFolder(run.repoRoot);
|
|
132
|
+
|
|
133
|
+
const head: BenchSide = {
|
|
134
|
+
root: run.repoRoot,
|
|
135
|
+
config,
|
|
136
|
+
files,
|
|
137
|
+
reportPath: join(workFolder, 'head.jsonl'),
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
const [ran] = await Promise.allSettled([runHeadSide(head, run, deps)]);
|
|
141
|
+
const [cleaned] = await Promise.allSettled([deps.removeFolder(workFolder)]);
|
|
142
|
+
|
|
143
|
+
const result = pickCleanedUpResult(ran, cleaned);
|
|
82
144
|
|
|
145
|
+
if (result.kind === 'throw') {
|
|
146
|
+
throw result.error;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return pickExitCodeAfterTests(result.value, run.signal.aborted);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function runHeadSide(head: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
|
|
153
|
+
const exitCode = await runBenchSide(head, run, deps);
|
|
154
|
+
const report = await readBenchReport(head, deps);
|
|
155
|
+
|
|
156
|
+
return writeBenchVerdict(pickHeadVerdict(report, exitCode), deps);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Runs one side's bench files against that side's root, so a base checkout builds its own programs
|
|
160
|
+
// and starts its own lane.
|
|
161
|
+
async function runBenchSide(side: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
|
|
162
|
+
if (side.files.length === 0) {
|
|
83
163
|
return 0;
|
|
84
164
|
}
|
|
85
165
|
|
|
86
|
-
const
|
|
166
|
+
const sideRun: RigRun = { ...run, repoRoot: side.root };
|
|
167
|
+
|
|
168
|
+
const buildExitCode = await runProgramBuilds(side.config.programs, sideRun, deps);
|
|
87
169
|
|
|
88
|
-
|
|
170
|
+
if (buildExitCode !== 0) {
|
|
171
|
+
return buildExitCode;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const hooks = buildProgramLaneHooks(
|
|
175
|
+
makeBenchHooks(side, sideRun, deps),
|
|
176
|
+
side.root,
|
|
177
|
+
side.config.programs,
|
|
178
|
+
);
|
|
89
179
|
|
|
90
|
-
return
|
|
180
|
+
return runLaneHooks(side.files, hooks);
|
|
91
181
|
}
|
|
92
182
|
|
|
93
|
-
function makeBenchHooks(
|
|
94
|
-
const lane = config.lanes.find((candidate) => candidate.name === config.benchLane);
|
|
183
|
+
function makeBenchHooks(side: BenchSide, run: RigRun, deps: BenchDeps): LaneHooks {
|
|
184
|
+
const lane = side.config.lanes.find((candidate) => candidate.name === side.config.benchLane);
|
|
95
185
|
|
|
96
186
|
const laneHooks: Pick<LaneHooks, 'start' | 'stop'> =
|
|
97
187
|
lane === undefined
|
|
@@ -119,6 +209,8 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
|
|
|
119
209
|
'bun',
|
|
120
210
|
deps.benchScript,
|
|
121
211
|
join(run.repoRoot, file),
|
|
212
|
+
'--report',
|
|
213
|
+
side.reportPath,
|
|
122
214
|
]),
|
|
123
215
|
{ cwd: run.repoRoot, environment, signal: run.signal },
|
|
124
216
|
);
|
|
@@ -130,3 +222,220 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
|
|
|
130
222
|
},
|
|
131
223
|
};
|
|
132
224
|
}
|
|
225
|
+
|
|
226
|
+
// A bench file that failed before it measured anything appends no report. Each entry's file
|
|
227
|
+
// becomes relative to the side's root, so the same file matches on the base and on the head.
|
|
228
|
+
async function readBenchReport(side: BenchSide, deps: BenchDeps): Promise<ParsedBenchReport> {
|
|
229
|
+
const text = await deps.readText(side.reportPath);
|
|
230
|
+
|
|
231
|
+
const report = parseBenchReport(text ?? '');
|
|
232
|
+
|
|
233
|
+
if (report.kind === 'refused') {
|
|
234
|
+
return report;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return {
|
|
238
|
+
kind: 'parsed',
|
|
239
|
+
entries: report.entries.map((entry) => ({
|
|
240
|
+
file: relative(side.root, entry.file),
|
|
241
|
+
name: entry.name,
|
|
242
|
+
concurrency: entry.concurrency,
|
|
243
|
+
medianMs: entry.medianMs,
|
|
244
|
+
spreadMs: entry.spreadMs,
|
|
245
|
+
budgetMs: entry.budgetMs,
|
|
246
|
+
})),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
interface BenchComparison {
|
|
251
|
+
readonly baseRef: string;
|
|
252
|
+
|
|
253
|
+
readonly name: string | null;
|
|
254
|
+
|
|
255
|
+
readonly headConfig: RigConfig;
|
|
256
|
+
|
|
257
|
+
readonly headFiles: readonly string[];
|
|
258
|
+
|
|
259
|
+
// Every bench file of the head, for the names a missing benchmark lists.
|
|
260
|
+
readonly headBenchFiles: readonly string[];
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
interface BaseCheckout {
|
|
264
|
+
readonly workFolder: string;
|
|
265
|
+
|
|
266
|
+
// The top folder of the checkout, which git adds and removes.
|
|
267
|
+
readonly checkoutRoot: string;
|
|
268
|
+
|
|
269
|
+
// The folder inside the checkout that matches the folder rig started in.
|
|
270
|
+
readonly sideRoot: string;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function runBenchComparison(
|
|
274
|
+
comparison: BenchComparison,
|
|
275
|
+
run: RigRun,
|
|
276
|
+
deps: BenchDeps,
|
|
277
|
+
): Promise<number> {
|
|
278
|
+
const gitPrefix = await readGitPrefix(run.repoRoot, run.environment, deps);
|
|
279
|
+
const workFolder = await deps.createWorkFolder(run.repoRoot);
|
|
280
|
+
|
|
281
|
+
const checkoutRoot = join(workFolder, 'base');
|
|
282
|
+
|
|
283
|
+
const base: BaseCheckout = {
|
|
284
|
+
workFolder,
|
|
285
|
+
checkoutRoot,
|
|
286
|
+
sideRoot: buildBaseSideRoot(checkoutRoot, gitPrefix),
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
// An abort makes the add report failure even when git finished the checkout, so every path,
|
|
290
|
+
// a failed add included, runs the same removal.
|
|
291
|
+
const [compared] = await Promise.allSettled([runCheckoutComparison(comparison, base, run, deps)]);
|
|
292
|
+
const [cleaned] = await Promise.allSettled([removeComparisonCheckout(base, run, deps)]);
|
|
293
|
+
|
|
294
|
+
const result = pickCleanedUpResult(compared, cleaned);
|
|
295
|
+
|
|
296
|
+
if (result.kind === 'throw') {
|
|
297
|
+
throw result.error;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return pickExitCodeAfterTests(Math.max(result.value, result.cleanup), run.signal.aborted);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function runCheckoutComparison(
|
|
304
|
+
comparison: BenchComparison,
|
|
305
|
+
checkout: BaseCheckout,
|
|
306
|
+
run: RigRun,
|
|
307
|
+
deps: BenchDeps,
|
|
308
|
+
): Promise<number> {
|
|
309
|
+
const addExitCode = await deps.runCommand(
|
|
310
|
+
['git', 'worktree', 'add', '--detach', checkout.checkoutRoot, comparison.baseRef],
|
|
311
|
+
{ cwd: run.repoRoot, environment: run.environment, signal: run.signal },
|
|
312
|
+
);
|
|
313
|
+
|
|
314
|
+
const addFailure = findBaseSetupVerdict('checkout', comparison.baseRef, addExitCode);
|
|
315
|
+
|
|
316
|
+
if (addFailure !== null) {
|
|
317
|
+
return writeBenchVerdict(addFailure, deps);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return runCheckedOutComparison(comparison, checkout, run, deps);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async function runCheckedOutComparison(
|
|
324
|
+
comparison: BenchComparison,
|
|
325
|
+
checkout: BaseCheckout,
|
|
326
|
+
run: RigRun,
|
|
327
|
+
deps: BenchDeps,
|
|
328
|
+
): Promise<number> {
|
|
329
|
+
const installExitCode = await deps.runCommand(['bun', 'install', '--frozen-lockfile'], {
|
|
330
|
+
cwd: checkout.checkoutRoot,
|
|
331
|
+
environment: run.environment,
|
|
332
|
+
signal: run.signal,
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
const installFailure = findBaseSetupVerdict('install', comparison.baseRef, installExitCode);
|
|
336
|
+
|
|
337
|
+
if (installFailure !== null) {
|
|
338
|
+
return writeBenchVerdict(installFailure, deps);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const baseSide = await readBaseSide(checkout.sideRoot, deps);
|
|
342
|
+
|
|
343
|
+
const missing = findMissingBenchVerdict(
|
|
344
|
+
comparison.name,
|
|
345
|
+
comparison.headBenchFiles,
|
|
346
|
+
baseSide.benchFiles,
|
|
347
|
+
);
|
|
348
|
+
|
|
349
|
+
if (missing !== null) {
|
|
350
|
+
return writeBenchVerdict(missing, deps);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const base: BenchSide = {
|
|
354
|
+
root: checkout.sideRoot,
|
|
355
|
+
config: baseSide.config,
|
|
356
|
+
files: pickNamedBenchFiles(baseSide.benchFiles, comparison.name),
|
|
357
|
+
reportPath: join(checkout.workFolder, 'base.jsonl'),
|
|
358
|
+
};
|
|
359
|
+
|
|
360
|
+
const head: BenchSide = {
|
|
361
|
+
root: run.repoRoot,
|
|
362
|
+
config: comparison.headConfig,
|
|
363
|
+
files: comparison.headFiles,
|
|
364
|
+
reportPath: join(checkout.workFolder, 'head.jsonl'),
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
const baseExitCode = await runBenchSide(base, run, deps);
|
|
368
|
+
const headExitCode = await runBenchSide(head, run, deps);
|
|
369
|
+
const baseReport = await readBenchReport(base, deps);
|
|
370
|
+
const headReport = await readBenchReport(head, deps);
|
|
371
|
+
|
|
372
|
+
return writeBenchVerdict(
|
|
373
|
+
pickComparisonVerdict(
|
|
374
|
+
{ exitCode: baseExitCode, report: baseReport },
|
|
375
|
+
{ exitCode: headExitCode, report: headReport },
|
|
376
|
+
),
|
|
377
|
+
deps,
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
interface BaseSide {
|
|
382
|
+
readonly config: RigConfig;
|
|
383
|
+
|
|
384
|
+
readonly benchFiles: readonly string[];
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// A package that only the head has gives an empty base side, which runs nothing, so the table
|
|
388
|
+
// shows `-` for the base.
|
|
389
|
+
async function readBaseSide(root: string, deps: BenchDeps): Promise<BaseSide> {
|
|
390
|
+
const isPresent = await deps.probeFolder(root);
|
|
391
|
+
|
|
392
|
+
if (!isPresent) {
|
|
393
|
+
return { config: defineRig({}), benchFiles: [] };
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const config = await deps.readConfig(root);
|
|
397
|
+
const benchFiles = await readBenchFiles(root, deps);
|
|
398
|
+
|
|
399
|
+
return { config, benchFiles };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function removeComparisonCheckout(
|
|
403
|
+
base: BaseCheckout,
|
|
404
|
+
run: RigRun,
|
|
405
|
+
deps: BenchDeps,
|
|
406
|
+
): Promise<number> {
|
|
407
|
+
const checkout = await removeBaseCheckout(base.checkoutRoot, run, deps);
|
|
408
|
+
|
|
409
|
+
const cleanup = pickCheckoutCleanup(base.checkoutRoot, checkout);
|
|
410
|
+
|
|
411
|
+
writeBenchOutput(cleanup.output, deps);
|
|
412
|
+
|
|
413
|
+
if (!cleanup.isWorkFolderKept) {
|
|
414
|
+
await deps.removeFolder(base.workFolder);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
return cleanup.exitCode;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// An aborted run still removes the checkout, so the removal gets a signal of its own.
|
|
421
|
+
async function removeBaseCheckout(
|
|
422
|
+
checkoutRoot: string,
|
|
423
|
+
run: RigRun,
|
|
424
|
+
deps: BenchDeps,
|
|
425
|
+
): Promise<CheckoutState> {
|
|
426
|
+
const removeExitCode = await deps.runCommand(
|
|
427
|
+
['git', 'worktree', 'remove', '--force', checkoutRoot],
|
|
428
|
+
{ cwd: run.repoRoot, environment: run.environment, signal: new AbortController().signal },
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
if (removeExitCode === 0) {
|
|
432
|
+
return 'removed';
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const listing = await deps.readCommandOutput(['git', 'worktree', 'list', '--porcelain', '-z'], {
|
|
436
|
+
cwd: run.repoRoot,
|
|
437
|
+
environment: run.environment,
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
return pickUnremovedCheckoutState(listing, checkoutRoot);
|
|
441
|
+
}
|
package/src/run-cli.ts
CHANGED
|
@@ -17,9 +17,7 @@ import type { UnitDeps } from './run-unit.ts';
|
|
|
17
17
|
import type { RigConfig, RigRun } from './types.ts';
|
|
18
18
|
|
|
19
19
|
export interface CLIDeps
|
|
20
|
-
extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
|
|
21
|
-
readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
|
|
22
|
-
}
|
|
20
|
+
extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {}
|
|
23
21
|
|
|
24
22
|
export async function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number> {
|
|
25
23
|
const args = await readCLIArgs(argv);
|
package/src/run-lane.ts
CHANGED
|
@@ -7,7 +7,10 @@ import { buildTestSelection } from './build-test-selection.ts';
|
|
|
7
7
|
import { collectUnprogrammedE2EDirs } from './collect-unprogrammed-e2e-dirs.ts';
|
|
8
8
|
import { formatNoMatchMessage } from './format-no-match-message.ts';
|
|
9
9
|
import { formatNoTestsMessage } from './format-no-tests-message.ts';
|
|
10
|
+
import { hasLaneSuffix } from './has-lane-suffix.ts';
|
|
11
|
+
import { isTestFile } from './is-test-file.ts';
|
|
10
12
|
import type { LaneHooks } from './lane-hooks.ts';
|
|
13
|
+
import { makeIgnoreCheck } from './make-ignore-check.ts';
|
|
11
14
|
import { makeRepoLaneHooks } from './make-repo-lane-hooks.ts';
|
|
12
15
|
import { parseTestArgs } from './parse-test-args.ts';
|
|
13
16
|
import { pickE2EDirs } from './pick-e2e-dirs.ts';
|
|
@@ -66,7 +69,12 @@ export async function runLane(request: LaneRequest, run: RigRun, deps: LaneDeps)
|
|
|
66
69
|
return 0;
|
|
67
70
|
}
|
|
68
71
|
|
|
69
|
-
const e2eDirs =
|
|
72
|
+
const e2eDirs = collectRunnableE2EDirs(
|
|
73
|
+
pickE2EDirs(trackedFiles, isWorkspaceRepo),
|
|
74
|
+
trackedFiles,
|
|
75
|
+
request.config,
|
|
76
|
+
);
|
|
77
|
+
|
|
70
78
|
const testArgs = pickLaneTestArgs(request.config, lane, selection, e2eDirs);
|
|
71
79
|
const programs = lane.name === 'e2e' ? request.config.programs : {};
|
|
72
80
|
|
|
@@ -97,6 +105,23 @@ export async function runLane(request: LaneRequest, run: RigRun, deps: LaneDeps)
|
|
|
97
105
|
return pickExitCodeAfterTests(exitCode, run.signal.aborted);
|
|
98
106
|
}
|
|
99
107
|
|
|
108
|
+
function collectRunnableE2EDirs(
|
|
109
|
+
e2eDirs: readonly string[],
|
|
110
|
+
trackedFiles: readonly string[],
|
|
111
|
+
config: RigConfig,
|
|
112
|
+
): readonly string[] {
|
|
113
|
+
const isIgnored = makeIgnoreCheck(config.ignore);
|
|
114
|
+
|
|
115
|
+
const isE2ETest = (file: string) =>
|
|
116
|
+
isTestFile(file) &&
|
|
117
|
+
!isIgnored(file) &&
|
|
118
|
+
!config.lanes.some((lane) => lane.suffix !== null && hasLaneSuffix(file, lane.suffix));
|
|
119
|
+
|
|
120
|
+
return e2eDirs.filter((directory) =>
|
|
121
|
+
trackedFiles.some((file) => file.startsWith(`${directory}/`) && isE2ETest(file)),
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
100
125
|
function pickLaneTestArgs(
|
|
101
126
|
config: RigConfig,
|
|
102
127
|
lane: ResolvedLane,
|
|
@@ -110,25 +135,27 @@ function pickLaneTestArgs(
|
|
|
110
135
|
.with({ kind: 'lane' }, (laneRun) => [
|
|
111
136
|
timeoutArg,
|
|
112
137
|
...pickLaneSelection(lane, e2eDirs),
|
|
113
|
-
...
|
|
138
|
+
...pickLaneIgnoreArgs(config, lane),
|
|
114
139
|
...laneRun.flags,
|
|
115
140
|
])
|
|
116
141
|
.exhaustive();
|
|
117
142
|
}
|
|
118
143
|
|
|
119
144
|
// A folder selection takes every test file under it, so the e2e lane leaves out the files that
|
|
120
|
-
// belong to a lane with a suffix.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
145
|
+
// belong to a lane with a suffix. A --path-ignore-patterns argument replaces bunfig.toml's
|
|
146
|
+
// pathIgnorePatterns, so every lane passes the config's ignore list itself.
|
|
147
|
+
function pickLaneIgnoreArgs(config: RigConfig, lane: ResolvedLane): readonly string[] {
|
|
148
|
+
const otherLanePatterns =
|
|
149
|
+
lane.suffix === null
|
|
150
|
+
? config.lanes.flatMap((other) =>
|
|
151
|
+
other.suffix === null
|
|
152
|
+
? []
|
|
153
|
+
: [`**/*${other.suffix}.test.ts`, `**/*${other.suffix}.test.tsx`],
|
|
154
|
+
)
|
|
155
|
+
: [];
|
|
156
|
+
|
|
157
|
+
return [...otherLanePatterns, ...config.ignore].map(
|
|
158
|
+
(pattern) => `--path-ignore-patterns=${pattern}`,
|
|
132
159
|
);
|
|
133
160
|
}
|
|
134
161
|
|
package/src/types.ts
CHANGED
|
@@ -36,6 +36,8 @@ export interface BenchDeclaration {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export interface RigDeclaration {
|
|
39
|
+
readonly ignore?: readonly string[];
|
|
40
|
+
|
|
39
41
|
readonly lanes?: Readonly<Record<string, LaneDeclaration>>;
|
|
40
42
|
|
|
41
43
|
readonly programs?: Readonly<Record<string, ProgramDeclaration>>;
|
|
@@ -64,6 +66,8 @@ export interface ResolvedLane {
|
|
|
64
66
|
}
|
|
65
67
|
|
|
66
68
|
export interface RigConfig {
|
|
69
|
+
readonly ignore: readonly string[];
|
|
70
|
+
|
|
67
71
|
readonly lanes: readonly ResolvedLane[];
|
|
68
72
|
|
|
69
73
|
readonly programs: Readonly<Record<string, ProgramDeclaration>>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function pickMedian(samples: readonly number[]): number;
|
package/src/bench/pick-median.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export function pickMedian(samples: readonly number[]): number {
|
|
2
|
-
const sorted = samples.toSorted((left, right) => left - right);
|
|
3
|
-
const middle = Math.floor(sorted.length / 2);
|
|
4
|
-
|
|
5
|
-
if (sorted.length % 2 === 1) {
|
|
6
|
-
return sorted[middle] ?? 0;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
|
|
10
|
-
}
|