@zgeoff/rig 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/bench/bench-report-entry-schema.d.ts +10 -0
  2. package/dist/bench/build-base-side-root.d.ts +1 -0
  3. package/dist/bench/collect-bench-comparison.d.ts +3 -0
  4. package/dist/bench/define-bench.d.ts +2 -0
  5. package/dist/bench/find-base-setup-verdict.d.ts +3 -0
  6. package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
  7. package/dist/bench/format-bench-comparison.d.ts +2 -0
  8. package/dist/bench/format-bench-report.d.ts +3 -0
  9. package/dist/bench/format-benchmark-result.d.ts +1 -1
  10. package/dist/bench/is-concurrency-list.d.ts +1 -0
  11. package/dist/bench/parse-bench-report.d.ts +9 -0
  12. package/dist/bench/pick-bench-exit-code.d.ts +2 -0
  13. package/dist/bench/pick-bench-plan.d.ts +13 -0
  14. package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
  15. package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
  16. package/dist/bench/pick-comparison-verdict.d.ts +7 -0
  17. package/dist/bench/pick-head-verdict.d.ts +3 -0
  18. package/dist/bench/pick-named-bench-files.d.ts +1 -0
  19. package/dist/bench/pick-quantile.d.ts +1 -0
  20. package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
  21. package/dist/bench/read-bench-args.d.ts +9 -0
  22. package/dist/bench/read-bench-file-args.d.ts +9 -0
  23. package/dist/bench/run-benchmark.d.ts +3 -3
  24. package/dist/bench/to-bench-name.d.ts +1 -0
  25. package/dist/bench/types.d.ts +20 -0
  26. package/dist/build-bench-folder-path.d.ts +1 -0
  27. package/dist/create-bench-work-folder.d.ts +6 -0
  28. package/dist/format-parse-error.d.ts +1 -0
  29. package/dist/read-command-output.d.ts +8 -0
  30. package/dist/read-git-common-dir.d.ts +6 -0
  31. package/dist/read-git-prefix.d.ts +6 -0
  32. package/dist/run-bench.d.ts +7 -0
  33. package/dist/run-cli.d.ts +1 -2
  34. package/package.json +8 -8
  35. package/src/bench/bench-report-entry-schema.ts +17 -0
  36. package/src/bench/build-base-side-root.ts +7 -0
  37. package/src/bench/collect-bench-comparison.ts +43 -0
  38. package/src/bench/define-bench.ts +15 -0
  39. package/src/bench/find-base-setup-verdict.ts +21 -0
  40. package/src/bench/find-missing-bench-verdict.ts +33 -0
  41. package/src/bench/format-bench-comparison.ts +72 -0
  42. package/src/bench/format-bench-report.ts +24 -0
  43. package/src/bench/format-benchmark-result.ts +7 -3
  44. package/src/bench/is-concurrency-list.ts +10 -0
  45. package/src/bench/parse-bench-definition.ts +21 -3
  46. package/src/bench/parse-bench-report.ts +42 -0
  47. package/src/bench/pick-bench-exit-code.ts +15 -0
  48. package/src/bench/pick-bench-plan.ts +25 -0
  49. package/src/bench/pick-checkout-cleanup.ts +29 -0
  50. package/src/bench/pick-cleaned-up-result.ts +30 -0
  51. package/src/bench/pick-comparison-verdict.ts +36 -0
  52. package/src/bench/pick-head-verdict.ts +11 -0
  53. package/src/bench/pick-named-bench-files.ts +8 -0
  54. package/src/bench/pick-quantile.ts +9 -0
  55. package/src/bench/pick-unremoved-checkout-state.ts +30 -0
  56. package/src/bench/read-bench-args.ts +45 -0
  57. package/src/bench/read-bench-file-args.ts +38 -0
  58. package/src/bench/run-benchmark.ts +88 -21
  59. package/src/bench/to-bench-name.ts +5 -0
  60. package/src/bench/types.ts +37 -0
  61. package/src/build-bench-folder-path.ts +7 -0
  62. package/src/cli.ts +31 -0
  63. package/src/create-bench-work-folder.ts +26 -0
  64. package/src/format-parse-error.ts +9 -0
  65. package/src/read-cli-args.ts +1 -10
  66. package/src/read-command-output.ts +30 -0
  67. package/src/read-git-common-dir.ts +27 -0
  68. package/src/read-git-prefix.ts +27 -0
  69. package/src/run-bench-file.ts +23 -13
  70. package/src/run-bench.ts +348 -39
  71. package/src/run-cli.ts +1 -3
  72. package/dist/bench/pick-median.d.ts +0 -1
  73. package/src/bench/pick-median.ts +0 -10
@@ -1,5 +1,6 @@
1
1
  import { cli } from 'gunshi/bone';
2
2
  import { define } from 'gunshi/definition';
3
+ import { formatParseError } from './format-parse-error.ts';
3
4
 
4
5
  export interface CLIArgs {
5
6
  readonly command: string;
@@ -55,16 +56,6 @@ function dropFirstSeparator(args: readonly string[]): readonly string[] {
55
56
  return separator === -1 ? args : args.toSpliced(separator, 1);
56
57
  }
57
58
 
58
- // The parser throws one error that holds every reason the command line is refused, so each reason
59
- // prints on a line of its own.
60
- function formatParseError(error: unknown): string {
61
- if (error instanceof AggregateError) {
62
- return error.errors.map((reason) => formatParseError(reason)).join('\n');
63
- }
64
-
65
- return error instanceof Error ? error.message : String(error);
66
- }
67
-
68
59
  const forwardedArgSchema = { isolate: { type: 'boolean' } } as const;
69
60
 
70
61
  // The forwarded arguments hold `bun test` options that rig never declares, so this parse is not
@@ -0,0 +1,30 @@
1
+ import type { ProcessRuntime, SpawnOptions } from './browser/process-runtime.ts';
2
+
3
+ export interface CommandOutput {
4
+ readonly exitCode: number;
5
+
6
+ readonly stdout: string;
7
+ }
8
+
9
+ export type ReadCommandOptions = Omit<SpawnOptions, 'stdout'>;
10
+
11
+ export type ReadCommandOutput = (
12
+ command: readonly string[],
13
+ options: ReadCommandOptions,
14
+ ) => Promise<CommandOutput>;
15
+
16
+ // The command's stderr goes to rig's own, so a failure prints git's reason.
17
+ export async function readCommandOutput(
18
+ command: readonly string[],
19
+ options: ReadCommandOptions,
20
+ runtime: Pick<ProcessRuntime, 'spawn'>,
21
+ ): Promise<CommandOutput> {
22
+ const listing = runtime.spawn(command, { ...options, stdout: 'pipe' });
23
+
24
+ const [exitCode, stdout] = await Promise.all([
25
+ listing.exited,
26
+ new Response(listing.stdout).text(),
27
+ ]);
28
+
29
+ return { exitCode, stdout };
30
+ }
@@ -0,0 +1,27 @@
1
+ import type { SpawnOptions } from './browser/process-runtime.ts';
2
+ import type { ReadCommandOutput } from './read-command-output.ts';
3
+
4
+ export interface GitCommonDirDeps {
5
+ readonly readCommandOutput: ReadCommandOutput;
6
+ }
7
+
8
+ // Returns git's own answer: in the main checkout it is relative to the folder git runs in, such as
9
+ // ../../.git from a package folder, and in a linked worktree it is absolute.
10
+ export async function readGitCommonDir(
11
+ repoRoot: string,
12
+ environment: SpawnOptions['environment'],
13
+ deps: GitCommonDirDeps,
14
+ ): Promise<string> {
15
+ const output = await deps.readCommandOutput(['git', 'rev-parse', '--git-common-dir'], {
16
+ cwd: repoRoot,
17
+ environment,
18
+ });
19
+
20
+ if (output.exitCode !== 0) {
21
+ throw new Error(
22
+ `git rev-parse --git-common-dir failed in ${repoRoot} with exit code ${output.exitCode}`,
23
+ );
24
+ }
25
+
26
+ return output.stdout.trim();
27
+ }
@@ -0,0 +1,27 @@
1
+ import type { SpawnOptions } from './browser/process-runtime.ts';
2
+ import type { ReadCommandOutput } from './read-command-output.ts';
3
+
4
+ export interface GitPrefixDeps {
5
+ readonly readCommandOutput: ReadCommandOutput;
6
+ }
7
+
8
+ // Returns the folder's path inside its repo as git prints it, with a trailing slash, or an empty
9
+ // string at the repo's top level.
10
+ export async function readGitPrefix(
11
+ folder: string,
12
+ environment: SpawnOptions['environment'],
13
+ deps: GitPrefixDeps,
14
+ ): Promise<string> {
15
+ const output = await deps.readCommandOutput(['git', 'rev-parse', '--show-prefix'], {
16
+ cwd: folder,
17
+ environment,
18
+ });
19
+
20
+ if (output.exitCode !== 0) {
21
+ throw new Error(
22
+ `git rev-parse --show-prefix failed in ${folder} with exit code ${output.exitCode}`,
23
+ );
24
+ }
25
+
26
+ return output.stdout.trim();
27
+ }
@@ -1,25 +1,35 @@
1
+ import { appendFile } from 'node:fs/promises';
2
+ import { formatBenchReport } from './bench/format-bench-report.ts';
1
3
  import { formatBenchmarkResult } from './bench/format-benchmark-result.ts';
2
4
  import { parseBenchDefinition } from './bench/parse-bench-definition.ts';
5
+ import { readBenchFileArgs } from './bench/read-bench-file-args.ts';
3
6
  import { runBenchmark } from './bench/run-benchmark.ts';
4
7
 
5
- // `rig bench` runs this once per bench file, in the environment the benchmarks get.
6
- const file = process.argv[2] ?? '';
8
+ // `rig bench` runs this once per bench file, in the environment the benchmarks get, and judges the
9
+ // budget from the report this appends, so a bench that ran exits 0 whatever its median.
10
+ const args = await readBenchFileArgs(process.argv.slice(2));
7
11
 
8
- const loaded: unknown = await import(file);
12
+ if (args.kind === 'refused') {
13
+ process.stderr.write(`rig bench: ${args.reason}\n`);
9
14
 
10
- const defaultExport: unknown =
11
- typeof loaded === 'object' && loaded !== null ? Reflect.get(loaded, 'default') : null;
15
+ process.exitCode = 2;
16
+ } else {
17
+ const loaded: unknown = await import(args.file);
12
18
 
13
- const parsed = parseBenchDefinition(defaultExport);
19
+ const defaultExport: unknown =
20
+ typeof loaded === 'object' && loaded !== null ? Reflect.get(loaded, 'default') : null;
14
21
 
15
- if (parsed.kind === 'refused') {
16
- process.stderr.write(`rig bench: ${file}: ${parsed.reason}\n`);
22
+ const parsed = parseBenchDefinition(defaultExport);
17
23
 
18
- process.exitCode = 2;
19
- } else {
20
- const result = await runBenchmark(parsed.definition, () => performance.now());
24
+ if (parsed.kind === 'refused') {
25
+ process.stderr.write(`rig bench: ${args.file}: ${parsed.reason}\n`);
26
+
27
+ process.exitCode = 2;
28
+ } else {
29
+ const results = await runBenchmark(parsed.definition, () => performance.now());
21
30
 
22
- process.stdout.write(formatBenchmarkResult(parsed.definition, result));
31
+ process.stdout.write(formatBenchmarkResult(parsed.definition, results));
23
32
 
24
- process.exitCode = result.medianMs > parsed.definition.budgetMs ? 1 : 0;
33
+ await appendFile(args.reportPath, formatBenchReport(args.file, parsed.definition, results));
34
+ }
25
35
  }
package/src/run-bench.ts CHANGED
@@ -1,10 +1,28 @@
1
- import { basename, join } from 'node:path';
1
+ import { join, relative } from 'node:path';
2
+ import { match } from 'ts-pattern';
3
+ import { buildBaseSideRoot } from './bench/build-base-side-root.ts';
4
+ import { findBaseSetupVerdict } from './bench/find-base-setup-verdict.ts';
5
+ import { findMissingBenchVerdict } from './bench/find-missing-bench-verdict.ts';
6
+ import { parseBenchReport } from './bench/parse-bench-report.ts';
7
+ import type { ParsedBenchReport } from './bench/parse-bench-report.ts';
8
+ import { pickBenchPlan } from './bench/pick-bench-plan.ts';
9
+ import { pickCheckoutCleanup } from './bench/pick-checkout-cleanup.ts';
10
+ import { pickCleanedUpResult } from './bench/pick-cleaned-up-result.ts';
11
+ import { pickComparisonVerdict } from './bench/pick-comparison-verdict.ts';
12
+ import { pickHeadVerdict } from './bench/pick-head-verdict.ts';
13
+ import { pickNamedBenchFiles } from './bench/pick-named-bench-files.ts';
14
+ import { pickUnremovedCheckoutState } from './bench/pick-unremoved-checkout-state.ts';
15
+ import { readBenchArgs } from './bench/read-bench-args.ts';
16
+ import type { BenchOutput, BenchVerdict, CheckoutState } from './bench/types.ts';
2
17
  import { pickExitCodeAfterTests } from './browser/pick-exit-code-after-tests.ts';
3
18
  import { buildDietCommand } from './build-diet-command.ts';
4
19
  import { buildProgramLaneHooks } from './build-program-lane-hooks.ts';
5
20
  import { collectBenchFiles } from './collect-bench-files.ts';
21
+ import { defineRig } from './define-rig.ts';
6
22
  import type { LaneHooks } from './lane-hooks.ts';
7
23
  import { makeRepoLaneHooks } from './make-repo-lane-hooks.ts';
24
+ import type { ReadCommandOutput } from './read-command-output.ts';
25
+ import { readGitPrefix } from './read-git-prefix.ts';
8
26
  import { runLaneHooks } from './run-lane-hooks.ts';
9
27
  import type { ProgramBuildDeps } from './run-program-builds.ts';
10
28
  import { runProgramBuilds } from './run-program-builds.ts';
@@ -16,82 +34,154 @@ export interface BenchDeps extends ProgramBuildDeps {
16
34
  // The script that runs one bench file.
17
35
  readonly benchScript: string;
18
36
 
37
+ readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
38
+
19
39
  readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
20
40
 
21
41
  readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
22
42
 
23
43
  readonly importModule: (path: string) => Promise<unknown>;
44
+
45
+ // Makes a new folder for one run's reports and base checkout.
46
+ readonly createWorkFolder: (repoRoot: string) => Promise<string>;
47
+
48
+ readonly readText: (path: string) => Promise<string | null>;
49
+
50
+ readonly probeFolder: (path: string) => Promise<boolean>;
51
+
52
+ readonly removeFolder: (path: string) => Promise<void>;
53
+
54
+ readonly readCommandOutput: ReadCommandOutput;
24
55
  }
25
56
 
26
57
  // Builds each program once, as `rig e2e` does, then runs each bench file in a process of its own,
27
- // between the start and stop hooks of the bench lane when the config names one. The bench process
28
- // gets RIG_PROGRAM_<NAME> for each program, so resolveProgram gives the built artifact.
58
+ // between the start and stop hooks of the bench lane when the config names one. A base ref runs
59
+ // the same way from the same folder of a checkout of that ref, with that folder's config.
29
60
  export async function runBench(
30
61
  args: readonly string[],
31
62
  config: RigConfig,
32
63
  run: RigRun,
33
64
  deps: BenchDeps,
34
65
  ): Promise<number> {
35
- if (args.length > 1) {
36
- deps.output.err('rig bench: takes at most one benchmark name\n');
66
+ const parsed = await readBenchArgs(args);
37
67
 
38
- return 2;
68
+ if (parsed.kind === 'refused') {
69
+ return writeBenchVerdict(
70
+ { output: [{ stream: 'err', text: `rig bench: ${parsed.reason}\n` }], exitCode: 2 },
71
+ deps,
72
+ );
39
73
  }
40
74
 
41
- const isWorkspaceRepo = await deps.probeWorkspaceRepo(run.repoRoot);
42
- const trackedFiles = await deps.readTrackedFiles(run.repoRoot);
75
+ const headBenchFiles = await readBenchFiles(run.repoRoot, deps);
76
+
77
+ return match(pickBenchPlan(parsed.name, parsed.baseRef, headBenchFiles))
78
+ .with({ kind: 'finished' }, (plan) => writeBenchVerdict(plan.verdict, deps))
79
+ .with({ kind: 'head' }, (plan) => runHeadBench(plan.files, config, run, deps))
80
+ .with({ kind: 'comparison' }, (plan) =>
81
+ runBenchComparison(
82
+ {
83
+ baseRef: plan.baseRef,
84
+ name: parsed.name,
85
+ headConfig: config,
86
+ headFiles: plan.files,
87
+ headBenchFiles,
88
+ },
89
+ run,
90
+ deps,
91
+ ),
92
+ )
93
+ .exhaustive();
94
+ }
43
95
 
44
- const benchFiles = collectBenchFiles(trackedFiles, isWorkspaceRepo);
45
- const [name] = args;
96
+ function writeBenchVerdict(verdict: BenchVerdict, deps: BenchDeps): number {
97
+ writeBenchOutput(verdict.output, deps);
46
98
 
47
- const files =
48
- name === undefined ? benchFiles : benchFiles.filter((file) => toBenchName(file) === name);
99
+ return verdict.exitCode;
100
+ }
49
101
 
50
- if (files.length === 0) {
51
- return reportNoBenchmark(name, benchFiles, deps);
102
+ function writeBenchOutput(output: readonly BenchOutput[], deps: BenchDeps): void {
103
+ for (const entry of output) {
104
+ deps.output[entry.stream](entry.text);
52
105
  }
106
+ }
53
107
 
54
- const buildExitCode = await runProgramBuilds(config.programs, run, deps);
108
+ async function readBenchFiles(root: string, deps: BenchDeps): Promise<readonly string[]> {
109
+ const isWorkspaceRepo = await deps.probeWorkspaceRepo(root);
110
+ const trackedFiles = await deps.readTrackedFiles(root);
55
111
 
56
- if (buildExitCode !== 0) {
57
- return buildExitCode;
58
- }
112
+ return collectBenchFiles(trackedFiles, isWorkspaceRepo);
113
+ }
59
114
 
60
- const hooks = buildProgramLaneHooks(
61
- makeBenchHooks(config, run, deps),
62
- run.repoRoot,
63
- config.programs,
64
- );
115
+ interface BenchSide {
116
+ readonly root: string;
65
117
 
66
- const exitCode = await runLaneHooks(files, hooks);
118
+ readonly config: RigConfig;
67
119
 
68
- return pickExitCodeAfterTests(exitCode, run.signal.aborted);
69
- }
120
+ readonly files: readonly string[];
70
121
 
71
- function toBenchName(file: string): string {
72
- return basename(file, '.bench.ts');
122
+ readonly reportPath: string;
73
123
  }
74
124
 
75
- function reportNoBenchmark(
76
- name: string | undefined,
77
- benchFiles: readonly string[],
125
+ async function runHeadBench(
126
+ files: readonly string[],
127
+ config: RigConfig,
128
+ run: RigRun,
78
129
  deps: BenchDeps,
79
- ): number {
80
- if (name === undefined) {
81
- deps.output.out('rig: the repo has no benchmarks\n');
130
+ ): Promise<number> {
131
+ const workFolder = await deps.createWorkFolder(run.repoRoot);
132
+
133
+ const head: BenchSide = {
134
+ root: run.repoRoot,
135
+ config,
136
+ files,
137
+ reportPath: join(workFolder, 'head.jsonl'),
138
+ };
139
+
140
+ const [ran] = await Promise.allSettled([runHeadSide(head, run, deps)]);
141
+ const [cleaned] = await Promise.allSettled([deps.removeFolder(workFolder)]);
142
+
143
+ const result = pickCleanedUpResult(ran, cleaned);
82
144
 
145
+ if (result.kind === 'throw') {
146
+ throw result.error;
147
+ }
148
+
149
+ return pickExitCodeAfterTests(result.value, run.signal.aborted);
150
+ }
151
+
152
+ async function runHeadSide(head: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
153
+ const exitCode = await runBenchSide(head, run, deps);
154
+ const report = await readBenchReport(head, deps);
155
+
156
+ return writeBenchVerdict(pickHeadVerdict(report, exitCode), deps);
157
+ }
158
+
159
+ // Runs one side's bench files against that side's root, so a base checkout builds its own programs
160
+ // and starts its own lane.
161
+ async function runBenchSide(side: BenchSide, run: RigRun, deps: BenchDeps): Promise<number> {
162
+ if (side.files.length === 0) {
83
163
  return 0;
84
164
  }
85
165
 
86
- const names = benchFiles.map((file) => toBenchName(file)).join(', ');
166
+ const sideRun: RigRun = { ...run, repoRoot: side.root };
167
+
168
+ const buildExitCode = await runProgramBuilds(side.config.programs, sideRun, deps);
87
169
 
88
- deps.output.err(`rig bench: no benchmark named "${name}"; the benchmarks are ${names}\n`);
170
+ if (buildExitCode !== 0) {
171
+ return buildExitCode;
172
+ }
173
+
174
+ const hooks = buildProgramLaneHooks(
175
+ makeBenchHooks(side, sideRun, deps),
176
+ side.root,
177
+ side.config.programs,
178
+ );
89
179
 
90
- return 1;
180
+ return runLaneHooks(side.files, hooks);
91
181
  }
92
182
 
93
- function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHooks {
94
- const lane = config.lanes.find((candidate) => candidate.name === config.benchLane);
183
+ function makeBenchHooks(side: BenchSide, run: RigRun, deps: BenchDeps): LaneHooks {
184
+ const lane = side.config.lanes.find((candidate) => candidate.name === side.config.benchLane);
95
185
 
96
186
  const laneHooks: Pick<LaneHooks, 'start' | 'stop'> =
97
187
  lane === undefined
@@ -119,6 +209,8 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
119
209
  'bun',
120
210
  deps.benchScript,
121
211
  join(run.repoRoot, file),
212
+ '--report',
213
+ side.reportPath,
122
214
  ]),
123
215
  { cwd: run.repoRoot, environment, signal: run.signal },
124
216
  );
@@ -130,3 +222,220 @@ function makeBenchHooks(config: RigConfig, run: RigRun, deps: BenchDeps): LaneHo
130
222
  },
131
223
  };
132
224
  }
225
+
226
+ // A bench file that failed before it measured anything appends no report. Each entry's file
227
+ // becomes relative to the side's root, so the same file matches on the base and on the head.
228
+ async function readBenchReport(side: BenchSide, deps: BenchDeps): Promise<ParsedBenchReport> {
229
+ const text = await deps.readText(side.reportPath);
230
+
231
+ const report = parseBenchReport(text ?? '');
232
+
233
+ if (report.kind === 'refused') {
234
+ return report;
235
+ }
236
+
237
+ return {
238
+ kind: 'parsed',
239
+ entries: report.entries.map((entry) => ({
240
+ file: relative(side.root, entry.file),
241
+ name: entry.name,
242
+ concurrency: entry.concurrency,
243
+ medianMs: entry.medianMs,
244
+ spreadMs: entry.spreadMs,
245
+ budgetMs: entry.budgetMs,
246
+ })),
247
+ };
248
+ }
249
+
250
+ interface BenchComparison {
251
+ readonly baseRef: string;
252
+
253
+ readonly name: string | null;
254
+
255
+ readonly headConfig: RigConfig;
256
+
257
+ readonly headFiles: readonly string[];
258
+
259
+ // Every bench file of the head, for the names a missing benchmark lists.
260
+ readonly headBenchFiles: readonly string[];
261
+ }
262
+
263
+ interface BaseCheckout {
264
+ readonly workFolder: string;
265
+
266
+ // The top folder of the checkout, which git adds and removes.
267
+ readonly checkoutRoot: string;
268
+
269
+ // The folder inside the checkout that matches the folder rig started in.
270
+ readonly sideRoot: string;
271
+ }
272
+
273
+ async function runBenchComparison(
274
+ comparison: BenchComparison,
275
+ run: RigRun,
276
+ deps: BenchDeps,
277
+ ): Promise<number> {
278
+ const gitPrefix = await readGitPrefix(run.repoRoot, run.environment, deps);
279
+ const workFolder = await deps.createWorkFolder(run.repoRoot);
280
+
281
+ const checkoutRoot = join(workFolder, 'base');
282
+
283
+ const base: BaseCheckout = {
284
+ workFolder,
285
+ checkoutRoot,
286
+ sideRoot: buildBaseSideRoot(checkoutRoot, gitPrefix),
287
+ };
288
+
289
+ // An abort makes the add report failure even when git finished the checkout, so every path,
290
+ // a failed add included, runs the same removal.
291
+ const [compared] = await Promise.allSettled([runCheckoutComparison(comparison, base, run, deps)]);
292
+ const [cleaned] = await Promise.allSettled([removeComparisonCheckout(base, run, deps)]);
293
+
294
+ const result = pickCleanedUpResult(compared, cleaned);
295
+
296
+ if (result.kind === 'throw') {
297
+ throw result.error;
298
+ }
299
+
300
+ return pickExitCodeAfterTests(Math.max(result.value, result.cleanup), run.signal.aborted);
301
+ }
302
+
303
+ async function runCheckoutComparison(
304
+ comparison: BenchComparison,
305
+ checkout: BaseCheckout,
306
+ run: RigRun,
307
+ deps: BenchDeps,
308
+ ): Promise<number> {
309
+ const addExitCode = await deps.runCommand(
310
+ ['git', 'worktree', 'add', '--detach', checkout.checkoutRoot, comparison.baseRef],
311
+ { cwd: run.repoRoot, environment: run.environment, signal: run.signal },
312
+ );
313
+
314
+ const addFailure = findBaseSetupVerdict('checkout', comparison.baseRef, addExitCode);
315
+
316
+ if (addFailure !== null) {
317
+ return writeBenchVerdict(addFailure, deps);
318
+ }
319
+
320
+ return runCheckedOutComparison(comparison, checkout, run, deps);
321
+ }
322
+
323
+ async function runCheckedOutComparison(
324
+ comparison: BenchComparison,
325
+ checkout: BaseCheckout,
326
+ run: RigRun,
327
+ deps: BenchDeps,
328
+ ): Promise<number> {
329
+ const installExitCode = await deps.runCommand(['bun', 'install', '--frozen-lockfile'], {
330
+ cwd: checkout.checkoutRoot,
331
+ environment: run.environment,
332
+ signal: run.signal,
333
+ });
334
+
335
+ const installFailure = findBaseSetupVerdict('install', comparison.baseRef, installExitCode);
336
+
337
+ if (installFailure !== null) {
338
+ return writeBenchVerdict(installFailure, deps);
339
+ }
340
+
341
+ const baseSide = await readBaseSide(checkout.sideRoot, deps);
342
+
343
+ const missing = findMissingBenchVerdict(
344
+ comparison.name,
345
+ comparison.headBenchFiles,
346
+ baseSide.benchFiles,
347
+ );
348
+
349
+ if (missing !== null) {
350
+ return writeBenchVerdict(missing, deps);
351
+ }
352
+
353
+ const base: BenchSide = {
354
+ root: checkout.sideRoot,
355
+ config: baseSide.config,
356
+ files: pickNamedBenchFiles(baseSide.benchFiles, comparison.name),
357
+ reportPath: join(checkout.workFolder, 'base.jsonl'),
358
+ };
359
+
360
+ const head: BenchSide = {
361
+ root: run.repoRoot,
362
+ config: comparison.headConfig,
363
+ files: comparison.headFiles,
364
+ reportPath: join(checkout.workFolder, 'head.jsonl'),
365
+ };
366
+
367
+ const baseExitCode = await runBenchSide(base, run, deps);
368
+ const headExitCode = await runBenchSide(head, run, deps);
369
+ const baseReport = await readBenchReport(base, deps);
370
+ const headReport = await readBenchReport(head, deps);
371
+
372
+ return writeBenchVerdict(
373
+ pickComparisonVerdict(
374
+ { exitCode: baseExitCode, report: baseReport },
375
+ { exitCode: headExitCode, report: headReport },
376
+ ),
377
+ deps,
378
+ );
379
+ }
380
+
381
+ interface BaseSide {
382
+ readonly config: RigConfig;
383
+
384
+ readonly benchFiles: readonly string[];
385
+ }
386
+
387
+ // A package that only the head has gives an empty base side, which runs nothing, so the table
388
+ // shows `-` for the base.
389
+ async function readBaseSide(root: string, deps: BenchDeps): Promise<BaseSide> {
390
+ const isPresent = await deps.probeFolder(root);
391
+
392
+ if (!isPresent) {
393
+ return { config: defineRig({}), benchFiles: [] };
394
+ }
395
+
396
+ const config = await deps.readConfig(root);
397
+ const benchFiles = await readBenchFiles(root, deps);
398
+
399
+ return { config, benchFiles };
400
+ }
401
+
402
+ async function removeComparisonCheckout(
403
+ base: BaseCheckout,
404
+ run: RigRun,
405
+ deps: BenchDeps,
406
+ ): Promise<number> {
407
+ const checkout = await removeBaseCheckout(base.checkoutRoot, run, deps);
408
+
409
+ const cleanup = pickCheckoutCleanup(base.checkoutRoot, checkout);
410
+
411
+ writeBenchOutput(cleanup.output, deps);
412
+
413
+ if (!cleanup.isWorkFolderKept) {
414
+ await deps.removeFolder(base.workFolder);
415
+ }
416
+
417
+ return cleanup.exitCode;
418
+ }
419
+
420
+ // An aborted run still removes the checkout, so the removal gets a signal of its own.
421
+ async function removeBaseCheckout(
422
+ checkoutRoot: string,
423
+ run: RigRun,
424
+ deps: BenchDeps,
425
+ ): Promise<CheckoutState> {
426
+ const removeExitCode = await deps.runCommand(
427
+ ['git', 'worktree', 'remove', '--force', checkoutRoot],
428
+ { cwd: run.repoRoot, environment: run.environment, signal: new AbortController().signal },
429
+ );
430
+
431
+ if (removeExitCode === 0) {
432
+ return 'removed';
433
+ }
434
+
435
+ const listing = await deps.readCommandOutput(['git', 'worktree', 'list', '--porcelain', '-z'], {
436
+ cwd: run.repoRoot,
437
+ environment: run.environment,
438
+ });
439
+
440
+ return pickUnremovedCheckoutState(listing, checkoutRoot);
441
+ }
package/src/run-cli.ts CHANGED
@@ -17,9 +17,7 @@ import type { UnitDeps } from './run-unit.ts';
17
17
  import type { RigConfig, RigRun } from './types.ts';
18
18
 
19
19
  export interface CLIDeps
20
- extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
21
- readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
22
- }
20
+ extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {}
23
21
 
24
22
  export async function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number> {
25
23
  const args = await readCLIArgs(argv);
@@ -1 +0,0 @@
1
- export declare function pickMedian(samples: readonly number[]): number;
@@ -1,10 +0,0 @@
1
- export function pickMedian(samples: readonly number[]): number {
2
- const sorted = samples.toSorted((left, right) => left - right);
3
- const middle = Math.floor(sorted.length / 2);
4
-
5
- if (sorted.length % 2 === 1) {
6
- return sorted[middle] ?? 0;
7
- }
8
-
9
- return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
10
- }