@zgeoff/rig 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bench/bench-report-entry-schema.d.ts +10 -0
- package/dist/bench/build-base-side-root.d.ts +1 -0
- package/dist/bench/collect-bench-comparison.d.ts +3 -0
- package/dist/bench/define-bench.d.ts +2 -0
- package/dist/bench/find-base-setup-verdict.d.ts +3 -0
- package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
- package/dist/bench/format-bench-comparison.d.ts +2 -0
- package/dist/bench/format-bench-report.d.ts +3 -0
- package/dist/bench/format-benchmark-result.d.ts +1 -1
- package/dist/bench/is-concurrency-list.d.ts +1 -0
- package/dist/bench/parse-bench-report.d.ts +9 -0
- package/dist/bench/pick-bench-exit-code.d.ts +2 -0
- package/dist/bench/pick-bench-plan.d.ts +13 -0
- package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
- package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
- package/dist/bench/pick-comparison-verdict.d.ts +7 -0
- package/dist/bench/pick-head-verdict.d.ts +3 -0
- package/dist/bench/pick-named-bench-files.d.ts +1 -0
- package/dist/bench/pick-quantile.d.ts +1 -0
- package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
- package/dist/bench/read-bench-args.d.ts +9 -0
- package/dist/bench/read-bench-file-args.d.ts +9 -0
- package/dist/bench/run-benchmark.d.ts +3 -3
- package/dist/bench/to-bench-name.d.ts +1 -0
- package/dist/bench/types.d.ts +20 -0
- package/dist/build-bench-folder-path.d.ts +1 -0
- package/dist/create-bench-work-folder.d.ts +6 -0
- package/dist/format-parse-error.d.ts +1 -0
- package/dist/read-command-output.d.ts +8 -0
- package/dist/read-git-common-dir.d.ts +6 -0
- package/dist/read-git-prefix.d.ts +6 -0
- package/dist/run-bench.d.ts +7 -0
- package/dist/run-cli.d.ts +1 -2
- package/package.json +8 -8
- package/src/bench/bench-report-entry-schema.ts +17 -0
- package/src/bench/build-base-side-root.ts +7 -0
- package/src/bench/collect-bench-comparison.ts +43 -0
- package/src/bench/define-bench.ts +15 -0
- package/src/bench/find-base-setup-verdict.ts +21 -0
- package/src/bench/find-missing-bench-verdict.ts +33 -0
- package/src/bench/format-bench-comparison.ts +72 -0
- package/src/bench/format-bench-report.ts +24 -0
- package/src/bench/format-benchmark-result.ts +7 -3
- package/src/bench/is-concurrency-list.ts +10 -0
- package/src/bench/parse-bench-definition.ts +21 -3
- package/src/bench/parse-bench-report.ts +42 -0
- package/src/bench/pick-bench-exit-code.ts +15 -0
- package/src/bench/pick-bench-plan.ts +25 -0
- package/src/bench/pick-checkout-cleanup.ts +29 -0
- package/src/bench/pick-cleaned-up-result.ts +30 -0
- package/src/bench/pick-comparison-verdict.ts +36 -0
- package/src/bench/pick-head-verdict.ts +11 -0
- package/src/bench/pick-named-bench-files.ts +8 -0
- package/src/bench/pick-quantile.ts +9 -0
- package/src/bench/pick-unremoved-checkout-state.ts +30 -0
- package/src/bench/read-bench-args.ts +45 -0
- package/src/bench/read-bench-file-args.ts +38 -0
- package/src/bench/run-benchmark.ts +88 -21
- package/src/bench/to-bench-name.ts +5 -0
- package/src/bench/types.ts +37 -0
- package/src/build-bench-folder-path.ts +7 -0
- package/src/cli.ts +31 -0
- package/src/create-bench-work-folder.ts +26 -0
- package/src/format-parse-error.ts +9 -0
- package/src/read-cli-args.ts +1 -10
- package/src/read-command-output.ts +30 -0
- package/src/read-git-common-dir.ts +27 -0
- package/src/read-git-prefix.ts +27 -0
- package/src/run-bench-file.ts +23 -13
- package/src/run-bench.ts +348 -39
- package/src/run-cli.ts +1 -3
- package/dist/bench/pick-median.d.ts +0 -1
- package/src/bench/pick-median.ts +0 -10
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { BenchDefinition } from './define-bench.ts';
|
|
2
|
+
import { isConcurrencyList } from './is-concurrency-list.ts';
|
|
2
3
|
|
|
3
4
|
export type ParsedBenchDefinition =
|
|
4
5
|
| { readonly kind: 'parsed'; readonly definition: BenchDefinition }
|
|
@@ -15,7 +16,15 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
|
|
|
15
16
|
const budgetMs: unknown = Reflect.get(value, 'budgetMs');
|
|
16
17
|
const prepare: unknown = Reflect.get(value, 'prepare');
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
// A bench file on a base ref can import an older rig, whose result has no concurrency.
|
|
20
|
+
const concurrency: unknown = Reflect.get(value, 'concurrency') ?? [1];
|
|
21
|
+
|
|
22
|
+
if (
|
|
23
|
+
typeof name !== 'string' ||
|
|
24
|
+
typeof budgetMs !== 'number' ||
|
|
25
|
+
!isFunction(prepare) ||
|
|
26
|
+
!isConcurrencyList(concurrency)
|
|
27
|
+
) {
|
|
19
28
|
return { kind: 'refused', reason: 'the default export is no defineBench result' };
|
|
20
29
|
}
|
|
21
30
|
|
|
@@ -24,6 +33,7 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
|
|
|
24
33
|
definition: {
|
|
25
34
|
name,
|
|
26
35
|
budgetMs,
|
|
36
|
+
concurrency,
|
|
27
37
|
prepare: async () => {
|
|
28
38
|
const prepared: unknown = await prepare();
|
|
29
39
|
|
|
@@ -40,8 +50,10 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
|
|
|
40
50
|
}
|
|
41
51
|
|
|
42
52
|
return {
|
|
43
|
-
step:
|
|
44
|
-
|
|
53
|
+
step: () => {
|
|
54
|
+
const result: unknown = step();
|
|
55
|
+
|
|
56
|
+
return result === undefined ? undefined : settleStep(result);
|
|
45
57
|
},
|
|
46
58
|
dispose: async () => {
|
|
47
59
|
await dispose();
|
|
@@ -55,3 +67,9 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
|
|
|
55
67
|
function isFunction(value: unknown): value is (...args: unknown[]) => unknown {
|
|
56
68
|
return typeof value === 'function';
|
|
57
69
|
}
|
|
70
|
+
|
|
71
|
+
// `instanceof Promise` misses a thenable and a promise from another realm, such as a `node:vm`
|
|
72
|
+
// context, so any value but `undefined` goes through `await`, which adopts both.
|
|
73
|
+
async function settleStep(result: unknown): Promise<void> {
|
|
74
|
+
await result;
|
|
75
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { BenchReportEntry } from './bench-report-entry-schema.ts';
|
|
2
|
+
import { benchReportEntrySchema } from './bench-report-entry-schema.ts';
|
|
3
|
+
|
|
4
|
+
export type ParsedBenchReport =
|
|
5
|
+
| { readonly kind: 'parsed'; readonly entries: readonly BenchReportEntry[] }
|
|
6
|
+
| { readonly kind: 'refused'; readonly reason: string };
|
|
7
|
+
|
|
8
|
+
// A report holds what other processes appended, one JSON entry per line, so each line is checked
|
|
9
|
+
// instead of trusted.
|
|
10
|
+
export function parseBenchReport(text: string): ParsedBenchReport {
|
|
11
|
+
const entries: BenchReportEntry[] = [];
|
|
12
|
+
|
|
13
|
+
for (const line of text.split('\n')) {
|
|
14
|
+
if (line.trim() === '') {
|
|
15
|
+
continue;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const entry = parseEntry(line);
|
|
19
|
+
|
|
20
|
+
if (entry === null) {
|
|
21
|
+
return { kind: 'refused', reason: `the bench report holds a line that is no entry: ${line}` };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
entries.push(entry);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return { kind: 'parsed', entries };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function parseEntry(line: string): BenchReportEntry | null {
|
|
31
|
+
let value: unknown;
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
value = JSON.parse(line);
|
|
35
|
+
} catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const parsed = benchReportEntrySchema.safeParse(value);
|
|
40
|
+
|
|
41
|
+
return parsed.success ? parsed.data : null;
|
|
42
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { BenchReportEntry } from './bench-report-entry-schema.ts';
|
|
2
|
+
|
|
3
|
+
// The budget holds on the head only, so a base slower than its budget still lets the head pass.
|
|
4
|
+
export function pickBenchExitCode(
|
|
5
|
+
sideExitCodes: readonly number[],
|
|
6
|
+
headEntries: readonly BenchReportEntry[],
|
|
7
|
+
): number {
|
|
8
|
+
const failedExitCode = sideExitCodes.find((exitCode) => exitCode !== 0);
|
|
9
|
+
|
|
10
|
+
if (failedExitCode !== undefined) {
|
|
11
|
+
return failedExitCode;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
return headEntries.some((entry) => entry.medianMs > entry.budgetMs) ? 1 : 0;
|
|
15
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { findMissingBenchVerdict } from './find-missing-bench-verdict.ts';
|
|
2
|
+
import { pickNamedBenchFiles } from './pick-named-bench-files.ts';
|
|
3
|
+
import type { BenchVerdict } from './types.ts';
|
|
4
|
+
|
|
5
|
+
export type BenchPlan =
|
|
6
|
+
| { readonly kind: 'finished'; readonly verdict: BenchVerdict }
|
|
7
|
+
| { readonly kind: 'head'; readonly files: readonly string[] }
|
|
8
|
+
| { readonly kind: 'comparison'; readonly baseRef: string; readonly files: readonly string[] };
|
|
9
|
+
|
|
10
|
+
// With a base, whether a benchmark is missing waits for the base's files.
|
|
11
|
+
export function pickBenchPlan(
|
|
12
|
+
name: string | null,
|
|
13
|
+
baseRef: string | null,
|
|
14
|
+
headBenchFiles: readonly string[],
|
|
15
|
+
): BenchPlan {
|
|
16
|
+
const files = pickNamedBenchFiles(headBenchFiles, name);
|
|
17
|
+
|
|
18
|
+
if (baseRef !== null) {
|
|
19
|
+
return { kind: 'comparison', baseRef, files };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const missing = findMissingBenchVerdict(name, headBenchFiles, []);
|
|
23
|
+
|
|
24
|
+
return missing === null ? { kind: 'head', files } : { kind: 'finished', verdict: missing };
|
|
25
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { BenchOutput, CheckoutState } from './types.ts';
|
|
2
|
+
|
|
3
|
+
export interface CheckoutCleanup {
|
|
4
|
+
readonly output: readonly BenchOutput[];
|
|
5
|
+
|
|
6
|
+
// A checkout that git still registers keeps its work folder, so git's record of the checkout
|
|
7
|
+
// still points at a folder.
|
|
8
|
+
readonly isWorkFolderKept: boolean;
|
|
9
|
+
|
|
10
|
+
// A checkout that git still registers fails a comparison that passed.
|
|
11
|
+
readonly exitCode: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function pickCheckoutCleanup(baseRoot: string, checkout: CheckoutState): CheckoutCleanup {
|
|
15
|
+
if (checkout !== 'left') {
|
|
16
|
+
return { output: [], isWorkFolderKept: false, exitCode: 0 };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
return {
|
|
20
|
+
output: [
|
|
21
|
+
{
|
|
22
|
+
stream: 'err',
|
|
23
|
+
text: `rig bench: could not remove the checkout at ${baseRoot}; remove it with git worktree remove --force ${baseRoot}\n`,
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
isWorkFolderKept: true,
|
|
27
|
+
exitCode: 1,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type CleanedUpResult<T, C> =
|
|
2
|
+
| { readonly kind: 'fulfilled'; readonly value: T; readonly cleanup: C }
|
|
3
|
+
| { readonly kind: 'throw'; readonly error: unknown };
|
|
4
|
+
|
|
5
|
+
// Takes how a run settled and how its cleanup settled. A cleanup that throws after a run that threw
|
|
6
|
+
// keeps both errors, the run's first, so the run's cause still reaches the caller.
|
|
7
|
+
export function pickCleanedUpResult<T, C>(
|
|
8
|
+
ran: PromiseSettledResult<T>,
|
|
9
|
+
cleaned: PromiseSettledResult<C>,
|
|
10
|
+
): CleanedUpResult<T, C> {
|
|
11
|
+
if (ran.status === 'rejected' && cleaned.status === 'rejected') {
|
|
12
|
+
return {
|
|
13
|
+
kind: 'throw',
|
|
14
|
+
error: new AggregateError(
|
|
15
|
+
[ran.reason, cleaned.reason],
|
|
16
|
+
'the run failed, and its cleanup failed too',
|
|
17
|
+
),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (ran.status === 'rejected') {
|
|
22
|
+
return { kind: 'throw', error: ran.reason };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (cleaned.status === 'rejected') {
|
|
26
|
+
return { kind: 'throw', error: cleaned.reason };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
return { kind: 'fulfilled', value: ran.value, cleanup: cleaned.value };
|
|
30
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { collectBenchComparison } from './collect-bench-comparison.ts';
|
|
2
|
+
import { formatBenchComparison } from './format-bench-comparison.ts';
|
|
3
|
+
import type { ParsedBenchReport } from './parse-bench-report.ts';
|
|
4
|
+
import { pickBenchExitCode } from './pick-bench-exit-code.ts';
|
|
5
|
+
import type { BenchVerdict } from './types.ts';
|
|
6
|
+
|
|
7
|
+
export interface BenchSideOutcome {
|
|
8
|
+
readonly exitCode: number;
|
|
9
|
+
|
|
10
|
+
readonly report: ParsedBenchReport;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function pickComparisonVerdict(
|
|
14
|
+
base: BenchSideOutcome,
|
|
15
|
+
head: BenchSideOutcome,
|
|
16
|
+
): BenchVerdict {
|
|
17
|
+
if (base.report.kind === 'refused') {
|
|
18
|
+
return { output: [{ stream: 'err', text: `rig bench: ${base.report.reason}\n` }], exitCode: 1 };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (head.report.kind === 'refused') {
|
|
22
|
+
return { output: [{ stream: 'err', text: `rig bench: ${head.report.reason}\n` }], exitCode: 1 };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return {
|
|
26
|
+
output: [
|
|
27
|
+
{
|
|
28
|
+
stream: 'out',
|
|
29
|
+
text: formatBenchComparison(
|
|
30
|
+
collectBenchComparison(base.report.entries, head.report.entries),
|
|
31
|
+
),
|
|
32
|
+
},
|
|
33
|
+
],
|
|
34
|
+
exitCode: pickBenchExitCode([head.exitCode, base.exitCode], head.report.entries),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ParsedBenchReport } from './parse-bench-report.ts';
|
|
2
|
+
import { pickBenchExitCode } from './pick-bench-exit-code.ts';
|
|
3
|
+
import type { BenchVerdict } from './types.ts';
|
|
4
|
+
|
|
5
|
+
export function pickHeadVerdict(report: ParsedBenchReport, headExitCode: number): BenchVerdict {
|
|
6
|
+
if (report.kind === 'refused') {
|
|
7
|
+
return { output: [{ stream: 'err', text: `rig bench: ${report.reason}\n` }], exitCode: 1 };
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
return { output: [], exitCode: pickBenchExitCode([headExitCode], report.entries) };
|
|
11
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { toBenchName } from './to-bench-name.ts';
|
|
2
|
+
|
|
3
|
+
export function pickNamedBenchFiles(
|
|
4
|
+
benchFiles: readonly string[],
|
|
5
|
+
name: string | null,
|
|
6
|
+
): readonly string[] {
|
|
7
|
+
return name === null ? benchFiles : benchFiles.filter((file) => toBenchName(file) === name);
|
|
8
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function pickQuantile(samples: readonly number[], quantile: number): number {
|
|
2
|
+
const sorted = samples.toSorted((left, right) => left - right);
|
|
3
|
+
const position = (sorted.length - 1) * quantile;
|
|
4
|
+
const lower = Math.floor(position);
|
|
5
|
+
const lowerSample = sorted[lower] ?? 0;
|
|
6
|
+
const upperSample = sorted[Math.ceil(position)] ?? lowerSample;
|
|
7
|
+
|
|
8
|
+
return lowerSample + (upperSample - lowerSample) * (position - lower);
|
|
9
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import type { CommandOutput } from '../read-command-output.ts';
|
|
3
|
+
import type { CheckoutState } from './types.ts';
|
|
4
|
+
|
|
5
|
+
// Takes what `git worktree list --porcelain -z` gave after a failed removal. A listing that failed
|
|
6
|
+
// cannot show the checkout is gone, so the checkout counts as left.
|
|
7
|
+
export function pickUnremovedCheckoutState(
|
|
8
|
+
listing: CommandOutput,
|
|
9
|
+
baseRoot: string,
|
|
10
|
+
): Exclude<CheckoutState, 'removed'> {
|
|
11
|
+
if (listing.exitCode !== 0) {
|
|
12
|
+
return 'left';
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const isListed = collectWorktreePaths(listing.stdout).some(
|
|
16
|
+
(path) => resolve(path) === resolve(baseRoot),
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
return isListed ? 'left' : 'absent';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Each field ends in NUL and each record ends in an empty field, so a path that holds a newline
|
|
23
|
+
// stays whole.
|
|
24
|
+
function collectWorktreePaths(stdout: string): string[] {
|
|
25
|
+
return stdout
|
|
26
|
+
.split('\0\0')
|
|
27
|
+
.map((record) => record.split('\0')[0] ?? '')
|
|
28
|
+
.filter((field) => field.startsWith('worktree '))
|
|
29
|
+
.map((field) => field.slice('worktree '.length));
|
|
30
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { cli } from 'gunshi/bone';
|
|
2
|
+
import { define } from 'gunshi/definition';
|
|
3
|
+
import { formatParseError } from '../format-parse-error.ts';
|
|
4
|
+
|
|
5
|
+
export type BenchArgs =
|
|
6
|
+
| { readonly kind: 'parsed'; readonly name: string | null; readonly baseRef: string | null }
|
|
7
|
+
| { readonly kind: 'refused'; readonly reason: string };
|
|
8
|
+
|
|
9
|
+
const argSchema = {
|
|
10
|
+
base: { type: 'string' },
|
|
11
|
+
name: { type: 'positional', required: false },
|
|
12
|
+
} as const;
|
|
13
|
+
|
|
14
|
+
export async function readBenchArgs(args: readonly string[]): Promise<BenchArgs> {
|
|
15
|
+
let parsed: BenchArgs | null = null;
|
|
16
|
+
|
|
17
|
+
const command = define({
|
|
18
|
+
name: 'rig bench',
|
|
19
|
+
args: argSchema,
|
|
20
|
+
run: (ctx) => {
|
|
21
|
+
// The parser refuses `--base` with no value through an error whose own message is empty.
|
|
22
|
+
if (ctx.validationError !== undefined) {
|
|
23
|
+
parsed = { kind: 'refused', reason: '--base needs a ref' };
|
|
24
|
+
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (ctx.positionals.length > 1) {
|
|
29
|
+
parsed = { kind: 'refused', reason: 'takes at most one benchmark name' };
|
|
30
|
+
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
parsed = { kind: 'parsed', name: ctx.values.name ?? null, baseRef: ctx.values.base ?? null };
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
await cli([...args], command, { strict: true });
|
|
40
|
+
} catch (error) {
|
|
41
|
+
return { kind: 'refused', reason: formatParseError(error) };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return parsed ?? { kind: 'refused', reason: 'the command line could not be parsed' };
|
|
45
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { cli } from 'gunshi/bone';
|
|
2
|
+
import { define } from 'gunshi/definition';
|
|
3
|
+
import { formatParseError } from '../format-parse-error.ts';
|
|
4
|
+
|
|
5
|
+
export type BenchFileArgs =
|
|
6
|
+
| { readonly kind: 'parsed'; readonly file: string; readonly reportPath: string }
|
|
7
|
+
| { readonly kind: 'refused'; readonly reason: string };
|
|
8
|
+
|
|
9
|
+
const argSchema = {
|
|
10
|
+
file: { type: 'positional' },
|
|
11
|
+
report: { type: 'string', required: true },
|
|
12
|
+
} as const;
|
|
13
|
+
|
|
14
|
+
export async function readBenchFileArgs(args: readonly string[]): Promise<BenchFileArgs> {
|
|
15
|
+
let parsed: BenchFileArgs | null = null;
|
|
16
|
+
|
|
17
|
+
const command = define({
|
|
18
|
+
name: 'run-bench-file',
|
|
19
|
+
args: argSchema,
|
|
20
|
+
run: (ctx) => {
|
|
21
|
+
if (ctx.validationError !== undefined) {
|
|
22
|
+
parsed = { kind: 'refused', reason: formatParseError(ctx.validationError) };
|
|
23
|
+
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
parsed = { kind: 'parsed', file: ctx.values.file, reportPath: ctx.values.report };
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
await cli([...args], command, { strict: true });
|
|
33
|
+
} catch (error) {
|
|
34
|
+
return { kind: 'refused', reason: formatParseError(error) };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return parsed ?? { kind: 'refused', reason: 'the command line could not be parsed' };
|
|
38
|
+
}
|
|
@@ -1,38 +1,105 @@
|
|
|
1
|
-
import type { BenchDefinition } from './define-bench.ts';
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
export const warmUpRuns = 5;
|
|
5
|
-
export const measuredRuns = 20;
|
|
1
|
+
import type { BenchDefinition, PreparedBench } from './define-bench.ts';
|
|
2
|
+
import { pickCleanedUpResult } from './pick-cleaned-up-result.ts';
|
|
3
|
+
import { pickQuantile } from './pick-quantile.ts';
|
|
6
4
|
|
|
7
5
|
export interface BenchmarkResult {
|
|
6
|
+
readonly concurrency: number;
|
|
7
|
+
|
|
8
8
|
readonly medianMs: number;
|
|
9
|
+
|
|
10
|
+
// The interquartile range of the measured times.
|
|
11
|
+
readonly spreadMs: number;
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
// `prepare` runs once
|
|
12
|
-
//
|
|
14
|
+
// `prepare` runs once for every level, and the setup result is disposed of after the last level,
|
|
15
|
+
// even when a run throws.
|
|
13
16
|
export async function runBenchmark(
|
|
14
17
|
definition: BenchDefinition,
|
|
15
18
|
now: () => number,
|
|
16
|
-
): Promise<BenchmarkResult> {
|
|
19
|
+
): Promise<readonly BenchmarkResult[]> {
|
|
17
20
|
const prepared = await definition.prepare();
|
|
21
|
+
const [ran] = await Promise.allSettled([runLevels(definition, prepared, now)]);
|
|
22
|
+
const [cleaned] = await Promise.allSettled([prepared.dispose()]);
|
|
18
23
|
|
|
19
|
-
|
|
20
|
-
for (let index = 0; index < warmUpRuns; index += 1) {
|
|
21
|
-
await prepared.step();
|
|
22
|
-
}
|
|
24
|
+
const result = pickCleanedUpResult(ran, cleaned);
|
|
23
25
|
|
|
24
|
-
|
|
26
|
+
if (result.kind === 'throw') {
|
|
27
|
+
throw result.error;
|
|
28
|
+
}
|
|
25
29
|
|
|
26
|
-
|
|
27
|
-
|
|
30
|
+
return result.value;
|
|
31
|
+
}
|
|
28
32
|
|
|
29
|
-
|
|
33
|
+
async function runLevels(
|
|
34
|
+
definition: BenchDefinition,
|
|
35
|
+
prepared: PreparedBench,
|
|
36
|
+
now: () => number,
|
|
37
|
+
): Promise<readonly BenchmarkResult[]> {
|
|
38
|
+
const results: BenchmarkResult[] = [];
|
|
30
39
|
|
|
31
|
-
|
|
32
|
-
|
|
40
|
+
for (const concurrency of definition.concurrency) {
|
|
41
|
+
const result = await runLevel(prepared, concurrency, now);
|
|
33
42
|
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
|
|
43
|
+
results.push(result);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return results;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const warmUpRounds = 5;
|
|
50
|
+
const measuredRounds = 20;
|
|
51
|
+
|
|
52
|
+
async function runLevel(
|
|
53
|
+
prepared: PreparedBench,
|
|
54
|
+
concurrency: number,
|
|
55
|
+
now: () => number,
|
|
56
|
+
): Promise<BenchmarkResult> {
|
|
57
|
+
for (let round = 0; round < warmUpRounds; round += 1) {
|
|
58
|
+
await runRound(prepared, concurrency, now);
|
|
37
59
|
}
|
|
60
|
+
|
|
61
|
+
const samples: number[] = [];
|
|
62
|
+
|
|
63
|
+
for (let round = 0; round < measuredRounds; round += 1) {
|
|
64
|
+
const roundSamples = await runRound(prepared, concurrency, now);
|
|
65
|
+
|
|
66
|
+
samples.push(...roundSamples);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
concurrency,
|
|
71
|
+
medianMs: pickQuantile(samples, 0.5),
|
|
72
|
+
spreadMs: pickQuantile(samples, 0.75) - pickQuantile(samples, 0.25),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Each call is timed on its own, so the times show how long one call waits under the load of the
|
|
77
|
+
// others, not how long the round took. A rejected call still waits for the rest of its round, so
|
|
78
|
+
// the setup result is never disposed of under a call that runs.
|
|
79
|
+
async function runRound(
|
|
80
|
+
prepared: PreparedBench,
|
|
81
|
+
concurrency: number,
|
|
82
|
+
now: () => number,
|
|
83
|
+
): Promise<number[]> {
|
|
84
|
+
const settled = await Promise.allSettled(
|
|
85
|
+
Array.from({ length: concurrency }, async () => {
|
|
86
|
+
const startMs = now();
|
|
87
|
+
const result = prepared.step();
|
|
88
|
+
|
|
89
|
+
// A promise from another realm fails `instanceof Promise`, so only `undefined` skips the await.
|
|
90
|
+
if (result !== undefined) {
|
|
91
|
+
await result;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return now() - startMs;
|
|
95
|
+
}),
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
const rejected = settled.find((outcome) => outcome.status === 'rejected');
|
|
99
|
+
|
|
100
|
+
if (rejected !== undefined) {
|
|
101
|
+
throw rejected.reason;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return settled.flatMap((outcome) => (outcome.status === 'fulfilled' ? [outcome.value] : []));
|
|
38
105
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export interface BenchOutput {
|
|
2
|
+
readonly stream: 'out' | 'err';
|
|
3
|
+
|
|
4
|
+
readonly text: string;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
// What `rig bench` prints, in order, and the exit code it returns after.
|
|
8
|
+
export interface BenchVerdict {
|
|
9
|
+
readonly output: readonly BenchOutput[];
|
|
10
|
+
|
|
11
|
+
readonly exitCode: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// What became of the base checkout after its removal: `absent` when the removal failed and git
|
|
15
|
+
// registers no such worktree, `left` when git still registers it or its list failed.
|
|
16
|
+
export type CheckoutState = 'removed' | 'absent' | 'left';
|
|
17
|
+
|
|
18
|
+
export interface BenchSideTimes {
|
|
19
|
+
readonly medianMs: number;
|
|
20
|
+
|
|
21
|
+
readonly spreadMs: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// One benchmark at one level, with its times on the base and on the head, or null on a side that
|
|
25
|
+
// lacks it.
|
|
26
|
+
export interface BenchComparisonRow {
|
|
27
|
+
// The bench file, relative to the root of its side, so both sides give the same path.
|
|
28
|
+
readonly file: string;
|
|
29
|
+
|
|
30
|
+
readonly name: string;
|
|
31
|
+
|
|
32
|
+
readonly concurrency: number;
|
|
33
|
+
|
|
34
|
+
readonly base: BenchSideTimes | null;
|
|
35
|
+
|
|
36
|
+
readonly head: BenchSideTimes | null;
|
|
37
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { join, resolve } from 'node:path';
|
|
2
|
+
|
|
3
|
+
// The folder sits in the git folder, which git never lists as untracked, and in the common one, so
|
|
4
|
+
// a run from a linked worktree shares one place with the main checkout.
|
|
5
|
+
export function buildBenchFolderPath(repoRoot: string, gitCommonDir: string): string {
|
|
6
|
+
return join(resolve(repoRoot, gitCommonDir), 'rig-bench');
|
|
7
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
|
+
import { mkdir, mkdtemp } from 'node:fs/promises';
|
|
3
4
|
import { availableParallelism, totalmem } from 'node:os';
|
|
4
5
|
import { join } from 'node:path';
|
|
5
6
|
import { readOptionalText } from '@zgeoff/std';
|
|
7
|
+
import { processRuntime } from './browser/process-runtime.ts';
|
|
8
|
+
import { removePath } from './browser/remove-path.ts';
|
|
6
9
|
import { wireBrowserLaneHooks } from './browser/wire-browser-lane-hooks.ts';
|
|
10
|
+
import { createBenchWorkFolder } from './create-bench-work-folder.ts';
|
|
7
11
|
import { initFiles } from './init-files.ts';
|
|
8
12
|
import { probeWorkspaceRepo } from './probe-workspace-repo.ts';
|
|
13
|
+
import { readCommandOutput } from './read-command-output.ts';
|
|
14
|
+
import type { CommandOutput, ReadCommandOptions } from './read-command-output.ts';
|
|
15
|
+
import { readGitCommonDir } from './read-git-common-dir.ts';
|
|
9
16
|
import { readRigConfig } from './read-rig-config.ts';
|
|
10
17
|
import { readTrackedFiles } from './read-tracked-files.ts';
|
|
11
18
|
import { readWorkerCount } from './read-worker-count.ts';
|
|
@@ -37,6 +44,13 @@ const output = {
|
|
|
37
44
|
// `bun run` starts a package script in the folder of the package that defines it.
|
|
38
45
|
const repoRoot = process.cwd();
|
|
39
46
|
|
|
47
|
+
function readOutput(
|
|
48
|
+
command: readonly string[],
|
|
49
|
+
options: ReadCommandOptions,
|
|
50
|
+
): Promise<CommandOutput> {
|
|
51
|
+
return readCommandOutput(command, options, processRuntime);
|
|
52
|
+
}
|
|
53
|
+
|
|
40
54
|
try {
|
|
41
55
|
process.exitCode = await runCLI(
|
|
42
56
|
process.argv.slice(2),
|
|
@@ -66,6 +80,23 @@ try {
|
|
|
66
80
|
readText: readOptionalText,
|
|
67
81
|
}),
|
|
68
82
|
importModule: (path) => import(path),
|
|
83
|
+
createWorkFolder: (root) =>
|
|
84
|
+
createBenchWorkFolder(root, {
|
|
85
|
+
readGitCommonDir: (gitRoot) =>
|
|
86
|
+
readGitCommonDir(gitRoot, process.env, { readCommandOutput: readOutput }),
|
|
87
|
+
createFolder: async (path) => {
|
|
88
|
+
await mkdir(path, { recursive: true });
|
|
89
|
+
},
|
|
90
|
+
createTempFolder: (prefix) => mkdtemp(prefix),
|
|
91
|
+
}),
|
|
92
|
+
readText: async (path) => {
|
|
93
|
+
const text = await readOptionalText(path);
|
|
94
|
+
|
|
95
|
+
return text ?? null;
|
|
96
|
+
},
|
|
97
|
+
probeFolder: (path) => Promise.resolve(existsSync(path)),
|
|
98
|
+
removeFolder: removePath,
|
|
99
|
+
readCommandOutput: readOutput,
|
|
69
100
|
runCommand,
|
|
70
101
|
wireBrowserLaneHooks,
|
|
71
102
|
},
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { buildBenchFolderPath } from './build-bench-folder-path.ts';
|
|
3
|
+
|
|
4
|
+
export interface BenchWorkFolderDeps {
|
|
5
|
+
// Returns what `git rev-parse --git-common-dir` prints in the repo root.
|
|
6
|
+
readonly readGitCommonDir: (repoRoot: string) => Promise<string>;
|
|
7
|
+
|
|
8
|
+
// Makes the folder and its parents, and keeps a folder that exists.
|
|
9
|
+
readonly createFolder: (path: string) => Promise<void>;
|
|
10
|
+
|
|
11
|
+
// Makes a new folder whose path starts with the prefix, and returns its path.
|
|
12
|
+
readonly createTempFolder: (prefix: string) => Promise<string>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function createBenchWorkFolder(
|
|
16
|
+
repoRoot: string,
|
|
17
|
+
deps: BenchWorkFolderDeps,
|
|
18
|
+
): Promise<string> {
|
|
19
|
+
const gitCommonDir = await deps.readGitCommonDir(repoRoot);
|
|
20
|
+
|
|
21
|
+
const benchFolder = buildBenchFolderPath(repoRoot, gitCommonDir);
|
|
22
|
+
|
|
23
|
+
await deps.createFolder(benchFolder);
|
|
24
|
+
|
|
25
|
+
return deps.createTempFolder(join(benchFolder, 'run-'));
|
|
26
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// The parser throws one error that holds every reason the command line is refused, so each reason
|
|
2
|
+
// prints on a line of its own.
|
|
3
|
+
export function formatParseError(error: unknown): string {
|
|
4
|
+
if (error instanceof AggregateError) {
|
|
5
|
+
return error.errors.map((reason) => formatParseError(reason)).join('\n');
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
return error instanceof Error ? error.message : String(error);
|
|
9
|
+
}
|