@zgeoff/rig 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +3 -4
  2. package/dist/bench/bench-report-entry-schema.d.ts +10 -0
  3. package/dist/bench/build-base-side-root.d.ts +1 -0
  4. package/dist/bench/collect-bench-comparison.d.ts +3 -0
  5. package/dist/bench/define-bench.d.ts +2 -0
  6. package/dist/bench/find-base-setup-verdict.d.ts +3 -0
  7. package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
  8. package/dist/bench/format-bench-comparison.d.ts +2 -0
  9. package/dist/bench/format-bench-report.d.ts +3 -0
  10. package/dist/bench/format-benchmark-result.d.ts +1 -1
  11. package/dist/bench/is-concurrency-list.d.ts +1 -0
  12. package/dist/bench/parse-bench-report.d.ts +9 -0
  13. package/dist/bench/pick-bench-exit-code.d.ts +2 -0
  14. package/dist/bench/pick-bench-plan.d.ts +13 -0
  15. package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
  16. package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
  17. package/dist/bench/pick-comparison-verdict.d.ts +7 -0
  18. package/dist/bench/pick-head-verdict.d.ts +3 -0
  19. package/dist/bench/pick-named-bench-files.d.ts +1 -0
  20. package/dist/bench/pick-quantile.d.ts +1 -0
  21. package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
  22. package/dist/bench/read-bench-args.d.ts +9 -0
  23. package/dist/bench/read-bench-file-args.d.ts +9 -0
  24. package/dist/bench/run-benchmark.d.ts +3 -3
  25. package/dist/bench/to-bench-name.d.ts +1 -0
  26. package/dist/bench/types.d.ts +20 -0
  27. package/dist/build-bench-folder-path.d.ts +1 -0
  28. package/dist/create-bench-work-folder.d.ts +6 -0
  29. package/dist/format-parse-error.d.ts +1 -0
  30. package/dist/has-lane-suffix.d.ts +1 -0
  31. package/dist/index.js +33 -2
  32. package/dist/is-test-file.d.ts +1 -0
  33. package/dist/make-ignore-check.d.ts +1 -0
  34. package/dist/read-command-output.d.ts +8 -0
  35. package/dist/read-git-common-dir.d.ts +6 -0
  36. package/dist/read-git-prefix.d.ts +6 -0
  37. package/dist/run-bench.d.ts +7 -0
  38. package/dist/run-cli.d.ts +1 -2
  39. package/dist/types.d.ts +2 -0
  40. package/package.json +4 -4
  41. package/src/bench/bench-report-entry-schema.ts +17 -0
  42. package/src/bench/build-base-side-root.ts +7 -0
  43. package/src/bench/collect-bench-comparison.ts +43 -0
  44. package/src/bench/define-bench.ts +15 -0
  45. package/src/bench/find-base-setup-verdict.ts +21 -0
  46. package/src/bench/find-missing-bench-verdict.ts +33 -0
  47. package/src/bench/format-bench-comparison.ts +72 -0
  48. package/src/bench/format-bench-report.ts +24 -0
  49. package/src/bench/format-benchmark-result.ts +7 -3
  50. package/src/bench/is-concurrency-list.ts +10 -0
  51. package/src/bench/parse-bench-definition.ts +21 -3
  52. package/src/bench/parse-bench-report.ts +42 -0
  53. package/src/bench/pick-bench-exit-code.ts +15 -0
  54. package/src/bench/pick-bench-plan.ts +25 -0
  55. package/src/bench/pick-checkout-cleanup.ts +29 -0
  56. package/src/bench/pick-cleaned-up-result.ts +30 -0
  57. package/src/bench/pick-comparison-verdict.ts +36 -0
  58. package/src/bench/pick-head-verdict.ts +11 -0
  59. package/src/bench/pick-named-bench-files.ts +8 -0
  60. package/src/bench/pick-quantile.ts +9 -0
  61. package/src/bench/pick-unremoved-checkout-state.ts +30 -0
  62. package/src/bench/read-bench-args.ts +45 -0
  63. package/src/bench/read-bench-file-args.ts +38 -0
  64. package/src/bench/run-benchmark.ts +88 -21
  65. package/src/bench/to-bench-name.ts +5 -0
  66. package/src/bench/types.ts +37 -0
  67. package/src/build-bench-folder-path.ts +7 -0
  68. package/src/build-test-selection.ts +5 -9
  69. package/src/build-unit-command.ts +3 -2
  70. package/src/cli.ts +31 -0
  71. package/src/create-bench-work-folder.ts +26 -0
  72. package/src/define-rig.ts +23 -1
  73. package/src/format-parse-error.ts +9 -0
  74. package/src/has-lane-suffix.ts +3 -0
  75. package/src/is-test-file.ts +5 -0
  76. package/src/make-ignore-check.ts +15 -0
  77. package/src/parse-rig-config.ts +29 -1
  78. package/src/read-cli-args.ts +1 -10
  79. package/src/read-command-output.ts +30 -0
  80. package/src/read-git-common-dir.ts +27 -0
  81. package/src/read-git-prefix.ts +27 -0
  82. package/src/resolve-related-tests.ts +4 -4
  83. package/src/run-bench-file.ts +23 -13
  84. package/src/run-bench.ts +348 -39
  85. package/src/run-cli.ts +1 -3
  86. package/src/run-lane.ts +41 -14
  87. package/src/types.ts +4 -0
  88. package/dist/bench/pick-median.d.ts +0 -1
  89. package/src/bench/pick-median.ts +0 -10
@@ -0,0 +1,72 @@
1
+ import type { BenchComparisonRow, BenchSideTimes } from './types.ts';
2
+
3
+ const header = [
4
+ 'bench',
5
+ 'concurrency',
6
+ 'base median',
7
+ 'base spread',
8
+ 'head median',
9
+ 'head spread',
10
+ 'change',
11
+ ] as const;
12
+
13
+ export function formatBenchComparison(rows: readonly BenchComparisonRow[]): string {
14
+ const cells: readonly (readonly string[])[] = [
15
+ header,
16
+ ...rows.map((row) => formatCells(row, rows)),
17
+ ];
18
+
19
+ const widths = header.map((_, column) =>
20
+ Math.max(...cells.map((line) => line[column]?.length ?? 0)),
21
+ );
22
+
23
+ return cells
24
+ .map((line) =>
25
+ line
26
+ .map((cell, column) =>
27
+ column === line.length - 1 ? cell : cell.padEnd((widths[column] ?? 0) + 2),
28
+ )
29
+ .join(''),
30
+ )
31
+ .map((line) => `${line}\n`)
32
+ .join('');
33
+ }
34
+
35
+ function formatCells(
36
+ row: BenchComparisonRow,
37
+ rows: readonly BenchComparisonRow[],
38
+ ): readonly string[] {
39
+ return [
40
+ formatLabel(row, rows),
41
+ String(row.concurrency),
42
+ formatTime(row.base?.medianMs),
43
+ formatTime(row.base?.spreadMs),
44
+ formatTime(row.head?.medianMs),
45
+ formatTime(row.head?.spreadMs),
46
+ formatChange(row.base, row.head),
47
+ ];
48
+ }
49
+
50
+ // The name alone reads best, so the file shows only where two rows would look the same.
51
+ function formatLabel(row: BenchComparisonRow, rows: readonly BenchComparisonRow[]): string {
52
+ const isShared = rows.some(
53
+ (other) => other !== row && other.name === row.name && other.concurrency === row.concurrency,
54
+ );
55
+
56
+ return isShared ? `${row.name} (${row.file})` : row.name;
57
+ }
58
+
59
+ function formatTime(timeMs: number | undefined): string {
60
+ return timeMs === undefined ? '-' : `${timeMs.toFixed(2)}ms`;
61
+ }
62
+
63
+ // A base median of zero gives no ratio to show.
64
+ function formatChange(base: BenchSideTimes | null, head: BenchSideTimes | null): string {
65
+ if (base === null || head === null || base.medianMs === 0) {
66
+ return '-';
67
+ }
68
+
69
+ const changePercent = ((head.medianMs - base.medianMs) / base.medianMs) * 100;
70
+
71
+ return `${changePercent < 0 ? '' : '+'}${changePercent.toFixed(1)}%`;
72
+ }
@@ -0,0 +1,24 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ import type { BenchDefinition } from './define-bench.ts';
3
+ import type { BenchmarkResult } from './run-benchmark.ts';
4
+
5
+ export function formatBenchReport(
6
+ file: string,
7
+ definition: BenchDefinition,
8
+ results: readonly BenchmarkResult[],
9
+ ): string {
10
+ return results
11
+ .map((result) => {
12
+ const entry: BenchReportEntry = {
13
+ file,
14
+ name: definition.name,
15
+ concurrency: result.concurrency,
16
+ medianMs: result.medianMs,
17
+ spreadMs: result.spreadMs,
18
+ budgetMs: definition.budgetMs,
19
+ };
20
+
21
+ return `${JSON.stringify(entry)}\n`;
22
+ })
23
+ .join('');
24
+ }
@@ -3,9 +3,13 @@ import type { BenchmarkResult } from './run-benchmark.ts';
3
3
 
4
4
  export function formatBenchmarkResult(
5
5
  definition: BenchDefinition,
6
- result: BenchmarkResult,
6
+ results: readonly BenchmarkResult[],
7
7
  ): string {
8
- const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
8
+ return results
9
+ .map((result) => {
10
+ const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
9
11
 
10
- return `bench ${definition.name}: median ${result.medianMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
12
+ return `bench ${definition.name} at concurrency ${result.concurrency}: median ${result.medianMs.toFixed(2)}ms, spread ${result.spreadMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
13
+ })
14
+ .join('');
11
15
  }
@@ -0,0 +1,10 @@
1
+ export function isConcurrencyList(value: unknown): value is readonly number[] {
2
+ return (
3
+ Array.isArray(value) &&
4
+ value.length > 0 &&
5
+ value.every(
6
+ (level: unknown) => typeof level === 'number' && Number.isSafeInteger(level) && level > 0,
7
+ ) &&
8
+ new Set(value).size === value.length
9
+ );
10
+ }
@@ -1,4 +1,5 @@
1
1
  import type { BenchDefinition } from './define-bench.ts';
2
+ import { isConcurrencyList } from './is-concurrency-list.ts';
2
3
 
3
4
  export type ParsedBenchDefinition =
4
5
  | { readonly kind: 'parsed'; readonly definition: BenchDefinition }
@@ -15,7 +16,15 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
15
16
  const budgetMs: unknown = Reflect.get(value, 'budgetMs');
16
17
  const prepare: unknown = Reflect.get(value, 'prepare');
17
18
 
18
- if (typeof name !== 'string' || typeof budgetMs !== 'number' || !isFunction(prepare)) {
19
+ // A bench file on a base ref can import an older rig, whose result has no concurrency.
20
+ const concurrency: unknown = Reflect.get(value, 'concurrency') ?? [1];
21
+
22
+ if (
23
+ typeof name !== 'string' ||
24
+ typeof budgetMs !== 'number' ||
25
+ !isFunction(prepare) ||
26
+ !isConcurrencyList(concurrency)
27
+ ) {
19
28
  return { kind: 'refused', reason: 'the default export is no defineBench result' };
20
29
  }
21
30
 
@@ -24,6 +33,7 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
24
33
  definition: {
25
34
  name,
26
35
  budgetMs,
36
+ concurrency,
27
37
  prepare: async () => {
28
38
  const prepared: unknown = await prepare();
29
39
 
@@ -40,8 +50,10 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
40
50
  }
41
51
 
42
52
  return {
43
- step: async () => {
44
- await step();
53
+ step: () => {
54
+ const result: unknown = step();
55
+
56
+ return result === undefined ? undefined : settleStep(result);
45
57
  },
46
58
  dispose: async () => {
47
59
  await dispose();
@@ -55,3 +67,9 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
55
67
  function isFunction(value: unknown): value is (...args: unknown[]) => unknown {
56
68
  return typeof value === 'function';
57
69
  }
70
+
71
+ // `instanceof Promise` misses a thenable and a promise from another realm, such as a `node:vm`
72
+ // context, so any value but `undefined` goes through `await`, which adopts both.
73
+ async function settleStep(result: unknown): Promise<void> {
74
+ await result;
75
+ }
@@ -0,0 +1,42 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ import { benchReportEntrySchema } from './bench-report-entry-schema.ts';
3
+
4
+ export type ParsedBenchReport =
5
+ | { readonly kind: 'parsed'; readonly entries: readonly BenchReportEntry[] }
6
+ | { readonly kind: 'refused'; readonly reason: string };
7
+
8
+ // A report holds what other processes appended, one JSON entry per line, so each line is checked
9
+ // instead of trusted.
10
+ export function parseBenchReport(text: string): ParsedBenchReport {
11
+ const entries: BenchReportEntry[] = [];
12
+
13
+ for (const line of text.split('\n')) {
14
+ if (line.trim() === '') {
15
+ continue;
16
+ }
17
+
18
+ const entry = parseEntry(line);
19
+
20
+ if (entry === null) {
21
+ return { kind: 'refused', reason: `the bench report holds a line that is no entry: ${line}` };
22
+ }
23
+
24
+ entries.push(entry);
25
+ }
26
+
27
+ return { kind: 'parsed', entries };
28
+ }
29
+
30
+ function parseEntry(line: string): BenchReportEntry | null {
31
+ let value: unknown;
32
+
33
+ try {
34
+ value = JSON.parse(line);
35
+ } catch {
36
+ return null;
37
+ }
38
+
39
+ const parsed = benchReportEntrySchema.safeParse(value);
40
+
41
+ return parsed.success ? parsed.data : null;
42
+ }
@@ -0,0 +1,15 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+
3
+ // The budget holds on the head only, so a base slower than its budget still lets the head pass.
4
+ export function pickBenchExitCode(
5
+ sideExitCodes: readonly number[],
6
+ headEntries: readonly BenchReportEntry[],
7
+ ): number {
8
+ const failedExitCode = sideExitCodes.find((exitCode) => exitCode !== 0);
9
+
10
+ if (failedExitCode !== undefined) {
11
+ return failedExitCode;
12
+ }
13
+
14
+ return headEntries.some((entry) => entry.medianMs > entry.budgetMs) ? 1 : 0;
15
+ }
@@ -0,0 +1,25 @@
1
+ import { findMissingBenchVerdict } from './find-missing-bench-verdict.ts';
2
+ import { pickNamedBenchFiles } from './pick-named-bench-files.ts';
3
+ import type { BenchVerdict } from './types.ts';
4
+
5
+ export type BenchPlan =
6
+ | { readonly kind: 'finished'; readonly verdict: BenchVerdict }
7
+ | { readonly kind: 'head'; readonly files: readonly string[] }
8
+ | { readonly kind: 'comparison'; readonly baseRef: string; readonly files: readonly string[] };
9
+
10
+ // With a base, whether a benchmark is missing waits for the base's files.
11
+ export function pickBenchPlan(
12
+ name: string | null,
13
+ baseRef: string | null,
14
+ headBenchFiles: readonly string[],
15
+ ): BenchPlan {
16
+ const files = pickNamedBenchFiles(headBenchFiles, name);
17
+
18
+ if (baseRef !== null) {
19
+ return { kind: 'comparison', baseRef, files };
20
+ }
21
+
22
+ const missing = findMissingBenchVerdict(name, headBenchFiles, []);
23
+
24
+ return missing === null ? { kind: 'head', files } : { kind: 'finished', verdict: missing };
25
+ }
@@ -0,0 +1,29 @@
1
+ import type { BenchOutput, CheckoutState } from './types.ts';
2
+
3
+ export interface CheckoutCleanup {
4
+ readonly output: readonly BenchOutput[];
5
+
6
+ // A checkout that git still registers keeps its work folder, so git's record of the checkout
7
+ // still points at a folder.
8
+ readonly isWorkFolderKept: boolean;
9
+
10
+ // A checkout that git still registers fails a comparison that passed.
11
+ readonly exitCode: number;
12
+ }
13
+
14
+ export function pickCheckoutCleanup(baseRoot: string, checkout: CheckoutState): CheckoutCleanup {
15
+ if (checkout !== 'left') {
16
+ return { output: [], isWorkFolderKept: false, exitCode: 0 };
17
+ }
18
+
19
+ return {
20
+ output: [
21
+ {
22
+ stream: 'err',
23
+ text: `rig bench: could not remove the checkout at ${baseRoot}; remove it with git worktree remove --force ${baseRoot}\n`,
24
+ },
25
+ ],
26
+ isWorkFolderKept: true,
27
+ exitCode: 1,
28
+ };
29
+ }
@@ -0,0 +1,30 @@
1
+ export type CleanedUpResult<T, C> =
2
+ | { readonly kind: 'fulfilled'; readonly value: T; readonly cleanup: C }
3
+ | { readonly kind: 'throw'; readonly error: unknown };
4
+
5
+ // Takes how a run settled and how its cleanup settled. A cleanup that throws after a run that threw
6
+ // keeps both errors, the run's first, so the run's cause still reaches the caller.
7
+ export function pickCleanedUpResult<T, C>(
8
+ ran: PromiseSettledResult<T>,
9
+ cleaned: PromiseSettledResult<C>,
10
+ ): CleanedUpResult<T, C> {
11
+ if (ran.status === 'rejected' && cleaned.status === 'rejected') {
12
+ return {
13
+ kind: 'throw',
14
+ error: new AggregateError(
15
+ [ran.reason, cleaned.reason],
16
+ 'the run failed, and its cleanup failed too',
17
+ ),
18
+ };
19
+ }
20
+
21
+ if (ran.status === 'rejected') {
22
+ return { kind: 'throw', error: ran.reason };
23
+ }
24
+
25
+ if (cleaned.status === 'rejected') {
26
+ return { kind: 'throw', error: cleaned.reason };
27
+ }
28
+
29
+ return { kind: 'fulfilled', value: ran.value, cleanup: cleaned.value };
30
+ }
@@ -0,0 +1,36 @@
1
+ import { collectBenchComparison } from './collect-bench-comparison.ts';
2
+ import { formatBenchComparison } from './format-bench-comparison.ts';
3
+ import type { ParsedBenchReport } from './parse-bench-report.ts';
4
+ import { pickBenchExitCode } from './pick-bench-exit-code.ts';
5
+ import type { BenchVerdict } from './types.ts';
6
+
7
+ export interface BenchSideOutcome {
8
+ readonly exitCode: number;
9
+
10
+ readonly report: ParsedBenchReport;
11
+ }
12
+
13
+ export function pickComparisonVerdict(
14
+ base: BenchSideOutcome,
15
+ head: BenchSideOutcome,
16
+ ): BenchVerdict {
17
+ if (base.report.kind === 'refused') {
18
+ return { output: [{ stream: 'err', text: `rig bench: ${base.report.reason}\n` }], exitCode: 1 };
19
+ }
20
+
21
+ if (head.report.kind === 'refused') {
22
+ return { output: [{ stream: 'err', text: `rig bench: ${head.report.reason}\n` }], exitCode: 1 };
23
+ }
24
+
25
+ return {
26
+ output: [
27
+ {
28
+ stream: 'out',
29
+ text: formatBenchComparison(
30
+ collectBenchComparison(base.report.entries, head.report.entries),
31
+ ),
32
+ },
33
+ ],
34
+ exitCode: pickBenchExitCode([head.exitCode, base.exitCode], head.report.entries),
35
+ };
36
+ }
@@ -0,0 +1,11 @@
1
+ import type { ParsedBenchReport } from './parse-bench-report.ts';
2
+ import { pickBenchExitCode } from './pick-bench-exit-code.ts';
3
+ import type { BenchVerdict } from './types.ts';
4
+
5
+ export function pickHeadVerdict(report: ParsedBenchReport, headExitCode: number): BenchVerdict {
6
+ if (report.kind === 'refused') {
7
+ return { output: [{ stream: 'err', text: `rig bench: ${report.reason}\n` }], exitCode: 1 };
8
+ }
9
+
10
+ return { output: [], exitCode: pickBenchExitCode([headExitCode], report.entries) };
11
+ }
@@ -0,0 +1,8 @@
1
+ import { toBenchName } from './to-bench-name.ts';
2
+
3
+ export function pickNamedBenchFiles(
4
+ benchFiles: readonly string[],
5
+ name: string | null,
6
+ ): readonly string[] {
7
+ return name === null ? benchFiles : benchFiles.filter((file) => toBenchName(file) === name);
8
+ }
@@ -0,0 +1,9 @@
1
+ export function pickQuantile(samples: readonly number[], quantile: number): number {
2
+ const sorted = samples.toSorted((left, right) => left - right);
3
+ const position = (sorted.length - 1) * quantile;
4
+ const lower = Math.floor(position);
5
+ const lowerSample = sorted[lower] ?? 0;
6
+ const upperSample = sorted[Math.ceil(position)] ?? lowerSample;
7
+
8
+ return lowerSample + (upperSample - lowerSample) * (position - lower);
9
+ }
@@ -0,0 +1,30 @@
1
+ import { resolve } from 'node:path';
2
+ import type { CommandOutput } from '../read-command-output.ts';
3
+ import type { CheckoutState } from './types.ts';
4
+
5
+ // Takes what `git worktree list --porcelain -z` gave after a failed removal. A listing that failed
6
+ // cannot show the checkout is gone, so the checkout counts as left.
7
+ export function pickUnremovedCheckoutState(
8
+ listing: CommandOutput,
9
+ baseRoot: string,
10
+ ): Exclude<CheckoutState, 'removed'> {
11
+ if (listing.exitCode !== 0) {
12
+ return 'left';
13
+ }
14
+
15
+ const isListed = collectWorktreePaths(listing.stdout).some(
16
+ (path) => resolve(path) === resolve(baseRoot),
17
+ );
18
+
19
+ return isListed ? 'left' : 'absent';
20
+ }
21
+
22
+ // Each field ends in NUL and each record ends in an empty field, so a path that holds a newline
23
+ // stays whole.
24
+ function collectWorktreePaths(stdout: string): string[] {
25
+ return stdout
26
+ .split('\0\0')
27
+ .map((record) => record.split('\0')[0] ?? '')
28
+ .filter((field) => field.startsWith('worktree '))
29
+ .map((field) => field.slice('worktree '.length));
30
+ }
@@ -0,0 +1,45 @@
1
+ import { cli } from 'gunshi/bone';
2
+ import { define } from 'gunshi/definition';
3
+ import { formatParseError } from '../format-parse-error.ts';
4
+
5
+ export type BenchArgs =
6
+ | { readonly kind: 'parsed'; readonly name: string | null; readonly baseRef: string | null }
7
+ | { readonly kind: 'refused'; readonly reason: string };
8
+
9
+ const argSchema = {
10
+ base: { type: 'string' },
11
+ name: { type: 'positional', required: false },
12
+ } as const;
13
+
14
+ export async function readBenchArgs(args: readonly string[]): Promise<BenchArgs> {
15
+ let parsed: BenchArgs | null = null;
16
+
17
+ const command = define({
18
+ name: 'rig bench',
19
+ args: argSchema,
20
+ run: (ctx) => {
21
+ // The parser refuses `--base` with no value through an error whose own message is empty.
22
+ if (ctx.validationError !== undefined) {
23
+ parsed = { kind: 'refused', reason: '--base needs a ref' };
24
+
25
+ return;
26
+ }
27
+
28
+ if (ctx.positionals.length > 1) {
29
+ parsed = { kind: 'refused', reason: 'takes at most one benchmark name' };
30
+
31
+ return;
32
+ }
33
+
34
+ parsed = { kind: 'parsed', name: ctx.values.name ?? null, baseRef: ctx.values.base ?? null };
35
+ },
36
+ });
37
+
38
+ try {
39
+ await cli([...args], command, { strict: true });
40
+ } catch (error) {
41
+ return { kind: 'refused', reason: formatParseError(error) };
42
+ }
43
+
44
+ return parsed ?? { kind: 'refused', reason: 'the command line could not be parsed' };
45
+ }
@@ -0,0 +1,38 @@
1
+ import { cli } from 'gunshi/bone';
2
+ import { define } from 'gunshi/definition';
3
+ import { formatParseError } from '../format-parse-error.ts';
4
+
5
+ export type BenchFileArgs =
6
+ | { readonly kind: 'parsed'; readonly file: string; readonly reportPath: string }
7
+ | { readonly kind: 'refused'; readonly reason: string };
8
+
9
+ const argSchema = {
10
+ file: { type: 'positional' },
11
+ report: { type: 'string', required: true },
12
+ } as const;
13
+
14
+ export async function readBenchFileArgs(args: readonly string[]): Promise<BenchFileArgs> {
15
+ let parsed: BenchFileArgs | null = null;
16
+
17
+ const command = define({
18
+ name: 'run-bench-file',
19
+ args: argSchema,
20
+ run: (ctx) => {
21
+ if (ctx.validationError !== undefined) {
22
+ parsed = { kind: 'refused', reason: formatParseError(ctx.validationError) };
23
+
24
+ return;
25
+ }
26
+
27
+ parsed = { kind: 'parsed', file: ctx.values.file, reportPath: ctx.values.report };
28
+ },
29
+ });
30
+
31
+ try {
32
+ await cli([...args], command, { strict: true });
33
+ } catch (error) {
34
+ return { kind: 'refused', reason: formatParseError(error) };
35
+ }
36
+
37
+ return parsed ?? { kind: 'refused', reason: 'the command line could not be parsed' };
38
+ }
@@ -1,38 +1,105 @@
1
- import type { BenchDefinition } from './define-bench.ts';
2
- import { pickMedian } from './pick-median.ts';
3
-
4
- export const warmUpRuns = 5;
5
- export const measuredRuns = 20;
1
+ import type { BenchDefinition, PreparedBench } from './define-bench.ts';
2
+ import { pickCleanedUpResult } from './pick-cleaned-up-result.ts';
3
+ import { pickQuantile } from './pick-quantile.ts';
6
4
 
7
5
  export interface BenchmarkResult {
6
+ readonly concurrency: number;
7
+
8
8
  readonly medianMs: number;
9
+
10
+ // The interquartile range of the measured times.
11
+ readonly spreadMs: number;
9
12
  }
10
13
 
11
- // `prepare` runs once, then `run` warms up before the measured runs, and the median of those is the
12
- // result. The setup result is disposed of after the measured runs, even when a run throws.
14
+ // `prepare` runs once for every level, and the setup result is disposed of after the last level,
15
+ // even when a run throws.
13
16
  export async function runBenchmark(
14
17
  definition: BenchDefinition,
15
18
  now: () => number,
16
- ): Promise<BenchmarkResult> {
19
+ ): Promise<readonly BenchmarkResult[]> {
17
20
  const prepared = await definition.prepare();
21
+ const [ran] = await Promise.allSettled([runLevels(definition, prepared, now)]);
22
+ const [cleaned] = await Promise.allSettled([prepared.dispose()]);
18
23
 
19
- try {
20
- for (let index = 0; index < warmUpRuns; index += 1) {
21
- await prepared.step();
22
- }
24
+ const result = pickCleanedUpResult(ran, cleaned);
23
25
 
24
- const samples: number[] = [];
26
+ if (result.kind === 'throw') {
27
+ throw result.error;
28
+ }
25
29
 
26
- for (let index = 0; index < measuredRuns; index += 1) {
27
- const start = now();
30
+ return result.value;
31
+ }
28
32
 
29
- await prepared.step();
33
+ async function runLevels(
34
+ definition: BenchDefinition,
35
+ prepared: PreparedBench,
36
+ now: () => number,
37
+ ): Promise<readonly BenchmarkResult[]> {
38
+ const results: BenchmarkResult[] = [];
30
39
 
31
- samples.push(now() - start);
32
- }
40
+ for (const concurrency of definition.concurrency) {
41
+ const result = await runLevel(prepared, concurrency, now);
33
42
 
34
- return { medianMs: pickMedian(samples) };
35
- } finally {
36
- await prepared.dispose();
43
+ results.push(result);
44
+ }
45
+
46
+ return results;
47
+ }
48
+
49
+ const warmUpRounds = 5;
50
+ const measuredRounds = 20;
51
+
52
+ async function runLevel(
53
+ prepared: PreparedBench,
54
+ concurrency: number,
55
+ now: () => number,
56
+ ): Promise<BenchmarkResult> {
57
+ for (let round = 0; round < warmUpRounds; round += 1) {
58
+ await runRound(prepared, concurrency, now);
37
59
  }
60
+
61
+ const samples: number[] = [];
62
+
63
+ for (let round = 0; round < measuredRounds; round += 1) {
64
+ const roundSamples = await runRound(prepared, concurrency, now);
65
+
66
+ samples.push(...roundSamples);
67
+ }
68
+
69
+ return {
70
+ concurrency,
71
+ medianMs: pickQuantile(samples, 0.5),
72
+ spreadMs: pickQuantile(samples, 0.75) - pickQuantile(samples, 0.25),
73
+ };
74
+ }
75
+
76
+ // Each call is timed on its own, so the times show how long one call waits under the load of the
77
+ // others, not how long the round took. A rejected call still waits for the rest of its round, so
78
+ // the setup result is never disposed of under a call that runs.
79
+ async function runRound(
80
+ prepared: PreparedBench,
81
+ concurrency: number,
82
+ now: () => number,
83
+ ): Promise<number[]> {
84
+ const settled = await Promise.allSettled(
85
+ Array.from({ length: concurrency }, async () => {
86
+ const startMs = now();
87
+ const result = prepared.step();
88
+
89
+ // A promise from another realm fails `instanceof Promise`, so only `undefined` skips the await.
90
+ if (result !== undefined) {
91
+ await result;
92
+ }
93
+
94
+ return now() - startMs;
95
+ }),
96
+ );
97
+
98
+ const rejected = settled.find((outcome) => outcome.status === 'rejected');
99
+
100
+ if (rejected !== undefined) {
101
+ throw rejected.reason;
102
+ }
103
+
104
+ return settled.flatMap((outcome) => (outcome.status === 'fulfilled' ? [outcome.value] : []));
38
105
  }
@@ -0,0 +1,5 @@
1
+ import { basename } from 'node:path';
2
+
3
+ export function toBenchName(file: string): string {
4
+ return basename(file, '.bench.ts');
5
+ }