@zgeoff/rig 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/bench/bench-report-entry-schema.d.ts +10 -0
  2. package/dist/bench/build-base-side-root.d.ts +1 -0
  3. package/dist/bench/collect-bench-comparison.d.ts +3 -0
  4. package/dist/bench/define-bench.d.ts +2 -0
  5. package/dist/bench/find-base-setup-verdict.d.ts +3 -0
  6. package/dist/bench/find-missing-bench-verdict.d.ts +2 -0
  7. package/dist/bench/format-bench-comparison.d.ts +2 -0
  8. package/dist/bench/format-bench-report.d.ts +3 -0
  9. package/dist/bench/format-benchmark-result.d.ts +1 -1
  10. package/dist/bench/is-concurrency-list.d.ts +1 -0
  11. package/dist/bench/parse-bench-report.d.ts +9 -0
  12. package/dist/bench/pick-bench-exit-code.d.ts +2 -0
  13. package/dist/bench/pick-bench-plan.d.ts +13 -0
  14. package/dist/bench/pick-checkout-cleanup.d.ts +7 -0
  15. package/dist/bench/pick-cleaned-up-result.d.ts +9 -0
  16. package/dist/bench/pick-comparison-verdict.d.ts +7 -0
  17. package/dist/bench/pick-head-verdict.d.ts +3 -0
  18. package/dist/bench/pick-named-bench-files.d.ts +1 -0
  19. package/dist/bench/pick-quantile.d.ts +1 -0
  20. package/dist/bench/pick-unremoved-checkout-state.d.ts +3 -0
  21. package/dist/bench/read-bench-args.d.ts +9 -0
  22. package/dist/bench/read-bench-file-args.d.ts +9 -0
  23. package/dist/bench/run-benchmark.d.ts +3 -3
  24. package/dist/bench/to-bench-name.d.ts +1 -0
  25. package/dist/bench/types.d.ts +20 -0
  26. package/dist/build-bench-folder-path.d.ts +1 -0
  27. package/dist/create-bench-work-folder.d.ts +6 -0
  28. package/dist/format-parse-error.d.ts +1 -0
  29. package/dist/read-command-output.d.ts +8 -0
  30. package/dist/read-git-common-dir.d.ts +6 -0
  31. package/dist/read-git-prefix.d.ts +6 -0
  32. package/dist/run-bench.d.ts +7 -0
  33. package/dist/run-cli.d.ts +1 -2
  34. package/package.json +4 -4
  35. package/src/bench/bench-report-entry-schema.ts +17 -0
  36. package/src/bench/build-base-side-root.ts +7 -0
  37. package/src/bench/collect-bench-comparison.ts +43 -0
  38. package/src/bench/define-bench.ts +15 -0
  39. package/src/bench/find-base-setup-verdict.ts +21 -0
  40. package/src/bench/find-missing-bench-verdict.ts +33 -0
  41. package/src/bench/format-bench-comparison.ts +72 -0
  42. package/src/bench/format-bench-report.ts +24 -0
  43. package/src/bench/format-benchmark-result.ts +7 -3
  44. package/src/bench/is-concurrency-list.ts +10 -0
  45. package/src/bench/parse-bench-definition.ts +21 -3
  46. package/src/bench/parse-bench-report.ts +42 -0
  47. package/src/bench/pick-bench-exit-code.ts +15 -0
  48. package/src/bench/pick-bench-plan.ts +25 -0
  49. package/src/bench/pick-checkout-cleanup.ts +29 -0
  50. package/src/bench/pick-cleaned-up-result.ts +30 -0
  51. package/src/bench/pick-comparison-verdict.ts +36 -0
  52. package/src/bench/pick-head-verdict.ts +11 -0
  53. package/src/bench/pick-named-bench-files.ts +8 -0
  54. package/src/bench/pick-quantile.ts +9 -0
  55. package/src/bench/pick-unremoved-checkout-state.ts +30 -0
  56. package/src/bench/read-bench-args.ts +45 -0
  57. package/src/bench/read-bench-file-args.ts +38 -0
  58. package/src/bench/run-benchmark.ts +88 -21
  59. package/src/bench/to-bench-name.ts +5 -0
  60. package/src/bench/types.ts +37 -0
  61. package/src/build-bench-folder-path.ts +7 -0
  62. package/src/cli.ts +31 -0
  63. package/src/create-bench-work-folder.ts +26 -0
  64. package/src/format-parse-error.ts +9 -0
  65. package/src/read-cli-args.ts +1 -10
  66. package/src/read-command-output.ts +30 -0
  67. package/src/read-git-common-dir.ts +27 -0
  68. package/src/read-git-prefix.ts +27 -0
  69. package/src/run-bench-file.ts +23 -13
  70. package/src/run-bench.ts +348 -39
  71. package/src/run-cli.ts +1 -3
  72. package/dist/bench/pick-median.d.ts +0 -1
  73. package/src/bench/pick-median.ts +0 -10
@@ -0,0 +1,10 @@
1
+ import { z } from 'zod';
2
+ export declare const benchReportEntrySchema: z.ZodReadonly<z.ZodObject<{
3
+ file: z.ZodString;
4
+ name: z.ZodString;
5
+ concurrency: z.ZodNumber;
6
+ medianMs: z.ZodNumber;
7
+ spreadMs: z.ZodNumber;
8
+ budgetMs: z.ZodNumber;
9
+ }, z.core.$strip>>;
10
+ export type BenchReportEntry = z.infer<typeof benchReportEntrySchema>;
@@ -0,0 +1 @@
1
+ export declare function buildBaseSideRoot(checkoutRoot: string, gitPrefix: string): string;
@@ -0,0 +1,3 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ import type { BenchComparisonRow } from './types.ts';
3
+ export declare function collectBenchComparison(baseEntries: readonly BenchReportEntry[], headEntries: readonly BenchReportEntry[]): BenchComparisonRow[];
@@ -1,6 +1,7 @@
1
1
  interface BenchBaseOptions {
2
2
  readonly name: string;
3
3
  readonly budget: string;
4
+ readonly concurrency?: readonly number[];
4
5
  }
5
6
  interface BenchSetupOptions<Context> extends BenchBaseOptions {
6
7
  readonly setup: () => Promise<Context>;
@@ -18,6 +19,7 @@ export interface PreparedBench {
18
19
  export interface BenchDefinition {
19
20
  readonly name: string;
20
21
  readonly budgetMs: number;
22
+ readonly concurrency: readonly number[];
21
23
  readonly prepare: () => Promise<PreparedBench>;
22
24
  }
23
25
  export declare function defineBench<Context>(options: BenchOptions<Context>): BenchDefinition;
@@ -0,0 +1,3 @@
1
+ import type { BenchVerdict } from './types.ts';
2
+ export type BaseSetupStep = 'checkout' | 'install';
3
+ export declare function findBaseSetupVerdict(step: BaseSetupStep, baseRef: string, exitCode: number): BenchVerdict | null;
@@ -0,0 +1,2 @@
1
+ import type { BenchVerdict } from './types.ts';
2
+ export declare function findMissingBenchVerdict(name: string | null, headBenchFiles: readonly string[], baseBenchFiles: readonly string[]): BenchVerdict | null;
@@ -0,0 +1,2 @@
1
+ import type { BenchComparisonRow } from './types.ts';
2
+ export declare function formatBenchComparison(rows: readonly BenchComparisonRow[]): string;
@@ -0,0 +1,3 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ import type { BenchmarkResult } from './run-benchmark.ts';
3
+ export declare function formatBenchReport(file: string, definition: BenchDefinition, results: readonly BenchmarkResult[]): string;
@@ -1,3 +1,3 @@
1
1
  import type { BenchDefinition } from './define-bench.ts';
2
2
  import type { BenchmarkResult } from './run-benchmark.ts';
3
- export declare function formatBenchmarkResult(definition: BenchDefinition, result: BenchmarkResult): string;
3
+ export declare function formatBenchmarkResult(definition: BenchDefinition, results: readonly BenchmarkResult[]): string;
@@ -0,0 +1 @@
1
+ export declare function isConcurrencyList(value: unknown): value is readonly number[];
@@ -0,0 +1,9 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ export type ParsedBenchReport = {
3
+ readonly kind: 'parsed';
4
+ readonly entries: readonly BenchReportEntry[];
5
+ } | {
6
+ readonly kind: 'refused';
7
+ readonly reason: string;
8
+ };
9
+ export declare function parseBenchReport(text: string): ParsedBenchReport;
@@ -0,0 +1,2 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ export declare function pickBenchExitCode(sideExitCodes: readonly number[], headEntries: readonly BenchReportEntry[]): number;
@@ -0,0 +1,13 @@
1
+ import type { BenchVerdict } from './types.ts';
2
+ export type BenchPlan = {
3
+ readonly kind: 'finished';
4
+ readonly verdict: BenchVerdict;
5
+ } | {
6
+ readonly kind: 'head';
7
+ readonly files: readonly string[];
8
+ } | {
9
+ readonly kind: 'comparison';
10
+ readonly baseRef: string;
11
+ readonly files: readonly string[];
12
+ };
13
+ export declare function pickBenchPlan(name: string | null, baseRef: string | null, headBenchFiles: readonly string[]): BenchPlan;
@@ -0,0 +1,7 @@
1
+ import type { BenchOutput, CheckoutState } from './types.ts';
2
+ export interface CheckoutCleanup {
3
+ readonly output: readonly BenchOutput[];
4
+ readonly isWorkFolderKept: boolean;
5
+ readonly exitCode: number;
6
+ }
7
+ export declare function pickCheckoutCleanup(baseRoot: string, checkout: CheckoutState): CheckoutCleanup;
@@ -0,0 +1,9 @@
1
+ export type CleanedUpResult<T, C> = {
2
+ readonly kind: 'fulfilled';
3
+ readonly value: T;
4
+ readonly cleanup: C;
5
+ } | {
6
+ readonly kind: 'throw';
7
+ readonly error: unknown;
8
+ };
9
+ export declare function pickCleanedUpResult<T, C>(ran: PromiseSettledResult<T>, cleaned: PromiseSettledResult<C>): CleanedUpResult<T, C>;
@@ -0,0 +1,7 @@
1
+ import type { ParsedBenchReport } from './parse-bench-report.ts';
2
+ import type { BenchVerdict } from './types.ts';
3
+ export interface BenchSideOutcome {
4
+ readonly exitCode: number;
5
+ readonly report: ParsedBenchReport;
6
+ }
7
+ export declare function pickComparisonVerdict(base: BenchSideOutcome, head: BenchSideOutcome): BenchVerdict;
@@ -0,0 +1,3 @@
1
+ import type { ParsedBenchReport } from './parse-bench-report.ts';
2
+ import type { BenchVerdict } from './types.ts';
3
+ export declare function pickHeadVerdict(report: ParsedBenchReport, headExitCode: number): BenchVerdict;
@@ -0,0 +1 @@
1
+ export declare function pickNamedBenchFiles(benchFiles: readonly string[], name: string | null): readonly string[];
@@ -0,0 +1 @@
1
+ export declare function pickQuantile(samples: readonly number[], quantile: number): number;
@@ -0,0 +1,3 @@
1
+ import type { CommandOutput } from '../read-command-output.ts';
2
+ import type { CheckoutState } from './types.ts';
3
+ export declare function pickUnremovedCheckoutState(listing: CommandOutput, baseRoot: string): Exclude<CheckoutState, 'removed'>;
@@ -0,0 +1,9 @@
1
+ export type BenchArgs = {
2
+ readonly kind: 'parsed';
3
+ readonly name: string | null;
4
+ readonly baseRef: string | null;
5
+ } | {
6
+ readonly kind: 'refused';
7
+ readonly reason: string;
8
+ };
9
+ export declare function readBenchArgs(args: readonly string[]): Promise<BenchArgs>;
@@ -0,0 +1,9 @@
1
+ export type BenchFileArgs = {
2
+ readonly kind: 'parsed';
3
+ readonly file: string;
4
+ readonly reportPath: string;
5
+ } | {
6
+ readonly kind: 'refused';
7
+ readonly reason: string;
8
+ };
9
+ export declare function readBenchFileArgs(args: readonly string[]): Promise<BenchFileArgs>;
@@ -1,7 +1,7 @@
1
1
  import type { BenchDefinition } from './define-bench.ts';
2
- export declare const warmUpRuns = 5;
3
- export declare const measuredRuns = 20;
4
2
  export interface BenchmarkResult {
3
+ readonly concurrency: number;
5
4
  readonly medianMs: number;
5
+ readonly spreadMs: number;
6
6
  }
7
- export declare function runBenchmark(definition: BenchDefinition, now: () => number): Promise<BenchmarkResult>;
7
+ export declare function runBenchmark(definition: BenchDefinition, now: () => number): Promise<readonly BenchmarkResult[]>;
@@ -0,0 +1 @@
1
+ export declare function toBenchName(file: string): string;
@@ -0,0 +1,20 @@
1
+ export interface BenchOutput {
2
+ readonly stream: 'out' | 'err';
3
+ readonly text: string;
4
+ }
5
+ export interface BenchVerdict {
6
+ readonly output: readonly BenchOutput[];
7
+ readonly exitCode: number;
8
+ }
9
+ export type CheckoutState = 'removed' | 'absent' | 'left';
10
+ export interface BenchSideTimes {
11
+ readonly medianMs: number;
12
+ readonly spreadMs: number;
13
+ }
14
+ export interface BenchComparisonRow {
15
+ readonly file: string;
16
+ readonly name: string;
17
+ readonly concurrency: number;
18
+ readonly base: BenchSideTimes | null;
19
+ readonly head: BenchSideTimes | null;
20
+ }
@@ -0,0 +1 @@
1
+ export declare function buildBenchFolderPath(repoRoot: string, gitCommonDir: string): string;
@@ -0,0 +1,6 @@
1
+ export interface BenchWorkFolderDeps {
2
+ readonly readGitCommonDir: (repoRoot: string) => Promise<string>;
3
+ readonly createFolder: (path: string) => Promise<void>;
4
+ readonly createTempFolder: (prefix: string) => Promise<string>;
5
+ }
6
+ export declare function createBenchWorkFolder(repoRoot: string, deps: BenchWorkFolderDeps): Promise<string>;
@@ -0,0 +1 @@
1
+ export declare function formatParseError(error: unknown): string;
@@ -0,0 +1,8 @@
1
+ import type { ProcessRuntime, SpawnOptions } from './browser/process-runtime.ts';
2
+ export interface CommandOutput {
3
+ readonly exitCode: number;
4
+ readonly stdout: string;
5
+ }
6
+ export type ReadCommandOptions = Omit<SpawnOptions, 'stdout'>;
7
+ export type ReadCommandOutput = (command: readonly string[], options: ReadCommandOptions) => Promise<CommandOutput>;
8
+ export declare function readCommandOutput(command: readonly string[], options: ReadCommandOptions, runtime: Pick<ProcessRuntime, 'spawn'>): Promise<CommandOutput>;
@@ -0,0 +1,6 @@
1
+ import type { SpawnOptions } from './browser/process-runtime.ts';
2
+ import type { ReadCommandOutput } from './read-command-output.ts';
3
+ export interface GitCommonDirDeps {
4
+ readonly readCommandOutput: ReadCommandOutput;
5
+ }
6
+ export declare function readGitCommonDir(repoRoot: string, environment: SpawnOptions['environment'], deps: GitCommonDirDeps): Promise<string>;
@@ -0,0 +1,6 @@
1
+ import type { SpawnOptions } from './browser/process-runtime.ts';
2
+ import type { ReadCommandOutput } from './read-command-output.ts';
3
+ export interface GitPrefixDeps {
4
+ readonly readCommandOutput: ReadCommandOutput;
5
+ }
6
+ export declare function readGitPrefix(folder: string, environment: SpawnOptions['environment'], deps: GitPrefixDeps): Promise<string>;
@@ -1,10 +1,17 @@
1
+ import type { ReadCommandOutput } from './read-command-output.ts';
1
2
  import type { ProgramBuildDeps } from './run-program-builds.ts';
2
3
  import type { RigConfig, RigRun } from './types.ts';
3
4
  export interface BenchDeps extends ProgramBuildDeps {
4
5
  readonly dietScript: string;
5
6
  readonly benchScript: string;
7
+ readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
6
8
  readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
7
9
  readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
8
10
  readonly importModule: (path: string) => Promise<unknown>;
11
+ readonly createWorkFolder: (repoRoot: string) => Promise<string>;
12
+ readonly readText: (path: string) => Promise<string | null>;
13
+ readonly probeFolder: (path: string) => Promise<boolean>;
14
+ readonly removeFolder: (path: string) => Promise<void>;
15
+ readonly readCommandOutput: ReadCommandOutput;
9
16
  }
10
17
  export declare function runBench(args: readonly string[], config: RigConfig, run: RigRun, deps: BenchDeps): Promise<number>;
package/dist/run-cli.d.ts CHANGED
@@ -4,8 +4,7 @@ import type { InitDeps } from './run-init.ts';
4
4
  import type { LaneDeps } from './run-lane.ts';
5
5
  import type { RelatedTestDeps } from './run-related-tests.ts';
6
6
  import type { UnitDeps } from './run-unit.ts';
7
- import type { RigConfig, RigRun } from './types.ts';
7
+ import type { RigRun } from './types.ts';
8
8
  export interface CLIDeps extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
9
- readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
10
9
  }
11
10
  export declare function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zgeoff/rig",
3
- "version": "0.1.4",
3
+ "version": "0.1.5",
4
4
  "description": "Run every test lane of a repo from one declaration of its limits, with a browser lane built in",
5
5
  "keywords": [
6
6
  "bun",
@@ -50,7 +50,8 @@
50
50
  "@zgeoff/diet": "0.2.1",
51
51
  "@zgeoff/std": "0.4.2",
52
52
  "gunshi": "1.0.0-rc.1",
53
- "ts-pattern": "5.9.2"
53
+ "ts-pattern": "5.9.2",
54
+ "zod": "4.6.7"
54
55
  },
55
56
  "devDependencies": {
56
57
  "@types/bun": "1.4.4",
@@ -58,8 +59,7 @@
58
59
  "@zgeoff/testing": "0.2.1",
59
60
  "tiny-invariant": "1.3.3",
60
61
  "tsrs": "0.18.0-ts7.1.0-dev.20260929",
61
- "typescript": "7.0.2",
62
- "zod": "4.6.7"
62
+ "typescript": "7.0.2"
63
63
  },
64
64
  "engines": {
65
65
  "bun": ">=1.3"
@@ -0,0 +1,17 @@
1
+ import { z } from 'zod';
2
+
3
+ // One line of the report that a bench file process appends for `rig bench` to read: one benchmark
4
+ // at one concurrency level.
5
+ export const benchReportEntrySchema = z
6
+ .object({
7
+ // Two bench files can define benchmarks of the same name, so the file tells them apart.
8
+ file: z.string(),
9
+ name: z.string(),
10
+ concurrency: z.number(),
11
+ medianMs: z.number(),
12
+ spreadMs: z.number(),
13
+ budgetMs: z.number(),
14
+ })
15
+ .readonly();
16
+
17
+ export type BenchReportEntry = z.infer<typeof benchReportEntrySchema>;
@@ -0,0 +1,7 @@
1
+ import { join } from 'node:path';
2
+
3
+ // The base side runs from the same folder inside its checkout as the head side runs inside the
4
+ // repo, so both sides read the config and bench files of the same package.
5
+ export function buildBaseSideRoot(checkoutRoot: string, gitPrefix: string): string {
6
+ return join(checkoutRoot, gitPrefix.replace(/\/$/u, ''));
7
+ }
@@ -0,0 +1,43 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ import type { BenchComparisonRow, BenchSideTimes } from './types.ts';
3
+
4
+ // Each side's entries name their bench file relative to that side's root, so the same file matches
5
+ // on both sides.
6
+ export function collectBenchComparison(
7
+ baseEntries: readonly BenchReportEntry[],
8
+ headEntries: readonly BenchReportEntry[],
9
+ ): BenchComparisonRow[] {
10
+ const headRows = headEntries.map((head) => {
11
+ const base = baseEntries.find((candidate) => isSameLevel(candidate, head));
12
+
13
+ return {
14
+ file: head.file,
15
+ name: head.name,
16
+ concurrency: head.concurrency,
17
+ base: base === undefined ? null : toSideTimes(base),
18
+ head: toSideTimes(head),
19
+ };
20
+ });
21
+
22
+ const baseOnlyRows = baseEntries
23
+ .filter((base) => !headEntries.some((head) => isSameLevel(head, base)))
24
+ .map((base) => ({
25
+ file: base.file,
26
+ name: base.name,
27
+ concurrency: base.concurrency,
28
+ base: toSideTimes(base),
29
+ head: null,
30
+ }));
31
+
32
+ return [...headRows, ...baseOnlyRows];
33
+ }
34
+
35
+ function isSameLevel(left: BenchReportEntry, right: BenchReportEntry): boolean {
36
+ return (
37
+ left.file === right.file && left.name === right.name && left.concurrency === right.concurrency
38
+ );
39
+ }
40
+
41
+ function toSideTimes(entry: BenchReportEntry): BenchSideTimes {
42
+ return { medianMs: entry.medianMs, spreadMs: entry.spreadMs };
43
+ }
@@ -1,10 +1,14 @@
1
1
  import { parseDuration } from '../parse-duration.ts';
2
+ import { isConcurrencyList } from './is-concurrency-list.ts';
2
3
 
3
4
  interface BenchBaseOptions {
4
5
  readonly name: string;
5
6
 
6
7
  // A duration such as `'50ms'` or `'2s'` that the median of a measured run must not pass.
7
8
  readonly budget: string;
9
+
10
+ // The numbers of `run` calls to keep in flight at once, each measured in turn.
11
+ readonly concurrency?: readonly number[];
8
12
  }
9
13
 
10
14
  interface BenchSetupOptions<Context> extends BenchBaseOptions {
@@ -34,6 +38,8 @@ export interface BenchDefinition {
34
38
 
35
39
  readonly budgetMs: number;
36
40
 
41
+ readonly concurrency: readonly number[];
42
+
37
43
  // Runs `setup` and returns the step to measure, which holds the setup result, and its disposal.
38
44
  readonly prepare: () => Promise<PreparedBench>;
39
45
  }
@@ -48,9 +54,18 @@ export function defineBench<Context>(options: BenchOptions<Context>): BenchDefin
48
54
  );
49
55
  }
50
56
 
57
+ const concurrency = options.concurrency ?? [1];
58
+
59
+ if (!isConcurrencyList(concurrency)) {
60
+ throw new Error(
61
+ `defineBench: the benchmark "${options.name}" has the concurrency [${(options.concurrency ?? []).join(', ')}], which is no list of distinct positive whole numbers`,
62
+ );
63
+ }
64
+
51
65
  return {
52
66
  name: options.name,
53
67
  budgetMs,
68
+ concurrency,
54
69
  prepare: async () => {
55
70
  if (options.setup === undefined) {
56
71
  return { step: () => options.run(), dispose: () => Promise.resolve() };
@@ -0,0 +1,21 @@
1
+ import { match } from 'ts-pattern';
2
+ import type { BenchVerdict } from './types.ts';
3
+
4
+ export type BaseSetupStep = 'checkout' | 'install';
5
+
6
+ export function findBaseSetupVerdict(
7
+ step: BaseSetupStep,
8
+ baseRef: string,
9
+ exitCode: number,
10
+ ): BenchVerdict | null {
11
+ if (exitCode === 0) {
12
+ return null;
13
+ }
14
+
15
+ const text = match(step)
16
+ .with('checkout', () => `rig bench: could not check out "${baseRef}"\n`)
17
+ .with('install', () => `rig bench: bun install failed in the checkout of "${baseRef}"\n`)
18
+ .exhaustive();
19
+
20
+ return { output: [{ stream: 'err', text }], exitCode: 1 };
21
+ }
@@ -0,0 +1,33 @@
1
+ import { pickNamedBenchFiles } from './pick-named-bench-files.ts';
2
+ import { toBenchName } from './to-bench-name.ts';
3
+ import type { BenchVerdict } from './types.ts';
4
+
5
+ // A benchmark that only the base has still compares, so the run stops only when neither side has
6
+ // a file to run. A run with no base passes no base files.
7
+ export function findMissingBenchVerdict(
8
+ name: string | null,
9
+ headBenchFiles: readonly string[],
10
+ baseBenchFiles: readonly string[],
11
+ ): BenchVerdict | null {
12
+ const benchFiles = [...headBenchFiles, ...baseBenchFiles];
13
+
14
+ if (pickNamedBenchFiles(benchFiles, name).length > 0) {
15
+ return null;
16
+ }
17
+
18
+ if (name === null) {
19
+ return { output: [{ stream: 'out', text: 'rig: the repo has no benchmarks\n' }], exitCode: 0 };
20
+ }
21
+
22
+ const names = [...new Set(benchFiles.map((file) => toBenchName(file)))].join(', ');
23
+
24
+ return {
25
+ output: [
26
+ {
27
+ stream: 'err',
28
+ text: `rig bench: no benchmark named "${name}"; the benchmarks are ${names}\n`,
29
+ },
30
+ ],
31
+ exitCode: 1,
32
+ };
33
+ }
@@ -0,0 +1,72 @@
1
+ import type { BenchComparisonRow, BenchSideTimes } from './types.ts';
2
+
3
+ const header = [
4
+ 'bench',
5
+ 'concurrency',
6
+ 'base median',
7
+ 'base spread',
8
+ 'head median',
9
+ 'head spread',
10
+ 'change',
11
+ ] as const;
12
+
13
+ export function formatBenchComparison(rows: readonly BenchComparisonRow[]): string {
14
+ const cells: readonly (readonly string[])[] = [
15
+ header,
16
+ ...rows.map((row) => formatCells(row, rows)),
17
+ ];
18
+
19
+ const widths = header.map((_, column) =>
20
+ Math.max(...cells.map((line) => line[column]?.length ?? 0)),
21
+ );
22
+
23
+ return cells
24
+ .map((line) =>
25
+ line
26
+ .map((cell, column) =>
27
+ column === line.length - 1 ? cell : cell.padEnd((widths[column] ?? 0) + 2),
28
+ )
29
+ .join(''),
30
+ )
31
+ .map((line) => `${line}\n`)
32
+ .join('');
33
+ }
34
+
35
+ function formatCells(
36
+ row: BenchComparisonRow,
37
+ rows: readonly BenchComparisonRow[],
38
+ ): readonly string[] {
39
+ return [
40
+ formatLabel(row, rows),
41
+ String(row.concurrency),
42
+ formatTime(row.base?.medianMs),
43
+ formatTime(row.base?.spreadMs),
44
+ formatTime(row.head?.medianMs),
45
+ formatTime(row.head?.spreadMs),
46
+ formatChange(row.base, row.head),
47
+ ];
48
+ }
49
+
50
+ // The name alone reads best, so the file shows only where two rows would look the same.
51
+ function formatLabel(row: BenchComparisonRow, rows: readonly BenchComparisonRow[]): string {
52
+ const isShared = rows.some(
53
+ (other) => other !== row && other.name === row.name && other.concurrency === row.concurrency,
54
+ );
55
+
56
+ return isShared ? `${row.name} (${row.file})` : row.name;
57
+ }
58
+
59
+ function formatTime(timeMs: number | undefined): string {
60
+ return timeMs === undefined ? '-' : `${timeMs.toFixed(2)}ms`;
61
+ }
62
+
63
+ // A base median of zero gives no ratio to show.
64
+ function formatChange(base: BenchSideTimes | null, head: BenchSideTimes | null): string {
65
+ if (base === null || head === null || base.medianMs === 0) {
66
+ return '-';
67
+ }
68
+
69
+ const changePercent = ((head.medianMs - base.medianMs) / base.medianMs) * 100;
70
+
71
+ return `${changePercent < 0 ? '' : '+'}${changePercent.toFixed(1)}%`;
72
+ }
@@ -0,0 +1,24 @@
1
+ import type { BenchReportEntry } from './bench-report-entry-schema.ts';
2
+ import type { BenchDefinition } from './define-bench.ts';
3
+ import type { BenchmarkResult } from './run-benchmark.ts';
4
+
5
+ export function formatBenchReport(
6
+ file: string,
7
+ definition: BenchDefinition,
8
+ results: readonly BenchmarkResult[],
9
+ ): string {
10
+ return results
11
+ .map((result) => {
12
+ const entry: BenchReportEntry = {
13
+ file,
14
+ name: definition.name,
15
+ concurrency: result.concurrency,
16
+ medianMs: result.medianMs,
17
+ spreadMs: result.spreadMs,
18
+ budgetMs: definition.budgetMs,
19
+ };
20
+
21
+ return `${JSON.stringify(entry)}\n`;
22
+ })
23
+ .join('');
24
+ }
@@ -3,9 +3,13 @@ import type { BenchmarkResult } from './run-benchmark.ts';
3
3
 
4
4
  export function formatBenchmarkResult(
5
5
  definition: BenchDefinition,
6
- result: BenchmarkResult,
6
+ results: readonly BenchmarkResult[],
7
7
  ): string {
8
- const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
8
+ return results
9
+ .map((result) => {
10
+ const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
9
11
 
10
- return `bench ${definition.name}: median ${result.medianMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
12
+ return `bench ${definition.name} at concurrency ${result.concurrency}: median ${result.medianMs.toFixed(2)}ms, spread ${result.spreadMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
13
+ })
14
+ .join('');
11
15
  }
@@ -0,0 +1,10 @@
1
+ export function isConcurrencyList(value: unknown): value is readonly number[] {
2
+ return (
3
+ Array.isArray(value) &&
4
+ value.length > 0 &&
5
+ value.every(
6
+ (level: unknown) => typeof level === 'number' && Number.isSafeInteger(level) && level > 0,
7
+ ) &&
8
+ new Set(value).size === value.length
9
+ );
10
+ }
@@ -1,4 +1,5 @@
1
1
  import type { BenchDefinition } from './define-bench.ts';
2
+ import { isConcurrencyList } from './is-concurrency-list.ts';
2
3
 
3
4
  export type ParsedBenchDefinition =
4
5
  | { readonly kind: 'parsed'; readonly definition: BenchDefinition }
@@ -15,7 +16,15 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
15
16
  const budgetMs: unknown = Reflect.get(value, 'budgetMs');
16
17
  const prepare: unknown = Reflect.get(value, 'prepare');
17
18
 
18
- if (typeof name !== 'string' || typeof budgetMs !== 'number' || !isFunction(prepare)) {
19
+ // A bench file on a base ref can import an older rig, whose result has no concurrency.
20
+ const concurrency: unknown = Reflect.get(value, 'concurrency') ?? [1];
21
+
22
+ if (
23
+ typeof name !== 'string' ||
24
+ typeof budgetMs !== 'number' ||
25
+ !isFunction(prepare) ||
26
+ !isConcurrencyList(concurrency)
27
+ ) {
19
28
  return { kind: 'refused', reason: 'the default export is no defineBench result' };
20
29
  }
21
30
 
@@ -24,6 +33,7 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
24
33
  definition: {
25
34
  name,
26
35
  budgetMs,
36
+ concurrency,
27
37
  prepare: async () => {
28
38
  const prepared: unknown = await prepare();
29
39
 
@@ -40,8 +50,10 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
40
50
  }
41
51
 
42
52
  return {
43
- step: async () => {
44
- await step();
53
+ step: () => {
54
+ const result: unknown = step();
55
+
56
+ return result === undefined ? undefined : settleStep(result);
45
57
  },
46
58
  dispose: async () => {
47
59
  await dispose();
@@ -55,3 +67,9 @@ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
55
67
  function isFunction(value: unknown): value is (...args: unknown[]) => unknown {
56
68
  return typeof value === 'function';
57
69
  }
70
+
71
+ // `instanceof Promise` misses a thenable and a promise from another realm, such as a `node:vm`
72
+ // context, so any value but `undefined` goes through `await`, which adopts both.
73
+ async function settleStep(result: unknown): Promise<void> {
74
+ await result;
75
+ }