@zgeoff/rig 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +14 -2
  2. package/dist/bench/define-bench.d.ts +24 -0
  3. package/dist/bench/format-benchmark-result.d.ts +3 -0
  4. package/dist/bench/parse-bench-definition.d.ts +9 -0
  5. package/dist/bench/pick-median.d.ts +1 -0
  6. package/dist/bench/run-benchmark.d.ts +7 -0
  7. package/dist/bench.d.ts +2 -0
  8. package/dist/build-init-scripts.d.ts +1 -1
  9. package/dist/build-program-environment.d.ts +3 -0
  10. package/dist/build-program-lane-hooks.d.ts +3 -0
  11. package/dist/build-test-selection.d.ts +1 -0
  12. package/dist/collect-bench-files.d.ts +1 -0
  13. package/dist/collect-eval-files.d.ts +1 -0
  14. package/dist/collect-unprogrammed-e2e-dirs.d.ts +2 -0
  15. package/dist/e2e.d.ts +2 -0
  16. package/dist/index.d.ts +1 -1
  17. package/dist/index.js +83 -3
  18. package/dist/is-command-name.d.ts +1 -1
  19. package/dist/pick-e2e-dirs.d.ts +1 -1
  20. package/dist/probe-workspace-repo.d.ts +1 -0
  21. package/dist/programs/resolve-program.d.ts +12 -0
  22. package/dist/programs/resolve-repo-root.d.ts +1 -0
  23. package/dist/run-bench-file.d.ts +1 -0
  24. package/dist/run-bench.d.ts +10 -0
  25. package/dist/run-cli.d.ts +3 -1
  26. package/dist/run-eval.d.ts +8 -0
  27. package/dist/run-init.d.ts +1 -0
  28. package/dist/run-lane.d.ts +4 -5
  29. package/dist/run-program-builds.d.ts +7 -0
  30. package/dist/run-unit-tests.d.ts +1 -0
  31. package/dist/types.d.ts +13 -0
  32. package/package.json +15 -3
  33. package/src/bench/define-bench.ts +78 -0
  34. package/src/bench/format-benchmark-result.ts +11 -0
  35. package/src/bench/parse-bench-definition.ts +57 -0
  36. package/src/bench/pick-median.ts +10 -0
  37. package/src/bench/run-benchmark.ts +38 -0
  38. package/src/bench.ts +2 -0
  39. package/src/build-init-scripts.ts +17 -3
  40. package/src/build-program-environment.ts +20 -0
  41. package/src/build-program-lane-hooks.ts +24 -0
  42. package/src/build-test-selection.ts +8 -7
  43. package/src/build-unit-command.ts +15 -1
  44. package/src/cli.ts +4 -0
  45. package/src/collect-bench-files.ts +23 -0
  46. package/src/collect-eval-files.ts +5 -0
  47. package/src/collect-unprogrammed-e2e-dirs.ts +21 -0
  48. package/src/define-rig.ts +65 -2
  49. package/src/e2e.ts +2 -0
  50. package/src/index.ts +2 -0
  51. package/src/is-command-name.ts +1 -1
  52. package/src/parse-rig-config.ts +64 -2
  53. package/src/pick-e2e-dirs.ts +8 -5
  54. package/src/probe-workspace-repo.ts +15 -0
  55. package/src/programs/resolve-program.ts +92 -0
  56. package/src/programs/resolve-repo-root.ts +18 -0
  57. package/src/run-bench-file.ts +25 -0
  58. package/src/run-bench.ts +132 -0
  59. package/src/run-cli.ts +8 -1
  60. package/src/run-eval.ts +74 -0
  61. package/src/run-init.ts +6 -1
  62. package/src/run-lane.ts +39 -10
  63. package/src/run-program-builds.ts +35 -0
  64. package/src/run-unit-tests.ts +4 -1
  65. package/src/run-unit.ts +1 -0
  66. package/src/types.ts +30 -0
package/README.md CHANGED
@@ -35,11 +35,15 @@ so a run that outgrows its memory budget fails alone when `diet` finds a
35
35
 
36
36
  ## Commands
37
37
 
38
- - `rig unit` runs the unit tests in random order with the host's worker count.
38
+ - `rig unit` runs the unit tests in random order with the host's worker count, with `--no-isolate`
39
+ so the files of a worker share one module graph. Pass `--isolate` to turn that off.
39
40
  - `rig related <paths...>` runs the unit tests beside the changed files, for a pre-push hook.
40
41
  - `rig <lane>` runs one lane: `e2e`, `browser`, `docker`, `host`, `agent-sdk`, `model-api`, or a
41
42
  lane the repo declares.
42
- - `rig init` writes the `test`, `test:related` and `test:<lane>` scripts and a starter config.
43
+ - `rig bench [name]` and `rig eval [name]` run the benchmarks and the evaluation suites, or the one
44
+ named `name`.
45
+ - `rig init` writes the `test`, `test:related`, `test:<lane>`, `bench` and `eval` scripts the repo
46
+ has files for, and a starter config.
43
47
 
44
48
  `rig unit` and the lane commands pass every argument after the command on to `bun test`, so
45
49
  `bun run test --seed=1` replays a run's order, and rig drops one `--` among them. The exception is
@@ -47,6 +51,9 @@ so a run that outgrows its memory budget fails alone when `diet` finds a
47
51
  `--timeout` or `--timeout=<ms>` argument and runs nothing. `rig related` takes changed file paths
48
52
  only, and exits 2 on an option.
49
53
 
54
+ An `e2e/` folder is the e2e lane's in a folder that holds a `package.json`, and at the repo root
55
+ only when the root `package.json` has no `workspaces` field.
56
+
50
57
  Among those arguments, a path keeps only the lane's own files that it selects: a file, the files
51
58
  under a folder, or, for a bare word, the files whose path holds it. rig resolves a path, with its
52
59
  `.` and `..` segments, against the repo root, and a path outside the repo selects nothing. When the
@@ -70,6 +77,11 @@ export default defineRig({
70
77
  Oxlint's plugin reads this file under Node 22.18 or later, or under Bun, because it imports the
71
78
  config to take the limits from it.
72
79
 
80
+ A repo declares the programs its `e2e/` folders and benchmarks run in `programs`, which `rig e2e`
81
+ and `rig bench` build first; `resolveProgram` gives each one's command. `rig bench` and `rig eval`
82
+ run the benchmarks and evaluation suites. [Programs in e2e tests](../../docs/guides/e2e-programs.md)
83
+ and [benchmarks and evaluations](../../docs/guides/benchmarks-and-evals.md) describe each.
84
+
73
85
  ## Testing in a page
74
86
 
75
87
  A `.browser.test.ts` suite runs in Chrome through `Bun.WebView`. `runInPage` runs a real function in
@@ -0,0 +1,24 @@
1
+ interface BenchBaseOptions {
2
+ readonly name: string;
3
+ readonly budget: string;
4
+ }
5
+ interface BenchSetupOptions<Context> extends BenchBaseOptions {
6
+ readonly setup: () => Promise<Context>;
7
+ readonly run: (context: Context) => Promise<void> | void;
8
+ }
9
+ interface BenchPlainOptions extends BenchBaseOptions {
10
+ readonly setup?: undefined;
11
+ readonly run: () => Promise<void> | void;
12
+ }
13
+ export type BenchOptions<Context> = BenchSetupOptions<Context> | BenchPlainOptions;
14
+ export interface PreparedBench {
15
+ readonly step: () => Promise<void> | void;
16
+ readonly dispose: () => Promise<void>;
17
+ }
18
+ export interface BenchDefinition {
19
+ readonly name: string;
20
+ readonly budgetMs: number;
21
+ readonly prepare: () => Promise<PreparedBench>;
22
+ }
23
+ export declare function defineBench<Context>(options: BenchOptions<Context>): BenchDefinition;
24
+ export {};
@@ -0,0 +1,3 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ import type { BenchmarkResult } from './run-benchmark.ts';
3
+ export declare function formatBenchmarkResult(definition: BenchDefinition, result: BenchmarkResult): string;
@@ -0,0 +1,9 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ export type ParsedBenchDefinition = {
3
+ readonly kind: 'parsed';
4
+ readonly definition: BenchDefinition;
5
+ } | {
6
+ readonly kind: 'refused';
7
+ readonly reason: string;
8
+ };
9
+ export declare function parseBenchDefinition(value: unknown): ParsedBenchDefinition;
@@ -0,0 +1 @@
1
+ export declare function pickMedian(samples: readonly number[]): number;
@@ -0,0 +1,7 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ export declare const warmUpRuns = 5;
3
+ export declare const measuredRuns = 20;
4
+ export interface BenchmarkResult {
5
+ readonly medianMs: number;
6
+ }
7
+ export declare function runBenchmark(definition: BenchDefinition, now: () => number): Promise<BenchmarkResult>;
@@ -0,0 +1,2 @@
1
+ export { defineBench } from './bench/define-bench.ts';
2
+ export type { BenchDefinition, BenchOptions } from './bench/define-bench.ts';
@@ -1,3 +1,3 @@
1
1
  import type { RigConfig } from './types.ts';
2
2
  export type PackageScripts = Readonly<Record<string, string>>;
3
- export declare function buildInitScripts(oldScripts: PackageScripts, config: RigConfig, trackedFiles: readonly string[]): PackageScripts;
3
+ export declare function buildInitScripts(oldScripts: PackageScripts, config: RigConfig, trackedFiles: readonly string[], isWorkspaceRepo: boolean): PackageScripts;
@@ -0,0 +1,3 @@
1
+ import type { ProgramDeclaration } from './types.ts';
2
+ export declare function buildProgramVariable(name: string): string;
3
+ export declare function buildProgramEnvironment(repoRoot: string, programs: Readonly<Record<string, ProgramDeclaration>>): Readonly<Record<string, string>>;
@@ -0,0 +1,3 @@
1
+ import type { LaneHooks } from './lane-hooks.ts';
2
+ import type { ProgramDeclaration } from './types.ts';
3
+ export declare function buildProgramLaneHooks(hooks: LaneHooks, repoRoot: string, programs: Readonly<Record<string, ProgramDeclaration>>): LaneHooks;
@@ -17,6 +17,7 @@ export interface TestSelectionRequest {
17
17
  readonly config: RigConfig;
18
18
  readonly lane: Pick<ResolvedLane, 'name' | 'suffix'>;
19
19
  readonly repoRoot: string;
20
+ readonly isWorkspaceRepo: boolean;
20
21
  readonly trackedFiles: readonly string[];
21
22
  readonly args: TestArgs;
22
23
  }
@@ -0,0 +1 @@
1
+ export declare function collectBenchFiles(trackedFiles: readonly string[], isWorkspaceRepo: boolean): readonly string[];
@@ -0,0 +1 @@
1
+ export declare function collectEvalFiles(trackedFiles: readonly string[]): readonly string[];
@@ -0,0 +1,2 @@
1
+ import type { ProgramDeclaration } from './types.ts';
2
+ export declare function collectUnprogrammedE2EDirs(e2eDirs: readonly string[], programs: Readonly<Record<string, ProgramDeclaration>>): readonly string[];
package/dist/e2e.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ export { resolveProgram } from './programs/resolve-program.ts';
2
+ export type { ProgramLaunch } from './programs/resolve-program.ts';
package/dist/index.d.ts CHANGED
@@ -3,4 +3,4 @@ export { parseRigConfig } from './parse-rig-config.ts';
3
3
  export type { ParsedRigConfig } from './parse-rig-config.ts';
4
4
  export { runInPage } from './browser/run-in-page.ts';
5
5
  export type { PageResultSchema } from './browser/run-in-page.ts';
6
- export type { LaneContext, LaneDeclaration, LaneStart, ResolvedLane, RigConfig, RigDeclaration, } from './types.ts';
6
+ export type { LaneContext, BenchDeclaration, LaneDeclaration, ProgramDeclaration, LaneStart, ResolvedLane, RigConfig, RigDeclaration, } from './types.ts';
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  // src/is-command-name.ts
2
- var commandNames = ["init", "related"];
2
+ var commandNames = ["bench", "eval", "init", "related"];
3
3
  function isCommandName(name) {
4
4
  return commandNames.some((command) => command === name);
5
5
  }
@@ -43,7 +43,11 @@ function defineRig(declaration) {
43
43
  ...declared.filter(([name]) => !standardLanes.some((standard) => standard.name === name)).map(([name, laneDeclaration]) => resolveProjectLane(name, laneDeclaration))
44
44
  ];
45
45
  assertDistinctSuffixes(lanes);
46
- return { lanes };
46
+ const programs = Object.fromEntries(Object.entries(declaration.programs ?? {}).map(([name, program]) => [
47
+ name,
48
+ readProgram(name, program)
49
+ ]));
50
+ return { lanes, programs, benchLane: readBenchLane(declaration.bench?.lane, lanes) };
47
51
  }
48
52
  function resolveStandardLane(standard, declaration) {
49
53
  const declared = declaration ?? {};
@@ -137,6 +141,41 @@ function assertDistinctSuffixes(lanes) {
137
141
  owners.set(lane.suffix, lane.name);
138
142
  }
139
143
  }
144
+ var programNamePattern = /^[a-z][a-z0-9-]*$/u;
145
+ var programPathFields = ["package", "source", "artifact"];
146
+ function readProgram(name, program) {
147
+ if (!programNamePattern.test(name)) {
148
+ throw new Error(`rig.config.ts: the program name "${name}" must be lowercase letters, digits and dashes, starting with a letter`);
149
+ }
150
+ for (const field of programPathFields) {
151
+ const path = program[field];
152
+ if (typeof path !== "string" || path === "") {
153
+ throw new Error(`rig.config.ts: program "${name}" needs a ${field}, a non-empty path`);
154
+ }
155
+ if (path.startsWith("/") || path.split("/").includes("..")) {
156
+ throw new Error(`rig.config.ts: program "${name}" has the ${field} "${path}", which must be a relative path with no ".." segment`);
157
+ }
158
+ }
159
+ const build = program.build;
160
+ if (typeof build !== "string" || build.trim() === "") {
161
+ throw new Error(`rig.config.ts: program "${name}" needs a build, a shell command`);
162
+ }
163
+ return {
164
+ package: program.package,
165
+ source: program.source,
166
+ build: program.build,
167
+ artifact: program.artifact
168
+ };
169
+ }
170
+ function readBenchLane(lane, lanes) {
171
+ if (lane === undefined) {
172
+ return null;
173
+ }
174
+ if (!lanes.some((candidate) => candidate.name === lane)) {
175
+ throw new Error(`rig.config.ts: bench.lane "${lane}" names no lane`);
176
+ }
177
+ return lane;
178
+ }
140
179
  // src/parse-rig-config.ts
141
180
  function parseRigConfig(value) {
142
181
  if (typeof value !== "object" || value === null || !("lanes" in value)) {
@@ -154,7 +193,18 @@ function parseRigConfig(value) {
154
193
  }
155
194
  resolved.push(parsed.lane);
156
195
  }
157
- return { kind: "parsed", config: { lanes: resolved } };
196
+ const programs = parsePrograms(Reflect.get(value, "programs"));
197
+ if (programs === null) {
198
+ return { kind: "refused", reason: "its programs are no record of program declarations" };
199
+ }
200
+ const benchLane = Reflect.get(value, "benchLane");
201
+ if (benchLane !== undefined && !isNullableString(benchLane)) {
202
+ return { kind: "refused", reason: `its benchLane is ${JSON.stringify(benchLane)}` };
203
+ }
204
+ return {
205
+ kind: "parsed",
206
+ config: { lanes: resolved, programs, benchLane: benchLane ?? null }
207
+ };
158
208
  }
159
209
  function parseResolvedLane(value, index) {
160
210
  if (typeof value !== "object" || value === null) {
@@ -202,6 +252,36 @@ function refuseField(lane, field, value) {
202
252
  function isParallelism(value) {
203
253
  return value === "workers" || typeof value === "number" && Number.isInteger(value) && value > 0;
204
254
  }
255
+ function parsePrograms(value) {
256
+ if (value === undefined) {
257
+ return {};
258
+ }
259
+ if (typeof value !== "object" || value === null) {
260
+ return null;
261
+ }
262
+ const programs = {};
263
+ for (const [name, program] of Object.entries(value)) {
264
+ const parsed = parseProgram(program);
265
+ if (parsed === null) {
266
+ return null;
267
+ }
268
+ programs[name] = parsed;
269
+ }
270
+ return programs;
271
+ }
272
+ function parseProgram(value) {
273
+ if (typeof value !== "object" || value === null) {
274
+ return null;
275
+ }
276
+ const packageFolder = Reflect.get(value, "package");
277
+ const source = Reflect.get(value, "source");
278
+ const build = Reflect.get(value, "build");
279
+ const artifact = Reflect.get(value, "artifact");
280
+ if (typeof packageFolder !== "string" || typeof source !== "string" || typeof build !== "string" || typeof artifact !== "string") {
281
+ return null;
282
+ }
283
+ return { package: packageFolder, source, build, artifact };
284
+ }
205
285
  // src/browser/run-in-page.ts
206
286
  async function runInPage(view, pageFunction, argument, schema) {
207
287
  const source = pageFunction.toString();
@@ -1,3 +1,3 @@
1
- export declare const commandNames: readonly ['init', 'related'];
1
+ export declare const commandNames: readonly ['bench', 'eval', 'init', 'related'];
2
2
  export type CommandName = (typeof commandNames)[number];
3
3
  export declare function isCommandName(name: string): name is CommandName;
@@ -1 +1 @@
1
- export declare function pickE2EDirs(trackedFiles: readonly string[]): readonly string[];
1
+ export declare function pickE2EDirs(trackedFiles: readonly string[], isWorkspaceRepo: boolean): readonly string[];
@@ -0,0 +1 @@
1
+ export declare function probeWorkspaceRepo(repoRoot: string): Promise<boolean>;
@@ -0,0 +1,12 @@
1
+ export interface ProgramLaunch {
2
+ readonly command: readonly string[];
3
+ readonly cwd: string;
4
+ }
5
+ export interface ProgramLaunchSources {
6
+ readonly environment: Readonly<Record<string, string | undefined>>;
7
+ readonly cwd: string;
8
+ readonly importModule: (path: string) => Promise<unknown>;
9
+ readonly probeFile: (path: string) => boolean;
10
+ }
11
+ export declare function readLaunchSources(): ProgramLaunchSources;
12
+ export declare function resolveProgram(name: string, sources?: ProgramLaunchSources): Promise<ProgramLaunch>;
@@ -0,0 +1 @@
1
+ export declare function resolveRepoRoot(cwd: string, probeFile: (path: string) => boolean): string;
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,10 @@
1
+ import type { ProgramBuildDeps } from './run-program-builds.ts';
2
+ import type { RigConfig, RigRun } from './types.ts';
3
+ export interface BenchDeps extends ProgramBuildDeps {
4
+ readonly dietScript: string;
5
+ readonly benchScript: string;
6
+ readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
7
+ readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
8
+ readonly importModule: (path: string) => Promise<unknown>;
9
+ }
10
+ export declare function runBench(args: readonly string[], config: RigConfig, run: RigRun, deps: BenchDeps): Promise<number>;
package/dist/run-cli.d.ts CHANGED
@@ -1,9 +1,11 @@
1
+ import type { BenchDeps } from './run-bench.ts';
2
+ import type { EvalDeps } from './run-eval.ts';
1
3
  import type { InitDeps } from './run-init.ts';
2
4
  import type { LaneDeps } from './run-lane.ts';
3
5
  import type { RelatedTestDeps } from './run-related-tests.ts';
4
6
  import type { UnitDeps } from './run-unit.ts';
5
7
  import type { RigConfig, RigRun } from './types.ts';
6
- export interface CLIDeps extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps {
8
+ export interface CLIDeps extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
7
9
  readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
8
10
  }
9
11
  export declare function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number>;
@@ -0,0 +1,8 @@
1
+ import type { CommandOptions } from './run-command.ts';
2
+ import type { RigOutput, RigRun } from './types.ts';
3
+ export interface EvalDeps {
4
+ readonly output: RigOutput;
5
+ readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
6
+ readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
7
+ }
8
+ export declare function runEval(args: readonly string[], run: RigRun, deps: EvalDeps): Promise<number>;
@@ -1,6 +1,7 @@
1
1
  import type { InitFiles } from './init-files.ts';
2
2
  import type { RigConfig, RigOutput } from './types.ts';
3
3
  export interface InitDeps {
4
+ readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
4
5
  readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
5
6
  readonly initFiles: InitFiles;
6
7
  readonly output: RigOutput;
@@ -1,13 +1,12 @@
1
1
  import type { BrowserLaneWiring } from './browser/wire-browser-lane-hooks.ts';
2
2
  import type { LaneHooks } from './lane-hooks.ts';
3
- import type { CommandOptions } from './run-command.ts';
4
- import type { ResolvedLane, RigConfig, RigOutput, RigRun } from './types.ts';
5
- export interface LaneDeps {
3
+ import type { ProgramBuildDeps } from './run-program-builds.ts';
4
+ import type { ResolvedLane, RigConfig, RigRun } from './types.ts';
5
+ export interface LaneDeps extends ProgramBuildDeps {
6
6
  readonly dietScript: string;
7
- readonly output: RigOutput;
7
+ readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
8
8
  readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
9
9
  readonly importModule: (path: string) => Promise<unknown>;
10
- readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
11
10
  readonly wireBrowserLaneHooks: (wiring: BrowserLaneWiring) => Promise<LaneHooks>;
12
11
  }
13
12
  export interface LaneRequest {
@@ -0,0 +1,7 @@
1
+ import type { CommandOptions } from './run-command.ts';
2
+ import type { ProgramDeclaration, RigOutput, RigRun } from './types.ts';
3
+ export interface ProgramBuildDeps {
4
+ readonly output: RigOutput;
5
+ readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
6
+ }
7
+ export declare function runProgramBuilds(programs: Readonly<Record<string, ProgramDeclaration>>, run: RigRun, deps: ProgramBuildDeps): Promise<number>;
@@ -6,6 +6,7 @@ export interface UnitRequest {
6
6
  }
7
7
  export interface UnitRunDeps {
8
8
  readonly dietScript: string;
9
+ readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
9
10
  readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
10
11
  readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
11
12
  }
package/dist/types.d.ts CHANGED
@@ -5,8 +5,19 @@ export interface LaneDeclaration {
5
5
  readonly start?: string;
6
6
  readonly stop?: string;
7
7
  }
8
+ export interface ProgramDeclaration {
9
+ readonly package: string;
10
+ readonly source: string;
11
+ readonly build: string;
12
+ readonly artifact: string;
13
+ }
14
+ export interface BenchDeclaration {
15
+ readonly lane?: string;
16
+ }
8
17
  export interface RigDeclaration {
9
18
  readonly lanes?: Readonly<Record<string, LaneDeclaration>>;
19
+ readonly programs?: Readonly<Record<string, ProgramDeclaration>>;
20
+ readonly bench?: BenchDeclaration;
10
21
  }
11
22
  export type LaneParallelism = 'workers' | number;
12
23
  export interface ResolvedLane {
@@ -20,6 +31,8 @@ export interface ResolvedLane {
20
31
  }
21
32
  export interface RigConfig {
22
33
  readonly lanes: readonly ResolvedLane[];
34
+ readonly programs: Readonly<Record<string, ProgramDeclaration>>;
35
+ readonly benchLane: string | null;
23
36
  }
24
37
  export interface LaneContext {
25
38
  readonly repoRoot: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zgeoff/rig",
3
- "version": "0.0.5",
3
+ "version": "0.0.6",
4
4
  "description": "Run every test lane of a repo from one declaration of its limits, with a browser lane built in",
5
5
  "keywords": [
6
6
  "bun",
@@ -19,7 +19,8 @@
19
19
  "files": [
20
20
  "src",
21
21
  "!src/**/*.test.ts",
22
- "dist"
22
+ "dist",
23
+ "!src/test-utils"
23
24
  ],
24
25
  "type": "module",
25
26
  "exports": {
@@ -27,7 +28,16 @@
27
28
  "types": "./dist/index.d.ts",
28
29
  "bun": "./src/index.ts",
29
30
  "default": "./dist/index.js"
30
- }
31
+ },
32
+ "./e2e": {
33
+ "types": "./dist/e2e.d.ts",
34
+ "bun": "./src/e2e.ts"
35
+ },
36
+ "./bench": {
37
+ "types": "./dist/bench.d.ts",
38
+ "bun": "./src/bench.ts"
39
+ },
40
+ "./test-utils/*": "./src/test-utils/*.ts"
31
41
  },
32
42
  "publishConfig": {
33
43
  "access": "public"
@@ -44,6 +54,8 @@
44
54
  },
45
55
  "devDependencies": {
46
56
  "@types/bun": "1.4.4",
57
+ "@zgeoff/std": "0.3.2",
58
+ "@zgeoff/testing": "0.1.0",
47
59
  "tiny-invariant": "1.3.3",
48
60
  "tsrs": "0.18.0-ts7.1.0-dev.20260929",
49
61
  "typescript": "7.0.2",
@@ -0,0 +1,78 @@
1
+ import { parseDuration } from '../parse-duration.ts';
2
+
3
+ interface BenchBaseOptions {
4
+ readonly name: string;
5
+
6
+ // A duration such as `'50ms'` or `'2s'` that the median of a measured run must not pass.
7
+ readonly budget: string;
8
+ }
9
+
10
+ interface BenchSetupOptions<Context> extends BenchBaseOptions {
11
+ readonly setup: () => Promise<Context>;
12
+
13
+ readonly run: (context: Context) => Promise<void> | void;
14
+ }
15
+
16
+ interface BenchPlainOptions extends BenchBaseOptions {
17
+ readonly setup?: undefined;
18
+
19
+ readonly run: () => Promise<void> | void;
20
+ }
21
+
22
+ export type BenchOptions<Context> = BenchSetupOptions<Context> | BenchPlainOptions;
23
+
24
+ // What `prepare` hands the runner: the step it measures, and `dispose`, which it calls once after the
25
+ // measured runs, even when a run throws.
26
+ export interface PreparedBench {
27
+ readonly step: () => Promise<void> | void;
28
+
29
+ readonly dispose: () => Promise<void>;
30
+ }
31
+
32
+ export interface BenchDefinition {
33
+ readonly name: string;
34
+
35
+ readonly budgetMs: number;
36
+
37
+ // Runs `setup` and returns the step to measure, which holds the setup result, and its disposal.
38
+ readonly prepare: () => Promise<PreparedBench>;
39
+ }
40
+
41
+ // A bench file default-exports the result, which `rig bench` imports and measures.
42
+ export function defineBench<Context>(options: BenchOptions<Context>): BenchDefinition {
43
+ const budgetMs = parseDuration(options.budget);
44
+
45
+ if (budgetMs === null) {
46
+ throw new Error(
47
+ `defineBench: the benchmark "${options.name}" has the budget "${options.budget}", which is no duration such as '50ms' or '2s'`,
48
+ );
49
+ }
50
+
51
+ return {
52
+ name: options.name,
53
+ budgetMs,
54
+ prepare: async () => {
55
+ if (options.setup === undefined) {
56
+ return { step: () => options.run(), dispose: () => Promise.resolve() };
57
+ }
58
+
59
+ const context = await options.setup();
60
+
61
+ return { step: () => options.run(context), dispose: () => runContextDispose(context) };
62
+ },
63
+ };
64
+ }
65
+
66
+ // A setup result that holds a resource, such as a handle to a program that setup started, is
67
+ // disposed of through its `Symbol.asyncDispose`, so the program stops.
68
+ async function runContextDispose(context: unknown): Promise<void> {
69
+ if ((typeof context !== 'object' && typeof context !== 'function') || context === null) {
70
+ return;
71
+ }
72
+
73
+ const dispose: unknown = Reflect.get(context, Symbol.asyncDispose);
74
+
75
+ if (typeof dispose === 'function') {
76
+ await Reflect.apply(dispose, context, []);
77
+ }
78
+ }
@@ -0,0 +1,11 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ import type { BenchmarkResult } from './run-benchmark.ts';
3
+
4
+ export function formatBenchmarkResult(
5
+ definition: BenchDefinition,
6
+ result: BenchmarkResult,
7
+ ): string {
8
+ const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
9
+
10
+ return `bench ${definition.name}: median ${result.medianMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
11
+ }
@@ -0,0 +1,57 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+
3
+ export type ParsedBenchDefinition =
4
+ | { readonly kind: 'parsed'; readonly definition: BenchDefinition }
5
+ | { readonly kind: 'refused'; readonly reason: string };
6
+
7
+ // A bench file is outside input, so the runner checks that its default export is a `defineBench`
8
+ // result instead of trusting its type.
9
+ export function parseBenchDefinition(value: unknown): ParsedBenchDefinition {
10
+ if (typeof value !== 'object' || value === null) {
11
+ return { kind: 'refused', reason: 'the default export is no defineBench result' };
12
+ }
13
+
14
+ const name: unknown = Reflect.get(value, 'name');
15
+ const budgetMs: unknown = Reflect.get(value, 'budgetMs');
16
+ const prepare: unknown = Reflect.get(value, 'prepare');
17
+
18
+ if (typeof name !== 'string' || typeof budgetMs !== 'number' || !isFunction(prepare)) {
19
+ return { kind: 'refused', reason: 'the default export is no defineBench result' };
20
+ }
21
+
22
+ return {
23
+ kind: 'parsed',
24
+ definition: {
25
+ name,
26
+ budgetMs,
27
+ prepare: async () => {
28
+ const prepared: unknown = await prepare();
29
+
30
+ const step: unknown =
31
+ typeof prepared === 'object' && prepared !== null ? Reflect.get(prepared, 'step') : null;
32
+
33
+ const dispose: unknown =
34
+ typeof prepared === 'object' && prepared !== null
35
+ ? Reflect.get(prepared, 'dispose')
36
+ : null;
37
+
38
+ if (!isFunction(step) || !isFunction(dispose)) {
39
+ throw new Error(`the benchmark "${name}" prepared no step to measure`);
40
+ }
41
+
42
+ return {
43
+ step: async () => {
44
+ await step();
45
+ },
46
+ dispose: async () => {
47
+ await dispose();
48
+ },
49
+ };
50
+ },
51
+ },
52
+ };
53
+ }
54
+
55
+ function isFunction(value: unknown): value is (...args: unknown[]) => unknown {
56
+ return typeof value === 'function';
57
+ }
@@ -0,0 +1,10 @@
1
+ export function pickMedian(samples: readonly number[]): number {
2
+ const sorted = samples.toSorted((left, right) => left - right);
3
+ const middle = Math.floor(sorted.length / 2);
4
+
5
+ if (sorted.length % 2 === 1) {
6
+ return sorted[middle] ?? 0;
7
+ }
8
+
9
+ return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
10
+ }
@@ -0,0 +1,38 @@
1
+ import type { BenchDefinition } from './define-bench.ts';
2
+ import { pickMedian } from './pick-median.ts';
3
+
4
+ export const warmUpRuns = 5;
5
+ export const measuredRuns = 20;
6
+
7
+ export interface BenchmarkResult {
8
+ readonly medianMs: number;
9
+ }
10
+
11
+ // `prepare` runs once, then `run` warms up before the measured runs, and the median of those is the
12
+ // result. The setup result is disposed of after the measured runs, even when a run throws.
13
+ export async function runBenchmark(
14
+ definition: BenchDefinition,
15
+ now: () => number,
16
+ ): Promise<BenchmarkResult> {
17
+ const prepared = await definition.prepare();
18
+
19
+ try {
20
+ for (let index = 0; index < warmUpRuns; index += 1) {
21
+ await prepared.step();
22
+ }
23
+
24
+ const samples: number[] = [];
25
+
26
+ for (let index = 0; index < measuredRuns; index += 1) {
27
+ const start = now();
28
+
29
+ await prepared.step();
30
+
31
+ samples.push(now() - start);
32
+ }
33
+
34
+ return { medianMs: pickMedian(samples) };
35
+ } finally {
36
+ await prepared.dispose();
37
+ }
38
+ }