@zgeoff/rig 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/bench/define-bench.d.ts +24 -0
- package/dist/bench/format-benchmark-result.d.ts +3 -0
- package/dist/bench/parse-bench-definition.d.ts +9 -0
- package/dist/bench/pick-median.d.ts +1 -0
- package/dist/bench/run-benchmark.d.ts +7 -0
- package/dist/bench.d.ts +2 -0
- package/dist/browser/collect-descendant-pids.d.ts +2 -0
- package/dist/browser/pick-descendant-stop-step.d.ts +15 -0
- package/dist/browser/pick-listing-read-step.d.ts +10 -0
- package/dist/browser/probe-process.d.ts +1 -1
- package/dist/browser/read-process-table.d.ts +1 -0
- package/dist/browser/stop-descendant-processes.d.ts +10 -0
- package/dist/build-init-scripts.d.ts +1 -1
- package/dist/build-program-environment.d.ts +3 -0
- package/dist/build-program-lane-hooks.d.ts +3 -0
- package/dist/build-test-selection.d.ts +1 -0
- package/dist/collect-bench-files.d.ts +1 -0
- package/dist/collect-eval-files.d.ts +1 -0
- package/dist/collect-unprogrammed-e2e-dirs.d.ts +2 -0
- package/dist/e2e.d.ts +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +83 -3
- package/dist/is-command-name.d.ts +1 -1
- package/dist/pick-e2e-dirs.d.ts +1 -1
- package/dist/probe-workspace-repo.d.ts +1 -0
- package/dist/programs/resolve-program.d.ts +12 -0
- package/dist/programs/resolve-repo-root.d.ts +1 -0
- package/dist/read-cli-args.d.ts +1 -0
- package/dist/run-bench-file.d.ts +1 -0
- package/dist/run-bench.d.ts +10 -0
- package/dist/run-cli.d.ts +3 -1
- package/dist/run-eval.d.ts +8 -0
- package/dist/run-init.d.ts +1 -0
- package/dist/run-lane.d.ts +4 -5
- package/dist/run-program-builds.d.ts +7 -0
- package/dist/run-unit-tests.d.ts +1 -0
- package/dist/test-run-args.d.ts +1 -0
- package/dist/types.d.ts +13 -0
- package/package.json +17 -5
- package/src/bench/define-bench.ts +78 -0
- package/src/bench/format-benchmark-result.ts +11 -0
- package/src/bench/parse-bench-definition.ts +57 -0
- package/src/bench/pick-median.ts +10 -0
- package/src/bench/run-benchmark.ts +38 -0
- package/src/bench.ts +2 -0
- package/src/browser/browser-preload.ts +18 -4
- package/src/browser/collect-descendant-pids.ts +22 -0
- package/src/browser/make-browser-lane-hooks.ts +2 -3
- package/src/browser/pick-descendant-stop-step.ts +40 -0
- package/src/browser/pick-listing-read-step.ts +17 -0
- package/src/browser/probe-host-process.ts +1 -3
- package/src/browser/probe-process.ts +1 -1
- package/src/browser/read-process-table.ts +10 -6
- package/src/browser/stop-descendant-processes.ts +90 -0
- package/src/build-init-scripts.ts +17 -3
- package/src/build-program-environment.ts +20 -0
- package/src/build-program-lane-hooks.ts +24 -0
- package/src/build-test-selection.ts +8 -7
- package/src/build-unit-command.ts +2 -1
- package/src/cli.ts +4 -0
- package/src/collect-bench-files.ts +23 -0
- package/src/collect-eval-files.ts +5 -0
- package/src/collect-unprogrammed-e2e-dirs.ts +21 -0
- package/src/define-rig.ts +65 -2
- package/src/e2e.ts +2 -0
- package/src/index.ts +2 -0
- package/src/is-command-name.ts +1 -1
- package/src/make-repo-lane-hooks.ts +2 -0
- package/src/parse-rig-config.ts +64 -2
- package/src/pick-e2e-dirs.ts +8 -5
- package/src/probe-workspace-repo.ts +15 -0
- package/src/programs/resolve-program.ts +92 -0
- package/src/programs/resolve-repo-root.ts +18 -0
- package/src/read-cli-args.ts +30 -1
- package/src/run-bench-file.ts +25 -0
- package/src/run-bench.ts +132 -0
- package/src/run-cli.ts +16 -1
- package/src/run-eval.ts +74 -0
- package/src/run-init.ts +6 -1
- package/src/run-lane.ts +39 -10
- package/src/run-program-builds.ts +35 -0
- package/src/run-unit-tests.ts +4 -1
- package/src/run-unit.ts +1 -0
- package/src/standard-lanes.ts +1 -1
- package/src/test-run-args.ts +3 -0
- package/src/types.ts +30 -0
package/README.md
CHANGED
|
@@ -39,14 +39,21 @@ so a run that outgrows its memory budget fails alone when `diet` finds a
|
|
|
39
39
|
- `rig related <paths...>` runs the unit tests beside the changed files, for a pre-push hook.
|
|
40
40
|
- `rig <lane>` runs one lane: `e2e`, `browser`, `docker`, `host`, `agent-sdk`, `model-api`, or a
|
|
41
41
|
lane the repo declares.
|
|
42
|
-
- `rig
|
|
42
|
+
- `rig bench [name]` and `rig eval [name]` run the benchmarks and the evaluation suites, or the one
|
|
43
|
+
named `name`.
|
|
44
|
+
- `rig init` writes the `test`, `test:related`, `test:<lane>`, `bench` and `eval` scripts the repo
|
|
45
|
+
has files for, and a starter config.
|
|
43
46
|
|
|
44
47
|
`rig unit` and the lane commands pass every argument after the command on to `bun test`, so
|
|
45
|
-
`bun run test --seed=1` replays a run's order, and rig drops one `--` among them. The
|
|
46
|
-
`--timeout
|
|
47
|
-
|
|
48
|
+
`bun run test --seed=1` replays a run's order, and rig drops one `--` among them. The exceptions are
|
|
49
|
+
`--timeout` and `--isolate`. The lane's limit in `rig.config.ts` is its one per-test timeout, and
|
|
50
|
+
every lane runs its files in random order with `--no-isolate`, so rig exits 2 on a `--timeout`,
|
|
51
|
+
`--timeout=<ms>` or `--isolate` argument and runs nothing. `rig related` takes changed file paths
|
|
48
52
|
only, and exits 2 on an option.
|
|
49
53
|
|
|
54
|
+
An `e2e/` folder is the e2e lane's in a folder that holds a `package.json`, and at the repo root
|
|
55
|
+
only when the root `package.json` has no `workspaces` field.
|
|
56
|
+
|
|
50
57
|
Among those arguments, a path keeps only the lane's own files that it selects: a file, the files
|
|
51
58
|
under a folder, or, for a bare word, the files whose path holds it. rig resolves a path, with its
|
|
52
59
|
`.` and `..` segments, against the repo root, and a path outside the repo selects nothing. When the
|
|
@@ -70,6 +77,11 @@ export default defineRig({
|
|
|
70
77
|
Oxlint's plugin reads this file under Node 22.18 or later, or under Bun, because it imports the
|
|
71
78
|
config to take the limits from it.
|
|
72
79
|
|
|
80
|
+
A repo declares the programs its `e2e/` folders and benchmarks run in `programs`, which `rig e2e`
|
|
81
|
+
and `rig bench` build first; `resolveProgram` gives each one's command. `rig bench` and `rig eval`
|
|
82
|
+
run the benchmarks and evaluation suites. [Programs in e2e tests](../../docs/guides/e2e-programs.md)
|
|
83
|
+
and [benchmarks and evaluations](../../docs/guides/benchmarks-and-evals.md) describe each.
|
|
84
|
+
|
|
73
85
|
## Testing in a page
|
|
74
86
|
|
|
75
87
|
A `.browser.test.ts` suite runs in Chrome through `Bun.WebView`. `runInPage` runs a real function in
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
interface BenchBaseOptions {
|
|
2
|
+
readonly name: string;
|
|
3
|
+
readonly budget: string;
|
|
4
|
+
}
|
|
5
|
+
interface BenchSetupOptions<Context> extends BenchBaseOptions {
|
|
6
|
+
readonly setup: () => Promise<Context>;
|
|
7
|
+
readonly run: (context: Context) => Promise<void> | void;
|
|
8
|
+
}
|
|
9
|
+
interface BenchPlainOptions extends BenchBaseOptions {
|
|
10
|
+
readonly setup?: undefined;
|
|
11
|
+
readonly run: () => Promise<void> | void;
|
|
12
|
+
}
|
|
13
|
+
export type BenchOptions<Context> = BenchSetupOptions<Context> | BenchPlainOptions;
|
|
14
|
+
export interface PreparedBench {
|
|
15
|
+
readonly step: () => Promise<void> | void;
|
|
16
|
+
readonly dispose: () => Promise<void>;
|
|
17
|
+
}
|
|
18
|
+
export interface BenchDefinition {
|
|
19
|
+
readonly name: string;
|
|
20
|
+
readonly budgetMs: number;
|
|
21
|
+
readonly prepare: () => Promise<PreparedBench>;
|
|
22
|
+
}
|
|
23
|
+
export declare function defineBench<Context>(options: BenchOptions<Context>): BenchDefinition;
|
|
24
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { BenchDefinition } from './define-bench.ts';
|
|
2
|
+
export type ParsedBenchDefinition = {
|
|
3
|
+
readonly kind: 'parsed';
|
|
4
|
+
readonly definition: BenchDefinition;
|
|
5
|
+
} | {
|
|
6
|
+
readonly kind: 'refused';
|
|
7
|
+
readonly reason: string;
|
|
8
|
+
};
|
|
9
|
+
export declare function parseBenchDefinition(value: unknown): ParsedBenchDefinition;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function pickMedian(samples: readonly number[]): number;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { BenchDefinition } from './define-bench.ts';
|
|
2
|
+
export declare const warmUpRuns = 5;
|
|
3
|
+
export declare const measuredRuns = 20;
|
|
4
|
+
export interface BenchmarkResult {
|
|
5
|
+
readonly medianMs: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function runBenchmark(definition: BenchDefinition, now: () => number): Promise<BenchmarkResult>;
|
package/dist/bench.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ProcessEntry } from './read-process-table.ts';
|
|
2
|
+
export type DescendantStopStep = {
|
|
3
|
+
readonly kind: 'stopped';
|
|
4
|
+
} | {
|
|
5
|
+
readonly kind: 'wait';
|
|
6
|
+
} | {
|
|
7
|
+
readonly kind: 'failed';
|
|
8
|
+
readonly running: readonly number[];
|
|
9
|
+
};
|
|
10
|
+
export interface DescendantStopState {
|
|
11
|
+
readonly descendants: readonly number[];
|
|
12
|
+
readonly listing: readonly ProcessEntry[] | null;
|
|
13
|
+
readonly isPastDeadline: boolean;
|
|
14
|
+
}
|
|
15
|
+
export declare function pickDescendantStopStep(state: DescendantStopState): DescendantStopStep;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ProcessEntry } from './read-process-table.ts';
|
|
2
|
+
export type ListingReadStep = {
|
|
3
|
+
readonly kind: 'listed';
|
|
4
|
+
readonly listing: readonly ProcessEntry[];
|
|
5
|
+
} | {
|
|
6
|
+
readonly kind: 'retry';
|
|
7
|
+
} | {
|
|
8
|
+
readonly kind: 'failed';
|
|
9
|
+
};
|
|
10
|
+
export declare function pickListingReadStep(listing: readonly ProcessEntry[] | null, isPastDeadline: boolean): ListingReadStep;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ProcessEntry } from './read-process-table.ts';
|
|
2
2
|
export interface ProcessProbeDependencies {
|
|
3
|
-
readonly signalProcess: (pid: number, signal: number) =>
|
|
3
|
+
readonly signalProcess: (pid: number, signal: number) => boolean;
|
|
4
4
|
readonly readProcessTable: () => readonly ProcessEntry[] | null;
|
|
5
5
|
}
|
|
6
6
|
export declare function probeProcess(pid: number, dependencies: ProcessProbeDependencies): boolean;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Clock } from '../system-clock.ts';
|
|
2
|
+
import type { ProcessEntry } from './read-process-table.ts';
|
|
3
|
+
export interface DescendantStop {
|
|
4
|
+
readonly rootPID: number;
|
|
5
|
+
readonly timeoutMs: number;
|
|
6
|
+
readonly clock: Clock;
|
|
7
|
+
readonly readProcessTable: () => readonly ProcessEntry[] | null;
|
|
8
|
+
readonly signalProcess: (pid: number, signal: NodeJS.Signals) => boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function stopDescendantProcesses(stop: DescendantStop): Promise<void>;
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { RigConfig } from './types.ts';
|
|
2
2
|
export type PackageScripts = Readonly<Record<string, string>>;
|
|
3
|
-
export declare function buildInitScripts(oldScripts: PackageScripts, config: RigConfig, trackedFiles: readonly string[]): PackageScripts;
|
|
3
|
+
export declare function buildInitScripts(oldScripts: PackageScripts, config: RigConfig, trackedFiles: readonly string[], isWorkspaceRepo: boolean): PackageScripts;
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { ProgramDeclaration } from './types.ts';
|
|
2
|
+
export declare function buildProgramVariable(name: string): string;
|
|
3
|
+
export declare function buildProgramEnvironment(repoRoot: string, programs: Readonly<Record<string, ProgramDeclaration>>): Readonly<Record<string, string>>;
|
|
@@ -17,6 +17,7 @@ export interface TestSelectionRequest {
|
|
|
17
17
|
readonly config: RigConfig;
|
|
18
18
|
readonly lane: Pick<ResolvedLane, 'name' | 'suffix'>;
|
|
19
19
|
readonly repoRoot: string;
|
|
20
|
+
readonly isWorkspaceRepo: boolean;
|
|
20
21
|
readonly trackedFiles: readonly string[];
|
|
21
22
|
readonly args: TestArgs;
|
|
22
23
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function collectBenchFiles(trackedFiles: readonly string[], isWorkspaceRepo: boolean): readonly string[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function collectEvalFiles(trackedFiles: readonly string[]): readonly string[];
|
package/dist/e2e.d.ts
ADDED
package/dist/index.d.ts
CHANGED
|
@@ -3,4 +3,4 @@ export { parseRigConfig } from './parse-rig-config.ts';
|
|
|
3
3
|
export type { ParsedRigConfig } from './parse-rig-config.ts';
|
|
4
4
|
export { runInPage } from './browser/run-in-page.ts';
|
|
5
5
|
export type { PageResultSchema } from './browser/run-in-page.ts';
|
|
6
|
-
export type { LaneContext, LaneDeclaration, LaneStart, ResolvedLane, RigConfig, RigDeclaration, } from './types.ts';
|
|
6
|
+
export type { LaneContext, BenchDeclaration, LaneDeclaration, ProgramDeclaration, LaneStart, ResolvedLane, RigConfig, RigDeclaration, } from './types.ts';
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/is-command-name.ts
|
|
2
|
-
var commandNames = ["init", "related"];
|
|
2
|
+
var commandNames = ["bench", "eval", "init", "related"];
|
|
3
3
|
function isCommandName(name) {
|
|
4
4
|
return commandNames.some((command) => command === name);
|
|
5
5
|
}
|
|
@@ -43,7 +43,11 @@ function defineRig(declaration) {
|
|
|
43
43
|
...declared.filter(([name]) => !standardLanes.some((standard) => standard.name === name)).map(([name, laneDeclaration]) => resolveProjectLane(name, laneDeclaration))
|
|
44
44
|
];
|
|
45
45
|
assertDistinctSuffixes(lanes);
|
|
46
|
-
|
|
46
|
+
const programs = Object.fromEntries(Object.entries(declaration.programs ?? {}).map(([name, program]) => [
|
|
47
|
+
name,
|
|
48
|
+
readProgram(name, program)
|
|
49
|
+
]));
|
|
50
|
+
return { lanes, programs, benchLane: readBenchLane(declaration.bench?.lane, lanes) };
|
|
47
51
|
}
|
|
48
52
|
function resolveStandardLane(standard, declaration) {
|
|
49
53
|
const declared = declaration ?? {};
|
|
@@ -137,6 +141,41 @@ function assertDistinctSuffixes(lanes) {
|
|
|
137
141
|
owners.set(lane.suffix, lane.name);
|
|
138
142
|
}
|
|
139
143
|
}
|
|
144
|
+
var programNamePattern = /^[a-z][a-z0-9-]*$/u;
|
|
145
|
+
var programPathFields = ["package", "source", "artifact"];
|
|
146
|
+
function readProgram(name, program) {
|
|
147
|
+
if (!programNamePattern.test(name)) {
|
|
148
|
+
throw new Error(`rig.config.ts: the program name "${name}" must be lowercase letters, digits and dashes, starting with a letter`);
|
|
149
|
+
}
|
|
150
|
+
for (const field of programPathFields) {
|
|
151
|
+
const path = program[field];
|
|
152
|
+
if (typeof path !== "string" || path === "") {
|
|
153
|
+
throw new Error(`rig.config.ts: program "${name}" needs a ${field}, a non-empty path`);
|
|
154
|
+
}
|
|
155
|
+
if (path.startsWith("/") || path.split("/").includes("..")) {
|
|
156
|
+
throw new Error(`rig.config.ts: program "${name}" has the ${field} "${path}", which must be a relative path with no ".." segment`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const build = program.build;
|
|
160
|
+
if (typeof build !== "string" || build.trim() === "") {
|
|
161
|
+
throw new Error(`rig.config.ts: program "${name}" needs a build, a shell command`);
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
package: program.package,
|
|
165
|
+
source: program.source,
|
|
166
|
+
build: program.build,
|
|
167
|
+
artifact: program.artifact
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
function readBenchLane(lane, lanes) {
|
|
171
|
+
if (lane === undefined) {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
if (!lanes.some((candidate) => candidate.name === lane)) {
|
|
175
|
+
throw new Error(`rig.config.ts: bench.lane "${lane}" names no lane`);
|
|
176
|
+
}
|
|
177
|
+
return lane;
|
|
178
|
+
}
|
|
140
179
|
// src/parse-rig-config.ts
|
|
141
180
|
function parseRigConfig(value) {
|
|
142
181
|
if (typeof value !== "object" || value === null || !("lanes" in value)) {
|
|
@@ -154,7 +193,18 @@ function parseRigConfig(value) {
|
|
|
154
193
|
}
|
|
155
194
|
resolved.push(parsed.lane);
|
|
156
195
|
}
|
|
157
|
-
|
|
196
|
+
const programs = parsePrograms(Reflect.get(value, "programs"));
|
|
197
|
+
if (programs === null) {
|
|
198
|
+
return { kind: "refused", reason: "its programs are no record of program declarations" };
|
|
199
|
+
}
|
|
200
|
+
const benchLane = Reflect.get(value, "benchLane");
|
|
201
|
+
if (benchLane !== undefined && !isNullableString(benchLane)) {
|
|
202
|
+
return { kind: "refused", reason: `its benchLane is ${JSON.stringify(benchLane)}` };
|
|
203
|
+
}
|
|
204
|
+
return {
|
|
205
|
+
kind: "parsed",
|
|
206
|
+
config: { lanes: resolved, programs, benchLane: benchLane ?? null }
|
|
207
|
+
};
|
|
158
208
|
}
|
|
159
209
|
function parseResolvedLane(value, index) {
|
|
160
210
|
if (typeof value !== "object" || value === null) {
|
|
@@ -202,6 +252,36 @@ function refuseField(lane, field, value) {
|
|
|
202
252
|
function isParallelism(value) {
|
|
203
253
|
return value === "workers" || typeof value === "number" && Number.isInteger(value) && value > 0;
|
|
204
254
|
}
|
|
255
|
+
function parsePrograms(value) {
|
|
256
|
+
if (value === undefined) {
|
|
257
|
+
return {};
|
|
258
|
+
}
|
|
259
|
+
if (typeof value !== "object" || value === null) {
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
const programs = {};
|
|
263
|
+
for (const [name, program] of Object.entries(value)) {
|
|
264
|
+
const parsed = parseProgram(program);
|
|
265
|
+
if (parsed === null) {
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
programs[name] = parsed;
|
|
269
|
+
}
|
|
270
|
+
return programs;
|
|
271
|
+
}
|
|
272
|
+
function parseProgram(value) {
|
|
273
|
+
if (typeof value !== "object" || value === null) {
|
|
274
|
+
return null;
|
|
275
|
+
}
|
|
276
|
+
const packageFolder = Reflect.get(value, "package");
|
|
277
|
+
const source = Reflect.get(value, "source");
|
|
278
|
+
const build = Reflect.get(value, "build");
|
|
279
|
+
const artifact = Reflect.get(value, "artifact");
|
|
280
|
+
if (typeof packageFolder !== "string" || typeof source !== "string" || typeof build !== "string" || typeof artifact !== "string") {
|
|
281
|
+
return null;
|
|
282
|
+
}
|
|
283
|
+
return { package: packageFolder, source, build, artifact };
|
|
284
|
+
}
|
|
205
285
|
// src/browser/run-in-page.ts
|
|
206
286
|
async function runInPage(view, pageFunction, argument, schema) {
|
|
207
287
|
const source = pageFunction.toString();
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export declare const commandNames: readonly ['init', 'related'];
|
|
1
|
+
export declare const commandNames: readonly ['bench', 'eval', 'init', 'related'];
|
|
2
2
|
export type CommandName = (typeof commandNames)[number];
|
|
3
3
|
export declare function isCommandName(name: string): name is CommandName;
|
package/dist/pick-e2e-dirs.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare function pickE2EDirs(trackedFiles: readonly string[]): readonly string[];
|
|
1
|
+
export declare function pickE2EDirs(trackedFiles: readonly string[], isWorkspaceRepo: boolean): readonly string[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function probeWorkspaceRepo(repoRoot: string): Promise<boolean>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface ProgramLaunch {
|
|
2
|
+
readonly command: readonly string[];
|
|
3
|
+
readonly cwd: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ProgramLaunchSources {
|
|
6
|
+
readonly environment: Readonly<Record<string, string | undefined>>;
|
|
7
|
+
readonly cwd: string;
|
|
8
|
+
readonly importModule: (path: string) => Promise<unknown>;
|
|
9
|
+
readonly probeFile: (path: string) => boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function readLaunchSources(): ProgramLaunchSources;
|
|
12
|
+
export declare function resolveProgram(name: string, sources?: ProgramLaunchSources): Promise<ProgramLaunch>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function resolveRepoRoot(cwd: string, probeFile: (path: string) => boolean): string;
|
package/dist/read-cli-args.d.ts
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ProgramBuildDeps } from './run-program-builds.ts';
|
|
2
|
+
import type { RigConfig, RigRun } from './types.ts';
|
|
3
|
+
export interface BenchDeps extends ProgramBuildDeps {
|
|
4
|
+
readonly dietScript: string;
|
|
5
|
+
readonly benchScript: string;
|
|
6
|
+
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
7
|
+
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
8
|
+
readonly importModule: (path: string) => Promise<unknown>;
|
|
9
|
+
}
|
|
10
|
+
export declare function runBench(args: readonly string[], config: RigConfig, run: RigRun, deps: BenchDeps): Promise<number>;
|
package/dist/run-cli.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import type { BenchDeps } from './run-bench.ts';
|
|
2
|
+
import type { EvalDeps } from './run-eval.ts';
|
|
1
3
|
import type { InitDeps } from './run-init.ts';
|
|
2
4
|
import type { LaneDeps } from './run-lane.ts';
|
|
3
5
|
import type { RelatedTestDeps } from './run-related-tests.ts';
|
|
4
6
|
import type { UnitDeps } from './run-unit.ts';
|
|
5
7
|
import type { RigConfig, RigRun } from './types.ts';
|
|
6
|
-
export interface CLIDeps extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps {
|
|
8
|
+
export interface CLIDeps extends LaneDeps, UnitDeps, RelatedTestDeps, InitDeps, BenchDeps, EvalDeps {
|
|
7
9
|
readonly readConfig: (repoRoot: string) => Promise<RigConfig>;
|
|
8
10
|
}
|
|
9
11
|
export declare function runCLI(argv: readonly string[], run: RigRun, deps: CLIDeps): Promise<number>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { CommandOptions } from './run-command.ts';
|
|
2
|
+
import type { RigOutput, RigRun } from './types.ts';
|
|
3
|
+
export interface EvalDeps {
|
|
4
|
+
readonly output: RigOutput;
|
|
5
|
+
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
6
|
+
readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
|
|
7
|
+
}
|
|
8
|
+
export declare function runEval(args: readonly string[], run: RigRun, deps: EvalDeps): Promise<number>;
|
package/dist/run-init.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { InitFiles } from './init-files.ts';
|
|
2
2
|
import type { RigConfig, RigOutput } from './types.ts';
|
|
3
3
|
export interface InitDeps {
|
|
4
|
+
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
4
5
|
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
5
6
|
readonly initFiles: InitFiles;
|
|
6
7
|
readonly output: RigOutput;
|
package/dist/run-lane.d.ts
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { BrowserLaneWiring } from './browser/wire-browser-lane-hooks.ts';
|
|
2
2
|
import type { LaneHooks } from './lane-hooks.ts';
|
|
3
|
-
import type {
|
|
4
|
-
import type { ResolvedLane, RigConfig,
|
|
5
|
-
export interface LaneDeps {
|
|
3
|
+
import type { ProgramBuildDeps } from './run-program-builds.ts';
|
|
4
|
+
import type { ResolvedLane, RigConfig, RigRun } from './types.ts';
|
|
5
|
+
export interface LaneDeps extends ProgramBuildDeps {
|
|
6
6
|
readonly dietScript: string;
|
|
7
|
-
readonly
|
|
7
|
+
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
8
8
|
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
9
9
|
readonly importModule: (path: string) => Promise<unknown>;
|
|
10
|
-
readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
|
|
11
10
|
readonly wireBrowserLaneHooks: (wiring: BrowserLaneWiring) => Promise<LaneHooks>;
|
|
12
11
|
}
|
|
13
12
|
export interface LaneRequest {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CommandOptions } from './run-command.ts';
|
|
2
|
+
import type { ProgramDeclaration, RigOutput, RigRun } from './types.ts';
|
|
3
|
+
export interface ProgramBuildDeps {
|
|
4
|
+
readonly output: RigOutput;
|
|
5
|
+
readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
|
|
6
|
+
}
|
|
7
|
+
export declare function runProgramBuilds(programs: Readonly<Record<string, ProgramDeclaration>>, run: RigRun, deps: ProgramBuildDeps): Promise<number>;
|
package/dist/run-unit-tests.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export interface UnitRequest {
|
|
|
6
6
|
}
|
|
7
7
|
export interface UnitRunDeps {
|
|
8
8
|
readonly dietScript: string;
|
|
9
|
+
readonly probeWorkspaceRepo: (repoRoot: string) => Promise<boolean>;
|
|
9
10
|
readonly readTrackedFiles: (repoRoot: string) => Promise<readonly string[]>;
|
|
10
11
|
readonly runCommand: (command: readonly string[], options: CommandOptions) => Promise<number>;
|
|
11
12
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const testRunArgs: readonly string[];
|
package/dist/types.d.ts
CHANGED
|
@@ -5,8 +5,19 @@ export interface LaneDeclaration {
|
|
|
5
5
|
readonly start?: string;
|
|
6
6
|
readonly stop?: string;
|
|
7
7
|
}
|
|
8
|
+
export interface ProgramDeclaration {
|
|
9
|
+
readonly package: string;
|
|
10
|
+
readonly source: string;
|
|
11
|
+
readonly build: string;
|
|
12
|
+
readonly artifact: string;
|
|
13
|
+
}
|
|
14
|
+
export interface BenchDeclaration {
|
|
15
|
+
readonly lane?: string;
|
|
16
|
+
}
|
|
8
17
|
export interface RigDeclaration {
|
|
9
18
|
readonly lanes?: Readonly<Record<string, LaneDeclaration>>;
|
|
19
|
+
readonly programs?: Readonly<Record<string, ProgramDeclaration>>;
|
|
20
|
+
readonly bench?: BenchDeclaration;
|
|
10
21
|
}
|
|
11
22
|
export type LaneParallelism = 'workers' | number;
|
|
12
23
|
export interface ResolvedLane {
|
|
@@ -20,6 +31,8 @@ export interface ResolvedLane {
|
|
|
20
31
|
}
|
|
21
32
|
export interface RigConfig {
|
|
22
33
|
readonly lanes: readonly ResolvedLane[];
|
|
34
|
+
readonly programs: Readonly<Record<string, ProgramDeclaration>>;
|
|
35
|
+
readonly benchLane: string | null;
|
|
23
36
|
}
|
|
24
37
|
export interface LaneContext {
|
|
25
38
|
readonly repoRoot: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zgeoff/rig",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "Run every test lane of a repo from one declaration of its limits, with a browser lane built in",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bun",
|
|
@@ -19,7 +19,8 @@
|
|
|
19
19
|
"files": [
|
|
20
20
|
"src",
|
|
21
21
|
"!src/**/*.test.ts",
|
|
22
|
-
"dist"
|
|
22
|
+
"dist",
|
|
23
|
+
"!src/test-utils"
|
|
23
24
|
],
|
|
24
25
|
"type": "module",
|
|
25
26
|
"exports": {
|
|
@@ -27,7 +28,16 @@
|
|
|
27
28
|
"types": "./dist/index.d.ts",
|
|
28
29
|
"bun": "./src/index.ts",
|
|
29
30
|
"default": "./dist/index.js"
|
|
30
|
-
}
|
|
31
|
+
},
|
|
32
|
+
"./e2e": {
|
|
33
|
+
"types": "./dist/e2e.d.ts",
|
|
34
|
+
"bun": "./src/e2e.ts"
|
|
35
|
+
},
|
|
36
|
+
"./bench": {
|
|
37
|
+
"types": "./dist/bench.d.ts",
|
|
38
|
+
"bun": "./src/bench.ts"
|
|
39
|
+
},
|
|
40
|
+
"./test-utils/*": "./src/test-utils/*.ts"
|
|
31
41
|
},
|
|
32
42
|
"publishConfig": {
|
|
33
43
|
"access": "public"
|
|
@@ -37,13 +47,15 @@
|
|
|
37
47
|
"typecheck": "tsrs"
|
|
38
48
|
},
|
|
39
49
|
"dependencies": {
|
|
40
|
-
"@zgeoff/diet": "0.
|
|
41
|
-
"@zgeoff/std": "0.
|
|
50
|
+
"@zgeoff/diet": "0.2.0",
|
|
51
|
+
"@zgeoff/std": "0.4.0",
|
|
42
52
|
"gunshi": "1.0.0-rc.1",
|
|
43
53
|
"ts-pattern": "5.9.2"
|
|
44
54
|
},
|
|
45
55
|
"devDependencies": {
|
|
46
56
|
"@types/bun": "1.4.4",
|
|
57
|
+
"@zgeoff/std": "0.4.0",
|
|
58
|
+
"@zgeoff/testing": "0.2.0",
|
|
47
59
|
"tiny-invariant": "1.3.3",
|
|
48
60
|
"tsrs": "0.18.0-ts7.1.0-dev.20260929",
|
|
49
61
|
"typescript": "7.0.2",
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { parseDuration } from '../parse-duration.ts';
|
|
2
|
+
|
|
3
|
+
interface BenchBaseOptions {
|
|
4
|
+
readonly name: string;
|
|
5
|
+
|
|
6
|
+
// A duration such as `'50ms'` or `'2s'` that the median of a measured run must not pass.
|
|
7
|
+
readonly budget: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface BenchSetupOptions<Context> extends BenchBaseOptions {
|
|
11
|
+
readonly setup: () => Promise<Context>;
|
|
12
|
+
|
|
13
|
+
readonly run: (context: Context) => Promise<void> | void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface BenchPlainOptions extends BenchBaseOptions {
|
|
17
|
+
readonly setup?: undefined;
|
|
18
|
+
|
|
19
|
+
readonly run: () => Promise<void> | void;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export type BenchOptions<Context> = BenchSetupOptions<Context> | BenchPlainOptions;
|
|
23
|
+
|
|
24
|
+
// What `prepare` hands the runner: the step it measures, and `dispose`, which it calls once after the
|
|
25
|
+
// measured runs, even when a run throws.
|
|
26
|
+
export interface PreparedBench {
|
|
27
|
+
readonly step: () => Promise<void> | void;
|
|
28
|
+
|
|
29
|
+
readonly dispose: () => Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface BenchDefinition {
|
|
33
|
+
readonly name: string;
|
|
34
|
+
|
|
35
|
+
readonly budgetMs: number;
|
|
36
|
+
|
|
37
|
+
// Runs `setup` and returns the step to measure, which holds the setup result, and its disposal.
|
|
38
|
+
readonly prepare: () => Promise<PreparedBench>;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A bench file default-exports the result, which `rig bench` imports and measures.
|
|
42
|
+
export function defineBench<Context>(options: BenchOptions<Context>): BenchDefinition {
|
|
43
|
+
const budgetMs = parseDuration(options.budget);
|
|
44
|
+
|
|
45
|
+
if (budgetMs === null) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
`defineBench: the benchmark "${options.name}" has the budget "${options.budget}", which is no duration such as '50ms' or '2s'`,
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
name: options.name,
|
|
53
|
+
budgetMs,
|
|
54
|
+
prepare: async () => {
|
|
55
|
+
if (options.setup === undefined) {
|
|
56
|
+
return { step: () => options.run(), dispose: () => Promise.resolve() };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const context = await options.setup();
|
|
60
|
+
|
|
61
|
+
return { step: () => options.run(context), dispose: () => runContextDispose(context) };
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// A setup result that holds a resource, such as a handle to a program that setup started, is
|
|
67
|
+
// disposed of through its `Symbol.asyncDispose`, so the program stops.
|
|
68
|
+
async function runContextDispose(context: unknown): Promise<void> {
|
|
69
|
+
if ((typeof context !== 'object' && typeof context !== 'function') || context === null) {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const dispose: unknown = Reflect.get(context, Symbol.asyncDispose);
|
|
74
|
+
|
|
75
|
+
if (typeof dispose === 'function') {
|
|
76
|
+
await Reflect.apply(dispose, context, []);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { BenchDefinition } from './define-bench.ts';
|
|
2
|
+
import type { BenchmarkResult } from './run-benchmark.ts';
|
|
3
|
+
|
|
4
|
+
export function formatBenchmarkResult(
|
|
5
|
+
definition: BenchDefinition,
|
|
6
|
+
result: BenchmarkResult,
|
|
7
|
+
): string {
|
|
8
|
+
const verdict = result.medianMs > definition.budgetMs ? 'over budget' : 'within budget';
|
|
9
|
+
|
|
10
|
+
return `bench ${definition.name}: median ${result.medianMs.toFixed(2)}ms, budget ${definition.budgetMs}ms, ${verdict}\n`;
|
|
11
|
+
}
|