@omni-oss/task-bench 0.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +328 -0
- package/dist/bench/index.d.ts +82 -0
- package/dist/bench/index.d.ts.map +1 -0
- package/dist/bench/install.d.ts +5 -0
- package/dist/bench/install.d.ts.map +1 -0
- package/dist/bench/report.d.ts +9 -0
- package/dist/bench/report.d.ts.map +1 -0
- package/dist/bench/stats.d.ts +12 -0
- package/dist/bench/stats.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/config.d.ts +91 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/generate/index.d.ts +19 -0
- package/dist/generate/index.d.ts.map +1 -0
- package/dist/generate/templates.d.ts +20 -0
- package/dist/generate/templates.d.ts.map +1 -0
- package/dist/graph.d.ts +21 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.mjs +2 -0
- package/dist/src-D3XyMXAu.mjs +1167 -0
- package/dist/suite/index.d.ts +49 -0
- package/dist/suite/index.d.ts.map +1 -0
- package/dist/suite/preset.d.ts +93 -0
- package/dist/suite/preset.d.ts.map +1 -0
- package/dist/suite/report.d.ts +7 -0
- package/dist/suite/report.d.ts.map +1 -0
- package/dist/task-bench-cli.mjs +107 -0
- package/dist/tools/index.d.ts +18 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/moon.d.ts +10 -0
- package/dist/tools/moon.d.ts.map +1 -0
- package/dist/tools/nx.d.ts +7 -0
- package/dist/tools/nx.d.ts.map +1 -0
- package/dist/tools/omni.d.ts +7 -0
- package/dist/tools/omni.d.ts.map +1 -0
- package/dist/tools/turbo.d.ts +5 -0
- package/dist/tools/turbo.d.ts.map +1 -0
- package/dist/tools/types.d.ts +77 -0
- package/dist/tools/types.d.ts.map +1 -0
- package/package.json +41 -0
- package/project.omni.yaml +33 -0
- package/src/bench/index.ts +323 -0
- package/src/bench/install.ts +12 -0
- package/src/bench/report.ts +142 -0
- package/src/bench/stats.spec.ts +35 -0
- package/src/bench/stats.ts +38 -0
- package/src/cli/index.ts +410 -0
- package/src/config.ts +138 -0
- package/src/generate/index.ts +215 -0
- package/src/generate/templates.ts +87 -0
- package/src/graph.spec.ts +119 -0
- package/src/graph.ts +120 -0
- package/src/index.ts +31 -0
- package/src/suite/index.ts +113 -0
- package/src/suite/preset.spec.ts +95 -0
- package/src/suite/preset.ts +253 -0
- package/src/suite/report.ts +135 -0
- package/src/tools/adapters.spec.ts +95 -0
- package/src/tools/config.spec.ts +73 -0
- package/src/tools/index.ts +76 -0
- package/src/tools/moon.ts +106 -0
- package/src/tools/nx.ts +106 -0
- package/src/tools/omni.ts +96 -0
- package/src/tools/turbo.ts +78 -0
- package/src/tools/types.ts +116 -0
- package/tsconfig.json +4 -0
- package/tsconfig.project.json +6 -0
- package/tsconfig.types.json +4 -0
- package/vite.config.ts +29 -0
- package/vitest.config.unit.ts +13 -0
|
@@ -0,0 +1,1167 @@
|
|
|
1
|
+
import { existsSync as e, readFileSync as t, writeFileSync as n } from "node:fs";
|
|
2
|
+
import { cpus as r, tmpdir as i } from "node:os";
|
|
3
|
+
import { dirname as a, join as o } from "node:path";
|
|
4
|
+
import { mkdir as s, readFile as c, rm as l, writeFile as u } from "node:fs/promises";
|
|
5
|
+
import { performance as d } from "node:perf_hooks";
|
|
6
|
+
import { execa as f } from "execa";
|
|
7
|
+
import { z as p } from "zod";
|
|
8
|
+
import { satisfies as m } from "semver";
|
|
9
|
+
import { stringify as h } from "yaml";
|
|
10
|
+
//#region package.json
|
|
11
|
+
var g = "@omni-oss/task-bench", _ = "Generate configurable test workspaces and benchmark omni/turbo/nx task execution performance", v = "0.0.0-beta.1", y = [
|
|
12
|
+
"isolated",
|
|
13
|
+
"chain",
|
|
14
|
+
"fan-out",
|
|
15
|
+
"layered",
|
|
16
|
+
"random"
|
|
17
|
+
], b = p.enum(y), x = [
|
|
18
|
+
"omni",
|
|
19
|
+
"turbo",
|
|
20
|
+
"nx",
|
|
21
|
+
"moon"
|
|
22
|
+
], S = p.enum(x), C = p.object({
|
|
23
|
+
strategy: b.default("layered").describe("Shape of the inter-project dependency graph."),
|
|
24
|
+
layers: p.number().int().positive().default(5).describe("Number of layers for the `layered` strategy."),
|
|
25
|
+
fanout: p.number().int().nonnegative().default(3).describe("Maximum number of upstream dependencies per project (cap for `layered`/`random`)."),
|
|
26
|
+
edgeProbability: p.number().min(0).max(1).default(.35).describe("Edge inclusion probability for the `random` strategy.")
|
|
27
|
+
}).prefault({}), w = p.object({
|
|
28
|
+
logLines: p.number().int().nonnegative().default(25).describe("How many log lines each task prints to stdout."),
|
|
29
|
+
workIterations: p.number().int().nonnegative().default(15e4).describe("Iterations of cheap CPU work per task. Keep small so caching dominates."),
|
|
30
|
+
outputFiles: p.number().int().positive().default(1).describe("Number of output files each task writes into dist/."),
|
|
31
|
+
chainWithinProject: p.boolean().default(!0).describe("Whether task `tN` depends on `t(N-1)` within a project."),
|
|
32
|
+
fanUpstream: p.boolean().default(!0).describe("Whether task `tN` depends on `tN` of upstream projects (^tN).")
|
|
33
|
+
}).prefault({}), T = p.object({
|
|
34
|
+
turbo: p.string().default("2.10.3"),
|
|
35
|
+
nx: p.string().default("23.0.1"),
|
|
36
|
+
moon: p.string().default("2.3.5"),
|
|
37
|
+
bun: p.string().default("1.3.14")
|
|
38
|
+
}).prefault({}), E = p.object({
|
|
39
|
+
seed: p.number().int().nonnegative().default(1).describe("Seed for deterministic graph generation."),
|
|
40
|
+
projectPrefix: p.string().regex(/^[a-z][a-z0-9-]*$/).default("bench-p").describe("Prefix used for generated package names."),
|
|
41
|
+
projects: p.number().int().positive().default(50).describe("Number of projects to generate."),
|
|
42
|
+
tasksPerProject: p.number().int().positive().default(3).describe("Number of tasks (t0..tN-1) per project."),
|
|
43
|
+
dependency: C,
|
|
44
|
+
task: w,
|
|
45
|
+
tools: p.array(S).min(1).default([...x]).describe("Which runners to configure and benchmark."),
|
|
46
|
+
versions: T
|
|
47
|
+
});
|
|
48
|
+
function D(e) {
|
|
49
|
+
return E.parse(e ?? {});
|
|
50
|
+
}
|
|
51
|
+
//#endregion
|
|
52
|
+
//#region src/graph.ts
|
|
53
|
+
function ee(e) {
|
|
54
|
+
let t = e >>> 0;
|
|
55
|
+
return () => {
|
|
56
|
+
t = t + 1831565813 | 0;
|
|
57
|
+
let e = Math.imul(t ^ t >>> 15, 1 | t);
|
|
58
|
+
return e = e + Math.imul(e ^ e >>> 7, 61 | e) ^ e, ((e ^ e >>> 14) >>> 0) / 4294967296;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
function te(e) {
|
|
62
|
+
return Math.max(4, String(e - 1).length);
|
|
63
|
+
}
|
|
64
|
+
function O(e, t) {
|
|
65
|
+
let n = te(e.projects);
|
|
66
|
+
return `${e.projectPrefix}${String(t).padStart(n, "0")}`;
|
|
67
|
+
}
|
|
68
|
+
function ne(e, t, n) {
|
|
69
|
+
let r = t - e + 1;
|
|
70
|
+
if (r <= 0 || n <= 0) return [];
|
|
71
|
+
if (n >= r) return Array.from({ length: r }, (t, n) => e + n);
|
|
72
|
+
let i = [];
|
|
73
|
+
for (let t = 0; t < n; t++) {
|
|
74
|
+
let a = Math.round(t * (r - 1) / (n - 1 || 1));
|
|
75
|
+
i.push(e + a);
|
|
76
|
+
}
|
|
77
|
+
return [...new Set(i)];
|
|
78
|
+
}
|
|
79
|
+
function re(e, t, n) {
|
|
80
|
+
let { strategy: r, layers: i, fanout: a, edgeProbability: o } = e.dependency;
|
|
81
|
+
switch (r) {
|
|
82
|
+
case "isolated": return [];
|
|
83
|
+
case "chain": return t > 0 ? [t - 1] : [];
|
|
84
|
+
case "fan-out": return t > 0 ? [0] : [];
|
|
85
|
+
case "layered": {
|
|
86
|
+
let n = Math.ceil(e.projects / i), r = Math.floor(t / n);
|
|
87
|
+
return r === 0 ? [] : ne((r - 1) * n, Math.min(r * n, e.projects) - 1, a);
|
|
88
|
+
}
|
|
89
|
+
case "random": {
|
|
90
|
+
if (t === 0) return [];
|
|
91
|
+
let e = [];
|
|
92
|
+
for (let r = 0; r < t; r++) n() < o && e.push(r);
|
|
93
|
+
return a > 0 && e.length > a ? e.slice(e.length - a) : e;
|
|
94
|
+
}
|
|
95
|
+
default: return [];
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function k(e) {
|
|
99
|
+
let t = ee(e.seed), n = [];
|
|
100
|
+
for (let r = 0; r < e.projects; r++) {
|
|
101
|
+
let i = O(e, r);
|
|
102
|
+
n.push({
|
|
103
|
+
index: r,
|
|
104
|
+
name: i,
|
|
105
|
+
dir: `packages/${i}`,
|
|
106
|
+
dependencies: re(e, r, t)
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
return n;
|
|
110
|
+
}
|
|
111
|
+
function A(e) {
|
|
112
|
+
return Array.from({ length: e.tasksPerProject }, (e, t) => `t${t}`);
|
|
113
|
+
}
|
|
114
|
+
//#endregion
|
|
115
|
+
//#region src/tools/types.ts
|
|
116
|
+
function j(t, n) {
|
|
117
|
+
let r = o(t, "node_modules", ".bin", n);
|
|
118
|
+
return e(r) ? r : n;
|
|
119
|
+
}
|
|
120
|
+
async function M(e) {
|
|
121
|
+
await Promise.all(e.projectDirs.map((t) => l(o(e.rootDir, t, "dist"), {
|
|
122
|
+
recursive: !0,
|
|
123
|
+
force: !0
|
|
124
|
+
})));
|
|
125
|
+
}
|
|
126
|
+
function N(e, t) {
|
|
127
|
+
let n = [];
|
|
128
|
+
return e.task.chainWithinProject && t > 0 && n.push(`t${t - 1}`), e.task.fanUpstream && n.push(`^t${t}`), n;
|
|
129
|
+
}
|
|
130
|
+
function ie(e, t) {
|
|
131
|
+
return e.dependencies.map((e) => t[e]?.name ?? "");
|
|
132
|
+
}
|
|
133
|
+
//#endregion
|
|
134
|
+
//#region src/tools/moon.ts
|
|
135
|
+
var ae = "https://moonrepo.dev/schemas/workspace.json", oe = "https://moonrepo.dev/schemas/toolchain.json", se = "https://moonrepo.dev/schemas/project.json";
|
|
136
|
+
function ce(e, t) {
|
|
137
|
+
let n = [];
|
|
138
|
+
return e.task.chainWithinProject && t > 0 && n.push(`~:t${t - 1}`), e.task.fanUpstream && n.push(`^:t${t}`), n;
|
|
139
|
+
}
|
|
140
|
+
function le() {
|
|
141
|
+
return h({
|
|
142
|
+
$schema: ae,
|
|
143
|
+
projects: ["packages/*"]
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
function ue() {
|
|
147
|
+
return h({ $schema: oe });
|
|
148
|
+
}
|
|
149
|
+
function de(e, t, n) {
|
|
150
|
+
let r = {};
|
|
151
|
+
return A(e).forEach((t, n) => {
|
|
152
|
+
let i = ce(e, n);
|
|
153
|
+
r[t] = {
|
|
154
|
+
command: `node ./task.mjs ${t}`,
|
|
155
|
+
...i.length ? { deps: i } : {},
|
|
156
|
+
inputs: [
|
|
157
|
+
"package.json",
|
|
158
|
+
"task.mjs",
|
|
159
|
+
"src/**/*"
|
|
160
|
+
],
|
|
161
|
+
outputs: [`dist/${t}.*`]
|
|
162
|
+
};
|
|
163
|
+
}), h({
|
|
164
|
+
$schema: se,
|
|
165
|
+
id: t.name,
|
|
166
|
+
layer: "library",
|
|
167
|
+
language: "javascript",
|
|
168
|
+
...t.dependencies.length ? { dependsOn: ie(t, n) } : {},
|
|
169
|
+
tasks: r
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
var fe = {
|
|
173
|
+
tool: "moon",
|
|
174
|
+
hasDaemon: !1,
|
|
175
|
+
supportedVersions: ["^2.0.0"],
|
|
176
|
+
pinnedVersion: (e) => e.versions.moon,
|
|
177
|
+
devDependencies: (e) => ({ "@moonrepo/cli": e.versions.moon }),
|
|
178
|
+
setup: async (e) => {
|
|
179
|
+
await e.write(".moon/workspace.yml", le()), await e.write(".moon/toolchain.yml", ue());
|
|
180
|
+
for (let t of e.projects) await e.write(`${t.dir}/moon.yml`, de(e.config, t, e.projects));
|
|
181
|
+
},
|
|
182
|
+
run: (e, t) => ({
|
|
183
|
+
file: j(t.rootDir, "moon"),
|
|
184
|
+
args: [
|
|
185
|
+
"run",
|
|
186
|
+
`:${e}`,
|
|
187
|
+
"--concurrency",
|
|
188
|
+
String(t.concurrency)
|
|
189
|
+
]
|
|
190
|
+
}),
|
|
191
|
+
env: () => ({}),
|
|
192
|
+
clearCaches: async (e) => {
|
|
193
|
+
await M(e), await l(o(e.rootDir, ".moon", "cache"), {
|
|
194
|
+
recursive: !0,
|
|
195
|
+
force: !0
|
|
196
|
+
});
|
|
197
|
+
},
|
|
198
|
+
stopDaemon: async () => {}
|
|
199
|
+
};
|
|
200
|
+
//#endregion
|
|
201
|
+
//#region src/tools/nx.ts
|
|
202
|
+
function pe(e) {
|
|
203
|
+
let t = {};
|
|
204
|
+
return A(e).forEach((n, r) => {
|
|
205
|
+
t[n] = {
|
|
206
|
+
dependsOn: N(e, r),
|
|
207
|
+
cache: !0,
|
|
208
|
+
outputs: [`{projectRoot}/dist/${n}.*`],
|
|
209
|
+
inputs: [
|
|
210
|
+
"{projectRoot}/package.json",
|
|
211
|
+
"{projectRoot}/task.mjs",
|
|
212
|
+
"{projectRoot}/src/**/*"
|
|
213
|
+
]
|
|
214
|
+
};
|
|
215
|
+
}), `${JSON.stringify({
|
|
216
|
+
$schema: "./node_modules/nx/schemas/nx-schema.json",
|
|
217
|
+
targetDefaults: t
|
|
218
|
+
}, null, 2)}\n`;
|
|
219
|
+
}
|
|
220
|
+
function me(e, t) {
|
|
221
|
+
let n = {};
|
|
222
|
+
for (let r of A(e)) n[r] = {
|
|
223
|
+
executor: "nx:run-commands",
|
|
224
|
+
options: {
|
|
225
|
+
command: `node ./task.mjs ${r}`,
|
|
226
|
+
cwd: t.dir
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
return `${JSON.stringify({
|
|
230
|
+
name: t.name,
|
|
231
|
+
$schema: "../../node_modules/nx/schemas/project-schema.json",
|
|
232
|
+
targets: n
|
|
233
|
+
}, null, 2)}\n`;
|
|
234
|
+
}
|
|
235
|
+
var he = {
|
|
236
|
+
tool: "nx",
|
|
237
|
+
hasDaemon: !0,
|
|
238
|
+
supportedVersions: [">=21.0.0 <24.0.0"],
|
|
239
|
+
pinnedVersion: (e) => e.versions.nx,
|
|
240
|
+
devDependencies: (e) => ({ nx: e.versions.nx }),
|
|
241
|
+
setup: async (e) => {
|
|
242
|
+
await e.write("nx.json", pe(e.config));
|
|
243
|
+
for (let t of e.projects) await e.write(`${t.dir}/project.json`, me(e.config, t));
|
|
244
|
+
},
|
|
245
|
+
run: (e, t) => ({
|
|
246
|
+
file: j(t.rootDir, "nx"),
|
|
247
|
+
args: [
|
|
248
|
+
"run-many",
|
|
249
|
+
"-t",
|
|
250
|
+
e,
|
|
251
|
+
`--parallel=${t.concurrency}`
|
|
252
|
+
]
|
|
253
|
+
}),
|
|
254
|
+
env: (e) => ({
|
|
255
|
+
NX_DAEMON: e.daemon ? "true" : "false",
|
|
256
|
+
NX_TUI: "false"
|
|
257
|
+
}),
|
|
258
|
+
clearCaches: async (e) => {
|
|
259
|
+
await M(e), await f(j(e.rootDir, "nx"), ["reset"], {
|
|
260
|
+
cwd: e.rootDir,
|
|
261
|
+
reject: !1,
|
|
262
|
+
stdio: "ignore",
|
|
263
|
+
env: {
|
|
264
|
+
NX_DAEMON: e.daemon ? "true" : "false",
|
|
265
|
+
NX_TUI: "false"
|
|
266
|
+
}
|
|
267
|
+
});
|
|
268
|
+
},
|
|
269
|
+
stopDaemon: async (e) => {
|
|
270
|
+
await f(j(e.rootDir, "nx"), ["reset"], {
|
|
271
|
+
cwd: e.rootDir,
|
|
272
|
+
reject: !1,
|
|
273
|
+
stdio: "ignore",
|
|
274
|
+
env: { NX_DAEMON: "false" }
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
}, ge = "# yaml-language-server: $schema=https://raw.githubusercontent.com/omni-oss/json-schemas/refs/heads/main/project.json", _e = "# yaml-language-server: $schema=https://raw.githubusercontent.com/omni-oss/json-schemas/refs/heads/main/workspace.json";
|
|
278
|
+
function ve() {
|
|
279
|
+
return `${_e}\n${h({
|
|
280
|
+
ui: "stream",
|
|
281
|
+
projects: ["packages/*"]
|
|
282
|
+
})}`;
|
|
283
|
+
}
|
|
284
|
+
function ye(e, t, n) {
|
|
285
|
+
let r = {};
|
|
286
|
+
return A(e).forEach((t, n) => {
|
|
287
|
+
let i = N(e, n);
|
|
288
|
+
r[t] = {
|
|
289
|
+
exec: `node ./task.mjs ${t}`,
|
|
290
|
+
...i.length ? { dependencies: i } : {},
|
|
291
|
+
cache: { output: { files: [`dist/${t}.*`] } }
|
|
292
|
+
};
|
|
293
|
+
}), `${ge}\n${h({
|
|
294
|
+
name: t.name,
|
|
295
|
+
...t.dependencies.length ? { dependencies: ie(t, n) } : {},
|
|
296
|
+
cache: { key: { files: [
|
|
297
|
+
"package.json",
|
|
298
|
+
"task.mjs",
|
|
299
|
+
"src/**/*.js"
|
|
300
|
+
] } },
|
|
301
|
+
tasks: r
|
|
302
|
+
})}`;
|
|
303
|
+
}
|
|
304
|
+
var be = {
|
|
305
|
+
tool: "omni",
|
|
306
|
+
hasDaemon: !1,
|
|
307
|
+
supportedVersions: [">=0.16.0"],
|
|
308
|
+
pinnedVersion: () => null,
|
|
309
|
+
detectVersion: async (e) => {
|
|
310
|
+
let t = await f("omni", ["--version"], {
|
|
311
|
+
cwd: e,
|
|
312
|
+
reject: !1
|
|
313
|
+
});
|
|
314
|
+
return `${t.stdout} ${t.stderr}`.match(/(\d+\.\d+\.\d+[\w.-]*)/)?.[1] ?? null;
|
|
315
|
+
},
|
|
316
|
+
devDependencies: () => ({}),
|
|
317
|
+
setup: async (e) => {
|
|
318
|
+
await e.write("workspace.omni.yaml", ve());
|
|
319
|
+
for (let t of e.projects) await e.write(`${t.dir}/project.omni.yaml`, ye(e.config, t, e.projects));
|
|
320
|
+
},
|
|
321
|
+
run: (e, t) => ({
|
|
322
|
+
file: "omni",
|
|
323
|
+
args: [
|
|
324
|
+
"run",
|
|
325
|
+
e,
|
|
326
|
+
"-u",
|
|
327
|
+
"stream",
|
|
328
|
+
"-c",
|
|
329
|
+
String(t.concurrency)
|
|
330
|
+
]
|
|
331
|
+
}),
|
|
332
|
+
env: () => ({}),
|
|
333
|
+
clearCaches: async (e) => {
|
|
334
|
+
await M(e), await l(o(e.rootDir, ".omni", "cache"), {
|
|
335
|
+
recursive: !0,
|
|
336
|
+
force: !0
|
|
337
|
+
});
|
|
338
|
+
},
|
|
339
|
+
stopDaemon: async () => {}
|
|
340
|
+
};
|
|
341
|
+
//#endregion
|
|
342
|
+
//#region src/tools/turbo.ts
|
|
343
|
+
function xe(e) {
|
|
344
|
+
let t = {};
|
|
345
|
+
return A(e).forEach((n, r) => {
|
|
346
|
+
t[n] = {
|
|
347
|
+
dependsOn: N(e, r),
|
|
348
|
+
outputs: [`dist/${n}.*`],
|
|
349
|
+
inputs: [
|
|
350
|
+
"package.json",
|
|
351
|
+
"task.mjs",
|
|
352
|
+
"src/**"
|
|
353
|
+
]
|
|
354
|
+
};
|
|
355
|
+
}), `${JSON.stringify({
|
|
356
|
+
$schema: "https://turbo.build/schema.json",
|
|
357
|
+
globalPassThroughEnv: ["TASK_BENCH_EXEC_LOG"],
|
|
358
|
+
tasks: t
|
|
359
|
+
}, null, 2)}\n`;
|
|
360
|
+
}
|
|
361
|
+
//#endregion
|
|
362
|
+
//#region src/tools/index.ts
|
|
363
|
+
var Se = {
|
|
364
|
+
omni: be,
|
|
365
|
+
turbo: {
|
|
366
|
+
tool: "turbo",
|
|
367
|
+
hasDaemon: !0,
|
|
368
|
+
supportedVersions: ["^2.0.0"],
|
|
369
|
+
pinnedVersion: (e) => e.versions.turbo,
|
|
370
|
+
devDependencies: (e) => ({ turbo: e.versions.turbo }),
|
|
371
|
+
setup: async (e) => {
|
|
372
|
+
await e.write("turbo.json", xe(e.config));
|
|
373
|
+
},
|
|
374
|
+
run: (e, t) => ({
|
|
375
|
+
file: j(t.rootDir, "turbo"),
|
|
376
|
+
args: [
|
|
377
|
+
"run",
|
|
378
|
+
e,
|
|
379
|
+
"--log-order=stream",
|
|
380
|
+
`--concurrency=${t.concurrency}`,
|
|
381
|
+
t.daemon ? "--daemon" : "--no-daemon"
|
|
382
|
+
]
|
|
383
|
+
}),
|
|
384
|
+
env: () => ({}),
|
|
385
|
+
clearCaches: async (e) => {
|
|
386
|
+
await M(e), await l(o(e.rootDir, ".turbo"), {
|
|
387
|
+
recursive: !0,
|
|
388
|
+
force: !0
|
|
389
|
+
}), await l(o(e.rootDir, "node_modules", ".cache", "turbo"), {
|
|
390
|
+
recursive: !0,
|
|
391
|
+
force: !0
|
|
392
|
+
});
|
|
393
|
+
},
|
|
394
|
+
stopDaemon: async (e) => {
|
|
395
|
+
await f(j(e.rootDir, "turbo"), ["daemon", "stop"], {
|
|
396
|
+
cwd: e.rootDir,
|
|
397
|
+
reject: !1,
|
|
398
|
+
stdio: "ignore"
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
},
|
|
402
|
+
nx: he,
|
|
403
|
+
moon: fe
|
|
404
|
+
};
|
|
405
|
+
function P(e) {
|
|
406
|
+
return Se[e];
|
|
407
|
+
}
|
|
408
|
+
function F(e) {
|
|
409
|
+
return e.map(P);
|
|
410
|
+
}
|
|
411
|
+
function I(e, t) {
|
|
412
|
+
if (!e.supportedVersions.some((e) => m(t, e, {
|
|
413
|
+
includePrerelease: !0,
|
|
414
|
+
loose: !0
|
|
415
|
+
}))) throw Error(`${e.tool} version "${t}" is not supported by task-bench (supported: ${e.supportedVersions.join(" || ")}). Adjust versions.${e.tool} in your config or upgrade/downgrade the tool.`);
|
|
416
|
+
}
|
|
417
|
+
async function L(e, t, n = e.tools) {
|
|
418
|
+
let r = /* @__PURE__ */ new Map();
|
|
419
|
+
for (let i of n) {
|
|
420
|
+
let n = P(i), a = n.pinnedVersion(e);
|
|
421
|
+
a === null && n.detectVersion && (a = await n.detectVersion(t)), a !== null && I(n, a), r.set(i, a);
|
|
422
|
+
}
|
|
423
|
+
return r;
|
|
424
|
+
}
|
|
425
|
+
//#endregion
|
|
426
|
+
//#region src/bench/stats.ts
|
|
427
|
+
function R(e) {
|
|
428
|
+
if (e.length === 0) return {
|
|
429
|
+
samples: e,
|
|
430
|
+
min: 0,
|
|
431
|
+
max: 0,
|
|
432
|
+
mean: 0,
|
|
433
|
+
median: 0,
|
|
434
|
+
stddev: 0
|
|
435
|
+
};
|
|
436
|
+
let t = [...e].sort((e, t) => e - t), n = t.reduce((e, t) => e + t, 0) / t.length, r = Math.floor(t.length / 2), i = t.length % 2 == 0 ? ((t[r - 1] ?? 0) + (t[r] ?? 0)) / 2 : t[r] ?? 0, a = t.reduce((e, t) => e + (t - n) ** 2, 0) / t.length;
|
|
437
|
+
return {
|
|
438
|
+
samples: e,
|
|
439
|
+
min: t[0] ?? 0,
|
|
440
|
+
max: t[t.length - 1] ?? 0,
|
|
441
|
+
mean: n,
|
|
442
|
+
median: i,
|
|
443
|
+
stddev: Math.sqrt(a)
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
function z(e) {
|
|
447
|
+
return e >= 1e3 ? `${(e / 1e3).toFixed(2)}s` : `${e.toFixed(0)}ms`;
|
|
448
|
+
}
|
|
449
|
+
//#endregion
|
|
450
|
+
//#region src/bench/index.ts
|
|
451
|
+
function Ce(e) {
|
|
452
|
+
if (e.length === 0) return 0;
|
|
453
|
+
let t = [...e].sort((e, t) => e - t), n = Math.floor(t.length / 2);
|
|
454
|
+
return t.length % 2 == 0 ? ((t[n - 1] ?? 0) + (t[n] ?? 0)) / 2 : t[n] ?? 0;
|
|
455
|
+
}
|
|
456
|
+
function we(e) {
|
|
457
|
+
try {
|
|
458
|
+
let n = t(e, "utf8");
|
|
459
|
+
if (n.length === 0) return 0;
|
|
460
|
+
let r = 0;
|
|
461
|
+
for (let e = 0; e < n.length; e++) n[e] === "\n" && r++;
|
|
462
|
+
return r;
|
|
463
|
+
} catch {
|
|
464
|
+
return 0;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
async function B(e, t, r, i, a) {
|
|
468
|
+
n(i, "");
|
|
469
|
+
let o = d.now(), s = await f(e, t, {
|
|
470
|
+
cwd: r,
|
|
471
|
+
reject: !1,
|
|
472
|
+
env: {
|
|
473
|
+
FORCE_COLOR: "0",
|
|
474
|
+
TURBO_TELEMETRY_DISABLED: "1",
|
|
475
|
+
DO_NOT_TRACK: "1",
|
|
476
|
+
NX_TUI: "false",
|
|
477
|
+
TASK_BENCH_EXEC_LOG: i,
|
|
478
|
+
...a
|
|
479
|
+
}
|
|
480
|
+
}), c = d.now() - o, l = typeof s.stdout == "string" ? s.stdout : "", u = typeof s.stderr == "string" ? s.stderr : "";
|
|
481
|
+
return {
|
|
482
|
+
durationMs: c,
|
|
483
|
+
exitCode: s.exitCode ?? -1,
|
|
484
|
+
stdout: l,
|
|
485
|
+
stderr: u,
|
|
486
|
+
executed: we(i),
|
|
487
|
+
ok: s.exitCode === 0
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
function V(e) {
|
|
491
|
+
return {
|
|
492
|
+
runs: e.length,
|
|
493
|
+
failures: e.filter((e) => !e.ok).length,
|
|
494
|
+
stats: R(e.map((e) => e.durationMs)),
|
|
495
|
+
executedMedian: Ce(e.map((e) => e.executed))
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
async function H(e, t = {}) {
|
|
499
|
+
let n = JSON.parse(await c(o(e, "bench.config.json"), "utf8")), a = E.parse(n), s = k(a), l = A(a), u = t.tools ?? a.tools, d = t.task ?? l[l.length - 1] ?? "t0", f = t.coldRuns ?? 3, p = t.warmRuns ?? 5, m = t.concurrency ?? Math.max(1, r().length), h = t.daemon ?? !0, g = t.onEvent ?? (() => {}), _ = o(i(), `task-bench-exec-${process.pid}-${Date.now()}.log`), v = {
|
|
500
|
+
rootDir: e,
|
|
501
|
+
projectDirs: s.map((e) => e.dir),
|
|
502
|
+
concurrency: m,
|
|
503
|
+
daemon: h
|
|
504
|
+
}, y = await L(a, e, u), b = {};
|
|
505
|
+
for (let [e, t] of y) b[e] = t;
|
|
506
|
+
let x = [];
|
|
507
|
+
for (let t of u) {
|
|
508
|
+
g({
|
|
509
|
+
kind: "tool-start",
|
|
510
|
+
tool: t
|
|
511
|
+
});
|
|
512
|
+
let n = P(t), r = n.run(d, v), i = n.env(v);
|
|
513
|
+
try {
|
|
514
|
+
!h && n.hasDaemon && await n.stopDaemon(v);
|
|
515
|
+
let a = [];
|
|
516
|
+
for (let o = 1; o <= f; o++) {
|
|
517
|
+
await n.clearCaches(v), h && n.hasDaemon && await n.stopDaemon(v);
|
|
518
|
+
let s = await B(r.file, r.args, e, _, i);
|
|
519
|
+
a.push(s), g({
|
|
520
|
+
kind: "scenario",
|
|
521
|
+
tool: t,
|
|
522
|
+
scenario: "cold",
|
|
523
|
+
run: o,
|
|
524
|
+
total: f,
|
|
525
|
+
sample: s
|
|
526
|
+
}), s.exitCode !== 0 && g({
|
|
527
|
+
kind: "tool-unsuccessful",
|
|
528
|
+
tool: t,
|
|
529
|
+
sample: s
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
await B(r.file, r.args, e, _, i);
|
|
533
|
+
let o = [];
|
|
534
|
+
for (let n = 1; n <= p; n++) {
|
|
535
|
+
let a = await B(r.file, r.args, e, _, i);
|
|
536
|
+
o.push(a), g({
|
|
537
|
+
kind: "scenario",
|
|
538
|
+
tool: t,
|
|
539
|
+
scenario: "warm",
|
|
540
|
+
run: n,
|
|
541
|
+
total: p,
|
|
542
|
+
sample: a
|
|
543
|
+
}), a.exitCode !== 0 && g({
|
|
544
|
+
kind: "tool-unsuccessful",
|
|
545
|
+
tool: t,
|
|
546
|
+
sample: a
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
let s = Math.max(0, ...a.map((e) => e.executed), ...o.map((e) => e.executed));
|
|
550
|
+
x.push({
|
|
551
|
+
tool: t,
|
|
552
|
+
task: d,
|
|
553
|
+
taskGraphSize: s,
|
|
554
|
+
cold: V(a),
|
|
555
|
+
warm: V(o)
|
|
556
|
+
});
|
|
557
|
+
} catch (e) {
|
|
558
|
+
let n = e instanceof Error ? e.message : String(e);
|
|
559
|
+
g({
|
|
560
|
+
kind: "tool-error",
|
|
561
|
+
tool: t,
|
|
562
|
+
error: n
|
|
563
|
+
}), x.push({
|
|
564
|
+
tool: t,
|
|
565
|
+
task: d,
|
|
566
|
+
taskGraphSize: 0,
|
|
567
|
+
cold: V([]),
|
|
568
|
+
warm: V([]),
|
|
569
|
+
error: n
|
|
570
|
+
});
|
|
571
|
+
} finally {
|
|
572
|
+
await n.stopDaemon(v);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
return {
|
|
576
|
+
rootDir: e,
|
|
577
|
+
task: d,
|
|
578
|
+
projects: a.projects,
|
|
579
|
+
tasksPerProject: a.tasksPerProject,
|
|
580
|
+
concurrency: m,
|
|
581
|
+
daemon: h,
|
|
582
|
+
versions: b,
|
|
583
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
584
|
+
tools: x
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
//#endregion
|
|
588
|
+
//#region src/bench/install.ts
|
|
589
|
+
async function U(e, t = {}) {
|
|
590
|
+
await f("bun", ["install"], {
|
|
591
|
+
cwd: e,
|
|
592
|
+
stdio: t.quiet ? "ignore" : "inherit"
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
//#endregion
|
|
596
|
+
//#region src/bench/report.ts
|
|
597
|
+
function Te(e, t) {
|
|
598
|
+
return e.length >= t ? e : e + " ".repeat(t - e.length);
|
|
599
|
+
}
|
|
600
|
+
function W(e, t) {
|
|
601
|
+
if (e.samples.length === 0) return "—";
|
|
602
|
+
let n = `${z(e.median)} ±${z(e.stddev)}`;
|
|
603
|
+
return t > 0 ? `${n} ⚠${t}` : n;
|
|
604
|
+
}
|
|
605
|
+
function G(e) {
|
|
606
|
+
return e.taskGraphSize <= 0 || e.warm.stats.samples.length === 0 ? null : (e.taskGraphSize - e.warm.executedMedian) / e.taskGraphSize * 100;
|
|
607
|
+
}
|
|
608
|
+
function K(e) {
|
|
609
|
+
let t = G(e);
|
|
610
|
+
return t === null ? "—" : `${t.toFixed(0)}%`;
|
|
611
|
+
}
|
|
612
|
+
function Ee(e) {
|
|
613
|
+
let t = [], n = Math.max(0, ...e.tools.map((e) => e.taskGraphSize));
|
|
614
|
+
t.push(""), t.push(`task-bench: ${e.projects} projects × ${e.tasksPerProject} tasks (${n} task-graph nodes), running "${e.task}" at concurrency ${e.concurrency} (daemons ${e.daemon ? "on" : "off"})`), t.push(De(e)), t.push("");
|
|
615
|
+
let r = [
|
|
616
|
+
"tool",
|
|
617
|
+
"cold (median)",
|
|
618
|
+
"warm (median)",
|
|
619
|
+
"warm cache-hit",
|
|
620
|
+
"notes"
|
|
621
|
+
], i = e.tools.map((e) => {
|
|
622
|
+
let t = e.error ? `error: ${(e.error.split("\n")[0] ?? "").slice(0, 40)}` : "";
|
|
623
|
+
return [
|
|
624
|
+
e.tool,
|
|
625
|
+
W(e.cold.stats, e.cold.failures),
|
|
626
|
+
W(e.warm.stats, e.warm.failures),
|
|
627
|
+
K(e),
|
|
628
|
+
t
|
|
629
|
+
];
|
|
630
|
+
}), a = r.map((e, t) => Math.max(e.length, ...i.map((e) => (e[t] ?? "").length))), o = (e) => `| ${e.map((e, t) => Te(e, a[t] ?? 0)).join(" | ")} |`;
|
|
631
|
+
t.push(o(r)), t.push(`| ${a.map((e) => "-".repeat(e)).join(" | ")} |`);
|
|
632
|
+
for (let e of i) t.push(o(e));
|
|
633
|
+
t.push("");
|
|
634
|
+
let s = e.tools.filter((e) => {
|
|
635
|
+
let t = G(e);
|
|
636
|
+
return t !== null && t < 99.5;
|
|
637
|
+
});
|
|
638
|
+
for (let e of s) t.push(`⚠ ${e.tool}: warm runs were not fully cached (${K(e)} hit, ${e.warm.executedMedian}/${e.taskGraphSize} tasks re-ran) — treat its warm number with caution.`);
|
|
639
|
+
s.length && t.push("");
|
|
640
|
+
let c = e.tools.filter((e) => {
|
|
641
|
+
let t = G(e);
|
|
642
|
+
return !e.error && t !== null && t >= 99.5;
|
|
643
|
+
}).sort((e, t) => e.warm.stats.median - t.warm.stats.median), l = c[0], u = c[c.length - 1];
|
|
644
|
+
if (c.length > 1 && l && u) {
|
|
645
|
+
let e = u.warm.stats.median / l.warm.stats.median;
|
|
646
|
+
t.push(`Warm-cache overhead: ${l.tool} is fastest (${z(l.warm.stats.median)}), ${e.toFixed(2)}× faster than ${u.tool} (${z(u.warm.stats.median)}).\n`);
|
|
647
|
+
}
|
|
648
|
+
let d = e.tools.filter((e) => !e.error && e.cold.stats.samples.length > 0).sort((e, t) => e.cold.stats.median - t.cold.stats.median), f = d[0], p = d[d.length - 1];
|
|
649
|
+
if (d.length > 1 && f && p) {
|
|
650
|
+
let e = p.cold.stats.median / f.cold.stats.median;
|
|
651
|
+
t.push(`Cold-run overhead: ${f.tool} is fastest (${z(f.cold.stats.median)}), ${e.toFixed(2)}× faster than ${p.tool} (${z(p.cold.stats.median)}).`);
|
|
652
|
+
}
|
|
653
|
+
return (c.length > 1 || d.length > 1) && t.push(""), t.join("\n");
|
|
654
|
+
}
|
|
655
|
+
function De(e) {
|
|
656
|
+
return `versions: ${e.tools.map((t) => `\`${t.tool} ${e.versions[t.tool] ?? "?"}\``).join(", ")}`;
|
|
657
|
+
}
|
|
658
|
+
//#endregion
|
|
659
|
+
//#region src/generate/templates.ts
|
|
660
|
+
function Oe(e) {
|
|
661
|
+
return [
|
|
662
|
+
`// Auto-generated source for ${e.name}.`,
|
|
663
|
+
"// Edit the harness config, not this file.",
|
|
664
|
+
`export const id = ${JSON.stringify(e.name)};`,
|
|
665
|
+
`export const index = ${e.index};`,
|
|
666
|
+
"export const answer = 42;",
|
|
667
|
+
""
|
|
668
|
+
].join("\n");
|
|
669
|
+
}
|
|
670
|
+
function ke(e, t) {
|
|
671
|
+
let { logLines: n, workIterations: r, outputFiles: i } = e.task;
|
|
672
|
+
return `#!/usr/bin/env node
|
|
673
|
+
import { createHash } from "node:crypto";
|
|
674
|
+
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
675
|
+
import { dirname, join } from "node:path";
|
|
676
|
+
import { fileURLToPath } from "node:url";
|
|
677
|
+
|
|
678
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
679
|
+
const PROJECT = ${JSON.stringify(t.name)};
|
|
680
|
+
const LOG_LINES = ${n};
|
|
681
|
+
const WORK_ITERATIONS = ${r};
|
|
682
|
+
const OUTPUT_FILES = ${i};
|
|
683
|
+
|
|
684
|
+
const task = process.argv[2] ?? "task";
|
|
685
|
+
|
|
686
|
+
// Ground-truth execution marker: this line is only ever reached when the task
|
|
687
|
+
// actually runs. Cache hits skip the process entirely, so counting the lines
|
|
688
|
+
// in this out-of-tree log yields a tool-agnostic cache-hit rate. The log path
|
|
689
|
+
// lives outside the workspace so no runner hashes it as an input.
|
|
690
|
+
const execLog = process.env.TASK_BENCH_EXEC_LOG;
|
|
691
|
+
if (execLog) {
|
|
692
|
+
appendFileSync(execLog, \`\${PROJECT}\\t\${task}\\n\`);
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const source = readFileSync(join(HERE, "src", "index.js"), "utf8");
|
|
696
|
+
|
|
697
|
+
// Cheap, deterministic CPU work.
|
|
698
|
+
let acc = 0;
|
|
699
|
+
for (let i = 0; i < WORK_ITERATIONS; i++) {
|
|
700
|
+
acc = (acc + Math.imul(i ^ acc, 2654435761)) >>> 0;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
const digest = createHash("sha256")
|
|
704
|
+
.update(source)
|
|
705
|
+
.update(task)
|
|
706
|
+
.update(String(acc))
|
|
707
|
+
.digest("hex");
|
|
708
|
+
|
|
709
|
+
for (let i = 1; i <= LOG_LINES; i++) {
|
|
710
|
+
console.log(\`[\${PROJECT}] \${task}: step \${i}/\${LOG_LINES} digest=\${digest.slice(0, 12)}\`);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
const outDir = join(HERE, "dist");
|
|
714
|
+
mkdirSync(outDir, { recursive: true });
|
|
715
|
+
for (let f = 0; f < OUTPUT_FILES; f++) {
|
|
716
|
+
writeFileSync(
|
|
717
|
+
join(outDir, \`\${task}.\${f}.txt\`),
|
|
718
|
+
\`\${PROJECT}\\t\${task}\\t\${digest}\\n\`,
|
|
719
|
+
);
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
console.log(\`[\${PROJECT}] \${task}: complete (\${OUTPUT_FILES} output file(s))\`);
|
|
723
|
+
`;
|
|
724
|
+
}
|
|
725
|
+
//#endregion
|
|
726
|
+
//#region src/generate/index.ts
|
|
727
|
+
async function q(e, t, n, r) {
|
|
728
|
+
let i = o(e, t);
|
|
729
|
+
await s(a(i), { recursive: !0 }), await u(i, n), r.push(t);
|
|
730
|
+
}
|
|
731
|
+
function Ae(e, t) {
|
|
732
|
+
let n = {};
|
|
733
|
+
for (let r of t) Object.assign(n, r.devDependencies(e));
|
|
734
|
+
return `${JSON.stringify({
|
|
735
|
+
name: "task-bench-harness",
|
|
736
|
+
private: !0,
|
|
737
|
+
packageManager: `bun@${e.versions.bun}`,
|
|
738
|
+
workspaces: ["packages/*"],
|
|
739
|
+
...Object.keys(n).length ? { devDependencies: n } : {}
|
|
740
|
+
}, null, 2)}\n`;
|
|
741
|
+
}
|
|
742
|
+
function je(e, t, n) {
|
|
743
|
+
let r = {};
|
|
744
|
+
for (let t of A(e)) r[t] = `node ./task.mjs ${t}`;
|
|
745
|
+
let i = {};
|
|
746
|
+
for (let e of t.dependencies) {
|
|
747
|
+
let t = n[e];
|
|
748
|
+
t && (i[t.name] = "workspace:*");
|
|
749
|
+
}
|
|
750
|
+
return `${JSON.stringify({
|
|
751
|
+
name: t.name,
|
|
752
|
+
version: "0.0.0",
|
|
753
|
+
private: !0,
|
|
754
|
+
scripts: r,
|
|
755
|
+
...Object.keys(i).length ? { dependencies: i } : {}
|
|
756
|
+
}, null, 2)}\n`;
|
|
757
|
+
}
|
|
758
|
+
function Me() {
|
|
759
|
+
return [
|
|
760
|
+
"node_modules",
|
|
761
|
+
"dist",
|
|
762
|
+
".turbo",
|
|
763
|
+
".nx",
|
|
764
|
+
".omni",
|
|
765
|
+
".moon/cache",
|
|
766
|
+
"*.log",
|
|
767
|
+
""
|
|
768
|
+
].join("\n");
|
|
769
|
+
}
|
|
770
|
+
function Ne(e) {
|
|
771
|
+
return [
|
|
772
|
+
"# task-bench harness",
|
|
773
|
+
"",
|
|
774
|
+
"Auto-generated benchmark workspace. Do not edit by hand; regenerate via",
|
|
775
|
+
"`@omni-oss/task-bench` instead.",
|
|
776
|
+
"",
|
|
777
|
+
`- projects: **${e.projects}**`,
|
|
778
|
+
`- tasks per project: **${e.tasksPerProject}**`,
|
|
779
|
+
`- dependency strategy: **${e.dependency.strategy}**`,
|
|
780
|
+
`- tools: **${e.tools.join(", ")}**`,
|
|
781
|
+
"",
|
|
782
|
+
"The exact configuration is captured in `bench.config.json`.",
|
|
783
|
+
""
|
|
784
|
+
].join("\n");
|
|
785
|
+
}
|
|
786
|
+
async function Pe(e) {
|
|
787
|
+
let t = {
|
|
788
|
+
cwd: e,
|
|
789
|
+
reject: !1,
|
|
790
|
+
stdio: "ignore"
|
|
791
|
+
};
|
|
792
|
+
await f("git", ["init", "-q"], t), await f("git", ["add", "-A"], t), await f("git", [
|
|
793
|
+
"-c",
|
|
794
|
+
"user.email=bench@task-bench.local",
|
|
795
|
+
"-c",
|
|
796
|
+
"user.name=task-bench",
|
|
797
|
+
"commit",
|
|
798
|
+
"-qm",
|
|
799
|
+
"generated benchmark workspace"
|
|
800
|
+
], t);
|
|
801
|
+
}
|
|
802
|
+
async function Fe(e, t) {
|
|
803
|
+
let n = D(t), r = k(n), i = [], a = F(n.tools);
|
|
804
|
+
await l(e, {
|
|
805
|
+
recursive: !0,
|
|
806
|
+
force: !0
|
|
807
|
+
}), await s(e, { recursive: !0 });
|
|
808
|
+
let o = await L(n, e);
|
|
809
|
+
await q(e, "package.json", Ae(n, a), i), await q(e, ".gitignore", Me(), i), await q(e, "README.md", Ne(n), i), await q(e, "bench.config.json", `${JSON.stringify(n, null, 2)}\n`, i);
|
|
810
|
+
for (let t of r) {
|
|
811
|
+
let a = t.dir;
|
|
812
|
+
await q(e, `${a}/package.json`, je(n, t, r), i), await q(e, `${a}/src/index.js`, Oe(t), i), await q(e, `${a}/task.mjs`, ke(n, t), i);
|
|
813
|
+
}
|
|
814
|
+
let c = (t, n) => q(e, t, n, i);
|
|
815
|
+
for (let t of a) await t.setup({
|
|
816
|
+
rootDir: e,
|
|
817
|
+
config: n,
|
|
818
|
+
projects: r,
|
|
819
|
+
version: o.get(t.tool) ?? null,
|
|
820
|
+
write: c
|
|
821
|
+
});
|
|
822
|
+
return await Pe(e), {
|
|
823
|
+
rootDir: e,
|
|
824
|
+
config: n,
|
|
825
|
+
projects: r,
|
|
826
|
+
files: i
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
//#endregion
|
|
830
|
+
//#region src/suite/preset.ts
|
|
831
|
+
var J = p.object({
|
|
832
|
+
tools: p.array(S).optional(),
|
|
833
|
+
task: p.string().optional(),
|
|
834
|
+
coldRuns: p.number().int().nonnegative().optional(),
|
|
835
|
+
warmRuns: p.number().int().nonnegative().optional(),
|
|
836
|
+
concurrency: p.number().int().positive().optional(),
|
|
837
|
+
daemon: p.boolean().optional()
|
|
838
|
+
}).default({}), Ie = p.record(p.string(), p.unknown()).default({}), Le = p.object({
|
|
839
|
+
name: p.string().regex(/^[a-zA-Z0-9._-]+$/, "scenario name must be filesystem-safe (letters, digits, . _ -)"),
|
|
840
|
+
description: p.string().optional(),
|
|
841
|
+
config: Ie,
|
|
842
|
+
run: J
|
|
843
|
+
}), Y = p.object({
|
|
844
|
+
name: p.string().default("task-bench suite"),
|
|
845
|
+
description: p.string().optional(),
|
|
846
|
+
defaults: p.object({
|
|
847
|
+
config: Ie,
|
|
848
|
+
run: J
|
|
849
|
+
}).prefault({}),
|
|
850
|
+
scenarios: p.array(Le).min(1)
|
|
851
|
+
});
|
|
852
|
+
function Re(e) {
|
|
853
|
+
return typeof e == "object" && !!e && !Array.isArray(e);
|
|
854
|
+
}
|
|
855
|
+
function X(e, t) {
|
|
856
|
+
let n = { ...e };
|
|
857
|
+
for (let [e, r] of Object.entries(t)) {
|
|
858
|
+
let t = n[e];
|
|
859
|
+
n[e] = Re(t) && Re(r) ? X(t, r) : r;
|
|
860
|
+
}
|
|
861
|
+
return n;
|
|
862
|
+
}
|
|
863
|
+
function ze(e, t) {
|
|
864
|
+
let n = X(e.defaults.config, t.config), r = {
|
|
865
|
+
...e.defaults.run,
|
|
866
|
+
...t.run
|
|
867
|
+
};
|
|
868
|
+
return {
|
|
869
|
+
name: t.name,
|
|
870
|
+
description: t.description,
|
|
871
|
+
config: D(n),
|
|
872
|
+
run: r
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
var Z = [
|
|
876
|
+
"isolated",
|
|
877
|
+
"chain",
|
|
878
|
+
"fan-out",
|
|
879
|
+
"layered",
|
|
880
|
+
"random"
|
|
881
|
+
], Q = {
|
|
882
|
+
quick: {
|
|
883
|
+
name: "quick smoke suite",
|
|
884
|
+
description: "Tiny, fast sanity sweep across two shapes.",
|
|
885
|
+
defaults: {
|
|
886
|
+
config: {
|
|
887
|
+
projects: 30,
|
|
888
|
+
tasksPerProject: 2
|
|
889
|
+
},
|
|
890
|
+
run: {
|
|
891
|
+
concurrency: 8,
|
|
892
|
+
coldRuns: 1,
|
|
893
|
+
warmRuns: 2
|
|
894
|
+
}
|
|
895
|
+
},
|
|
896
|
+
scenarios: [{
|
|
897
|
+
name: "quick-isolated",
|
|
898
|
+
config: { dependency: { strategy: "isolated" } }
|
|
899
|
+
}, {
|
|
900
|
+
name: "quick-layered",
|
|
901
|
+
config: { dependency: { strategy: "layered" } }
|
|
902
|
+
}]
|
|
903
|
+
},
|
|
904
|
+
shapes: {
|
|
905
|
+
name: "dependency-shape sweep",
|
|
906
|
+
description: "How the dependency-graph shape affects discovery/scheduling overhead at a fixed scale.",
|
|
907
|
+
defaults: {
|
|
908
|
+
config: {
|
|
909
|
+
projects: 120,
|
|
910
|
+
tasksPerProject: 3
|
|
911
|
+
},
|
|
912
|
+
run: {
|
|
913
|
+
concurrency: 8,
|
|
914
|
+
coldRuns: 2,
|
|
915
|
+
warmRuns: 3
|
|
916
|
+
}
|
|
917
|
+
},
|
|
918
|
+
scenarios: Z.map((e) => ({
|
|
919
|
+
name: `shape-${e}`,
|
|
920
|
+
config: { dependency: { strategy: e } }
|
|
921
|
+
}))
|
|
922
|
+
},
|
|
923
|
+
scale: {
|
|
924
|
+
name: "scale sweep",
|
|
925
|
+
description: "How overhead grows with workspace size (layered graph).",
|
|
926
|
+
defaults: {
|
|
927
|
+
config: {
|
|
928
|
+
tasksPerProject: 3,
|
|
929
|
+
dependency: {
|
|
930
|
+
strategy: "layered",
|
|
931
|
+
layers: 8
|
|
932
|
+
}
|
|
933
|
+
},
|
|
934
|
+
run: {
|
|
935
|
+
concurrency: 8,
|
|
936
|
+
coldRuns: 2,
|
|
937
|
+
warmRuns: 3
|
|
938
|
+
}
|
|
939
|
+
},
|
|
940
|
+
scenarios: [
|
|
941
|
+
50,
|
|
942
|
+
150,
|
|
943
|
+
300,
|
|
944
|
+
600
|
|
945
|
+
].map((e) => ({
|
|
946
|
+
name: `scale-${e}`,
|
|
947
|
+
config: { projects: e }
|
|
948
|
+
}))
|
|
949
|
+
},
|
|
950
|
+
density: {
|
|
951
|
+
name: "task-density sweep",
|
|
952
|
+
description: "How the number of tasks per project affects overhead.",
|
|
953
|
+
defaults: {
|
|
954
|
+
config: {
|
|
955
|
+
projects: 120,
|
|
956
|
+
dependency: { strategy: "layered" }
|
|
957
|
+
},
|
|
958
|
+
run: {
|
|
959
|
+
concurrency: 8,
|
|
960
|
+
coldRuns: 2,
|
|
961
|
+
warmRuns: 3
|
|
962
|
+
}
|
|
963
|
+
},
|
|
964
|
+
scenarios: [
|
|
965
|
+
2,
|
|
966
|
+
5,
|
|
967
|
+
10
|
|
968
|
+
].map((e) => ({
|
|
969
|
+
name: `density-${e}`,
|
|
970
|
+
config: { tasksPerProject: e }
|
|
971
|
+
}))
|
|
972
|
+
},
|
|
973
|
+
daemon: {
|
|
974
|
+
name: "daemon on vs off",
|
|
975
|
+
description: "Whether each tool's persistent daemon changes warm/cold overhead.",
|
|
976
|
+
defaults: {
|
|
977
|
+
config: {
|
|
978
|
+
projects: 200,
|
|
979
|
+
tasksPerProject: 3,
|
|
980
|
+
dependency: { strategy: "layered" }
|
|
981
|
+
},
|
|
982
|
+
run: {
|
|
983
|
+
concurrency: 8,
|
|
984
|
+
coldRuns: 2,
|
|
985
|
+
warmRuns: 3
|
|
986
|
+
}
|
|
987
|
+
},
|
|
988
|
+
scenarios: [{
|
|
989
|
+
name: "daemon-on",
|
|
990
|
+
run: { daemon: !0 }
|
|
991
|
+
}, {
|
|
992
|
+
name: "daemon-off",
|
|
993
|
+
run: { daemon: !1 }
|
|
994
|
+
}]
|
|
995
|
+
},
|
|
996
|
+
full: {
|
|
997
|
+
name: "full suite",
|
|
998
|
+
description: "Shapes + scale + density in one run.",
|
|
999
|
+
defaults: { run: {
|
|
1000
|
+
concurrency: 8,
|
|
1001
|
+
coldRuns: 2,
|
|
1002
|
+
warmRuns: 3
|
|
1003
|
+
} },
|
|
1004
|
+
scenarios: [
|
|
1005
|
+
...Z.map((e) => ({
|
|
1006
|
+
name: `shape-${e}`,
|
|
1007
|
+
config: {
|
|
1008
|
+
projects: 120,
|
|
1009
|
+
tasksPerProject: 3,
|
|
1010
|
+
dependency: { strategy: e }
|
|
1011
|
+
}
|
|
1012
|
+
})),
|
|
1013
|
+
...[
|
|
1014
|
+
50,
|
|
1015
|
+
150,
|
|
1016
|
+
300
|
|
1017
|
+
].map((e) => ({
|
|
1018
|
+
name: `scale-${e}`,
|
|
1019
|
+
config: {
|
|
1020
|
+
projects: e,
|
|
1021
|
+
tasksPerProject: 3,
|
|
1022
|
+
dependency: {
|
|
1023
|
+
strategy: "layered",
|
|
1024
|
+
layers: 8
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
})),
|
|
1028
|
+
...[
|
|
1029
|
+
2,
|
|
1030
|
+
5,
|
|
1031
|
+
10
|
|
1032
|
+
].map((e) => ({
|
|
1033
|
+
name: `density-${e}`,
|
|
1034
|
+
config: {
|
|
1035
|
+
projects: 120,
|
|
1036
|
+
tasksPerProject: e,
|
|
1037
|
+
dependency: { strategy: "layered" }
|
|
1038
|
+
}
|
|
1039
|
+
}))
|
|
1040
|
+
]
|
|
1041
|
+
}
|
|
1042
|
+
};
|
|
1043
|
+
function Be() {
|
|
1044
|
+
return Object.keys(Q);
|
|
1045
|
+
}
|
|
1046
|
+
function Ve(e) {
|
|
1047
|
+
let t = Q[e];
|
|
1048
|
+
if (!t) throw Error(`unknown preset "${e}" (available: ${Be().join(", ")})`);
|
|
1049
|
+
return Y.parse(t);
|
|
1050
|
+
}
|
|
1051
|
+
function He(e) {
|
|
1052
|
+
return Y.parse(e);
|
|
1053
|
+
}
|
|
1054
|
+
//#endregion
|
|
1055
|
+
//#region src/suite/index.ts
|
|
1056
|
+
async function Ue(e, t) {
|
|
1057
|
+
let n = t.onEvent ?? (() => {}), r = [], i = e.scenarios.length;
|
|
1058
|
+
for (let a = 0; a < i; a++) {
|
|
1059
|
+
let s = e.scenarios[a];
|
|
1060
|
+
if (!s) continue;
|
|
1061
|
+
let c = ze(e, s), { config: u, run: d } = c, f = t.overrides?.tools ?? d.tools ?? u.tools;
|
|
1062
|
+
u.tools = f;
|
|
1063
|
+
let p = o(t.workdir, c.name);
|
|
1064
|
+
n({
|
|
1065
|
+
kind: "scenario-start",
|
|
1066
|
+
name: c.name,
|
|
1067
|
+
index: a,
|
|
1068
|
+
total: i
|
|
1069
|
+
}), await Fe(p, u), t.install !== !1 && await U(p, { quiet: !0 });
|
|
1070
|
+
let m = await H(p, {
|
|
1071
|
+
tools: f,
|
|
1072
|
+
task: t.overrides?.task ?? d.task,
|
|
1073
|
+
coldRuns: t.overrides?.coldRuns ?? d.coldRuns,
|
|
1074
|
+
warmRuns: t.overrides?.warmRuns ?? d.warmRuns,
|
|
1075
|
+
concurrency: t.overrides?.concurrency ?? d.concurrency,
|
|
1076
|
+
daemon: t.overrides?.daemon ?? d.daemon,
|
|
1077
|
+
onEvent: (e) => n({
|
|
1078
|
+
kind: "bench",
|
|
1079
|
+
name: c.name,
|
|
1080
|
+
event: e
|
|
1081
|
+
})
|
|
1082
|
+
});
|
|
1083
|
+
r.push({
|
|
1084
|
+
name: c.name,
|
|
1085
|
+
description: c.description,
|
|
1086
|
+
config: u,
|
|
1087
|
+
run: d,
|
|
1088
|
+
result: m
|
|
1089
|
+
}), t.keep || await l(p, {
|
|
1090
|
+
recursive: !0,
|
|
1091
|
+
force: !0
|
|
1092
|
+
}), n({
|
|
1093
|
+
kind: "scenario-done",
|
|
1094
|
+
name: c.name,
|
|
1095
|
+
index: a,
|
|
1096
|
+
total: i
|
|
1097
|
+
});
|
|
1098
|
+
}
|
|
1099
|
+
return {
|
|
1100
|
+
name: e.name,
|
|
1101
|
+
description: e.description,
|
|
1102
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1103
|
+
scenarios: r,
|
|
1104
|
+
taskBenchVersion: v
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
//#endregion
|
|
1108
|
+
//#region src/suite/report.ts
|
|
1109
|
+
function We(e) {
|
|
1110
|
+
let t = /* @__PURE__ */ new Set();
|
|
1111
|
+
for (let n of e.scenarios) for (let e of n.result.tools) t.add(e.tool);
|
|
1112
|
+
return x.filter((e) => t.has(e));
|
|
1113
|
+
}
|
|
1114
|
+
function Ge(e, t) {
|
|
1115
|
+
return `Tool versions:\n\n${t.map((t) => `\`${t} ${e.scenarios.map((e) => e.result.versions[t]).find((e) => e != null) ?? "?"}\``).join("\n\n")}`;
|
|
1116
|
+
}
|
|
1117
|
+
function Ke(e, t) {
|
|
1118
|
+
let n = e.result.tools.find((e) => e.tool === t);
|
|
1119
|
+
if (!n || n.error) return n?.error ? "err" : "—";
|
|
1120
|
+
let r = n.taskGraphSize > 0 ? (n.taskGraphSize - n.warm.executedMedian) / n.taskGraphSize : 1, i = z(n.warm.stats.median);
|
|
1121
|
+
return r < .995 ? `${i}⚠` : i;
|
|
1122
|
+
}
|
|
1123
|
+
function qe(e, t) {
|
|
1124
|
+
let n = e.result.tools.find((e) => e.tool === t);
|
|
1125
|
+
return !n || n.error ? n?.error ? "err" : "—" : z(n.cold.stats.median);
|
|
1126
|
+
}
|
|
1127
|
+
function $(e, t) {
|
|
1128
|
+
let n = e.map((e, n) => Math.max(e.length, ...t.map((e) => (e[n] ?? "").length))), r = (e, t) => e.padEnd(t), i = (e) => `| ${e.map((e, t) => r(e, n[t] ?? 0)).join(" | ")} |`;
|
|
1129
|
+
return [
|
|
1130
|
+
i(e),
|
|
1131
|
+
`| ${n.map((e) => "-".repeat(e)).join(" | ")} |`,
|
|
1132
|
+
...t.map(i)
|
|
1133
|
+
];
|
|
1134
|
+
}
|
|
1135
|
+
function Je(e) {
|
|
1136
|
+
let t = We(e), n = [];
|
|
1137
|
+
n.push(`# ${e.name}`), n.push(""), n.push(`TaskBench v${e.taskBenchVersion}`), n.push(""), e.description && (n.push(e.description), n.push("")), n.push(`Generated ${e.generatedAt} · ${e.scenarios.length} scenario(s).`), n.push(""), n.push(Ge(e, t)), n.push(""), n.push("`warm` = median wall time with a verified 100% cache hit (discovery + cache-restore overhead). `⚠` marks a scenario whose warm runs were not fully cached. Absolute times are hardware-dependent — read the ratios."), n.push(""), n.push("## Summary — warm median"), n.push(""), n.push(...$([
|
|
1138
|
+
"scenario",
|
|
1139
|
+
"nodes",
|
|
1140
|
+
"conc",
|
|
1141
|
+
"daemon",
|
|
1142
|
+
...t
|
|
1143
|
+
], e.scenarios.map((e) => [
|
|
1144
|
+
e.name,
|
|
1145
|
+
String(e.result.tools[0]?.taskGraphSize ?? 0),
|
|
1146
|
+
String(e.result.concurrency),
|
|
1147
|
+
e.result.daemon ? "on" : "off",
|
|
1148
|
+
...t.map((t) => Ke(e, t))
|
|
1149
|
+
]))), n.push(""), n.push("## Summary — cold median"), n.push(""), n.push(...$([
|
|
1150
|
+
"scenario",
|
|
1151
|
+
"nodes",
|
|
1152
|
+
"conc",
|
|
1153
|
+
...t
|
|
1154
|
+
], e.scenarios.map((e) => [
|
|
1155
|
+
e.name,
|
|
1156
|
+
String(e.result.tools[0]?.taskGraphSize ?? 0),
|
|
1157
|
+
String(e.result.concurrency),
|
|
1158
|
+
...t.map((t) => qe(e, t))
|
|
1159
|
+
]))), n.push(""), n.push("## Details"), n.push("");
|
|
1160
|
+
for (let t of e.scenarios) {
|
|
1161
|
+
let e = t.description ? `${t.name} — ${t.description}` : t.name;
|
|
1162
|
+
n.push(`### ${e}`), n.push(`Config: ${t.config.projects} projects × ${t.config.tasksPerProject} tasks, strategy \`${t.config.dependency.strategy}\`.`), n.push(Ee(t.result).trimEnd()), n.push("");
|
|
1163
|
+
}
|
|
1164
|
+
return n.join("\n");
|
|
1165
|
+
}
|
|
1166
|
+
//#endregion
|
|
1167
|
+
export { w as A, O as C, b as D, C as E, g as F, v as I, T as M, D as N, E as O, _ as P, k as S, y as T, z as _, Le as a, F as b, Ve as c, ze as d, Fe as f, R as g, H as h, J as i, S as j, x as k, Be as l, U as m, Ue as n, Y as o, Ee as p, Q as r, X as s, Je as t, He as u, I as v, A as w, L as x, P as y };
|