@cad0p/pi-memory-budget 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +21 -0
- package/README.md +142 -2
- package/docs/consumer-verification-section.md +28 -0
- package/docs/verification.md +55 -0
- package/package.json +63 -4
- package/src/args.mjs +199 -0
- package/src/budget.mjs +546 -0
- package/src/cli.mjs +96 -0
- package/src/config.mjs +180 -0
- package/src/fixture.mjs +491 -0
- package/src/gc-harness.js +67 -0
- package/src/index.mjs +13 -0
- package/src/orchestrate.mjs +164 -0
- package/src/pi.mjs +107 -0
- package/src/report.mjs +188 -0
- package/src/runner.mjs +347 -0
- package/src/sampler.mjs +162 -0
- package/src/stats.mjs +163 -0
package/src/budget.mjs
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
// Aggregation of runs and budget evaluation (pure, no I/O).
|
|
2
|
+
|
|
3
|
+
import { max, median, releaseSeconds } from "./stats.mjs";
|
|
4
|
+
|
|
5
|
+
/** Approved defaults (owner, 2026-11-19). MB, except spikeReleaseSeconds. */
|
|
6
|
+
export const DEFAULT_BUDGETS = Object.freeze({
|
|
7
|
+
steadyFresh: 20,
|
|
8
|
+
steadyLong: 20,
|
|
9
|
+
growth2x: 5,
|
|
10
|
+
spike: 100,
|
|
11
|
+
spikeReleaseSeconds: 30,
|
|
12
|
+
regression: 5,
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
export const BUDGET_KEYS = Object.keys(DEFAULT_BUDGETS);
|
|
16
|
+
export const CASE_ORDER = ["fresh", "long", "long2x"];
|
|
17
|
+
|
|
18
|
+
/** RSS noise allowance when judging a spike against the candidate's own steady level. */
|
|
19
|
+
const RELEASE_NOISE_MB = 5;
|
|
20
|
+
|
|
21
|
+
/** Samples at the threshold required for "released" (a single dip is not a release). */
|
|
22
|
+
const RELEASE_HOLD = 3;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @typedef {typeof DEFAULT_BUDGETS} Budgets
|
|
26
|
+
* @typedef {"PASS" | "FAIL" | "WAIVED" | "SKIP" | "INCONCLUSIVE"} Status
|
|
27
|
+
* @typedef {import("./runner.mjs").RunResult} RunResult
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Validates the budgets and waivers of a config file / CLI.
|
|
32
|
+
* @param {Record<string, unknown> | undefined} budgets
|
|
33
|
+
* @param {Record<string, unknown> | undefined} waivers
|
|
34
|
+
* @returns {{ budgets: Budgets, waivers: Record<string, string> }}
|
|
35
|
+
*/
|
|
36
|
+
export function resolveBudgets(budgets, waivers) {
|
|
37
|
+
const out = { ...DEFAULT_BUDGETS };
|
|
38
|
+
for (const [k, v] of Object.entries(budgets ?? {})) {
|
|
39
|
+
if (!BUDGET_KEYS.includes(k))
|
|
40
|
+
throw new Error(
|
|
41
|
+
`unknown budget "${k}" (known: ${BUDGET_KEYS.join(", ")})`,
|
|
42
|
+
);
|
|
43
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0)
|
|
44
|
+
throw new Error(`budget "${k}" must be a non-negative number`);
|
|
45
|
+
/** @type {Record<string, number>} */ (out)[k] = v;
|
|
46
|
+
}
|
|
47
|
+
/** @type {Record<string, string>} */
|
|
48
|
+
const w = {};
|
|
49
|
+
for (const [k, v] of Object.entries(waivers ?? {})) {
|
|
50
|
+
if (!BUDGET_KEYS.includes(k))
|
|
51
|
+
throw new Error(
|
|
52
|
+
`cannot waive unknown budget "${k}" (known: ${BUDGET_KEYS.join(", ")})`,
|
|
53
|
+
);
|
|
54
|
+
const reason =
|
|
55
|
+
typeof v === "string"
|
|
56
|
+
? v
|
|
57
|
+
: v && typeof v === "object"
|
|
58
|
+
? /** @type {{reason?: unknown}} */ (v).reason
|
|
59
|
+
: undefined;
|
|
60
|
+
if (typeof reason !== "string" || reason.trim() === "") {
|
|
61
|
+
throw new Error(
|
|
62
|
+
`waiver for "${k}" needs a non-empty reason (there is no silent skip)`,
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
w[k] = reason.trim();
|
|
66
|
+
}
|
|
67
|
+
return { budgets: /** @type {Budgets} */ (out), waivers: w };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* @typedef {object} Agg median over the runs of one configuration
|
|
72
|
+
* @property {number} runs
|
|
73
|
+
* @property {number | null} steadyMb median of per-run steady total RSS (null when no run settled)
|
|
74
|
+
* @property {number | null} steadyMinMb lowest per-run steady (null when no run settled)
|
|
75
|
+
* @property {number | null} steadyMaxMb highest per-run steady
|
|
76
|
+
* @property {number | null} steadySpreadMb max-min of per-run steady; null with fewer than 2 settled runs
|
|
77
|
+
* @property {number} spikeMb
|
|
78
|
+
* @property {number | null} releasedMb
|
|
79
|
+
* @property {number | null} mainSteadyMb
|
|
80
|
+
* @property {number | null} childrenSteadyMb
|
|
81
|
+
* @property {number | null} settleT median of per-run settle time (null when a run never settled)
|
|
82
|
+
* @property {boolean} settled whether every run settled
|
|
83
|
+
* @property {number} maxGapMs median of the per-run max ping gap
|
|
84
|
+
* @property {number} worstGapMs worst ping gap over all runs
|
|
85
|
+
* @property {{ heapUsedMb: number, externalMb: number, arrayBuffersMb: number } | null} gcDiag median post-GC heap numbers (diagnostic)
|
|
86
|
+
*/
|
|
87
|
+
|
|
88
|
+
/** @param {RunResult[]} runs */
|
|
89
|
+
function aggDiag(runs) {
|
|
90
|
+
const d = runs.map((r) => r.gcDiag).filter((x) => x);
|
|
91
|
+
if (!d.length) return null;
|
|
92
|
+
/** @param {"heapUsedMb"|"externalMb"|"arrayBuffersMb"} k */
|
|
93
|
+
const m = (k) =>
|
|
94
|
+
median(d.map((x) => /** @type {NonNullable<typeof x>} */ (x)[k]));
|
|
95
|
+
return {
|
|
96
|
+
heapUsedMb: m("heapUsedMb"),
|
|
97
|
+
externalMb: m("externalMb"),
|
|
98
|
+
arrayBuffersMb: m("arrayBuffersMb"),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* @param {RunResult[]} runs
|
|
104
|
+
* @returns {Agg}
|
|
105
|
+
*/
|
|
106
|
+
export function aggregate(runs) {
|
|
107
|
+
const settled = runs.every((r) => r.settled);
|
|
108
|
+
/** @param {(number | null)[]} v */
|
|
109
|
+
const med = (v) => {
|
|
110
|
+
const ok = /** @type {number[]} */ (
|
|
111
|
+
v.filter((x) => x !== null && !Number.isNaN(x))
|
|
112
|
+
);
|
|
113
|
+
return ok.length ? median(ok) : null;
|
|
114
|
+
};
|
|
115
|
+
const steady = /** @type {number[]} */ (
|
|
116
|
+
runs.map((r) => r.steadyMb).filter((x) => x !== null && !Number.isNaN(x))
|
|
117
|
+
);
|
|
118
|
+
const steadyMinMb = steady.length ? Math.min(...steady) : null;
|
|
119
|
+
const steadyMaxMb = steady.length ? Math.max(...steady) : null;
|
|
120
|
+
return {
|
|
121
|
+
runs: runs.length,
|
|
122
|
+
settled,
|
|
123
|
+
steadyMinMb,
|
|
124
|
+
steadyMaxMb,
|
|
125
|
+
steadySpreadMb:
|
|
126
|
+
steady.length >= 2 ? Math.max(...steady) - Math.min(...steady) : null,
|
|
127
|
+
settleT: settled
|
|
128
|
+
? median(runs.map((r) => /** @type {number} */ (r.settleT)))
|
|
129
|
+
: null,
|
|
130
|
+
steadyMb: med(runs.map((r) => r.steadyMb)),
|
|
131
|
+
spikeMb: median(runs.map((r) => r.spikeMb)),
|
|
132
|
+
releasedMb: med(runs.map((r) => r.releasedMb)),
|
|
133
|
+
mainSteadyMb: med(runs.map((r) => r.mainSteadyMb)),
|
|
134
|
+
childrenSteadyMb: med(runs.map((r) => r.childrenSteadyMb)),
|
|
135
|
+
maxGapMs: median(runs.map((r) => r.maxGapMs)),
|
|
136
|
+
worstGapMs: max(runs.map((r) => r.maxGapMs)),
|
|
137
|
+
gcDiag: aggDiag(runs),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* @typedef {object} Check
|
|
143
|
+
* @property {string} id unique id, e.g. "steadyLong" or "spike:fresh"
|
|
144
|
+
* @property {string} key budget key
|
|
145
|
+
* @property {string} case
|
|
146
|
+
* @property {string} label
|
|
147
|
+
* @property {number | null} value measured value (delta MB, or seconds), null when not available
|
|
148
|
+
* @property {number} limit
|
|
149
|
+
* @property {"MB" | "s"} unit
|
|
150
|
+
* @property {Status} status
|
|
151
|
+
* @property {string} [reason] waiver reason
|
|
152
|
+
* @property {string} [note]
|
|
153
|
+
* @property {Record<string, number | null>} detail raw numbers behind the value
|
|
154
|
+
*/
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* @typedef {Record<string, Record<string, RunResult[]>>} Measurements
|
|
158
|
+
* case -> configuration ("vanilla" | "candidate" | "baseline") -> runs
|
|
159
|
+
*/
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Explains a release that never happened.
|
|
163
|
+
* @param {RunResult[]} runs
|
|
164
|
+
* @param {number} vanillaSteadyMb
|
|
165
|
+
* @param {number} threshold
|
|
166
|
+
* @param {number} peakT
|
|
167
|
+
* @param {number} limitSec
|
|
168
|
+
*/
|
|
169
|
+
function releaseNote(runs, vanillaSteadyMb, threshold, peakT, limitSec) {
|
|
170
|
+
const endDelta =
|
|
171
|
+
median(
|
|
172
|
+
runs
|
|
173
|
+
.map((r) => r.releasedMb ?? Number.NaN)
|
|
174
|
+
.filter((x) => !Number.isNaN(x)),
|
|
175
|
+
) - vanillaSteadyMb;
|
|
176
|
+
let note = `still ${endDelta.toFixed(1)} MB over vanilla at the end of the window (allowed +${threshold.toFixed(1)} MB)`;
|
|
177
|
+
if (runs.some((r) => r.exitT !== null)) {
|
|
178
|
+
const t = median(runs.map((r) => r.exitT ?? 0));
|
|
179
|
+
note += `; pi exited ${(peakT + limitSec - t).toFixed(0)}s before the release could be observed (peak at ${peakT.toFixed(0)}s)`;
|
|
180
|
+
}
|
|
181
|
+
return note;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* @param {Measurements} m
|
|
186
|
+
* @param {{ budgets: Budgets, waivers: Record<string, string>, windowSec: number }} opts
|
|
187
|
+
* @returns {{ aggregates: Record<string, Record<string, Agg>>, checks: Check[], ok: boolean }}
|
|
188
|
+
*/
|
|
189
|
+
export function evaluate(m, opts) {
|
|
190
|
+
const { budgets, waivers } = opts;
|
|
191
|
+
/** @type {Record<string, Record<string, Agg>>} */
|
|
192
|
+
const aggregates = {};
|
|
193
|
+
for (const [c, configs] of Object.entries(m)) {
|
|
194
|
+
aggregates[c] = {};
|
|
195
|
+
for (const [name, runs] of Object.entries(configs))
|
|
196
|
+
aggregates[c][name] = aggregate(runs);
|
|
197
|
+
}
|
|
198
|
+
/** @type {Check[]} */
|
|
199
|
+
const checks = [];
|
|
200
|
+
|
|
201
|
+
/** @param {Omit<Check, "status" | "reason">} c @param {boolean | null | "inconclusive"} pass */
|
|
202
|
+
const push = (c, pass) => {
|
|
203
|
+
/** @type {Status} */
|
|
204
|
+
const status =
|
|
205
|
+
pass === null
|
|
206
|
+
? "SKIP"
|
|
207
|
+
: pass === "inconclusive"
|
|
208
|
+
? "INCONCLUSIVE"
|
|
209
|
+
: pass
|
|
210
|
+
? "PASS"
|
|
211
|
+
: "FAIL";
|
|
212
|
+
/** @type {Check} */
|
|
213
|
+
const check = { ...c, status };
|
|
214
|
+
if (waivers[c.key] && status !== "SKIP" && status !== "INCONCLUSIVE") {
|
|
215
|
+
check.reason = waivers[c.key];
|
|
216
|
+
check.note = `${c.note ? `${c.note}; ` : ""}would ${status}`;
|
|
217
|
+
check.status = "WAIVED";
|
|
218
|
+
}
|
|
219
|
+
checks.push(check);
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* When the reference configuration's own run-to-run steady spread exceeds
|
|
224
|
+
* the check's limit, the baseline is noisier than what the check can
|
|
225
|
+
* distinguish: the verdict is INCONCLUSIVE, never PASS nor FAIL.
|
|
226
|
+
* @param {Agg} ref
|
|
227
|
+
* @param {number} limitMb
|
|
228
|
+
* @param {string} name
|
|
229
|
+
* @returns {string | null}
|
|
230
|
+
*/
|
|
231
|
+
const spreadNote = (ref, limitMb, name) => {
|
|
232
|
+
const s = ref.steadySpreadMb;
|
|
233
|
+
if (s === null || s <= limitMb) return null;
|
|
234
|
+
return `${name} steady RSS spread across ${ref.runs} runs is ${s.toFixed(1)} MB (max-min), over the ${limitMb} MB budget of this check: too noisy for a verdict; rerun on a quieter machine or with more runs`;
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const cases = CASE_ORDER.filter((c) => m[c]);
|
|
238
|
+
for (const c of cases) {
|
|
239
|
+
const agg = aggregates[c];
|
|
240
|
+
const van = agg.vanilla;
|
|
241
|
+
const cand = agg.candidate;
|
|
242
|
+
if (!van || !cand) continue;
|
|
243
|
+
const steadyKey = c === "fresh" ? "steadyFresh" : "steadyLong";
|
|
244
|
+
|
|
245
|
+
const d =
|
|
246
|
+
cand.steadyMb === null || van.steadyMb === null
|
|
247
|
+
? null
|
|
248
|
+
: cand.steadyMb - van.steadyMb;
|
|
249
|
+
if (c === "fresh" || c === "long") {
|
|
250
|
+
// A run that never settles is not a passing number: FAIL with the reason.
|
|
251
|
+
// A settled candidate needs a settled, quiet vanilla to be judged: no
|
|
252
|
+
// baseline or a baseline noisier than the budget is INCONCLUSIVE.
|
|
253
|
+
const noise = spreadNote(van, budgets[steadyKey], "vanilla");
|
|
254
|
+
const note = !cand.settled
|
|
255
|
+
? "candidate never settled (no 10 s window with total-RSS range < 5 MB within the settle cap)"
|
|
256
|
+
: !van.settled
|
|
257
|
+
? "vanilla never settled: no steady baseline"
|
|
258
|
+
: (noise ?? undefined);
|
|
259
|
+
push(
|
|
260
|
+
{
|
|
261
|
+
id: steadyKey,
|
|
262
|
+
key: steadyKey,
|
|
263
|
+
case: c,
|
|
264
|
+
label: `steady added RSS, ${c}`,
|
|
265
|
+
value: d,
|
|
266
|
+
limit: budgets[steadyKey],
|
|
267
|
+
unit: "MB",
|
|
268
|
+
note,
|
|
269
|
+
detail: {
|
|
270
|
+
vanilla: van.steadyMb,
|
|
271
|
+
candidate: cand.steadyMb,
|
|
272
|
+
vanillaSpread: van.steadySpreadMb,
|
|
273
|
+
},
|
|
274
|
+
},
|
|
275
|
+
!cand.settled
|
|
276
|
+
? false
|
|
277
|
+
: !van.settled || noise !== null
|
|
278
|
+
? "inconclusive"
|
|
279
|
+
: d !== null && d <= budgets[steadyKey],
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (van.steadyMb === null) {
|
|
284
|
+
push(
|
|
285
|
+
{
|
|
286
|
+
id: `spike:${c}`,
|
|
287
|
+
key: "spike",
|
|
288
|
+
case: c,
|
|
289
|
+
label: `transient spike over vanilla steady, ${c}`,
|
|
290
|
+
value: null,
|
|
291
|
+
limit: budgets.spike,
|
|
292
|
+
unit: "MB",
|
|
293
|
+
note: "vanilla never settled: no reference",
|
|
294
|
+
detail: {},
|
|
295
|
+
},
|
|
296
|
+
"inconclusive",
|
|
297
|
+
);
|
|
298
|
+
} else if (spreadNote(van, budgets.spike, "vanilla")) {
|
|
299
|
+
push(
|
|
300
|
+
{
|
|
301
|
+
id: `spike:${c}`,
|
|
302
|
+
key: "spike",
|
|
303
|
+
case: c,
|
|
304
|
+
label: `transient spike over vanilla steady, ${c}`,
|
|
305
|
+
value: cand.spikeMb - van.steadyMb,
|
|
306
|
+
limit: budgets.spike,
|
|
307
|
+
unit: "MB",
|
|
308
|
+
note: /** @type {string} */ (
|
|
309
|
+
spreadNote(van, budgets.spike, "vanilla")
|
|
310
|
+
),
|
|
311
|
+
detail: {
|
|
312
|
+
vanilla: van.steadyMb,
|
|
313
|
+
candidate: cand.spikeMb,
|
|
314
|
+
vanillaSpread: van.steadySpreadMb,
|
|
315
|
+
},
|
|
316
|
+
},
|
|
317
|
+
"inconclusive",
|
|
318
|
+
);
|
|
319
|
+
} else {
|
|
320
|
+
const spike = cand.spikeMb - van.steadyMb;
|
|
321
|
+
push(
|
|
322
|
+
{
|
|
323
|
+
id: `spike:${c}`,
|
|
324
|
+
key: "spike",
|
|
325
|
+
case: c,
|
|
326
|
+
label: `transient spike over vanilla steady, ${c}`,
|
|
327
|
+
value: spike,
|
|
328
|
+
limit: budgets.spike,
|
|
329
|
+
unit: "MB",
|
|
330
|
+
detail: { vanilla: van.steadyMb, candidate: cand.spikeMb },
|
|
331
|
+
},
|
|
332
|
+
spike <= budgets.spike,
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (van.steadyMb === null) {
|
|
337
|
+
push(
|
|
338
|
+
{
|
|
339
|
+
id: `release:${c}`,
|
|
340
|
+
key: "spikeReleaseSeconds",
|
|
341
|
+
case: c,
|
|
342
|
+
label: `back under the ${steadyKey} budget after the spike, ${c}`,
|
|
343
|
+
value: null,
|
|
344
|
+
limit: budgets.spikeReleaseSeconds,
|
|
345
|
+
unit: "s",
|
|
346
|
+
note: "vanilla never settled: no reference",
|
|
347
|
+
detail: {},
|
|
348
|
+
},
|
|
349
|
+
"inconclusive",
|
|
350
|
+
);
|
|
351
|
+
} else {
|
|
352
|
+
// The spike must fall back under the steady budget. When the candidate's
|
|
353
|
+
// own settled level is already over that budget (the steady check
|
|
354
|
+
// reports it), a transient is judged against where it settles (+ RSS
|
|
355
|
+
// noise) instead: one steady problem is not reported twice, and a
|
|
356
|
+
// justified steady level does not make the check unsatisfiable.
|
|
357
|
+
const steadyDelta =
|
|
358
|
+
cand.steadyMb === null
|
|
359
|
+
? Number.POSITIVE_INFINITY
|
|
360
|
+
: cand.steadyMb - van.steadyMb;
|
|
361
|
+
const settles = steadyDelta + RELEASE_NOISE_MB > budgets[steadyKey];
|
|
362
|
+
const threshold =
|
|
363
|
+
settles && Number.isFinite(steadyDelta)
|
|
364
|
+
? steadyDelta + RELEASE_NOISE_MB
|
|
365
|
+
: budgets[steadyKey];
|
|
366
|
+
const noise = spreadNote(van, threshold, "vanilla");
|
|
367
|
+
const perRun = (m[c].candidate ?? []).map((r) =>
|
|
368
|
+
releaseSeconds(r.samples, van.steadyMb ?? 0, threshold, RELEASE_HOLD),
|
|
369
|
+
);
|
|
370
|
+
const runs = median(perRun.map((x) => x ?? Number.POSITIVE_INFINITY));
|
|
371
|
+
const never = !Number.isFinite(runs);
|
|
372
|
+
const peakT = median((m[c].candidate ?? []).map((r) => r.peakT));
|
|
373
|
+
// The runner extends the window so a release is observable; when the
|
|
374
|
+
// samples still end before peak + limit (process died, cap hit), the
|
|
375
|
+
// outcome is INCONCLUSIVE, never a plain FAIL.
|
|
376
|
+
const observable = (m[c].candidate ?? []).every(
|
|
377
|
+
(r) => r.windowEndSec >= r.peakT + budgets.spikeReleaseSeconds,
|
|
378
|
+
);
|
|
379
|
+
if (noise !== null) {
|
|
380
|
+
push(
|
|
381
|
+
{
|
|
382
|
+
id: `release:${c}`,
|
|
383
|
+
key: "spikeReleaseSeconds",
|
|
384
|
+
case: c,
|
|
385
|
+
label: `back under the ${steadyKey} budget after the spike, ${c}`,
|
|
386
|
+
value: never ? null : runs,
|
|
387
|
+
limit: budgets.spikeReleaseSeconds,
|
|
388
|
+
unit: "s",
|
|
389
|
+
note: noise,
|
|
390
|
+
detail: {
|
|
391
|
+
peakT,
|
|
392
|
+
thresholdMb: threshold,
|
|
393
|
+
vanillaSpread: van.steadySpreadMb,
|
|
394
|
+
},
|
|
395
|
+
},
|
|
396
|
+
"inconclusive",
|
|
397
|
+
);
|
|
398
|
+
} else if (never && !observable) {
|
|
399
|
+
push(
|
|
400
|
+
{
|
|
401
|
+
id: `release:${c}`,
|
|
402
|
+
key: "spikeReleaseSeconds",
|
|
403
|
+
case: c,
|
|
404
|
+
label: `back under the ${steadyKey} budget after the spike, ${c}`,
|
|
405
|
+
value: null,
|
|
406
|
+
limit: budgets.spikeReleaseSeconds,
|
|
407
|
+
unit: "s",
|
|
408
|
+
note: releaseNote(
|
|
409
|
+
m[c].candidate ?? [],
|
|
410
|
+
van.steadyMb,
|
|
411
|
+
threshold,
|
|
412
|
+
peakT,
|
|
413
|
+
budgets.spikeReleaseSeconds,
|
|
414
|
+
),
|
|
415
|
+
detail: { peakT, thresholdMb: threshold },
|
|
416
|
+
},
|
|
417
|
+
"inconclusive",
|
|
418
|
+
);
|
|
419
|
+
} else {
|
|
420
|
+
push(
|
|
421
|
+
{
|
|
422
|
+
id: `release:${c}`,
|
|
423
|
+
key: "spikeReleaseSeconds",
|
|
424
|
+
case: c,
|
|
425
|
+
label: `back under the ${steadyKey} budget after the spike, ${c}`,
|
|
426
|
+
value: never ? null : runs,
|
|
427
|
+
limit: budgets.spikeReleaseSeconds,
|
|
428
|
+
unit: "s",
|
|
429
|
+
note: never
|
|
430
|
+
? releaseNote(
|
|
431
|
+
m[c].candidate ?? [],
|
|
432
|
+
van.steadyMb,
|
|
433
|
+
threshold,
|
|
434
|
+
peakT,
|
|
435
|
+
budgets.spikeReleaseSeconds,
|
|
436
|
+
)
|
|
437
|
+
: settles && Number.isFinite(steadyDelta)
|
|
438
|
+
? `steady is over the budget: measured against its own steady level (+${threshold.toFixed(1)} MB)`
|
|
439
|
+
: undefined,
|
|
440
|
+
detail: { peakT, thresholdMb: threshold },
|
|
441
|
+
},
|
|
442
|
+
!never && runs <= budgets.spikeReleaseSeconds,
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
if (agg.baseline) {
|
|
448
|
+
const d =
|
|
449
|
+
cand.steadyMb === null || agg.baseline.steadyMb === null
|
|
450
|
+
? null
|
|
451
|
+
: cand.steadyMb - agg.baseline.steadyMb;
|
|
452
|
+
const noise = spreadNote(agg.baseline, budgets.regression, "baseline");
|
|
453
|
+
push(
|
|
454
|
+
{
|
|
455
|
+
id: `regression:${c}`,
|
|
456
|
+
key: "regression",
|
|
457
|
+
case: c,
|
|
458
|
+
label: `regression vs baseline extension, ${c}`,
|
|
459
|
+
value: d,
|
|
460
|
+
limit: budgets.regression,
|
|
461
|
+
unit: "MB",
|
|
462
|
+
note: !cand.settled
|
|
463
|
+
? "candidate never settled"
|
|
464
|
+
: !agg.baseline.settled
|
|
465
|
+
? "baseline never settled: no reference"
|
|
466
|
+
: (noise ?? undefined),
|
|
467
|
+
detail: {
|
|
468
|
+
baseline: agg.baseline.steadyMb,
|
|
469
|
+
candidate: cand.steadyMb,
|
|
470
|
+
baselineSpread: agg.baseline.steadySpreadMb,
|
|
471
|
+
},
|
|
472
|
+
},
|
|
473
|
+
!cand.settled
|
|
474
|
+
? false
|
|
475
|
+
: !agg.baseline.settled || noise !== null
|
|
476
|
+
? "inconclusive"
|
|
477
|
+
: d !== null && d <= budgets.regression,
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const a1 = aggregates.long;
|
|
483
|
+
const a2 = aggregates.long2x;
|
|
484
|
+
if (a1?.vanilla && a1.candidate && a2?.vanilla && a2.candidate) {
|
|
485
|
+
const settled = a1.candidate.settled && a2.candidate.settled;
|
|
486
|
+
const d1 =
|
|
487
|
+
a1.candidate.steadyMb !== null && a1.vanilla.steadyMb !== null
|
|
488
|
+
? a1.candidate.steadyMb - a1.vanilla.steadyMb
|
|
489
|
+
: null;
|
|
490
|
+
const d2 =
|
|
491
|
+
a2.candidate.steadyMb !== null && a2.vanilla.steadyMb !== null
|
|
492
|
+
? a2.candidate.steadyMb - a2.vanilla.steadyMb
|
|
493
|
+
: null;
|
|
494
|
+
const vanSettled = a1.vanilla.settled && a2.vanilla.settled;
|
|
495
|
+
const noise =
|
|
496
|
+
spreadNote(a1.vanilla, budgets.growth2x, "vanilla (long)") ??
|
|
497
|
+
spreadNote(a2.vanilla, budgets.growth2x, "vanilla (long2x)");
|
|
498
|
+
push(
|
|
499
|
+
{
|
|
500
|
+
id: "growth2x",
|
|
501
|
+
key: "growth2x",
|
|
502
|
+
case: "long2x",
|
|
503
|
+
label: "growth with session size (delta long2x minus delta long)",
|
|
504
|
+
value: d1 !== null && d2 !== null ? d2 - d1 : null,
|
|
505
|
+
limit: budgets.growth2x,
|
|
506
|
+
unit: "MB",
|
|
507
|
+
note: !settled
|
|
508
|
+
? "candidate never settled on a long case"
|
|
509
|
+
: !vanSettled
|
|
510
|
+
? "vanilla never settled on a long case: no baseline"
|
|
511
|
+
: (noise ?? undefined),
|
|
512
|
+
detail: {
|
|
513
|
+
deltaLong: d1,
|
|
514
|
+
deltaLong2x: d2,
|
|
515
|
+
vanillaSpreadLong: a1.vanilla.steadySpreadMb,
|
|
516
|
+
vanillaSpreadLong2x: a2.vanilla.steadySpreadMb,
|
|
517
|
+
},
|
|
518
|
+
},
|
|
519
|
+
!settled
|
|
520
|
+
? false
|
|
521
|
+
: !vanSettled || noise !== null
|
|
522
|
+
? "inconclusive"
|
|
523
|
+
: d1 !== null && d2 !== null && d2 - d1 <= budgets.growth2x,
|
|
524
|
+
);
|
|
525
|
+
} else if (m.long || m.long2x) {
|
|
526
|
+
push(
|
|
527
|
+
{
|
|
528
|
+
id: "growth2x",
|
|
529
|
+
key: "growth2x",
|
|
530
|
+
case: "long2x",
|
|
531
|
+
label: "growth with session size (delta long2x minus delta long)",
|
|
532
|
+
value: null,
|
|
533
|
+
limit: budgets.growth2x,
|
|
534
|
+
unit: "MB",
|
|
535
|
+
note: "needs both cases long and long2x",
|
|
536
|
+
detail: {},
|
|
537
|
+
},
|
|
538
|
+
null,
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
const ok = checks.every(
|
|
543
|
+
(c) => c.status === "PASS" || c.status === "WAIVED" || c.status === "SKIP",
|
|
544
|
+
);
|
|
545
|
+
return { aggregates, checks, ok };
|
|
546
|
+
}
|
package/src/cli.mjs
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// pi-memory-budget CLI entry point.
|
|
3
|
+
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { parseFixtureArgs, parseRunArgs, USAGE, UsageError } from "./args.mjs";
|
|
8
|
+
import { resolveOptions } from "./config.mjs";
|
|
9
|
+
import { writeFixture } from "./fixture.mjs";
|
|
10
|
+
import { runBudget } from "./orchestrate.mjs";
|
|
11
|
+
import { renderJson, renderMarkdown } from "./report.mjs";
|
|
12
|
+
import { installCleanupHooks } from "./runner.mjs";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @param {string[]} argv process arguments without node and script
|
|
16
|
+
* @returns {Promise<number>} exit code
|
|
17
|
+
*/
|
|
18
|
+
export async function main(argv) {
|
|
19
|
+
const [cmd, ...rest] = argv;
|
|
20
|
+
try {
|
|
21
|
+
if (!cmd || cmd === "--help" || cmd === "-h" || cmd === "help") {
|
|
22
|
+
process.stdout.write(USAGE);
|
|
23
|
+
return cmd ? 0 : 2;
|
|
24
|
+
}
|
|
25
|
+
if (cmd === "--version" || cmd === "-v") {
|
|
26
|
+
const pkg = JSON.parse(
|
|
27
|
+
fs.readFileSync(new URL("../package.json", import.meta.url), "utf8"),
|
|
28
|
+
);
|
|
29
|
+
process.stdout.write(`${pkg.version}\n`);
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
if (cmd === "fixture") {
|
|
33
|
+
const a = parseFixtureArgs(rest);
|
|
34
|
+
if (a.help) {
|
|
35
|
+
process.stdout.write(USAGE);
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
fs.mkdirSync(path.dirname(path.resolve(a.out)), { recursive: true });
|
|
39
|
+
const st = writeFixture(a.out, { mb: a.mb, seed: a.seed, cwd: a.cwd });
|
|
40
|
+
process.stdout.write(
|
|
41
|
+
`${a.out}: ${(st.bytes / 1048576).toFixed(1)} MiB, ${st.entries} entries, ${st.branches} branch points\n`,
|
|
42
|
+
);
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
if (cmd === "run") {
|
|
46
|
+
const a = parseRunArgs(rest);
|
|
47
|
+
if (a.help) {
|
|
48
|
+
process.stdout.write(USAGE);
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
51
|
+
installCleanupHooks();
|
|
52
|
+
const o = resolveOptions(a, os.tmpdir());
|
|
53
|
+
fs.mkdirSync(o.fixturesDir, { recursive: true });
|
|
54
|
+
const log = a.quiet
|
|
55
|
+
? () => {}
|
|
56
|
+
: (/** @type {string} */ m) =>
|
|
57
|
+
process.stderr.write(`[pi-memory-budget] ${m}\n`);
|
|
58
|
+
const { measurements, result, meta } = await runBudget(o, { log });
|
|
59
|
+
const input = { result, budgets: o.budgets, waivers: o.waivers, meta };
|
|
60
|
+
const md = renderMarkdown(input);
|
|
61
|
+
if (a.report) {
|
|
62
|
+
fs.mkdirSync(path.dirname(path.resolve(a.report)), { recursive: true });
|
|
63
|
+
fs.writeFileSync(a.report, md);
|
|
64
|
+
}
|
|
65
|
+
if (a.json) {
|
|
66
|
+
fs.mkdirSync(path.dirname(path.resolve(a.json)), { recursive: true });
|
|
67
|
+
fs.writeFileSync(
|
|
68
|
+
a.json,
|
|
69
|
+
`${JSON.stringify(renderJson({ ...input, measurements }), null, 2)}\n`,
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
process.stdout.write(md);
|
|
73
|
+
// 1 a budget FAILs; 3 no FAIL but an INCONCLUSIVE check (noisy baseline,
|
|
74
|
+
// unobserved release): a distinct non-zero so CI can tell them apart.
|
|
75
|
+
if (result.checks.some((c) => c.status === "FAIL")) return 1;
|
|
76
|
+
return result.ok ? 0 : 3;
|
|
77
|
+
}
|
|
78
|
+
throw new UsageError(`unknown command "${cmd}"\n\n${USAGE}`);
|
|
79
|
+
} catch (e) {
|
|
80
|
+
process.stderr.write(
|
|
81
|
+
`pi-memory-budget: ${e instanceof Error ? e.message : String(e)}\n`,
|
|
82
|
+
);
|
|
83
|
+
return 2;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
import { realpathSync } from "node:fs";
|
|
88
|
+
import { fileURLToPath } from "node:url";
|
|
89
|
+
|
|
90
|
+
// Run only when executed as a script (also through the node_modules/.bin symlink).
|
|
91
|
+
if (
|
|
92
|
+
process.argv[1] &&
|
|
93
|
+
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
|
|
94
|
+
) {
|
|
95
|
+
main(process.argv.slice(2)).then((code) => process.exit(code));
|
|
96
|
+
}
|