@cad0p/pi-memory-budget 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +21 -0
- package/README.md +142 -2
- package/docs/consumer-verification-section.md +28 -0
- package/docs/verification.md +55 -0
- package/package.json +63 -4
- package/src/args.mjs +199 -0
- package/src/budget.mjs +546 -0
- package/src/cli.mjs +96 -0
- package/src/config.mjs +180 -0
- package/src/fixture.mjs +491 -0
- package/src/gc-harness.js +67 -0
- package/src/index.mjs +13 -0
- package/src/orchestrate.mjs +164 -0
- package/src/pi.mjs +107 -0
- package/src/report.mjs +188 -0
- package/src/runner.mjs +347 -0
- package/src/sampler.mjs +162 -0
- package/src/stats.mjs +163 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// pi-memory-budget GC harness. The budget CLI loads this extension in EVERY
|
|
2
|
+
// measured pi process — vanilla, candidate and baseline alike — so its own
|
|
3
|
+
// footprint cancels out of the delta. It exists because an idle pi allocates
|
|
4
|
+
// almost nothing: left to itself V8 never runs a major GC and the settled RSS
|
|
5
|
+
// is whatever garbage happened to be uncollected when allocation stopped
|
|
6
|
+
// (issue #3: run-to-run swings of -216..+78 MB on 100 MB sessions). With pi
|
|
7
|
+
// started under --expose-gc (the runner adds it through NODE_OPTIONS) this
|
|
8
|
+
// harness forces full GCs until only retained memory is left.
|
|
9
|
+
|
|
10
|
+
// Marker grepped out of pi's stderr by the runner to confirm the harness is
|
|
11
|
+
// actually live in the measured process.
|
|
12
|
+
const MARK = "pmb-gc:";
|
|
13
|
+
|
|
14
|
+
/** One full, synchronous, compacting collection (V8 "major"). */
|
|
15
|
+
function collect() {
|
|
16
|
+
const gc = /** @type {any} */ (globalThis).gc;
|
|
17
|
+
try {
|
|
18
|
+
gc({ type: "major", execution: "sync" });
|
|
19
|
+
} catch {
|
|
20
|
+
// Older Node: gc() takes no options and is already a full GC.
|
|
21
|
+
gc();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export default function pmbGcHarness() {
|
|
26
|
+
if (typeof (/** @type {any} */ (globalThis).gc) !== "function") {
|
|
27
|
+
process.stderr.write(
|
|
28
|
+
MARK +
|
|
29
|
+
" unavailable: globalThis.gc missing, pi was not started with --expose-gc; steady RSS will include uncollected garbage\n",
|
|
30
|
+
);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
process.stderr.write(MARK + " ready\n");
|
|
34
|
+
|
|
35
|
+
// Two passes back to back: the second frees what the first pass orphaned.
|
|
36
|
+
const now = () => {
|
|
37
|
+
collect();
|
|
38
|
+
collect();
|
|
39
|
+
};
|
|
40
|
+
now();
|
|
41
|
+
|
|
42
|
+
// Re-arm a forced GC whenever the heap grew by STEP_MB since the last pass,
|
|
43
|
+
// and unconditionally every INTERVAL*10 ms for garbage heapUsed does not see
|
|
44
|
+
// (native buffers, external memory). STEP stays under the 5 MB quiet-window
|
|
45
|
+
// range so a step drop inside the steady window cannot break quiescence, and
|
|
46
|
+
// a process that keeps producing garbage simply never settles — the correct
|
|
47
|
+
// verdict for it anyway.
|
|
48
|
+
const STEP_MB = 4;
|
|
49
|
+
const INTERVAL_MS = 500;
|
|
50
|
+
let base = process.memoryUsage().heapUsed;
|
|
51
|
+
let ticks = 0;
|
|
52
|
+
const t = setInterval(() => {
|
|
53
|
+
ticks += 1;
|
|
54
|
+
const used = process.memoryUsage().heapUsed;
|
|
55
|
+
if (used - base >= STEP_MB * 1048576 || ticks % 10 === 0) {
|
|
56
|
+
now();
|
|
57
|
+
const m = process.memoryUsage();
|
|
58
|
+
base = m.heapUsed;
|
|
59
|
+
// Diagnostic only (post-GC numbers); budgets stay on total RSS.
|
|
60
|
+
process.stderr.write(
|
|
61
|
+
`${MARK} diag ${JSON.stringify({ heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers })}\n`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
}, INTERVAL_MS);
|
|
65
|
+
// Never keep pi alive for the harness alone.
|
|
66
|
+
t.unref();
|
|
67
|
+
}
|
package/src/index.mjs
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { parseFixtureArgs, parseRunArgs } from "./args.mjs";
|
|
2
|
+
export {
|
|
3
|
+
aggregate,
|
|
4
|
+
DEFAULT_BUDGETS,
|
|
5
|
+
evaluate,
|
|
6
|
+
resolveBudgets,
|
|
7
|
+
} from "./budget.mjs";
|
|
8
|
+
export { resolveOptions } from "./config.mjs";
|
|
9
|
+
export { ensureFixture, writeFixture } from "./fixture.mjs";
|
|
10
|
+
export { runBudget } from "./orchestrate.mjs";
|
|
11
|
+
export { GC_HARNESS } from "./pi.mjs";
|
|
12
|
+
export { renderJson, renderMarkdown } from "./report.mjs";
|
|
13
|
+
export { median, releaseSeconds, summarize } from "./stats.mjs";
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// Runs the (configuration x case x run) matrix sequentially: never more than
|
|
2
|
+
// one pi process at a time.
|
|
3
|
+
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import fs from "node:fs";
|
|
6
|
+
import os from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { evaluate } from "./budget.mjs";
|
|
9
|
+
import { ensureFixture } from "./fixture.mjs";
|
|
10
|
+
import { GC_HARNESS, piVersion, resolvePi } from "./pi.mjs";
|
|
11
|
+
import { measureRun } from "./runner.mjs";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @typedef {object} Hooks
|
|
15
|
+
* @property {(msg: string) => void} [log]
|
|
16
|
+
* @property {typeof measureRun} [measure] injectable for tests
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {string} command
|
|
21
|
+
* @param {string} cwd
|
|
22
|
+
* @param {NodeJS.ProcessEnv} env
|
|
23
|
+
* @returns {Promise<void>}
|
|
24
|
+
*/
|
|
25
|
+
function runSetup(command, cwd, env) {
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const p = spawn("sh", ["-c", command], {
|
|
28
|
+
cwd,
|
|
29
|
+
env,
|
|
30
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
31
|
+
});
|
|
32
|
+
p.on("error", reject);
|
|
33
|
+
p.on("exit", (code) =>
|
|
34
|
+
code === 0
|
|
35
|
+
? resolve()
|
|
36
|
+
: reject(new Error(`setup command failed (exit ${code}): ${command}`)),
|
|
37
|
+
);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {import("./config.mjs").ResolvedOptions} o
|
|
43
|
+
* @param {Hooks} [hooks]
|
|
44
|
+
*/
|
|
45
|
+
export async function runBudget(o, hooks = {}) {
|
|
46
|
+
const log = hooks.log ?? (() => {});
|
|
47
|
+
const measure = hooks.measure ?? measureRun;
|
|
48
|
+
const pi = resolvePi({ pi: o.pi, cwd: o.cwd });
|
|
49
|
+
const version = await piVersion(pi, o.cwd);
|
|
50
|
+
log(`pi ${version} (${pi.source}: ${pi.command})`);
|
|
51
|
+
|
|
52
|
+
/** @type {Record<string, string | null>} */
|
|
53
|
+
const sessions = {};
|
|
54
|
+
for (const c of o.cases) {
|
|
55
|
+
if (c === "fresh") sessions[c] = null;
|
|
56
|
+
else {
|
|
57
|
+
const mb = c === "long" ? o.sessionMb : o.sessionMb * 2;
|
|
58
|
+
const t = Date.now();
|
|
59
|
+
const { file, generated } = ensureFixture(o.fixturesDir, mb, o.seed);
|
|
60
|
+
log(
|
|
61
|
+
`fixture ${c}: ${file} (${(fs.statSync(file).size / 1048576).toFixed(0)} MiB, ${generated ? `generated in ${((Date.now() - t) / 1000).toFixed(1)}s` : "cached"})`,
|
|
62
|
+
);
|
|
63
|
+
sessions[c] = file;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// The GC harness is a normal -e extension loaded in every configuration
|
|
68
|
+
// alike, so its footprint cancels out of every delta.
|
|
69
|
+
const harness = o.gc ? [GC_HARNESS] : [];
|
|
70
|
+
if (o.gc)
|
|
71
|
+
log("gc normalization: --expose-gc + harness extension in every pi run");
|
|
72
|
+
|
|
73
|
+
/** @type {Record<string, string[]>} */
|
|
74
|
+
const configs = { vanilla: [], candidate: o.extensions };
|
|
75
|
+
if (o.baselineExtensions.length) configs.baseline = o.baselineExtensions;
|
|
76
|
+
|
|
77
|
+
/** @type {import("./budget.mjs").Measurements} */
|
|
78
|
+
const measurements = {};
|
|
79
|
+
for (const c of o.cases) {
|
|
80
|
+
measurements[c] = {};
|
|
81
|
+
for (const name of Object.keys(configs)) measurements[c][name] = [];
|
|
82
|
+
}
|
|
83
|
+
fs.mkdirSync(o.fixturesDir, { recursive: true });
|
|
84
|
+
const workDir = fs.mkdtempSync(path.join(o.fixturesDir, "work-"));
|
|
85
|
+
try {
|
|
86
|
+
// runs outermost, so a slow drift of the machine hits every configuration alike
|
|
87
|
+
for (let run = 1; run <= o.runs; run++) {
|
|
88
|
+
for (const c of o.cases) {
|
|
89
|
+
for (const [name, extensions] of Object.entries(configs)) {
|
|
90
|
+
let sessionFile = null;
|
|
91
|
+
const src = sessions[c];
|
|
92
|
+
if (src) {
|
|
93
|
+
// pi may append to the session file: every run gets its own copy
|
|
94
|
+
sessionFile = path.join(workDir, `${c}-${name}-${run}.jsonl`);
|
|
95
|
+
fs.copyFileSync(src, sessionFile, fs.constants.COPYFILE_FICLONE);
|
|
96
|
+
}
|
|
97
|
+
const env = { ...process.env, ...o.env };
|
|
98
|
+
const setup = o.setup[c] ?? o.setup["*"];
|
|
99
|
+
if (setup) {
|
|
100
|
+
await runSetup(setup, o.cwd, {
|
|
101
|
+
...env,
|
|
102
|
+
PMB_CASE: c,
|
|
103
|
+
PMB_CONFIG: name,
|
|
104
|
+
PMB_RUN: String(run),
|
|
105
|
+
PMB_SESSION_FILE: sessionFile ?? "",
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
log(`run ${run}/${o.runs} case=${c} config=${name} (${o.window}s)`);
|
|
109
|
+
const r = await measure({
|
|
110
|
+
pi,
|
|
111
|
+
extensions,
|
|
112
|
+
harnessExtensions: harness,
|
|
113
|
+
gc: o.gc,
|
|
114
|
+
sessionFile,
|
|
115
|
+
cwd: o.cwd,
|
|
116
|
+
env,
|
|
117
|
+
windowSec: o.window,
|
|
118
|
+
steadySec: Math.min(o.steadySeconds, o.window),
|
|
119
|
+
settleMaxSec: o.settleMax,
|
|
120
|
+
settleMinSec: o.settleMin,
|
|
121
|
+
releaseSec: o.budgets.spikeReleaseSeconds,
|
|
122
|
+
extraArgs: o.piArgs,
|
|
123
|
+
});
|
|
124
|
+
log(
|
|
125
|
+
` ${r.settled ? `steady ${/** @type {number} */ (r.steadyMb).toFixed(1)} MB (settled at ${/** @type {number} */ (r.settleT).toFixed(0)}s, main ${/** @type {number} */ (r.mainSteadyMb).toFixed(1)} + children ${/** @type {number} */ (r.childrenSteadyMb).toFixed(1)}` : "never settled"}, spike ${r.spikeMb.toFixed(1)} MB, max ping gap ${r.maxGapMs.toFixed(0)} ms`,
|
|
126
|
+
);
|
|
127
|
+
measurements[c][name].push(r);
|
|
128
|
+
if (o.gc && r.gcReady === false)
|
|
129
|
+
log(
|
|
130
|
+
" warning: GC harness did not report ready (pi without --expose-gc support?); steady may be garbage-timing noise",
|
|
131
|
+
);
|
|
132
|
+
if (sessionFile) fs.rmSync(sessionFile, { force: true });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
fs.rmSync(workDir, { recursive: true, force: true });
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const result = evaluate(measurements, {
|
|
141
|
+
budgets: o.budgets,
|
|
142
|
+
waivers: o.waivers,
|
|
143
|
+
windowSec: o.window,
|
|
144
|
+
});
|
|
145
|
+
const meta = {
|
|
146
|
+
piVersion: version,
|
|
147
|
+
piSource: pi.source,
|
|
148
|
+
platform: `${process.platform}-${process.arch}, node ${process.version}, ${os.cpus().length} cpus`,
|
|
149
|
+
runs: o.runs,
|
|
150
|
+
windowSec: o.window,
|
|
151
|
+
steadySec: Math.min(o.steadySeconds, o.window),
|
|
152
|
+
sessionMb: o.sessionMb,
|
|
153
|
+
settleMaxSec: o.settleMax,
|
|
154
|
+
settleMinSec: o.settleMin,
|
|
155
|
+
seed: o.seed,
|
|
156
|
+
gc: o.gc,
|
|
157
|
+
cases: o.cases,
|
|
158
|
+
extensions: o.extensions,
|
|
159
|
+
baselineExtensions: o.baselineExtensions,
|
|
160
|
+
cwd: o.cwd,
|
|
161
|
+
generatedAt: new Date().toISOString(),
|
|
162
|
+
};
|
|
163
|
+
return { measurements, result, meta };
|
|
164
|
+
}
|
package/src/pi.mjs
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// Resolving the pi binary and building the pi command line.
|
|
2
|
+
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
|
|
8
|
+
/** Extension that forces major GCs inside every measured pi (issue #3). */
|
|
9
|
+
export const GC_HARNESS = fileURLToPath(
|
|
10
|
+
new URL("./gc-harness.js", import.meta.url),
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Splits a --pi value such as "node /path/cli.js" into command and args.
|
|
15
|
+
* @param {string} cmd
|
|
16
|
+
* @returns {{ command: string, args: string[] }}
|
|
17
|
+
*/
|
|
18
|
+
export function splitCommand(cmd) {
|
|
19
|
+
const parts = cmd.trim().split(/\s+/).filter(Boolean);
|
|
20
|
+
if (parts.length === 0) throw new Error("--pi is empty");
|
|
21
|
+
return { command: parts[0], args: parts.slice(1) };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {string} name
|
|
26
|
+
* @param {string} [pathEnv]
|
|
27
|
+
* @returns {string | null}
|
|
28
|
+
*/
|
|
29
|
+
function which(name, pathEnv = process.env.PATH ?? "") {
|
|
30
|
+
for (const dir of pathEnv.split(path.delimiter)) {
|
|
31
|
+
if (!dir) continue;
|
|
32
|
+
const p = path.join(dir, name);
|
|
33
|
+
try {
|
|
34
|
+
fs.accessSync(p, fs.constants.X_OK);
|
|
35
|
+
if (fs.statSync(p).isFile()) return p;
|
|
36
|
+
} catch {
|
|
37
|
+
// not here
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Resolution order: --pi, then node_modules/.bin/pi of the cwd, then PATH.
|
|
45
|
+
* @param {{ pi?: string, cwd: string }} opts
|
|
46
|
+
* @returns {{ command: string, args: string[], source: string }}
|
|
47
|
+
*/
|
|
48
|
+
export function resolvePi(opts) {
|
|
49
|
+
if (opts.pi) return { ...splitCommand(opts.pi), source: "--pi" };
|
|
50
|
+
const local = path.join(opts.cwd, "node_modules", ".bin", "pi");
|
|
51
|
+
if (fs.existsSync(local))
|
|
52
|
+
return { command: local, args: [], source: "node_modules/.bin" };
|
|
53
|
+
const onPath = which("pi");
|
|
54
|
+
if (onPath) return { command: onPath, args: [], source: "PATH" };
|
|
55
|
+
throw new Error(
|
|
56
|
+
"could not find pi: pass --pi <cmd>, or install @earendil-works/pi-coding-agent in the project (node_modules/.bin/pi), or put pi on PATH",
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* @typedef {object} PiRunSpec
|
|
62
|
+
* @property {string[]} extensions absolute extension paths (-e)
|
|
63
|
+
* @property {string | null} sessionFile session file to open, null for --no-session
|
|
64
|
+
* @property {string[]} [harnessExtensions] loaded in every configuration alike, before the measured ones
|
|
65
|
+
* @property {string[]} [extraArgs]
|
|
66
|
+
*/
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* @param {PiRunSpec} spec
|
|
70
|
+
* @returns {string[]}
|
|
71
|
+
*/
|
|
72
|
+
export function buildPiArgs(spec) {
|
|
73
|
+
const args = ["--mode", "rpc"];
|
|
74
|
+
if (spec.sessionFile) args.push("--session", spec.sessionFile);
|
|
75
|
+
else args.push("--no-session");
|
|
76
|
+
// Isolation: the only difference between vanilla and candidate is the
|
|
77
|
+
// extension under test. -ne explicit-only extensions, -ns skills, -np prompt
|
|
78
|
+
// templates, --no-themes, --no-mcp (MCP servers would also show up as child
|
|
79
|
+
// processes), -nc AGENTS.md/CLAUDE.md discovery.
|
|
80
|
+
args.push("-ne", "-ns", "-np", "--no-themes", "--no-mcp", "-nc");
|
|
81
|
+
for (const e of [...(spec.harnessExtensions ?? []), ...spec.extensions])
|
|
82
|
+
args.push("-e", e);
|
|
83
|
+
args.push(...(spec.extraArgs ?? []));
|
|
84
|
+
return args;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* @param {{ command: string, args: string[] }} pi
|
|
89
|
+
* @param {string} cwd
|
|
90
|
+
* @returns {Promise<string>} output of `pi --version` (or "unknown")
|
|
91
|
+
*/
|
|
92
|
+
export function piVersion(pi, cwd) {
|
|
93
|
+
return new Promise((resolve) => {
|
|
94
|
+
execFile(
|
|
95
|
+
pi.command,
|
|
96
|
+
[...pi.args, "--version"],
|
|
97
|
+
{ cwd, encoding: "utf8", timeout: 30000 },
|
|
98
|
+
(err, stdout, stderr) => {
|
|
99
|
+
resolve(
|
|
100
|
+
err
|
|
101
|
+
? "unknown"
|
|
102
|
+
: (stdout || stderr).trim().split("\n").pop() || "unknown",
|
|
103
|
+
);
|
|
104
|
+
},
|
|
105
|
+
);
|
|
106
|
+
});
|
|
107
|
+
}
|
package/src/report.mjs
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
// Markdown and JSON rendering of the evaluation.
|
|
2
|
+
|
|
3
|
+
import { CASE_ORDER } from "./budget.mjs";
|
|
4
|
+
|
|
5
|
+
/** @param {number | null | undefined} n @param {number} [d] */
|
|
6
|
+
function fmt(n, d = 1) {
|
|
7
|
+
return n === null || n === undefined || Number.isNaN(n)
|
|
8
|
+
? "n/a"
|
|
9
|
+
: n.toFixed(d);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** @param {number | null | undefined} n */
|
|
13
|
+
function signed(n) {
|
|
14
|
+
return n === null || n === undefined || Number.isNaN(n)
|
|
15
|
+
? "n/a"
|
|
16
|
+
: `${n >= 0 ? "+" : ""}${n.toFixed(1)}`;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** @param {import("./budget.mjs").Check} c */
|
|
20
|
+
function statusCell(c) {
|
|
21
|
+
if (c.status === "WAIVED")
|
|
22
|
+
return `WAIVED (${(c.note ?? "").replace(/^.*would /, "would ")}): ${c.reason}`;
|
|
23
|
+
if (c.status === "SKIP") return `SKIP: ${c.note ?? "not applicable"}`;
|
|
24
|
+
if (c.status === "INCONCLUSIVE")
|
|
25
|
+
return `INCONCLUSIVE: ${c.note ?? "could not be measured"}`;
|
|
26
|
+
return c.status === "FAIL" && c.note ? `FAIL: ${c.note}` : c.status;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** @param {{ heapUsedMb: number, externalMb: number, arrayBuffersMb: number } | null | undefined} d */
|
|
30
|
+
function gcDiagCell(d) {
|
|
31
|
+
return d
|
|
32
|
+
? `${fmt(d.heapUsedMb)} / ${fmt(d.externalMb)} / ${fmt(d.arrayBuffersMb)}`
|
|
33
|
+
: "n/a";
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @typedef {object} ReportInput
|
|
38
|
+
* @property {ReturnType<typeof import("./budget.mjs").evaluate>} result
|
|
39
|
+
* @property {import("./budget.mjs").Budgets} budgets
|
|
40
|
+
* @property {Record<string, string>} waivers
|
|
41
|
+
* @property {Record<string, unknown>} meta
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @param {ReportInput} input
|
|
46
|
+
* @returns {string}
|
|
47
|
+
*/
|
|
48
|
+
export function renderMarkdown(input) {
|
|
49
|
+
const { result, meta } = input;
|
|
50
|
+
const out = [];
|
|
51
|
+
const verdict = result.checks.some((x) => x.status === "FAIL")
|
|
52
|
+
? "FAIL"
|
|
53
|
+
: result.ok
|
|
54
|
+
? "PASS"
|
|
55
|
+
: "INCONCLUSIVE";
|
|
56
|
+
out.push(`## Memory budget: ${verdict}`, "");
|
|
57
|
+
out.push(
|
|
58
|
+
`pi ${meta.piVersion}, ${meta.platform}, ${meta.runs} run(s) per configuration (median), ${meta.windowSec}s minimum window, steady = median of the settled ${meta.steadySec}s (total-RSS range < 5 MB, settle capped at ${meta.settleMaxSec}s). Deltas are candidate minus vanilla pi in the same job on TOTAL RSS: the pi pid plus every descendant. Values in MB.`,
|
|
59
|
+
"",
|
|
60
|
+
);
|
|
61
|
+
out.push(
|
|
62
|
+
meta.gc === false
|
|
63
|
+
? "GC normalization is OFF (--no-gc): settled RSS includes whatever garbage was uncollected when the window was sampled."
|
|
64
|
+
: "GC normalization is ON: every pi run (vanilla and candidate alike) starts with --expose-gc and a harness extension that forces major GCs, so steady measures retained memory, not GC timing. A check is INCONCLUSIVE when the run-to-run spread of the vanilla baseline alone exceeds the check's budget.",
|
|
65
|
+
"",
|
|
66
|
+
);
|
|
67
|
+
for (const c of CASE_ORDER) {
|
|
68
|
+
const agg = result.aggregates[c];
|
|
69
|
+
if (!agg) continue;
|
|
70
|
+
const checks = result.checks.filter(
|
|
71
|
+
(x) => x.case === c && x.id !== "growth2x",
|
|
72
|
+
);
|
|
73
|
+
out.push(`### Case: ${c}`, "");
|
|
74
|
+
out.push(
|
|
75
|
+
"| Check | Vanilla | Candidate | Delta | Budget | Result |",
|
|
76
|
+
"|---|---:|---:|---:|---:|---|",
|
|
77
|
+
);
|
|
78
|
+
for (const k of checks) {
|
|
79
|
+
const d = k.detail;
|
|
80
|
+
const unit = k.unit === "s" ? " s" : " MB";
|
|
81
|
+
if (k.key === "spikeReleaseSeconds") {
|
|
82
|
+
out.push(
|
|
83
|
+
`| ${k.label} | | | ${k.value === null ? "never" : `${fmt(k.value)} s`} | <= ${k.limit} s | ${statusCell(k)} |`,
|
|
84
|
+
);
|
|
85
|
+
} else if (k.key === "regression") {
|
|
86
|
+
out.push(
|
|
87
|
+
`| ${k.label} | baseline ${fmt(d.baseline)} | ${fmt(d.candidate)} | ${signed(k.value)} | <= +${k.limit}${unit} | ${statusCell(k)} |`,
|
|
88
|
+
);
|
|
89
|
+
} else {
|
|
90
|
+
out.push(
|
|
91
|
+
`| ${k.label} | ${fmt(d.vanilla)} | ${fmt(d.candidate)} | ${signed(k.value)} | <= +${k.limit}${unit} | ${statusCell(k)} |`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
out.push("");
|
|
96
|
+
out.push(
|
|
97
|
+
"| Configuration | settle at | steady | steady spread | spike | released | main steady | children steady | max ping gap | post-GC heap/external/arrayBuffers (diagnostic) |",
|
|
98
|
+
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
|
|
99
|
+
);
|
|
100
|
+
for (const [name, a] of Object.entries(agg)) {
|
|
101
|
+
const settle =
|
|
102
|
+
a.settleT === null
|
|
103
|
+
? a.settled
|
|
104
|
+
? "n/a"
|
|
105
|
+
: "never"
|
|
106
|
+
: `${a.settleT.toFixed(0)} s`;
|
|
107
|
+
const spread =
|
|
108
|
+
a.steadySpreadMb === null
|
|
109
|
+
? "n/a"
|
|
110
|
+
: a.steadySpreadMb === 0
|
|
111
|
+
? "0.0"
|
|
112
|
+
: `${fmt(a.steadySpreadMb)} (${fmt(a.steadyMinMb)}-${fmt(a.steadyMaxMb)})`;
|
|
113
|
+
out.push(
|
|
114
|
+
`| ${name} | ${settle} | ${a.steadyMb === null ? "not settled" : fmt(a.steadyMb)} | ${spread} | ${fmt(a.spikeMb)} | ${fmt(a.releasedMb)} | ${fmt(a.mainSteadyMb)} | ${fmt(a.childrenSteadyMb)} | ${fmt(a.maxGapMs, 0)} ms | ${gcDiagCell(a.gcDiag)} |`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
out.push("");
|
|
118
|
+
}
|
|
119
|
+
const growth = result.checks.find((x) => x.id === "growth2x");
|
|
120
|
+
if (growth) {
|
|
121
|
+
out.push("### Growth with session size", "");
|
|
122
|
+
out.push(
|
|
123
|
+
"| Check | Delta long | Delta long2x | Growth | Budget | Result |",
|
|
124
|
+
"|---|---:|---:|---:|---:|---|",
|
|
125
|
+
);
|
|
126
|
+
out.push(
|
|
127
|
+
`| ${growth.label} | ${signed(growth.detail.deltaLong)} | ${signed(growth.detail.deltaLong2x)} | ${signed(growth.value)} | <= +${growth.limit} MB | ${statusCell(growth)} |`,
|
|
128
|
+
);
|
|
129
|
+
out.push("");
|
|
130
|
+
}
|
|
131
|
+
if (Number(meta.runs) < 3) {
|
|
132
|
+
out.push(
|
|
133
|
+
"Note: fewer than 3 runs per configuration — the median cannot absorb noise; RSS varies by tens of MB on big sessions.",
|
|
134
|
+
"",
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const waived = result.checks.filter((x) => x.status === "WAIVED");
|
|
138
|
+
if (waived.length) {
|
|
139
|
+
out.push("Waived budgets (justified in the config, not skipped):", "");
|
|
140
|
+
for (const w of waived) out.push(`- ${w.id}: ${w.reason}`);
|
|
141
|
+
out.push("");
|
|
142
|
+
}
|
|
143
|
+
return `${out.join("\n")}\n`;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* @param {ReportInput & { measurements: import("./budget.mjs").Measurements }} input
|
|
148
|
+
*/
|
|
149
|
+
export function renderJson(input) {
|
|
150
|
+
/** @type {Record<string, Record<string, unknown>>} */
|
|
151
|
+
const raw = {};
|
|
152
|
+
for (const [c, configs] of Object.entries(input.measurements)) {
|
|
153
|
+
raw[c] = {};
|
|
154
|
+
for (const [name, runs] of Object.entries(configs)) {
|
|
155
|
+
raw[c][name] = runs.map((r) => ({
|
|
156
|
+
settleT: r.settleT,
|
|
157
|
+
settled: r.settled,
|
|
158
|
+
steadyMb: r.steadyMb,
|
|
159
|
+
mainSteadyMb: r.mainSteadyMb,
|
|
160
|
+
spikeMb: r.spikeMb,
|
|
161
|
+
releasedMb: r.releasedMb,
|
|
162
|
+
peakT: r.peakT,
|
|
163
|
+
childrenSteadyMb: r.childrenSteadyMb,
|
|
164
|
+
childrenSpikeMb: r.childrenSpikeMb,
|
|
165
|
+
gcReady: r.gcReady,
|
|
166
|
+
gcDiag: r.gcDiag,
|
|
167
|
+
maxGapMs: r.maxGapMs,
|
|
168
|
+
readyMs: r.readyMs,
|
|
169
|
+
pings: r.pings,
|
|
170
|
+
samples: r.samples.map((s) => [
|
|
171
|
+
Number(s.t.toFixed(2)),
|
|
172
|
+
Number(s.rss.toFixed(1)),
|
|
173
|
+
Number(s.kids.toFixed(1)),
|
|
174
|
+
]),
|
|
175
|
+
}));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
schema: "pi-memory-budget/1",
|
|
180
|
+
ok: input.result.ok,
|
|
181
|
+
meta: input.meta,
|
|
182
|
+
budgets: input.budgets,
|
|
183
|
+
waivers: input.waivers,
|
|
184
|
+
aggregates: input.result.aggregates,
|
|
185
|
+
checks: input.result.checks,
|
|
186
|
+
runs: raw,
|
|
187
|
+
};
|
|
188
|
+
}
|