@decohere/ci-tools 0.0.1-dev.109 → 0.0.1-dev.110

package/src/bench/history.mjs

@@ -0,0 +1,225 @@
+/**
+ * Benchmark history: the results of each `pnpm bench` run, kept per machine so runs are only
+ * compared like for like (#660). Pure functions over plain data; bench.mjs does the I/O.
+ *
+ * Shape (version 2):
+ *   { version: 2, contexts: { [contextId]: { context, runs: [{ date, sha, results }] } } }
+ * newest run first. A result is { id, label, unit, median, p90?, min, max, samples, files? }.
+ */
+
+import crypto from "node:crypto";
+
+export const HISTORY_VERSION = 2;
+/** Nightly runs: a month per machine, enough for a trend and a regression baseline. */
+export const MAX_RUNS = 30;
+/** Printed by each suite before its JSON array of results (projects/flux/bench/harness.ts). */
+export const RESULT_PREFIX = "BENCH_RESULT ";
+
+/** The machine a run happened on. Runs are compared only within one context. */
+export function getContext(os, env, nodeVersion, date = new Date()) {
+	const cpus = os.cpus();
+	const nodeMajor = nodeVersion.match(/^v(\d+)/)?.[1] ?? "?";
+	const rawEnv = env.BENCH_ENV ?? null;
+	return {
+		benchEnv: rawEnv ? `${rawEnv}-node${nodeMajor}` : "(local)",
+		date: date.toISOString(),
+		os: { platform: os.platform(), arch: os.arch() },
+		cpu: { model: cpus[0]?.model?.trim() ?? "unknown", count: cpus.length },
+		memoryGb: Number.parseFloat((os.totalmem() / 1024 ** 3).toFixed(1)),
+		node: nodeVersion,
+	};
+}
+
+/** 16-char SHA-256 prefix of platform|arch|cpu|cores|memGB|node. */
+export function contextId(ctx) {
+	const key = [
+		ctx.os.platform,
+		ctx.os.arch,
+		ctx.cpu.model,
+		ctx.cpu.count,
+		ctx.memoryGb,
+		ctx.node,
+	].join("|");
+	return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16);
+}
+
+/** The results a suite printed, from its whole output. Throws if it printed none. */
+export function parseResults(output, suite) {
+	const line = output
+		.split(/\r?\n/)
+		.reverse()
+		.find((candidate) => candidate.startsWith(RESULT_PREFIX));
+	if (line == null)
+		throw new Error(`suite "${suite}" printed no ${RESULT_PREFIX.trim()} line`);
+	return JSON.parse(line.slice(RESULT_PREFIX.length));
+}
+
+const FORGE_BENCHMARKS = [
+	["forge build es6 + types", "forge.build.local"],
+	["forge build es6 only", "forge.build.debug"],
+	["forge build types only", "forge.build.types-only"],
+];
+
+/**
+ * Results from `vitest bench --outputJson` for the forge build benchmark. Its names end in a
+ * file count, so they are mapped to stable ids; vitest reports no p90, so it is left out.
+ */
+export function fromVitest(vitestOutput) {
+	return vitestOutput.files
+		.flatMap((file) => file.groups)
+		.flatMap((group) => group.benchmarks)
+		.map((benchmark) => {
+			const id = FORGE_BENCHMARKS.find(([prefix]) => benchmark.name.startsWith(prefix))?.[1];
+			if (id == null) throw new Error(`unknown forge benchmark "${benchmark.name}"`);
+			const files = Number(benchmark.name.match(/\((\d+) files\)/)?.[1]);
+			return {
+				id,
+				label: benchmark.name.replace(/\s*\(\d+ files\)$/, ""),
+				unit: "ms",
+				median: benchmark.median ?? benchmark.mean,
+				min: benchmark.min,
+				max: benchmark.max,
+				samples: benchmark.sampleCount,
+				...(Number.isFinite(files) ? { files } : {}),
+			};
+		});
+}
+
+/** Throws when a result has no usable numbers: a suite that errored mid-run, not a slow one. */
+export function assertValid(results) {
+	const invalid = results.filter(
+		(result) => !Number.isFinite(result.median) || !(result.samples > 0),
+	);
+	if (invalid.length > 0) {
+		throw new Error(
+			`${invalid.length} benchmark(s) have no valid samples: ${invalid.map((r) => r.id).join(", ")}`,
+		);
+	}
+}
+
+/**
+ * Reads a stored history. Version 1 (forge build only, raw vitest output) is converted so its
+ * trend carries on; anything unrecognised starts empty rather than mixing formats.
+ */
+export function normalize(raw) {
+	if (raw?.version === HISTORY_VERSION && raw.contexts != null) return raw;
+	const contexts = {};
+	for (const [id, entry] of Object.entries(raw?.runs ?? {})) {
+		const runs = (entry.runs ?? [])
+			.filter((run) => run.vitestOutput != null)
+			.map((run) => ({ date: run.date, sha: null, results: fromVitest(run.vitestOutput) }));
+		if (runs.length > 0) contexts[id] = { context: entry.context, runs };
+	}
+	return { version: HISTORY_VERSION, contexts };
+}
+
+/** The previous run for this context, if any: what a new run is compared against. */
+export function previousRun(history, id) {
+	return history.contexts[id]?.runs[0];
+}
+
+/** A new history with `run` first in its context, trimmed to `max` runs. Does not mutate. */
+export function appendRun(history, ctx, run, max = MAX_RUNS) {
+	const id = contextId(ctx);
+	const runs = [run, ...(history.contexts[id]?.runs ?? [])].slice(0, max);
+	return {
+		version: HISTORY_VERSION,
+		contexts: { ...history.contexts, [id]: { context: ctx, runs } },
+	};
+}
+
+/** One row per current result, with the change in median against the previous run. */
+export function compare(results, previous) {
+	const before = new Map((previous?.results ?? []).map((result) => [result.id, result]));
+	return results.map((result) => {
+		const old = before.get(result.id);
+		if (old == null) return { id: result.id, current: result.median };
+		const delta = result.median - old.median;
+		return {
+			id: result.id,
+			current: result.median,
+			previous: old.median,
+			delta,
+			percent: old.median === 0 ? 0 : (delta / old.median) * 100,
+		};
+	});
+}
+
+/** The comparison as an aligned text table. */
+export function formatComparison(rows) {
+	const ms = (value) => (value == null ? "n/a" : `${value.toFixed(1)} ms`);
+	const signed = (value, suffix) =>
+		value == null ? "n/a" : `${value >= 0 ? "+" : ""}${value.toFixed(1)}${suffix}`;
+	const table = [
+		["Benchmark", "Median", "Previous", "Delta", "%"],
+		...rows.map((row) => [
+			row.id,
+			ms(row.current),
+			ms(row.previous),
+			signed(row.delta, " ms"),
+			signed(row.percent, "%"),
+		]),
+	];
+	const widths = table[0].map((_, column) =>
+		Math.max(...table.map((row) => row[column].length)),
+	);
+	const line = (row) =>
+		row
+			.map((cell, column) =>
+				(column === 0 ? cell.padEnd : cell.padStart).call(cell, widths[column]),
+			)
+			.join(" | ");
+	return [
+		line(table[0]),
+		widths.map((w) => "-".repeat(w)).join("-+-"),
+		...table.slice(1).map(line),
+	].join("\n");
+}
+
+/**
+ * When a benchmark counts as slower (#660): its median beat the baseline by more than
+ * `threshold` on each of the last `confirmRuns` runs, where the baseline is the median of the
+ * `baselineRuns` runs before those. Two nights in a row, so one noisy runner files nothing.
+ */
+export const REGRESSION = { threshold: 0.2, confirmRuns: 2, baselineRuns: 7, minBaseline: 3 };
+
+function medianOf(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	const middle = Math.floor(sorted.length / 2);
+	return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+/** The benchmarks that regressed on one machine, given its runs newest first. */
+export function findRegressions(runs, options = REGRESSION) {
+	const recent = runs.slice(0, options.confirmRuns);
+	const before = runs.slice(options.confirmRuns, options.confirmRuns + options.baselineRuns);
+	if (recent.length < options.confirmRuns) return [];
+	const medianIn = (run, id) => run.results.find((result) => result.id === id)?.median;
+	const regressions = [];
+	for (const latest of recent[0].results) {
+		const baselineValues = before
+			.map((run) => medianIn(run, latest.id))
+			.filter((value) => Number.isFinite(value));
+		if (baselineValues.length < options.minBaseline) continue;
+		const baseline = medianOf(baselineValues);
+		const recentValues = recent.map((run) => medianIn(run, latest.id));
+		const slower = recentValues.every(
+			(value) => Number.isFinite(value) && value > baseline * (1 + options.threshold),
+		);
+		if (!slower) continue;
+		regressions.push({
+			id: latest.id,
+			label: latest.label,
+			baseline,
+			percent: (recentValues[0] / baseline - 1) * 100,
+			recent: recent.map((run, index) => ({
+				date: run.date,
+				sha: run.sha,
+				median: recentValues[index],
+			})),
+			// The change landed between the newest baseline run and the oldest slow one.
+			lastGood: before[0] == null ? null : { date: before[0].date, sha: before[0].sha },
+		});
+	}
+	return regressions;
+}

package/src/bench/regression-issue.mjs

@@ -0,0 +1,51 @@
+/**
+ * The GitHub issue for a benchmark that got slower (#660): one open issue per benchmark,
+ * found again by its exact title and updated in place while the regression lasts.
+ */
+
+const PAGES_URL = "https://typesmiths.github.io/decohere/benchmark.html";
+
+export function issueTitle(regression) {
+	return `perf(bench): ${regression.id} is slower`;
+}
+
+function ms(value) {
+	return `${value.toFixed(1)} ms`;
+}
+
+function commitLink(repoUrl, sha) {
+	return sha == null ? "unknown" : `[\`${sha.slice(0, 7)}\`](${repoUrl}/commit/${sha})`;
+}
+
+export function issueBody(regression, { repoUrl, runUrl, machine, threshold }) {
+	const newest = regression.recent[0];
+	const oldestSlow = regression.recent.at(-1);
+	const range =
+		regression.lastGood?.sha != null && newest.sha != null
+			? `[${regression.lastGood.sha.slice(0, 7)}...${newest.sha.slice(0, 7)}](${repoUrl}/compare/${regression.lastGood.sha}...${newest.sha})`
+			: "unknown (no baseline commit recorded)";
+	return [
+		"## Problem",
+		"",
+		`The nightly benchmark **${regression.label}** (\`${regression.id}\`) has a median ${regression.percent.toFixed(0)}% above its baseline, on each of the last ${regression.recent.length} nightly runs. A benchmark counts as slower above ${Math.round(threshold * 100)}%.`,
+		"",
+		`- Baseline (median of the runs before): **${ms(regression.baseline)}**`,
+		...regression.recent.map(
+			(run) =>
+				`- ${run.date.slice(0, 10)}, ${commitLink(repoUrl, run.sha)}: **${ms(run.median)}**`,
+		),
+		`- Machine: ${machine}`,
+		"",
+		`The change landed after the last normal run (${regression.lastGood?.date.slice(0, 10) ?? "unknown"}) and by ${oldestSlow.date.slice(0, 10)}: commits ${range}.`,
+		"",
+		"## What to do",
+		"",
+		"1. Check the trend on the [benchmark page](" +
+			PAGES_URL +
+			"). A step that holds is a real change; a single machine type jumping may be the runner.",
+		"2. Reproduce locally: check out the last normal commit, `pnpm bench:save`, then the newest commit, `pnpm bench`, and read the comparison.",
+		"3. Fix the cause, or if the slowdown is intended, say why here and close the issue.",
+		"",
+		`Updated by the nightly bench workflow: ${runUrl}`,
+	].join("\n");
+}

package/src/pages/artifact.mjs

@@ -1,5 +1,8 @@
-import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { cp, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { normalize } from "../bench/history.mjs";
+import { buildBenchmarkHtml, buildStylesCss } from "./bench-page.mjs";
 
 const branch = process.env.GITHUB_REF_NAME ?? "local";
 const sha = process.env.GITHUB_SHA ?? "local";
@@ -8,201 +11,53 @@ const outputDir = process.env.PAGES_OUTPUT_DIR ?? "site";
 const badgesDir = "projects/ci-tools/dist/ci/badges";
 const benchHistoryPath = "projects/ci-tools/dist/ci/bench/bench-history.json";
 
-const numericFields = ["hz", "mean", "p99", "rme", "sampleCount"];
-
-function fmtMs(ms) {
-	if (typeof ms !== "number" || Number.isNaN(ms)) return "-";
-	if (ms >= 100) return `${Math.round(ms)} ms`;
-	return `${ms.toFixed(1)} ms`;
-}
-
-function fmtHz(hz) {
-	if (typeof hz !== "number" || Number.isNaN(hz)) return "-";
-	return `${hz.toFixed(2)}/s`;
-}
-
-function averageBenchmarks(runs) {
-	const first = runs[0]?.vitestOutput;
-	if (first == null) return [];
-
-	const latestBenchmarks = first.files
-		.flatMap((file) => file.groups)
-		.flatMap((group) => group.benchmarks);
-
-	return latestBenchmarks.map((benchmark) => {
-		const peers = runs
-			.flatMap((run) => run.vitestOutput?.files ?? [])
-			.flatMap((file) => file.groups ?? [])
-			.flatMap((group) => group.benchmarks ?? [])
-			.filter((candidate) => candidate.name === benchmark.name);
-
-		const averaged = { name: benchmark.name };
-		for (const field of numericFields) {
-			const values = peers
-				.map((peer) => peer[field])
-				.filter((value) => typeof value === "number");
-			if (values.length > 0) {
-				averaged[field] = values.reduce((sum, value) => sum + value, 0) / values.length;
-			}
-		}
-
-		return averaged;
-	});
-}
-
-export function buildBenchmarkHtml(benchHistory) {
-	const contexts = Object.entries(benchHistory.runs ?? {})
-		.filter(([, entry]) => Array.isArray(entry.runs) && entry.runs.length > 0)
-		.sort((a, b) => new Date(b[1].runs[0].date) - new Date(a[1].runs[0].date));
-
-	if (contexts.length === 0) {
-		return `<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Ark - Benchmarks</title>
-    <link rel="stylesheet" href="./styles.css" />
-  </head>
-  <body>
-    <main class="container">
-      <header class="hero">
-        <h1>Benchmarks</h1>
-        <p class="note">No benchmark history is available yet.</p>
-      </header>
-    </main>
-  </body>
-</html>
-`;
-	}
-
-	const rows = [];
-	for (const [, entry] of contexts) {
-		const benchmarks = averageBenchmarks(entry.runs);
-		const ctx = entry.context;
-		const environment = `${ctx.benchEnv ?? "unknown"}-${ctx.os.arch}`;
-		for (const benchmark of benchmarks) {
-			rows.push(`<tr>
-  <td>${environment}</td>
-  <td><code>${benchmark.name}</code></td>
-  <td>${fmtMs(benchmark.mean)}</td>
-  <td>${typeof benchmark.rme === "number" ? `±${benchmark.rme.toFixed(1)}%` : "-"}</td>
-  <td>${fmtMs(benchmark.p99)}</td>
-  <td>${fmtHz(benchmark.hz)}</td>
-  <td>${typeof benchmark.sampleCount === "number" ? Math.round(benchmark.sampleCount) : "-"}</td>
-</tr>`);
-		}
-	}
-
-	const details = contexts
-		.map(([, entry]) => {
-			const ctx = entry.context;
-			return `<li><strong>${ctx.benchEnv ?? "unknown"}-${ctx.os.arch}</strong> - ${ctx.os.platform} - ${ctx.cpu.model} x ${ctx.cpu.count} - ${ctx.memoryGb} GB - Node ${ctx.node} - rolling mean of ${entry.runs.length} run(s)</li>`;
-		})
-		.join("\n");
-
-	const updatedDate = contexts[0][1].runs[0].date.slice(0, 10);
-
-	return `<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Ark - Benchmarks</title>
-    <link rel="stylesheet" href="./styles.css" />
-  </head>
-  <body>
-    <main class="container">
-      <div class="hero">
-        <h1>Benchmarks</h1>
-        <p class="note">Updated ${updatedDate} · Branch: ${branch} · Commit: ${sha.slice(0, 12)}</p>
-      </div>
-      <section class="section">
-        <h2>Performance Table</h2>
-        <table>
-          <thead>
-            <tr>
-              <th>Environment</th>
-              <th>Benchmark</th>
-              <th>Mean</th>
-              <th>±RME</th>
-              <th>P99</th>
-              <th>Ops/s</th>
-              <th>Runs</th>
-            </tr>
-          </thead>
-          <tbody>
-            ${rows.join("\n")}
-          </tbody>
-        </table>
-      </section>
-      <section class="section">
-        <h2>Environment Notes</h2>
-        <ul>
-          ${details}
-        </ul>
-      </section>
-    </main>
-  </body>
-</html>
-`;
-}
-
-function buildStylesCss() {
-	return `:root {
-  --bg: #f4f7f3;
-  --surface: #ffffff;
-  --text: #132b24;
-  --muted: #4f6f65;
-  --accent: #0f9d72;
-  --accent-strong: #0d7f5c;
-  --border: #d6e3dd;
-  --shadow: 0 14px 40px rgba(12, 58, 45, 0.08);
-}
-* { box-sizing: border-box; }
-body { margin: 0; font-family: "Segoe UI", "Helvetica Neue", sans-serif; color: var(--text); background: radial-gradient(circle at top right, #dbefe7, var(--bg) 40%); }
-a { color: var(--accent-strong); }
-.container { max-width: 1100px; margin: 0 auto; padding: 28px 18px 60px; }
-.hero { background: linear-gradient(145deg, #183a31, #0e7f5c); color: #f3fff8; padding: 26px; border-radius: 18px; box-shadow: var(--shadow); }
-.hero h1 { margin: 6px 0 10px; }
-.note { color: #d8f5eb; margin: 0; }
-.section { margin-top: 26px; }
-table { width: 100%; border-collapse: collapse; background: var(--surface); border-radius: 12px; overflow: hidden; }
-th, td { border-bottom: 1px solid var(--border); padding: 9px 10px; text-align: left; }
-th { color: var(--muted); font-weight: 600; background: #f3faf7; }
-td code { color: var(--accent-strong); }
-@media (max-width: 700px) { .hero { padding: 20px; } .container { padding: 18px 12px 40px; } }
-`;
+/** Every badge file under `dir`, relative to it: the manifest restore-badges.mjs reads back. */
+async function listBadges(dir, prefix = "") {
+	const entries = await readdir(path.join(dir, prefix), { withFileTypes: true });
+	const nested = await Promise.all(
+		entries.map((entry) =>
+			entry.isDirectory()
+				? listBadges(dir, path.posix.join(prefix, entry.name))
+				: entry.name.endsWith(".svg")
+					? [path.posix.join(prefix, entry.name)]
+					: [],
+		),
+	);
+	return nested.flat().sort();
 }
 
 await rm(outputDir, { recursive: true, force: true });
-await mkdir(outputDir, { recursive: true });
+await mkdir(path.join(outputDir, "data"), { recursive: true });
 
 await writeFile(path.join(outputDir, "styles.css"), buildStylesCss());
 
 // index.html redirects to benchmark.html — no separate dashboard needed
 await writeFile(
 	path.join(outputDir, "index.html"),
-	`<!doctype html><html><head><meta charset="utf-8"/><meta http-equiv="refresh" content="0;url=./benchmark.html"/><title>Ark Pages</title></head><body><p><a href="./benchmark.html">Benchmarks</a></p></body></html>\n`,
+	`<!doctype html><html><head><meta charset="utf-8"/><meta http-equiv="refresh" content="0;url=./benchmark.html"/><title>Decohere Pages</title></head><body><p><a href="./benchmark.html">Benchmarks</a></p></body></html>\n`,
 );
 
-try {
-	const benchHistory = JSON.parse(await readFile(benchHistoryPath, "utf8"));
-	await writeFile(path.join(outputDir, "benchmark.html"), buildBenchmarkHtml(benchHistory));
-	const benchmarkDataDir = path.join(outputDir, "data");
-	await mkdir(benchmarkDataDir, { recursive: true });
-	await writeFile(
-		path.join(benchmarkDataDir, "bench-history.json"),
-		`${JSON.stringify(benchHistory, null, 2)}\n`,
-	);
-} catch {
-	await writeFile(path.join(outputDir, "benchmark.html"), buildBenchmarkHtml({ runs: {} }));
-}
+// No history yet (first deploy, or cleared) publishes the empty page, not a failure.
+const benchHistory = normalize(
+	existsSync(benchHistoryPath) ? JSON.parse(await readFile(benchHistoryPath, "utf8")) : null,
+);
+await writeFile(
+	path.join(outputDir, "benchmark.html"),
+	buildBenchmarkHtml(benchHistory, { branch, sha }),
+);
+await writeFile(
+	path.join(outputDir, "data", "bench-history.json"),
+	`${JSON.stringify(benchHistory, null, 2)}\n`,
+);
 
-try {
+// Badges are generated by earlier steps (CI status, coverage) or restored from the live site.
+if (existsSync(badgesDir)) {
 	await cp(badgesDir, path.join(outputDir, "badges"), { recursive: true });
-} catch {
-	// Badges may not exist yet for some runs.
+	const badges = await listBadges(badgesDir);
+	await writeFile(
+		path.join(outputDir, "badges", "manifest.json"),
+		`${JSON.stringify({ badges }, null, 2)}\n`,
+	);
 }
 
 await writeFile(path.join(outputDir, ".nojekyll"), "");