@decohere/ci-tools 0.0.1-dev.126 → 0.0.1-dev.14

package/src/bench/bench.mjs

@@ -1,22 +1,22 @@
 #!/usr/bin/env node
 
 /**
- * bench.mjs — run the benchmark suites and compare against the last saved run.
+ * bench.mjs — run the forge benchmark and compare against the last saved baseline.
  *
  * Usage:
- *   pnpm bench                          — run every suite and compare, do not save
- *   pnpm bench flux-load flux-serve     — run only these suites
- *   pnpm bench:save                     — run, compare, and append to the history
+ *   node bench.mjs              — run and compare, do not save
+ *   node bench.mjs save         — run, compare, and save as the new baseline
+ *   node bench.mjs report       — run and compare, do not save
+ *   BENCH_ITERATIONS=3 node bench.mjs
  *
- * Suites: forge-build, flux-load, flux-transpile, flux-serve. Each prints a BENCH_RESULT line
- * (see history.mjs). The history lives in projects/ci-tools/dist/ci/bench/bench-history.json;
- * the nightly bench workflow restores it from Pages, saves, and publishes the charts.
- *
- * Environment: BENCH_HISTORY_FILE overrides where `save` writes; BENCH_ENV names the machine class (CI sets gh-actions); BENCH_SAMPLES (default
- * 10) is the sample count for every benchmark; BENCH_LIBS and BENCH_FILES size the flux workspace.
+ * Context ID
+ * ----------
+ * 16-char SHA-256 prefix of: platform|arch|cpu|cores|memGB|node
+ * Ensures history is always compared like-for-like across machines.
  */
 
-import { execFileSync, spawnSync } from "node:child_process";
+import { spawnSync } from "node:child_process";
+import crypto from "node:crypto";
 import {
 	existsSync,
 	mkdirSync,
@@ -26,152 +26,386 @@ import {
 	writeFileSync,
 } from "node:fs";
 import os from "node:os";
-import { dirname, join } from "node:path";
-import {
-	appendRun,
-	assertValid,
-	compare,
-	contextId,
-	formatComparison,
-	fromVitest,
-	getContext,
-	normalize,
-	parseResults,
-	previousRun,
-} from "./history.mjs";
+import { join } from "node:path";
 
 const WORKSPACE_ROOT = process.cwd();
-// BENCH_HISTORY_FILE lets the nightly job save each comparison round to its own file.
-const HISTORY_FILE =
-	process.env.BENCH_HISTORY_FILE ??
-	join(WORKSPACE_ROOT, "projects/ci-tools/dist/ci/bench/bench-history.json");
-
-/** Runs a TypeScript benchmark from source, the way the repo runs its CLIs. */
-function fluxSuite(file) {
-	return () => {
-		const result = spawnSync(process.execPath, ["--import", "./tools/register.mjs", file], {
-			cwd: WORKSPACE_ROOT,
-			encoding: "utf8",
-			maxBuffer: 1024 * 1024 * 50,
-			env: { ...process.env, NODE_OPTIONS: "" },
+const HISTORY_FILE = join(
+	WORKSPACE_ROOT,
+	"projects/ci-tools/dist/ci/bench/bench-history.json",
+);
+const VITEST_BIN = join(WORKSPACE_ROOT, "node_modules/.bin/vitest");
+const BENCH_FILE = "projects/terminal/e2e/build.benchmark.bench.ts";
+const CONFIG = join(WORKSPACE_ROOT, "vitest.e2e.config.ts");
+const TRACE_LOG_DIR = join(WORKSPACE_ROOT, ".decohere", "logs");
+const MAX_RUNS = 10;
+const command = process.argv[2] ?? "run";
+
+// ── Context ───────────────────────────────────────────────────────────────────
+
+function getContext() {
+	const cpus = os.cpus();
+	const nodeMajor = process.version.match(/^v(\d+)/)?.[1] ?? "?";
+	const rawEnv = process.env.BENCH_ENV ?? null;
+	return {
+		benchEnv: rawEnv ? `${rawEnv}-node${nodeMajor}` : "(local)",
+		date: new Date().toISOString(),
+		os: { platform: os.platform(), arch: os.arch() },
+		cpu: { model: cpus[0]?.model?.trim() ?? "unknown", count: cpus.length },
+		memoryGb: parseFloat((os.totalmem() / 1024 ** 3).toFixed(1)),
+		node: process.version,
+	};
+}
+
+function contextId(ctx) {
+	const key = [
+		ctx.os.platform,
+		ctx.os.arch,
+		ctx.cpu.model,
+		ctx.cpu.count,
+		ctx.memoryGb,
+		ctx.node,
+	].join("|");
+	return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16);
+}
+
+function truthy(value, defaultValue = false) {
+	if (value == null) return defaultValue;
+	return !["0", "false", "no", "off"].includes(String(value).toLowerCase());
+}
+
+function toFileTimestamp(date = new Date()) {
+	const pad = (n) => String(n).padStart(2, "0");
+	return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
+}
+
+function toTraceSample(line, parsed, durationMs) {
+	return {
+		time: parsed.time,
+		name: parsed.name,
+		message: parsed.msg,
+		durationMs,
+		line,
+	};
+}
+
+function parseTraceSamples(rawOutput) {
+	const samples = [];
+	for (const rawLine of rawOutput.split(/\r?\n/)) {
+		const line = rawLine.trim();
+		if (!line.startsWith("{")) continue;
+
+		let parsed;
+		try {
+			parsed = JSON.parse(line);
+		} catch {
+			continue;
+		}
+
+		if (typeof parsed?.name !== "string" || typeof parsed?.msg !== "string") continue;
+		const match = parsed.msg.match(/^(.*):\s+([0-9]+(?:\.[0-9]+)?)ms$/);
+		if (match == null) continue;
+
+		const durationMs = Number(match[2]);
+		if (!Number.isFinite(durationMs)) continue;
+
+		samples.push({
+			className: parsed.name,
+			methodLabel: match[1],
+			methodName: match[1].split("(")[0],
+			durationMs,
+			sample: toTraceSample(rawLine, parsed, durationMs),
 		});
-		process.stderr.write(
-			result.stdout
-				.split("\n")
-				.filter((l) => !l.startsWith("BENCH_RESULT"))
-				.join("\n"),
-		);
-		if (result.status !== 0) {
-			throw new Error(`${file} exited ${result.status ?? result.signal}:\n${result.stderr}`);
+	}
+	return samples;
+}
+
+function aggregateSamples(samples, keyBuilder) {
+	const aggregates = new Map();
+	for (const sample of samples) {
+		const key = keyBuilder(sample);
+		const current = aggregates.get(key) ?? {
+			key,
+			hitCount: 0,
+			totalMs: 0,
+			maxMs: 0,
+		};
+		current.hitCount += 1;
+		current.totalMs += sample.durationMs;
+		if (sample.durationMs > current.maxMs) current.maxMs = sample.durationMs;
+		aggregates.set(key, current);
+	}
+	return [...aggregates.values()];
+}
+
+function analyzeTraceOutput(rawOutput) {
+	const samples = parseTraceSamples(rawOutput);
+	if (samples.length === 0) {
+		process.stderr.write("\n⚠  No structured trace samples found to analyze.\n");
+		return;
+	}
+
+	const byMethod = aggregateSamples(
+		samples,
+		(sample) => `${sample.className}.${sample.methodName}`,
+	);
+	const byClass = aggregateSamples(samples, (sample) => sample.className);
+	const byHits = [...byMethod].sort((a, b) => b.hitCount - a.hitCount).slice(0, 10);
+	const byTotal = [...byMethod].sort((a, b) => b.totalMs - a.totalMs).slice(0, 10);
+	const byMax = [...byMethod].sort((a, b) => b.maxMs - a.maxMs).slice(0, 10);
+	const byClassTotal = [...byClass].sort((a, b) => b.totalMs - a.totalMs).slice(0, 5);
+	const totalTraceMs = samples.reduce((sum, sample) => sum + sample.durationMs, 0);
+	const hottest = [...samples].sort((a, b) => b.durationMs - a.durationMs).slice(0, 5);
+
+	process.stderr.write(
+		"\n┌─ Trace Analysis ─────────────────────────────────────────────────────────\n",
+	);
+	process.stderr.write(`│  Samples      : ${samples.length}\n`);
+	process.stderr.write(`│  Unique funcs : ${byMethod.length}\n`);
+	process.stderr.write(`│  Total traced : ${totalTraceMs.toFixed(2)}ms\n`);
+	process.stderr.write(`│  Avg sample   : ${(totalTraceMs / samples.length).toFixed(3)}ms\n`);
+	process.stderr.write(
+		"└──────────────────────────────────────────────────────────────────────────\n",
+	);
+
+	const printTable = (title, rows, valueFormatter) => {
+		process.stderr.write(`\n${title}\n`);
+		for (const row of rows) {
+			process.stderr.write(
+				`  ${valueFormatter(row).padStart(10)}  ${row.key}  (hits=${row.hitCount}, max=${row.maxMs.toFixed(2)}ms)\n`,
+			);
 		}
-		return parseResults(result.stdout, file);
 	};
-}
 
-function forgeBuildSuite() {
-	const tmpDir = mkdtempSync(join(os.tmpdir(), "decohere-bench-"));
-	const outJson = join(tmpDir, "output.json");
-	try {
-		const result = spawnSync(
-			join(WORKSPACE_ROOT, "node_modules/.bin/vitest"),
-			[
-				"bench",
-				"--config",
-				join(WORKSPACE_ROOT, "vitest.config.ts"),
-				"--project",
-				"e2e",
-				"--run",
-				"--outputJson",
-				outJson,
-				"projects/cli/e2e/build.benchmark.bench.ts",
-			],
-			{
-				cwd: WORKSPACE_ROOT,
-				stdio: ["ignore", "inherit", "inherit"],
-				env: {
-					...process.env,
-					NODE_OPTIONS: "",
-				},
-			},
+	printTable("Top Functions By Hit Count", byHits, (row) => String(row.hitCount));
+	printTable("Top Functions By Total Time", byTotal, (row) => `${row.totalMs.toFixed(2)}ms`);
+	printTable(
+		"Top Functions By Worst Single Call",
+		byMax,
+		(row) => `${row.maxMs.toFixed(2)}ms`,
+	);
+	printTable(
+		"Top Classes By Total Trace Time",
+		byClassTotal,
+		(row) => `${row.totalMs.toFixed(2)}ms`,
+	);
+
+	process.stderr.write("\nSlowest Individual Samples\n");
+	for (const hot of hottest) {
+		process.stderr.write(
+			`  ${hot.durationMs.toFixed(2).padStart(10)}ms  ${hot.className}.${hot.methodLabel}\n`,
 		);
-		if (result.status !== 0) throw new Error(`vitest bench exited ${result.status}`);
-		// vitest swallows errors in bench callbacks, so a missing file means it failed.
-		if (!existsSync(outJson)) throw new Error("vitest bench wrote no output JSON");
-		return fromVitest(JSON.parse(readFileSync(outJson, "utf8")));
-	} finally {
-		rmSync(tmpDir, { recursive: true, force: true });
 	}
 }
 
-const SUITES = {
-	"forge-build": forgeBuildSuite,
-	"flux-load": fluxSuite("projects/flux/bench/load.bench.ts"),
-	"flux-transpile": fluxSuite("projects/flux/bench/transpile.bench.ts"),
-	"flux-serve": fluxSuite("projects/flux/bench/serve.bench.ts"),
-};
-
-function currentSha() {
-	if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA;
-	return execFileSync("git", ["rev-parse", "HEAD"], {
-		cwd: WORKSPACE_ROOT,
-		encoding: "utf8",
-	}).trim();
+function getBenchmarks(vitestOutput) {
+	return vitestOutput.files
+		.flatMap((file) => file.groups)
+		.flatMap((group) => group.benchmarks);
 }
 
-const [command = "run", ...requested] = process.argv.slice(2);
-if (command !== "run" && command !== "save") {
-	process.stderr.write(`Unknown command "${command}". Use "run" or "save".\n`);
-	process.exit(1);
-}
-const unknown = requested.filter((suite) => !(suite in SUITES));
-if (unknown.length > 0) {
-	process.stderr.write(
-		`Unknown suite(s): ${unknown.join(", ")}. Available: ${Object.keys(SUITES).join(", ")}\n`,
+function printBenchmarkComparison(currentRun, previousRun) {
+	if (previousRun == null) {
+		process.stderr.write("No previous saved run for this context; skipping comparison.\n");
+		return;
+	}
+
+	const currentBenchmarks = getBenchmarks(currentRun.vitestOutput);
+	const previousByName = new Map(
+		getBenchmarks(previousRun.vitestOutput).map((benchmark) => [benchmark.name, benchmark]),
 	);
-	process.exit(1);
+
+	process.stderr.write("Current Run vs Previous\n");
+	process.stderr.write(`  current : ${currentRun.date}\n`);
+	process.stderr.write(`  previous: ${previousRun.date}\n`);
+	process.stderr.write("\n");
+
+	const rows = currentBenchmarks.map((benchmark) => {
+		const previous = previousByName.get(benchmark.name);
+		if (previous == null) {
+			return {
+				name: benchmark.name,
+				current: `${benchmark.mean.toFixed(2)}ms`,
+				previous: "n/a",
+				delta: "n/a",
+				change: "n/a",
+			};
+		}
+
+		const delta = benchmark.mean - previous.mean;
+		const pct = previous.mean === 0 ? 0 : (delta / previous.mean) * 100;
+		const deltaSign = delta >= 0 ? "+" : "";
+		return {
+			name: benchmark.name,
+			current: `${benchmark.mean.toFixed(2)}ms`,
+			previous: `${previous.mean.toFixed(2)}ms`,
+			delta: `${deltaSign}${delta.toFixed(2)}ms`,
+			change: `${deltaSign}${pct.toFixed(2)}%`,
+		};
+	});
+
+	const headers = {
+		name: "Benchmark",
+		current: "Current",
+		previous: "Previous",
+		delta: "Delta",
+		change: "%",
+	};
+
+	const widths = {
+		name: Math.max(headers.name.length, ...rows.map((row) => row.name.length)),
+		current: Math.max(headers.current.length, ...rows.map((row) => row.current.length)),
+		previous: Math.max(headers.previous.length, ...rows.map((row) => row.previous.length)),
+		delta: Math.max(headers.delta.length, ...rows.map((row) => row.delta.length)),
+		change: Math.max(headers.change.length, ...rows.map((row) => row.change.length)),
+	};
+
+	const divider = [
+		"-".repeat(widths.name),
+		"-".repeat(widths.current),
+		"-".repeat(widths.previous),
+		"-".repeat(widths.delta),
+		"-".repeat(widths.change),
+	].join("-+-");
+
+	const formatRow = (row) =>
+		[
+			row.name.padEnd(widths.name),
+			row.current.padStart(widths.current),
+			row.previous.padStart(widths.previous),
+			row.delta.padStart(widths.delta),
+			row.change.padStart(widths.change),
+		].join(" | ");
+
+	process.stderr.write(`${formatRow(headers)}\n`);
+	process.stderr.write(`${divider}\n`);
+	for (const row of rows) {
+		process.stderr.write(`${formatRow(row)}\n`);
+	}
+
+	process.stderr.write("\n");
 }
-const suites = requested.length > 0 ? requested : Object.keys(SUITES);
 
-const ctx = getContext(os, process.env, process.version);
+// ── Run ───────────────────────────────────────────────────────────────────────
+
+const ctx = getContext();
 const id = contextId(ctx);
+const iterations = process.env.BENCH_ITERATIONS ?? "5";
+const traceEnabled = process.env.FORGE_TRACE != null;
+const shouldSave = truthy(process.env.BENCH_SAVE, command === "save");
+const shouldAnalyzeTrace = traceEnabled && truthy(process.env.BENCH_ANALYZE_TRACE, true);
+const shouldCollectTraceLog = traceEnabled && truthy(process.env.BENCH_TRACE_LOG, true);
+const traceLogPath = shouldCollectTraceLog
+	? join(TRACE_LOG_DIR, `perf-trace-${toFileTimestamp()}.log`)
+	: null;
+
+if (traceLogPath != null) {
+	mkdirSync(TRACE_LOG_DIR, { recursive: true });
+}
+
 process.stderr.write(
 	[
-		"",
-		`Context ${id} · ${ctx.benchEnv} · ${ctx.os.platform} ${ctx.os.arch}`,
-		`${ctx.cpu.model} × ${ctx.cpu.count} · ${ctx.memoryGb} GB · Node ${ctx.node}`,
-		`Suites: ${suites.join(", ")}`,
-		"",
-		"",
+		"\n┌─ Benchmark Environment ─────────────────────────────────────────────────",
+		`│  Context ID : ${id}`,
+		`│  BENCH_ENV  : ${ctx.benchEnv}`,
+		`│  Date       : ${ctx.date}`,
+		`│  OS         : ${ctx.os.platform} ${ctx.os.arch}`,
+		`│  CPU        : ${ctx.cpu.model} × ${ctx.cpu.count}`,
+		`│  Memory     : ${ctx.memoryGb} GB`,
+		`│  Node       : ${ctx.node}`,
+		"└──────────────────────────────────────────────────────────────────────────\n\n",
 	].join("\n"),
 );
 
-const results = [];
-for (const suite of suites) {
-	process.stderr.write(`── ${suite}\n`);
-	results.push(...SUITES[suite]());
-	process.stderr.write("\n");
+if (traceEnabled) {
+	process.stderr.write("Trace mode enabled with structured pino logs for analysis.\n\n");
 }
-assertValid(results);
 
-const history = normalize(
-	existsSync(HISTORY_FILE) ? JSON.parse(readFileSync(HISTORY_FILE, "utf8")) : null,
-);
-const previous = previousRun(history, id);
-process.stderr.write(
-	previous == null
-		? "No saved run for this machine yet; nothing to compare against.\n"
-		: `Compared with the saved run from ${previous.date}:\n`,
-);
-process.stderr.write(`${formatComparison(compare(results, previous))}\n\n`);
+const tmpDir = mkdtempSync(`${os.tmpdir()}/ark-bench-`);
+const outJson = join(tmpDir, "output.json");
 
-if (command === "save") {
-	const run = { date: ctx.date, sha: currentSha(), results };
-	const next = appendRun(history, ctx, run);
-	mkdirSync(dirname(HISTORY_FILE), { recursive: true });
-	writeFileSync(HISTORY_FILE, `${JSON.stringify(next, null, 2)}\n`);
-	process.stderr.write(
-		`Saved (context ${id}, ${next.contexts[id].runs.length} runs stored).\n`,
+try {
+	const result = spawnSync(
+		VITEST_BIN,
+		["bench", "--config", CONFIG, "--run", "--outputJson", outJson, BENCH_FILE],
+		{
+			cwd: WORKSPACE_ROOT,
+			stdio: traceEnabled ? "pipe" : "inherit",
+			encoding: traceEnabled ? "utf8" : undefined,
+			maxBuffer: 1024 * 1024 * 200,
+			env: {
+				...process.env,
+				NODE_OPTIONS: "",
+				BENCH_ITERATIONS: iterations,
+				...(traceEnabled
+					? { FORGE_TRACE_STRUCTURED: "1", LOG_LEVEL: process.env.LOG_LEVEL ?? "trace" }
+					: {}),
+			},
+		},
 	);
-} else {
-	process.stderr.write("Not saved. Run `pnpm bench:save` to record this run.\n");
+
+	if (traceEnabled) {
+		const stdout = typeof result.stdout === "string" ? result.stdout : "";
+		const stderr = typeof result.stderr === "string" ? result.stderr : "";
+		if (stdout.length > 0) process.stdout.write(stdout);
+		if (stderr.length > 0) process.stderr.write(stderr);
+
+		const combined = `${stdout}${stderr.length > 0 ? `\n${stderr}` : ""}`;
+		if (traceLogPath != null) {
+			writeFileSync(traceLogPath, combined);
+			process.stderr.write(`\n✓  Trace log written: ${traceLogPath}\n`);
+		}
+
+		if (shouldAnalyzeTrace) {
+			analyzeTraceOutput(combined);
+		}
+	}
+
+	if (result.status !== 0) process.exit(result.status ?? 1);
+
+	if (!existsSync(outJson)) {
+		process.stderr.write("✗  No output JSON — bench may have errored internally.\n");
+		process.exit(1);
+	}
+
+	const output = JSON.parse(readFileSync(outJson, "utf8"));
+	const benches = output.files.flatMap((f) => f.groups).flatMap((g) => g.benchmarks);
+	const currentRun = { date: ctx.date, context: ctx, vitestOutput: output };
+
+	// Fail loud on NaN — vitest swallows errors in bench callbacks silently
+	const invalid = benches.filter((b) => !Number.isFinite(b.mean) || !b.sampleCount);
+	if (invalid.length > 0) {
+		process.stderr.write(
+			`✗  ${invalid.length} benchmark(s) have NaN results — check for errors in the bench callback:\n` +
+				invalid.map((b) => `   • ${b.name}`).join("\n") +
+				"\n",
+		);
+		process.exit(1);
+	}
+
+	// ── Save ──────────────────────────────────────────────────────────────────
+
+	const history = existsSync(HISTORY_FILE)
+		? JSON.parse(readFileSync(HISTORY_FILE, "utf8"))
+		: { runs: {} };
+
+	if (!history.runs[id]) history.runs[id] = { context: ctx, runs: [] };
+	const previousRun = history.runs[id].runs[0];
+
+	printBenchmarkComparison(currentRun, previousRun);
+
+	if (!shouldSave) {
+		process.stderr.write(`✓  Not saved  (mode: ${command}, baseline unchanged)\n\n`);
+		process.exit(0);
+	}
+
+	history.runs[id].context = ctx;
+	history.runs[id].runs.unshift(currentRun);
+	if (history.runs[id].runs.length > MAX_RUNS) history.runs[id].runs.length = MAX_RUNS;
+
+	writeFileSync(HISTORY_FILE, JSON.stringify(history, null, 2));
+
+	const stored = history.runs[id].runs.length;
+	process.stderr.write(`\n✓  Saved  (context: ${id}, runs stored: ${stored}/${MAX_RUNS})\n\n`);
+} finally {
+	rmSync(tmpDir, { recursive: true, force: true });
 }

package/src/bench/trace.mjs

@@ -6,8 +6,8 @@ import { join } from "node:path";
 
 const WORKSPACE_ROOT = process.cwd();
 const VITEST_BIN = join(WORKSPACE_ROOT, "node_modules/.bin/vitest");
-const CONFIG = join(WORKSPACE_ROOT, "vitest.config.ts");
-const TRACE_SPEC = "projects/cli/e2e/build.trace.e2e.spec.ts";
+const CONFIG = join(WORKSPACE_ROOT, "vitest.e2e.config.ts");
+const TRACE_SPEC = "projects/terminal/e2e/build.trace.e2e.spec.ts";
 const TRACE_LOG_DIR = join(WORKSPACE_ROOT, ".decohere", "logs");
 
 function truthy(value, defaultValue = false) {
@@ -149,23 +149,19 @@ if (traceLogPath != null) {
 	mkdirSync(TRACE_LOG_DIR, { recursive: true });
 }
 
-const result = spawnSync(
-	VITEST_BIN,
-	["run", "--config", CONFIG, "--project", "e2e", TRACE_SPEC],
-	{
-		cwd: WORKSPACE_ROOT,
-		stdio: "pipe",
-		encoding: "utf8",
-		maxBuffer: 1024 * 1024 * 200,
-		env: {
-			...process.env,
-			NODE_OPTIONS: "",
-			FORGE_TRACE: "1",
-			FORGE_TRACE_STRUCTURED: "1",
-			LOG_LEVEL: process.env.LOG_LEVEL ?? "trace",
-		},
+const result = spawnSync(VITEST_BIN, ["run", "--config", CONFIG, TRACE_SPEC], {
+	cwd: WORKSPACE_ROOT,
+	stdio: "pipe",
+	encoding: "utf8",
+	maxBuffer: 1024 * 1024 * 200,
+	env: {
+		...process.env,
+		NODE_OPTIONS: "",
+		FORGE_TRACE: "1",
+		FORGE_TRACE_STRUCTURED: "1",
+		LOG_LEVEL: process.env.LOG_LEVEL ?? "trace",
 	},
-);
+});
 
 const stdout = typeof result.stdout === "string" ? result.stdout : "";
 const stderr = typeof result.stderr === "string" ? result.stderr : "";