@decohere/ci-tools 0.0.1-dev.20 → 0.0.1-dev.20261105.103

package/src/bench/history.mjs

@@ -0,0 +1,248 @@
+/**
+ * Benchmark history: the results of each `pnpm bench` run, kept per machine so runs are only
+ * compared like for like. Pure functions over plain data; bench.mjs does the I/O.
+ *
+ * Shape (version 2):
+ *   { version: 2, contexts: { [contextId]: { context, runs: [{ date, sha, results }] } } }
+ * newest run first. A result is { id, label, unit, median, p90?, min, max, samples, files? }.
+ */
+
+import crypto from "node:crypto";
+
+export const HISTORY_VERSION = 2;
+/** Nightly runs: a month per machine, enough for a trend and a regression baseline. */
+export const MAX_RUNS = 30;
+/** Printed by each suite before its JSON array of results (projects/flux/bench/harness.ts). */
+export const RESULT_PREFIX = "BENCH_RESULT ";
+
+/** The machine a run happened on. Runs are compared only within one context. */
+export function getContext(os, env, nodeVersion, date = new Date()) {
+	const cpus = os.cpus();
+	const nodeMajor = nodeVersion.match(/^v(\d+)/)?.[1] ?? "?";
+	const rawEnv = env.BENCH_ENV ?? null;
+	return {
+		benchEnv: rawEnv ? `${rawEnv}-node${nodeMajor}` : "(local)",
+		date: date.toISOString(),
+		os: { platform: os.platform(), arch: os.arch() },
+		cpu: { model: cpus[0]?.model?.trim() ?? "unknown", count: cpus.length },
+		memoryGb: Number.parseFloat((os.totalmem() / 1024 ** 3).toFixed(1)),
+		node: nodeVersion,
+	};
+}
+
+/** 16-char SHA-256 prefix of platform|arch|cpu|cores|memGB|node. */
+export function contextId(ctx) {
+	const key = [
+		ctx.os.platform,
+		ctx.os.arch,
+		ctx.cpu.model,
+		ctx.cpu.count,
+		ctx.memoryGb,
+		ctx.node,
+	].join("|");
+	return crypto.createHash("sha256").update(key).digest("hex").slice(0, 16);
+}
+
+/** The results a suite printed, from its whole output. Throws if it printed none. */
+export function parseResults(output, suite) {
+	const line = output
+		.split(/\r?\n/)
+		.reverse()
+		.find((candidate) => candidate.startsWith(RESULT_PREFIX));
+	if (line == null)
+		throw new Error(`suite "${suite}" printed no ${RESULT_PREFIX.trim()} line`);
+	return JSON.parse(line.slice(RESULT_PREFIX.length));
+}
+
+const FORGE_BENCHMARKS = [
+	["forge build es6 + types", "forge.build.local"],
+	["forge build es6 only", "forge.build.debug"],
+	["forge build types only", "forge.build.types-only"],
+];
+
+/**
+ * Results from `vitest bench --outputJson` for the forge build benchmark. Its names end in a
+ * file count, so they are mapped to stable ids; vitest reports no p90, so it is left out.
+ */
+export function fromVitest(vitestOutput) {
+	return vitestOutput.files
+		.flatMap((file) => file.groups)
+		.flatMap((group) => group.benchmarks)
+		.map((benchmark) => {
+			const id = FORGE_BENCHMARKS.find(([prefix]) => benchmark.name.startsWith(prefix))?.[1];
+			if (id == null) throw new Error(`unknown forge benchmark "${benchmark.name}"`);
+			const files = Number(benchmark.name.match(/\((\d+) files\)/)?.[1]);
+			return {
+				id,
+				label: benchmark.name.replace(/\s*\(\d+ files\)$/, ""),
+				unit: "ms",
+				median: benchmark.median ?? benchmark.mean,
+				min: benchmark.min,
+				max: benchmark.max,
+				samples: benchmark.sampleCount,
+				...(Number.isFinite(files) ? { files } : {}),
+			};
+		});
+}
+
+/** Throws when a result has no usable numbers: a suite that errored mid-run, not a slow one. */
+export function assertValid(results) {
+	const invalid = results.filter(
+		(result) => !Number.isFinite(result.median) || !(result.samples > 0),
+	);
+	if (invalid.length > 0) {
+		throw new Error(
+			`${invalid.length} benchmark(s) have no valid samples: ${invalid.map((r) => r.id).join(", ")}`,
+		);
+	}
+}
+
+/**
+ * Reads a stored history. Version 1 (forge build only, raw vitest output) is converted so its
+ * trend carries on; anything unrecognised starts empty rather than mixing formats.
+ */
+export function normalize(raw) {
+	if (raw?.version === HISTORY_VERSION && raw.contexts != null) return raw;
+	const contexts = {};
+	for (const [id, entry] of Object.entries(raw?.runs ?? {})) {
+		const runs = (entry.runs ?? [])
+			.filter((run) => run.vitestOutput != null)
+			.map((run) => ({ date: run.date, sha: null, results: fromVitest(run.vitestOutput) }));
+		if (runs.length > 0) contexts[id] = { context: entry.context, runs };
+	}
+	return { version: HISTORY_VERSION, contexts };
+}
+
+/** The previous run for this context, if any: what a new run is compared against. */
+export function previousRun(history, id) {
+	return history.contexts[id]?.runs[0];
+}
+
+/** Machine types with no run for this long are dropped: CI runner types come and go. */
+export const STALE_AFTER_DAYS = 30;
+
+/** The history without machine types whose newest run is older than `days` before `now`. */
+export function pruneStale(history, now, days = STALE_AFTER_DAYS) {
+	const cutoff = now.getTime() - days * 24 * 60 * 60 * 1000;
+	const contexts = Object.fromEntries(
+		Object.entries(history.contexts).filter(
+			([, entry]) => entry.runs[0] != null && new Date(entry.runs[0].date).getTime() >= cutoff,
+		),
+	);
+	return { version: HISTORY_VERSION, contexts };
+}
+
+/**
+ * A new history with `run` first in its context, trimmed to `max` runs, and machine types
+ * without a recent run dropped. Does not mutate.
+ */
+export function appendRun(history, ctx, run, max = MAX_RUNS) {
+	const id = contextId(ctx);
+	const runs = [run, ...(history.contexts[id]?.runs ?? [])].slice(0, max);
+	return pruneStale(
+		{
+			version: HISTORY_VERSION,
+			contexts: { ...history.contexts, [id]: { context: ctx, runs } },
+		},
+		new Date(run.date),
+	);
+}
+
+/** The newest run on any machine: the commit the previous nightly measured. */
+export function latestRun(history) {
+	return Object.values(history.contexts)
+		.map((entry) => entry.runs[0])
+		.filter((run) => run != null)
+		.sort((a, b) => new Date(b.date) - new Date(a.date))[0];
+}
+
+/** One row per current result, with the change in median against the previous run. */
+export function compare(results, previous) {
+	const before = new Map((previous?.results ?? []).map((result) => [result.id, result]));
+	return results.map((result) => {
+		const old = before.get(result.id);
+		if (old == null) return { id: result.id, current: result.median };
+		const delta = result.median - old.median;
+		return {
+			id: result.id,
+			current: result.median,
+			previous: old.median,
+			delta,
+			percent: old.median === 0 ? 0 : (delta / old.median) * 100,
+		};
+	});
+}
+
+/** The comparison as an aligned text table. */
+export function formatComparison(rows) {
+	const ms = (value) => (value == null ? "n/a" : `${value.toFixed(1)} ms`);
+	const signed = (value, suffix) =>
+		value == null ? "n/a" : `${value >= 0 ? "+" : ""}${value.toFixed(1)}${suffix}`;
+	const table = [
+		["Benchmark", "Median", "Previous", "Delta", "%"],
+		...rows.map((row) => [
+			row.id,
+			ms(row.current),
+			ms(row.previous),
+			signed(row.delta, " ms"),
+			signed(row.percent, "%"),
+		]),
+	];
+	const widths = table[0].map((_, column) =>
+		Math.max(...table.map((row) => row[column].length)),
+	);
+	const line = (row) =>
+		row
+			.map((cell, column) =>
+				(column === 0 ? cell.padEnd : cell.padStart).call(cell, widths[column]),
+			)
+			.join(" | ");
+	return [
+		line(table[0]),
+		widths.map((w) => "-".repeat(w)).join("-+-"),
+		...table.slice(1).map(line),
+	].join("\n");
+}
+
+/**
+ * When a benchmark counts as slower: the nightly job benchmarks tonight's commit (head)
+ * and the one the previous nightly measured (base) on the same runner, alternating head, base,
+ * head, base. A benchmark regressed when head's median beats base's by more than `threshold`
+ * in every round, so one noisy round files nothing.
+ */
+export const REGRESSION = { threshold: 0.2, rounds: 2 };
+
+/**
+ * The benchmarks slower on head than on base. `headRounds` and `baseRounds` are lists of result
+ * lists, one per round, run alternately on one machine. Benchmarks either side lacks are skipped.
+ */
+export function compareRounds(headRounds, baseRounds, options = REGRESSION) {
+	const rounds = Math.min(headRounds.length, baseRounds.length);
+	if (rounds < options.rounds) return [];
+	const medianIn = (results, id) => results.find((result) => result.id === id)?.median;
+	const regressions = [];
+	for (const head of headRounds[0]) {
+		const pairs = Array.from({ length: rounds }, (_, round) => ({
+			head: medianIn(headRounds[round], head.id),
+			base: medianIn(baseRounds[round], head.id),
+		}));
+		const comparable = pairs.every(
+			(pair) => Number.isFinite(pair.head) && Number.isFinite(pair.base) && pair.base > 0,
+		);
+		if (!comparable) continue;
+		const slower = pairs.every((pair) => pair.head > pair.base * (1 + options.threshold));
+		if (!slower) continue;
+		const mean = (values) => values.reduce((sum, value) => sum + value, 0) / values.length;
+		const headMs = mean(pairs.map((pair) => pair.head));
+		const baseMs = mean(pairs.map((pair) => pair.base));
+		regressions.push({
+			id: head.id,
+			label: head.label,
+			base: baseMs,
+			head: headMs,
+			percent: (headMs / baseMs - 1) * 100,
+			rounds: pairs,
+		});
+	}
+	return regressions;
+}

package/src/bench/nightly.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+
+/**
+ * nightly.mjs — the nightly benchmark run. Benchmarks tonight's commit (head) and the
+ * commit the previous nightly measured (base) on this runner, alternating head, base, head,
+ * base, so the comparison never mixes CPU models. Then:
+ *
+ * - appends head's first round to the history (projects/ci-tools/dist/ci/bench/), for the charts;
+ * - writes regressions.json next to it: the benchmarks slower on head in every round
+ *   (history.mjs `compareRounds`), which file-regressions.mjs turns into issues.
+ *
+ * Base runs from its own checkout (a git worktree) with its own dependencies and its own
+ * bench.mjs, so each commit is measured with its own code. With no earlier nightly, or no new
+ * commit since, it benchmarks head once and compares nothing.
+ *
+ *   node projects/ci-tools/src/bench/nightly.mjs   (from the repo root; needs the full git history)
+ */
+
+import { execFileSync } from "node:child_process";
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import { dirname, join } from "node:path";
+import {
+	appendRun,
+	compareRounds,
+	getContext,
+	latestRun,
+	normalize,
+	REGRESSION,
+} from "./history.mjs";
+
+const ROOT = process.cwd();
+const DIST = join(ROOT, "projects/ci-tools/dist/ci/bench");
+const HISTORY_FILE = join(DIST, "bench-history.json");
+const REGRESSIONS_FILE = join(DIST, "regressions.json");
+const BENCH = "projects/ci-tools/src/bench/bench.mjs";
+
+function run(command, args, cwd, env = {}) {
+	execFileSync(command, args, {
+		cwd,
+		stdio: ["ignore", "inherit", "inherit"],
+		env: { ...process.env, ...env },
+	});
+}
+
+function git(...args) {
+	return execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim();
+}
+
+/** Runs one round of every suite in `cwd` and returns its results, saved to its own history. */
+function benchRound(cwd, label, round, workDir) {
+	process.stderr.write(`\n══ ${label}, round ${round + 1}\n`);
+	const historyFile = join(workDir, `${label}-${round}.json`);
+	run("node", [BENCH, "save"], cwd, { BENCH_HISTORY_FILE: historyFile });
+	// Base's bench.mjs may predate BENCH_HISTORY_FILE and save to its checkout's default path.
+	const file = existsSync(historyFile)
+		? historyFile
+		: join(cwd, "projects/ci-tools/dist/ci/bench/bench-history.json");
+	const saved = normalize(JSON.parse(readFileSync(file, "utf8")));
+	const newest = latestRun(saved);
+	if (newest == null) throw new Error(`${label} round ${round + 1} saved no run`);
+	return newest.results;
+}
+
+const history = normalize(
+	existsSync(HISTORY_FILE) ? JSON.parse(readFileSync(HISTORY_FILE, "utf8")) : null,
+);
+const head = git("rev-parse", "HEAD");
+const base = latestRun(history)?.sha ?? null;
+const workDir = mkdtempSync(join(os.tmpdir(), "decohere-nightly-"));
+
+try {
+	const headRounds = [];
+	const baseRounds = [];
+	const compareWith = base != null && base !== head ? base : null;
+	if (compareWith == null) {
+		process.stderr.write(
+			base == null
+				? "No earlier nightly run: benchmarking head only.\n"
+				: `No new commit since the last nightly (${head.slice(0, 7)}): benchmarking head only.\n`,
+		);
+		headRounds.push(benchRound(ROOT, "head", 0, workDir));
+	} else {
+		const baseDir = join(workDir, "base");
+		run("git", ["worktree", "add", "--detach", baseDir, compareWith], ROOT);
+		run("pnpm", ["install", "--frozen-lockfile", "--prefer-offline"], baseDir);
+		for (let round = 0; round < REGRESSION.rounds; round++) {
+			headRounds.push(benchRound(ROOT, "head", round, workDir));
+			baseRounds.push(benchRound(baseDir, "base", round, workDir));
+		}
+	}
+
+	const ctx = getContext(os, process.env, process.version);
+	const next = appendRun(history, ctx, { date: ctx.date, sha: head, results: headRounds[0] });
+	mkdirSync(dirname(HISTORY_FILE), { recursive: true });
+	writeFileSync(HISTORY_FILE, `${JSON.stringify(next, null, 2)}\n`);
+
+	const regressions = compareWith == null ? [] : compareRounds(headRounds, baseRounds);
+	writeFileSync(
+		REGRESSIONS_FILE,
+		`${JSON.stringify({ base: compareWith, head, context: ctx, regressions }, null, 2)}\n`,
+	);
+	process.stderr.write(
+		`\nSaved head's run. ${compareWith == null ? "Nothing compared." : `Compared with ${compareWith.slice(0, 7)}: ${regressions.length} benchmark(s) slower in every round.`}\n`,
+	);
+} finally {
+	if (existsSync(join(workDir, "base"))) {
+		run("git", ["worktree", "remove", "--force", join(workDir, "base")], ROOT);
+	}
+	rmSync(workDir, { recursive: true, force: true });
+}

package/src/bench/regression-issue.mjs

@@ -0,0 +1,44 @@
+/**
+ * The GitHub issue for a benchmark that got slower: one open issue per benchmark,
+ * found again by its exact title and updated in place while the regression lasts.
+ */
+
+const PAGES_URL = "https://typesmiths.github.io/decohere/benchmark.html";
+
+export function issueTitle(regression) {
+	return `perf(bench): ${regression.id} is slower`;
+}
+
+function ms(value) {
+	return `${value.toFixed(1)} ms`;
+}
+
+function short(sha) {
+	return sha.slice(0, 7);
+}
+
+export function issueBody(regression, { repoUrl, runUrl, machine, threshold, base, head }) {
+	const range = `[${short(base)}...${short(head)}](${repoUrl}/compare/${base}...${head})`;
+	return [
+		"## Problem",
+		"",
+		`The nightly benchmark **${regression.label}** (\`${regression.id}\`) is ${regression.percent.toFixed(0)}% slower on [\`${short(head)}\`](${repoUrl}/commit/${head}) than on [\`${short(base)}\`](${repoUrl}/commit/${base}), the commit the previous nightly measured. Both ran on the same runner, alternately, and head was more than ${Math.round(threshold * 100)}% slower in every round.`,
+		"",
+		"| Round | Base | Head |",
+		"|---|---|---|",
+		...regression.rounds.map(
+			(round, index) => `| ${index + 1} | ${ms(round.base)} | ${ms(round.head)} |`,
+		),
+		"",
+		`- Machine: ${machine}`,
+		`- Commits: ${range}`,
+		"",
+		"## What to do",
+		"",
+		`1. Look through the commits in ${range} for the change.`,
+		`2. Reproduce locally: check out \`${short(base)}\`, run \`pnpm bench:save\`, then check out \`${short(head)}\` and run \`pnpm bench\` to compare.`,
+		`3. Fix the cause, or if the slowdown is intended, say why here and close the issue. Trend: [benchmark page](${PAGES_URL}).`,
+		"",
+		`Updated by the nightly bench workflow: ${runUrl}`,
+	].join("\n");
+}

package/src/bench/trace.mjs

@@ -6,8 +6,8 @@ import { join } from "node:path";
 
 const WORKSPACE_ROOT = process.cwd();
 const VITEST_BIN = join(WORKSPACE_ROOT, "node_modules/.bin/vitest");
-const CONFIG = join(WORKSPACE_ROOT, "vitest.e2e.config.ts");
-const TRACE_SPEC = "projects/terminal/e2e/build.trace.e2e.spec.ts";
+const CONFIG = join(WORKSPACE_ROOT, "vitest.config.ts");
+const TRACE_SPEC = "projects/cli/e2e/build.trace.e2e.spec.ts";
 const TRACE_LOG_DIR = join(WORKSPACE_ROOT, ".decohere", "logs");
 
 function truthy(value, defaultValue = false) {
@@ -149,19 +149,23 @@ if (traceLogPath != null) {
 	mkdirSync(TRACE_LOG_DIR, { recursive: true });
 }
 
-const result = spawnSync(VITEST_BIN, ["run", "--config", CONFIG, TRACE_SPEC], {
-	cwd: WORKSPACE_ROOT,
-	stdio: "pipe",
-	encoding: "utf8",
-	maxBuffer: 1024 * 1024 * 200,
-	env: {
-		...process.env,
-		NODE_OPTIONS: "",
-		FORGE_TRACE: "1",
-		FORGE_TRACE_STRUCTURED: "1",
-		LOG_LEVEL: process.env.LOG_LEVEL ?? "trace",
+const result = spawnSync(
+	VITEST_BIN,
+	["run", "--config", CONFIG, "--project", "e2e", TRACE_SPEC],
+	{
+		cwd: WORKSPACE_ROOT,
+		stdio: "pipe",
+		encoding: "utf8",
+		maxBuffer: 1024 * 1024 * 200,
+		env: {
+			...process.env,
+			NODE_OPTIONS: "",
+			FORGE_TRACE: "1",
+			FORGE_TRACE_STRUCTURED: "1",
+			LOG_LEVEL: process.env.LOG_LEVEL ?? "trace",
+		},
 	},
-});
+);
 
 const stdout = typeof result.stdout === "string" ? result.stdout : "";
 const stderr = typeof result.stderr === "string" ? result.stderr : "";

package/src/github/first-publish-check.mjs

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+/**
+ * Finds packages in this workspace that aren't on npm yet. A first publish needs an owner's npm
+ * login, so CI can't do it. On a PR that's only a notice, so a contributor can propose a package
+ * with no npm access. In the merge queue it fails, because Publish could not upload that package.
+ */
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { listPublishablePackages } from "./publish-verify.mjs";
+
+const REGISTRY = "https://registry.npmjs.org";
+
+/**
+ * @param {{
+ *   packages: string[],
+ *   blocking: boolean,
+ *   fetch: typeof globalThis.fetch,
+ *   log?: (message: string) => void,
+ * }} options
+ * @returns {Promise<number>} exit code
+ */
+export async function checkFirstPublish(options) {
+	const { packages, blocking, fetch } = options;
+	const log = options.log ?? console.log;
+
+	const results = await Promise.all(
+		packages.map(async (name) => {
+			// Not /-/package/<name>/dist-tags: for a name that does not exist it answers 401, not 404.
+			const url = `${REGISTRY}/${name.replace("/", "%2f")}`;
+			const response = await fetch(url, { signal: AbortSignal.timeout(30_000) });
+			// Anything else, like an outage, says nothing about whether the name exists.
+			if (!response.ok && response.status !== 404) {
+				throw new Error(`Could not check ${name} on npm: ${response.status}`);
+			}
+			return { name, missing: response.status === 404 };
+		}),
+	);
+
+	const missing = results.filter((result) => result.missing).map((result) => result.name);
+	if (missing.length === 0) {
+		log(`All ${packages.length} packages are on npm.`);
+		return 0;
+	}
+	const names = missing.join(" ");
+	if (blocking) {
+		log(
+			`::error::Not on npm yet: ${names}. An owner runs \`pnpm npm:bootstrap --otp <code> ${names}\`, ` +
+				"then adds this PR to the merge queue again.",
+		);
+		return 1;
+	}
+	log(
+		`::notice::New package, not on npm yet: ${names}. Contributors have nothing to do: a maintainer ` +
+			`runs \`pnpm npm:bootstrap --otp <code> ${names}\` before this merges.`,
+	);
+	return 0;
+}
+
+if (
+	process.argv[1] != null &&
+	import.meta.url === pathToFileURL(resolve(process.argv[1])).href
+) {
+	process.exit(
+		await checkFirstPublish({
+			packages: listPublishablePackages(process.cwd()).map((pkg) => pkg.name),
+			blocking: process.env.GITHUB_EVENT_NAME === "merge_group",
+			fetch: globalThis.fetch,
+		}),
+	);
+}