@decohere/ci-tools 0.0.1-dev.112 → 0.0.1-dev.113

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decohere/ci-tools",
-  "version": "0.0.1-dev.112",
+  "version": "0.0.1-dev.113",
   "description": "CI utility scripts for badge generation, coverage reporting, pages artifact assembly, and GitHub API tooling",
   "keywords": [
     "typescript",
@@ -16,6 +16,7 @@
   "exports": {
     "./lib/svg": "./src/lib/svg.mjs",
     "./lib/lcov": "./src/lib/lcov.mjs",
+    "./lib/github-issue": "./src/lib/github-issue.mjs",
     "./coverage/badge": "./src/coverage/badge.mjs",
     "./coverage/report": "./src/coverage/report.mjs",
     "./badges/ci-status": "./src/badges/ci-status.mjs",
@@ -25,7 +26,6 @@
     "./audit/summarize": "./src/audit/summarize.mjs",
     "./audit/enforce": "./src/audit/enforce.mjs",
     "./audit/upsert-issue": "./src/audit/upsert-issue.mjs",
-    "./audit/upsert-clickup": "./src/audit/upsert-clickup.mjs",
     "./bench/bench": "./src/bench/bench.mjs",
     "./bench/trace": "./src/bench/trace.mjs",
     "./changeset/aggregate-changelog": "./src/changeset/aggregate-changelog.mjs",

package/src/audit/upsert-issue.mjs

@@ -1,20 +1,10 @@
-import { spawnSync } from "node:child_process";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import os from "node:os";
-import path from "node:path";
+import { existsSync, readFileSync } from "node:fs";
+import { upsertIssue } from "../lib/github-issue.mjs";
 import { formatMarkdown, parseAudit } from "./format.mjs";
 
 const auditFilePath = process.argv[2] ?? "npm-audit.json";
 const ISSUE_TITLE = "security(ci): npm audit vulnerabilities detected";
 
-function runGh(args) {
-	const result = spawnSync("gh", args, { encoding: "utf8", env: process.env });
-	if (result.status !== 0) {
-		throw new Error(result.stderr || result.stdout || `gh command failed: ${args.join(" ")}`);
-	}
-	return result.stdout;
-}
-
 function runUrl() {
 	if (
 		process.env.GITHUB_SERVER_URL &&
@@ -26,21 +16,6 @@ function runUrl() {
 	return "(run URL unavailable)";
 }
 
-function findExistingIssueNumber() {
-	const output = runGh([
-		"issue",
-		"list",
-		"--state",
-		"open",
-		"--limit",
-		"200",
-		"--json",
-		"number,title",
-	]);
-	const issues = JSON.parse(output);
-	return issues.find((issue) => issue.title === ISSUE_TITLE)?.number ?? null;
-}
-
 if (!existsSync(auditFilePath)) {
 	console.log(`No audit file found at ${auditFilePath}; skipping issue update.`);
 	process.exit(0);
@@ -59,17 +34,8 @@ const body = [
 	`- Commit: \`${sha}\``,
 	`- Workflow run: ${runUrl()}`,
 	"",
-	"This issue is managed by CI and will be updated on subsequent main-branch runs while vulnerabilities remain.",
+	"This issue is managed by CI: main-branch runs and the nightly audit update it in place while vulnerabilities remain.",
 ].join("\n");
 
-const tempFile = path.join(os.tmpdir(), `npm-audit-issue-${Date.now()}.md`);
-writeFileSync(tempFile, body, "utf8");
-
-const existingIssueNumber = findExistingIssueNumber();
-if (existingIssueNumber) {
-	runGh(["issue", "comment", String(existingIssueNumber), "--body-file", tempFile]);
-	console.log(`Updated existing issue #${existingIssueNumber}.`);
-} else {
-	runGh(["issue", "create", "--title", ISSUE_TITLE, "--body-file", tempFile]);
-	console.log("Created new npm audit tracking issue.");
-}
+const result = upsertIssue({ title: ISSUE_TITLE, body });
+console.log(`${result.created ? "Opened" : "Updated"} issue #${result.number} ${result.url}`);

package/src/bench/bench.mjs

@@ -12,7 +12,7 @@
  * (see history.mjs). The history lives in projects/ci-tools/dist/ci/bench/bench-history.json;
  * the nightly bench workflow restores it from Pages, saves, and publishes the charts.
  *
- * Environment: BENCH_ENV names the machine class (CI sets gh-actions); BENCH_SAMPLES (default
+ * Environment: BENCH_HISTORY_FILE overrides where `save` writes; BENCH_ENV names the machine class (CI sets gh-actions); BENCH_SAMPLES (default
  * 10) is the sample count for every benchmark; BENCH_LIBS and BENCH_FILES size the flux workspace.
  */
 
@@ -41,10 +41,10 @@ import {
 } from "./history.mjs";
 
 const WORKSPACE_ROOT = process.cwd();
-const HISTORY_FILE = join(
-	WORKSPACE_ROOT,
-	"projects/ci-tools/dist/ci/bench/bench-history.json",
-);
+// BENCH_HISTORY_FILE lets the nightly job save each comparison round to its own file (#667).
+const HISTORY_FILE =
+	process.env.BENCH_HISTORY_FILE ??
+	join(WORKSPACE_ROOT, "projects/ci-tools/dist/ci/bench/bench-history.json");
 
 /** Runs a TypeScript benchmark from source, the way the repo runs its CLIs. */
 function fluxSuite(file) {

package/src/bench/file-regressions.mjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 /**
- * file-regressions.mjs — after `pnpm bench:save` in the nightly workflow, open an issue for each
- * benchmark that got slower on this machine (history.mjs `findRegressions`), or update the open
+ * file-regressions.mjs — after nightly.mjs, open an issue for each benchmark that got slower
+ * (regressions.json: head slower than base in every round on one runner), or update the open
  * one. Needs `gh` and GH_TOKEN with issues: write.
  *
  *   node projects/ci-tools/src/bench/file-regressions.mjs            # file issues
@@ -11,11 +11,10 @@
 
 import { spawnSync } from "node:child_process";
 import { readFileSync } from "node:fs";
-import os from "node:os";
-import { contextId, findRegressions, getContext, normalize, REGRESSION } from "./history.mjs";
+import { REGRESSION } from "./history.mjs";
 import { issueBody, issueTitle } from "./regression-issue.mjs";
 
-const HISTORY_FILE = "projects/ci-tools/dist/ci/bench/bench-history.json";
+const REGRESSIONS_FILE = "projects/ci-tools/dist/ci/bench/regressions.json";
 const dryRun = process.argv.includes("--dry-run");
 
 function gh(args, input) {
@@ -33,19 +32,19 @@ const runUrl = process.env.GITHUB_RUN_ID
 	? `${repoUrl}/actions/runs/${process.env.GITHUB_RUN_ID}`
 	: "(local run)";
 
-const ctx = getContext(os, process.env, process.version);
-const history = normalize(JSON.parse(readFileSync(HISTORY_FILE, "utf8")));
-const runs = history.contexts[contextId(ctx)]?.runs ?? [];
-const regressions = findRegressions(runs);
-const machine = `${ctx.benchEnv}, ${ctx.cpu.model} × ${ctx.cpu.count}`;
-
+const { base, head, context, regressions } = JSON.parse(
+	readFileSync(REGRESSIONS_FILE, "utf8"),
+);
+if (base == null) {
+	console.log("Nothing was compared tonight (no earlier nightly, or no new commit).");
+	process.exit(0);
+}
 if (regressions.length === 0) {
-	console.log(
-		`No benchmark regressions on this machine (${runs.length} runs in its history).`,
-	);
+	console.log(`No benchmark slower on ${head.slice(0, 7)} than on ${base.slice(0, 7)}.`);
 	process.exit(0);
 }
 
+const machine = `${context.benchEnv}, ${context.cpu.model} × ${context.cpu.count}`;
 const open = dryRun
 	? []
 	: JSON.parse(
@@ -59,6 +58,8 @@ for (const regression of regressions) {
 		runUrl,
 		machine,
 		threshold: REGRESSION.threshold,
+		base,
+		head,
 	});
 	if (dryRun) {
 		console.log(`# ${title}\n\n${body}\n`);

package/src/bench/history.mjs

@@ -118,14 +118,42 @@ export function previousRun(history, id) {
 	return history.contexts[id]?.runs[0];
 }
 
-/** A new history with `run` first in its context, trimmed to `max` runs. Does not mutate. */
+/** Machine types with no run for this long are dropped: CI runner types come and go (#667). */
+export const STALE_AFTER_DAYS = 30;
+
+/** The history without machine types whose newest run is older than `days` before `now`. */
+export function pruneStale(history, now, days = STALE_AFTER_DAYS) {
+	const cutoff = now.getTime() - days * 24 * 60 * 60 * 1000;
+	const contexts = Object.fromEntries(
+		Object.entries(history.contexts).filter(
+			([, entry]) => entry.runs[0] != null && new Date(entry.runs[0].date).getTime() >= cutoff,
+		),
+	);
+	return { version: HISTORY_VERSION, contexts };
+}
+
+/**
+ * A new history with `run` first in its context, trimmed to `max` runs, and machine types
+ * without a recent run dropped. Does not mutate.
+ */
 export function appendRun(history, ctx, run, max = MAX_RUNS) {
 	const id = contextId(ctx);
 	const runs = [run, ...(history.contexts[id]?.runs ?? [])].slice(0, max);
-	return {
-		version: HISTORY_VERSION,
-		contexts: { ...history.contexts, [id]: { context: ctx, runs } },
-	};
+	return pruneStale(
+		{
+			version: HISTORY_VERSION,
+			contexts: { ...history.contexts, [id]: { context: ctx, runs } },
+		},
+		new Date(run.date),
+	);
+}
+
+/** The newest run on any machine: the commit the previous nightly measured. */
+export function latestRun(history) {
+	return Object.values(history.contexts)
+		.map((entry) => entry.runs[0])
+		.filter((run) => run != null)
+		.sort((a, b) => new Date(b.date) - new Date(a.date))[0];
 }
 
 /** One row per current result, with the change in median against the previous run. */
@@ -177,48 +205,43 @@ export function formatComparison(rows) {
 }
 
 /**
- * When a benchmark counts as slower (#660): its median beat the baseline by more than
- * `threshold` on each of the last `confirmRuns` runs, where the baseline is the median of the
- * `baselineRuns` runs before those. Two nights in a row, so one noisy runner files nothing.
+ * When a benchmark counts as slower (#667): the nightly job benchmarks tonight's commit (head)
+ * and the one the previous nightly measured (base) on the same runner, alternating head, base,
+ * head, base. A benchmark regressed when head's median beats base's by more than `threshold`
+ * in every round, so one noisy round files nothing.
  */
-export const REGRESSION = { threshold: 0.2, confirmRuns: 2, baselineRuns: 7, minBaseline: 3 };
+export const REGRESSION = { threshold: 0.2, rounds: 2 };
 
-function medianOf(values) {
-	const sorted = [...values].sort((a, b) => a - b);
-	const middle = Math.floor(sorted.length / 2);
-	return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
-}
-
-/** The benchmarks that regressed on one machine, given its runs newest first. */
-export function findRegressions(runs, options = REGRESSION) {
-	const recent = runs.slice(0, options.confirmRuns);
-	const before = runs.slice(options.confirmRuns, options.confirmRuns + options.baselineRuns);
-	if (recent.length < options.confirmRuns) return [];
-	const medianIn = (run, id) => run.results.find((result) => result.id === id)?.median;
+/**
+ * The benchmarks slower on head than on base. `headRounds` and `baseRounds` are lists of result
+ * lists, one per round, run alternately on one machine. Benchmarks either side lacks are skipped.
+ */
+export function compareRounds(headRounds, baseRounds, options = REGRESSION) {
+	const rounds = Math.min(headRounds.length, baseRounds.length);
+	if (rounds < options.rounds) return [];
+	const medianIn = (results, id) => results.find((result) => result.id === id)?.median;
 	const regressions = [];
-	for (const latest of recent[0].results) {
-		const baselineValues = before
-			.map((run) => medianIn(run, latest.id))
-			.filter((value) => Number.isFinite(value));
-		if (baselineValues.length < options.minBaseline) continue;
-		const baseline = medianOf(baselineValues);
-		const recentValues = recent.map((run) => medianIn(run, latest.id));
-		const slower = recentValues.every(
-			(value) => Number.isFinite(value) && value > baseline * (1 + options.threshold),
+	for (const head of headRounds[0]) {
+		const pairs = Array.from({ length: rounds }, (_, round) => ({
+			head: medianIn(headRounds[round], head.id),
+			base: medianIn(baseRounds[round], head.id),
+		}));
+		const comparable = pairs.every(
+			(pair) => Number.isFinite(pair.head) && Number.isFinite(pair.base) && pair.base > 0,
 		);
+		if (!comparable) continue;
+		const slower = pairs.every((pair) => pair.head > pair.base * (1 + options.threshold));
 		if (!slower) continue;
+		const mean = (values) => values.reduce((sum, value) => sum + value, 0) / values.length;
+		const headMs = mean(pairs.map((pair) => pair.head));
+		const baseMs = mean(pairs.map((pair) => pair.base));
 		regressions.push({
-			id: latest.id,
-			label: latest.label,
-			baseline,
-			percent: (recentValues[0] / baseline - 1) * 100,
-			recent: recent.map((run, index) => ({
-				date: run.date,
-				sha: run.sha,
-				median: recentValues[index],
-			})),
-			// The change landed between the newest baseline run and the oldest slow one.
-			lastGood: before[0] == null ? null : { date: before[0].date, sha: before[0].sha },
+			id: head.id,
+			label: head.label,
+			base: baseMs,
+			head: headMs,
+			percent: (headMs / baseMs - 1) * 100,
+			rounds: pairs,
 		});
 	}
 	return regressions;

package/src/bench/nightly.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+
+/**
+ * nightly.mjs — the nightly benchmark run (#667). Benchmarks tonight's commit (head) and the
+ * commit the previous nightly measured (base) on this runner, alternating head, base, head,
+ * base, so the comparison never mixes CPU models. Then:
+ *
+ * - appends head's first round to the history (projects/ci-tools/dist/ci/bench/), for the charts;
+ * - writes regressions.json next to it: the benchmarks slower on head in every round
+ *   (history.mjs `compareRounds`), which file-regressions.mjs turns into issues.
+ *
+ * Base runs from its own checkout (a git worktree) with its own dependencies and its own
+ * bench.mjs, so each commit is measured with its own code. With no earlier nightly, or no new
+ * commit since, it benchmarks head once and compares nothing.
+ *
+ *   node projects/ci-tools/src/bench/nightly.mjs   (from the repo root; needs the full git history)
+ */
+
+import { execFileSync } from "node:child_process";
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import { dirname, join } from "node:path";
+import {
+	appendRun,
+	compareRounds,
+	getContext,
+	latestRun,
+	normalize,
+	REGRESSION,
+} from "./history.mjs";
+
+const ROOT = process.cwd();
+const DIST = join(ROOT, "projects/ci-tools/dist/ci/bench");
+const HISTORY_FILE = join(DIST, "bench-history.json");
+const REGRESSIONS_FILE = join(DIST, "regressions.json");
+const BENCH = "projects/ci-tools/src/bench/bench.mjs";
+
+function run(command, args, cwd, env = {}) {
+	execFileSync(command, args, {
+		cwd,
+		stdio: ["ignore", "inherit", "inherit"],
+		env: { ...process.env, ...env },
+	});
+}
+
+function git(...args) {
+	return execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim();
+}
+
+/** Runs one round of every suite in `cwd` and returns its results, saved to its own history. */
+function benchRound(cwd, label, round, workDir) {
+	process.stderr.write(`\n══ ${label}, round ${round + 1}\n`);
+	const historyFile = join(workDir, `${label}-${round}.json`);
+	run("node", [BENCH, "save"], cwd, { BENCH_HISTORY_FILE: historyFile });
+	// Base's bench.mjs may predate BENCH_HISTORY_FILE and save to its checkout's default path.
+	const file = existsSync(historyFile)
+		? historyFile
+		: join(cwd, "projects/ci-tools/dist/ci/bench/bench-history.json");
+	const saved = normalize(JSON.parse(readFileSync(file, "utf8")));
+	const newest = latestRun(saved);
+	if (newest == null) throw new Error(`${label} round ${round + 1} saved no run`);
+	return newest.results;
+}
+
+const history = normalize(
+	existsSync(HISTORY_FILE) ? JSON.parse(readFileSync(HISTORY_FILE, "utf8")) : null,
+);
+const head = git("rev-parse", "HEAD");
+const base = latestRun(history)?.sha ?? null;
+const workDir = mkdtempSync(join(os.tmpdir(), "decohere-nightly-"));
+
+try {
+	const headRounds = [];
+	const baseRounds = [];
+	const compareWith = base != null && base !== head ? base : null;
+	if (compareWith == null) {
+		process.stderr.write(
+			base == null
+				? "No earlier nightly run: benchmarking head only.\n"
+				: `No new commit since the last nightly (${head.slice(0, 7)}): benchmarking head only.\n`,
+		);
+		headRounds.push(benchRound(ROOT, "head", 0, workDir));
+	} else {
+		const baseDir = join(workDir, "base");
+		run("git", ["worktree", "add", "--detach", baseDir, compareWith], ROOT);
+		run("pnpm", ["install", "--frozen-lockfile", "--prefer-offline"], baseDir);
+		for (let round = 0; round < REGRESSION.rounds; round++) {
+			headRounds.push(benchRound(ROOT, "head", round, workDir));
+			baseRounds.push(benchRound(baseDir, "base", round, workDir));
+		}
+	}
+
+	const ctx = getContext(os, process.env, process.version);
+	const next = appendRun(history, ctx, { date: ctx.date, sha: head, results: headRounds[0] });
+	mkdirSync(dirname(HISTORY_FILE), { recursive: true });
+	writeFileSync(HISTORY_FILE, `${JSON.stringify(next, null, 2)}\n`);
+
+	const regressions = compareWith == null ? [] : compareRounds(headRounds, baseRounds);
+	writeFileSync(
+		REGRESSIONS_FILE,
+		`${JSON.stringify({ base: compareWith, head, context: ctx, regressions }, null, 2)}\n`,
+	);
+	process.stderr.write(
+		`\nSaved head's run. ${compareWith == null ? "Nothing compared." : `Compared with ${compareWith.slice(0, 7)}: ${regressions.length} benchmark(s) slower in every round.`}\n`,
+	);
+} finally {
+	if (existsSync(join(workDir, "base"))) {
+		run("git", ["worktree", "remove", "--force", join(workDir, "base")], ROOT);
+	}
+	rmSync(workDir, { recursive: true, force: true });
+}

package/src/bench/regression-issue.mjs

@@ -1,5 +1,5 @@
 /**
- * The GitHub issue for a benchmark that got slower (#660): one open issue per benchmark,
+ * The GitHub issue for a benchmark that got slower (#660, #667): one open issue per benchmark,
  * found again by its exact title and updated in place while the regression lasts.
  */
 
@@ -13,38 +13,31 @@ function ms(value) {
 	return `${value.toFixed(1)} ms`;
 }
 
-function commitLink(repoUrl, sha) {
-	return sha == null ? "unknown" : `[\`${sha.slice(0, 7)}\`](${repoUrl}/commit/${sha})`;
+function short(sha) {
+	return sha.slice(0, 7);
 }
 
-export function issueBody(regression, { repoUrl, runUrl, machine, threshold }) {
-	const newest = regression.recent[0];
-	const oldestSlow = regression.recent.at(-1);
-	const range =
-		regression.lastGood?.sha != null && newest.sha != null
-			? `[${regression.lastGood.sha.slice(0, 7)}...${newest.sha.slice(0, 7)}](${repoUrl}/compare/${regression.lastGood.sha}...${newest.sha})`
-			: "unknown (no baseline commit recorded)";
+export function issueBody(regression, { repoUrl, runUrl, machine, threshold, base, head }) {
+	const range = `[${short(base)}...${short(head)}](${repoUrl}/compare/${base}...${head})`;
 	return [
 		"## Problem",
 		"",
-		`The nightly benchmark **${regression.label}** (\`${regression.id}\`) has a median ${regression.percent.toFixed(0)}% above its baseline, on each of the last ${regression.recent.length} nightly runs. A benchmark counts as slower above ${Math.round(threshold * 100)}%.`,
+		`The nightly benchmark **${regression.label}** (\`${regression.id}\`) is ${regression.percent.toFixed(0)}% slower on [\`${short(head)}\`](${repoUrl}/commit/${head}) than on [\`${short(base)}\`](${repoUrl}/commit/${base}), the commit the previous nightly measured. Both ran on the same runner, alternately, and head was more than ${Math.round(threshold * 100)}% slower in every round.`,
 		"",
-		`- Baseline (median of the runs before): **${ms(regression.baseline)}**`,
-		...regression.recent.map(
-			(run) =>
-				`- ${run.date.slice(0, 10)}, ${commitLink(repoUrl, run.sha)}: **${ms(run.median)}**`,
+		"| Round | Base | Head |",
+		"|---|---|---|",
+		...regression.rounds.map(
+			(round, index) => `| ${index + 1} | ${ms(round.base)} | ${ms(round.head)} |`,
 		),
-		`- Machine: ${machine}`,
 		"",
-		`The change landed after the last normal run (${regression.lastGood?.date.slice(0, 10) ?? "unknown"}) and by ${oldestSlow.date.slice(0, 10)}: commits ${range}.`,
+		`- Machine: ${machine}`,
+		`- Commits: ${range}`,
 		"",
 		"## What to do",
 		"",
-		"1. Check the trend on the [benchmark page](" +
-			PAGES_URL +
-			"). A step that holds is a real change; a single machine type jumping may be the runner.",
-		"2. Reproduce locally: check out the last normal commit, `pnpm bench:save`, then the newest commit, `pnpm bench`, and read the comparison.",
-		"3. Fix the cause, or if the slowdown is intended, say why here and close the issue.",
+		`1. Look through the commits in ${range} for the change.`,
+		`2. Reproduce locally: check out \`${short(base)}\`, run \`pnpm bench:save\`, then check out \`${short(head)}\` and run \`pnpm bench\` to compare.`,
+		`3. Fix the cause, or if the slowdown is intended, say why here and close the issue. Trend: [benchmark page](${PAGES_URL}).`,
 		"",
 		`Updated by the nightly bench workflow: ${runUrl}`,
 	].join("\n");