@postedin/cms-client 0.1.0

package/bin/profile/interference.mjs

@@ -0,0 +1,604 @@
+#!/usr/bin/env node
+/**
+ * Puts a build and an editor on the same clock.
+ *
+ *   pnpm profile:interference
+ *   pnpm profile:interference --tag "m10 cluster"
+ *   pnpm profile:interference --idle 30 --collection posts
+ *
+ * "The admin is slower while a build runs" is the reported symptom and the one
+ * nothing measures. This polls a fixed set of REST requests every two seconds
+ * through three phases — idle, a profiled build, idle again — and reports what
+ * each one cost in each phase, joined to how many build requests were in
+ * flight at that moment.
+ *
+ * REST latency stands in for admin latency: the admin UI is served by the same
+ * functions against the same database. Timing the admin UI itself — list view,
+ * open document, autosave, publish — belongs on the CMS side.
+ *
+ * The default target is whatever `API_BASE_URL` resolves to, which is the demo
+ * CMS in this repository's `.env`. Pointing it at a client CMS means editing
+ * that variable, which is a deliberate act rather than a flag default.
+ */
+
+import { spawn } from 'node:child_process';
+import {
+	appendFileSync,
+	existsSync,
+	mkdirSync,
+	readFileSync,
+	writeFileSync,
+} from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadSiteEnv } from './site-env.mjs';
+import {
+	parsePollLog,
+	renderInterferenceMarkdown,
+	summarizeInterference,
+} from './interference-summary.mjs';
+import { LOG_DIR } from './report.mjs';
+import { parseLog } from './summarize.mjs';
+
+const here = dirname(fileURLToPath(import.meta.url));
+
+/** A page of the list view, as the admin pages it. */
+const LIST_LIMIT = 10;
+
+/** What the site fetches a collection at, and what the edit view resolves. */
+const DOCUMENT_DEPTH = 2;
+
+/**
+ * The CMS's own id for the request it just served, echoed on the response by a
+ * deployment running with `CMS_PROFILE=1` (`PROFILE_ID_HEADER` in the CMS's
+ * `src/profiling/requestProfile.ts`). Recording it is what lets a slow poll be
+ * looked up as a `cms.profile.request` line and opened into the commands and
+ * hooks behind it, rather than staying a number with no explanation. Null when
+ * the target is not profiling, which is the normal case.
+ */
+const PROFILE_ID_HEADER = 'x-cms-profile-id';
+
+const DEFAULTS = {
+	/** Every 2s. Short enough to see a burst, long enough not to be load. */
+	intervalMs: 2000,
+	/** Each idle phase. One minute of quiet on either side of the build. */
+	idleMs: 60_000,
+	/** The collection whose list and whose newest document stand in for the admin's. */
+	collection: 'posts',
+};
+
+const options = parseArgs(process.argv.slice(2));
+
+/**
+ * The phase the run is in: written by `phase`, read by the poll loop.
+ *
+ * A poll is labelled with the phase it started in rather than the one it
+ * finished in. A poll that goes out during the build and lands after it is a
+ * measurement of the build.
+ */
+let currentPhase = 'before';
+
+// Resolved in the mode the build will resolve it in — `astro build` runs Vite
+// in `production` — so the harness and the build it spawns are pointed at one
+// CMS even on a site that keeps an `.env.production`. `build.mjs` reads it the
+// same way for the same reason.
+const env = await loadSiteEnv(process.env.NODE_ENV ?? 'production');
+const { API_BASE_URL, CMS_API_KEY, PROJECT_SLUG } = env;
+
+if (!API_BASE_URL || !CMS_API_KEY || !PROJECT_SLUG) {
+	fail(
+		"API_BASE_URL, CMS_API_KEY and PROJECT_SLUG all have to be set. A local .env.local wins over .env in Vite's precedence, so check which one is in play.",
+	);
+}
+
+// The scheme names the collection the key belongs to, and the harness holds
+// the same credential the site builds with: an `api-keys` document, scoped to
+// one project and able to write nothing. `@postedin/cms-client`'s `api.ts` is where
+// that decision lives; sending `users` here would either fail to authenticate
+// or, with somebody's personal key, measure a principal no build ever uses.
+const CMS_API_KEY_COLLECTION = 'api-keys';
+
+const headers = {
+	Authorization: `${CMS_API_KEY_COLLECTION} API-Key ${CMS_API_KEY}`,
+	...(env.CMS_VERCEL_AUTOMATION_BYPASS_SECRET
+		? { 'x-vercel-protection-bypass': env.CMS_VERCEL_AUTOMATION_BYPASS_SECRET }
+		: {}),
+};
+
+const stamp = new Date().toISOString().slice(0, 19).replaceAll(':', '-');
+const pollLogPath = resolve(LOG_DIR, `interference-${stamp}.ndjson`);
+const buildLogPath = resolve(LOG_DIR, `interference-${stamp}-build.ndjson`);
+const reportPath = pollLogPath.replace(/\.ndjson$/, '.md');
+
+mkdirSync(dirname(pollLogPath), { recursive: true });
+
+await main();
+
+async function main() {
+	const probes = await buildProbes();
+
+	console.log(`Polling ${API_BASE_URL} every ${options.intervalMs / 1000}s.`);
+	console.log(`Log: ${pollLogPath}\n`);
+
+	writeLine({
+		kind: 'meta',
+		event: 'start',
+		startedAt: Date.now(),
+		cmsOrigin: new URL(API_BASE_URL).origin,
+		project: PROJECT_SLUG,
+		collection: options.collection,
+		documentId: probes.documentId,
+		tag: options.tag,
+		intervalMs: options.intervalMs,
+		idleMs: options.idleMs,
+		probes: probes.list.map(({ name, request, description }) => ({
+			name,
+			request,
+			description,
+		})),
+	});
+
+	const poller = startPolling(probes.list);
+
+	await phase('before', () => idle(options.idleMs));
+
+	const build = await phase('during', () => runBuild());
+
+	await phase('after', () => idle(options.idleMs));
+
+	poller.stop();
+	await poller.drained();
+
+	writeLine({
+		kind: 'meta',
+		event: 'end',
+		endedAt: Date.now(),
+		polls: poller.polls(),
+	});
+
+	report();
+
+	if (!build.ok) {
+		// A build that failed still produced latency worth reading, so the
+		// report is written either way — but the run did not measure what it
+		// set out to, and the exit code has to say so.
+		console.error(
+			`\nThe build failed: ${build.signal ? `signal ${build.signal}` : `exit ${build.exitCode}`}. The report covers what was polled, not a successful build.`,
+		);
+		process.exit(1);
+	}
+}
+
+/** Announces a phase, runs it, and labels every poll that starts inside it. */
+async function phase(name, run) {
+	currentPhase = name;
+	writeLine({ kind: 'meta', event: 'phase', phase: name, at: Date.now() });
+	console.log(`[${name}]`);
+
+	return await run();
+}
+
+/**
+ * The three requests the admin issues most, resolved against real data and
+ * checked before the run starts.
+ *
+ * The document probe needs an id, and the honest one is whatever an editor
+ * touched last — the same document the admin's list view puts at the top. It
+ * costs one request before the run, outside every phase, and so does the check
+ * each probe then gets.
+ *
+ * Every probe has to be a request this credential genuinely performs. An
+ * `api-keys` document is not a person, so `/api/users/me` answers it `200
+ * {"user":null}` — a shape that looks like a measurement and is really an
+ * unauthenticated stub, contributing a few hundred milliseconds of "admin
+ * latency" that no admin ever waits. The floor probe below is what replaced
+ * it: the same question — what does a request cost before it does any work —
+ * asked in a way that 403s when the credential is wrong.
+ */
+async function buildProbes() {
+	const listRequest = listUrl();
+	const newest = await probeJson(listRequest, 'the list-view probe');
+	const documentId = newest.docs?.[0]?.id;
+
+	if (!documentId) {
+		fail(
+			`No documents in \`${options.collection}\` for project \`${PROJECT_SLUG}\`, so there is no edit view to time. Pass --collection.`,
+		);
+	}
+
+	const documentRequest = `${API_BASE_URL}/api/${options.collection}/${documentId}?depth=${DOCUMENT_DEPTH}`;
+	const floorRequest = floorUrl();
+
+	const probes = [
+		{
+			name: 'list',
+			request: pathAndQuery(listRequest),
+			url: listRequest,
+			description: `The list view: ${LIST_LIMIT} documents, newest first, filtered to the project`,
+			// Already in hand: this is the request the document id came from.
+			answered: newest,
+			check: (body) =>
+				body.docs?.length ? null : 'it carried no `docs` to have cost anything',
+		},
+		{
+			name: 'document',
+			request: pathAndQuery(documentRequest),
+			url: documentRequest,
+			description: `The edit view: one document by id at depth ${DOCUMENT_DEPTH}, relationships resolved`,
+			check: (body) =>
+				body.id === documentId
+					? null
+					: `it carried \`${body.id ?? 'no id'}\` rather than the document that was asked for`,
+		},
+		{
+			name: 'floor',
+			request: pathAndQuery(floorRequest),
+			url: floorRequest,
+			description:
+				'The fixed cost of an authenticated, project-scoped read: one document, nothing resolved',
+			check: (body) =>
+				body.docs?.length ? null : 'it carried no `docs` to have cost anything',
+		},
+	];
+
+	for (const probe of probes) {
+		await verifyProbe(probe);
+	}
+
+	return { documentId, list: probes };
+}
+
+/**
+ * Refuses to start a run whose probe answers without measuring anything.
+ *
+ * A 200 is not enough: an endpoint can answer an unauthenticated caller with an
+ * empty shape, at full latency, and every table downstream will report those
+ * milliseconds as what an editor waits. The check is per probe because what
+ * counts as an answer differs — a page of documents, one document by id — and
+ * it runs once, before the first phase, so a misconfigured run costs a request
+ * rather than three minutes. A probe whose answer is already in hand — the
+ * list, which is where the document id came from — is not asked twice.
+ */
+async function verifyProbe(probe) {
+	const body =
+		probe.answered ??
+		(await probeJson(probe.url, `the \`${probe.name}\` probe`));
+	const complaint = probe.check(body);
+
+	if (complaint) {
+		fail(
+			`The \`${probe.name}\` probe (${probe.request}) answered 200 but ${complaint}. Timing that would measure an empty round trip, so the run stops here.`,
+		);
+	}
+}
+
+/**
+ * The list view's query, in the shape the admin sends it: a page of ten,
+ * newest edit first, scoped to one project. `qs-esm` is not imported here
+ * because this script has to run before anything is built.
+ *
+ * `depth=0` because a list view resolves no relationships. The edit view does,
+ * and it is a different endpoint — `/api/{collection}/{id}`, which takes no
+ * `where` — so it needs no builder at all.
+ */
+function listUrl() {
+	const params = new URLSearchParams({
+		limit: String(LIST_LIMIT),
+		depth: '0',
+		sort: '-updatedAt',
+		'where[and][0][project.slug][equals]': PROJECT_SLUG,
+	});
+
+	return `${API_BASE_URL}/api/${options.collection}?${params}`;
+}
+
+/**
+ * The same query asked for a single document, unsorted and unresolved.
+ *
+ * What is left when a read returns almost nothing is what every read pays
+ * whatever it asks for: the round trip, authenticating the key, resolving the
+ * project it is scoped to. Against the list probe's ten documents it is also
+ * the one comparison this harness can make on its own — how much of a list view
+ * is the page of documents, and how much is the request itself.
+ */
+function floorUrl() {
+	const params = new URLSearchParams({
+		limit: '1',
+		depth: '0',
+		'where[and][0][project.slug][equals]': PROJECT_SLUG,
+	});
+
+	return `${API_BASE_URL}/api/${options.collection}?${params}`;
+}
+
+/** Everything after the origin, which is what the report prints. */
+function pathAndQuery(url) {
+	const parsed = new URL(url);
+
+	return `${parsed.pathname}${parsed.search}`;
+}
+
+/**
+ * Polls every probe on a fixed cadence for the life of the run.
+ *
+ * A tick that is still in flight when the next one is due is not doubled up on:
+ * the harness is meant to measure the CMS under a build's load, not to add its
+ * own. The skip is logged, because it only happens when latency has grown past
+ * the interval and that is a fact about the run.
+ */
+function startPolling(probes) {
+	let stopped = false;
+	let ticks = 0;
+	let polls = 0;
+	let busy = false;
+	const settled = [];
+
+	const timer = setInterval(() => {
+		if (stopped) {
+			return;
+		}
+
+		if (busy) {
+			writeLine({
+				kind: 'meta',
+				event: 'skipped',
+				phase: currentPhase,
+				at: Date.now(),
+			});
+
+			return;
+		}
+
+		const tick = ++ticks;
+		const phase = currentPhase;
+
+		busy = true;
+
+		settled.push(
+			Promise.all(
+				probes.map((probe) =>
+					pollOnce(probe, tick, phase).then(() => {
+						polls += 1;
+					}),
+				),
+			).finally(() => {
+				busy = false;
+			}),
+		);
+	}, options.intervalMs);
+
+	// The harness's own clock must not be what holds the process open.
+	timer.unref?.();
+
+	return {
+		stop() {
+			stopped = true;
+			clearInterval(timer);
+		},
+		drained: () => Promise.allSettled(settled),
+		polls: () => polls,
+	};
+}
+
+/** One probe, one line: latency to the last byte, and what came back. */
+async function pollOnce(probe, tick, phase) {
+	const startedAt = Date.now();
+	const mark = performance.now();
+
+	try {
+		const response = await fetch(probe.url, { headers });
+		const ttfbMs = round(performance.now() - mark);
+		const body = await response.arrayBuffer();
+
+		writeLine({
+			kind: 'poll',
+			tick,
+			phase,
+			probe: probe.name,
+			startedAt,
+			endedAt: Date.now(),
+			ttfbMs,
+			ms: round(performance.now() - mark),
+			status: response.status,
+			ok: response.ok,
+			bytes: body.byteLength,
+			profileId: response.headers.get(PROFILE_ID_HEADER),
+		});
+	} catch (error) {
+		writeLine({
+			kind: 'poll',
+			tick,
+			phase,
+			probe: probe.name,
+			startedAt,
+			endedAt: Date.now(),
+			ttfbMs: null,
+			ms: round(performance.now() - mark),
+			status: null,
+			ok: false,
+			bytes: null,
+			profileId: null,
+			error: String(error?.message ?? error),
+		});
+	}
+}
+
+/**
+ * Runs the build `pnpm build:profile` runs, with its fetch log named here so
+ * the report can join the two by timestamp.
+ *
+ * `pnpm` is not spawned; `build:profile` is `cms-profile-build`, which is
+ * `build.mjs` beside this file,
+ * and calling it directly keeps the harness independent of PATH.
+ */
+function runBuild() {
+	const script = join(here, 'build.mjs');
+	const command = `node ${relativeToCwd(script)}`;
+	const startedAt = Date.now();
+
+	console.log(`Starting the build: ${command}\n`);
+
+	return new Promise((settle) => {
+		const build = spawn(process.execPath, [script, ...options.buildArgs], {
+			stdio: 'inherit',
+			env: {
+				...process.env,
+				PROFILE_FETCH_LOG: buildLogPath,
+			},
+		});
+
+		build.on('error', (error) => {
+			writeLine({
+				kind: 'meta',
+				event: 'build',
+				command,
+				logPath: buildLogPath,
+				startedAt,
+				endedAt: Date.now(),
+				exitCode: null,
+				signal: null,
+				error: String(error?.message ?? error),
+			});
+
+			settle({ ok: false, exitCode: null, signal: null });
+		});
+
+		build.on('exit', (exitCode, signal) => {
+			writeLine({
+				kind: 'meta',
+				event: 'build',
+				command,
+				logPath: buildLogPath,
+				startedAt,
+				endedAt: Date.now(),
+				exitCode,
+				signal,
+			});
+
+			settle({ ok: exitCode === 0 && !signal, exitCode, signal });
+		});
+	});
+}
+
+function idle(ms) {
+	return new Promise((settle) => {
+		setTimeout(settle, ms);
+	});
+}
+
+/** Renders the poll log against the build's fetch log. */
+function report() {
+	const { polls, meta } = parsePollLog(readFileSync(pollLogPath, 'utf-8'));
+	const buildRequests = existsSync(buildLogPath)
+		? parseLog(readFileSync(buildLogPath, 'utf-8')).requests
+		: [];
+
+	const markdown = renderInterferenceMarkdown(
+		summarizeInterference({ polls, meta, buildRequests }),
+		{ logPath: pollLogPath, generatedAt: new Date().toISOString() },
+	);
+
+	writeFileSync(reportPath, markdown);
+
+	console.log(`\nPoll log:   ${pollLogPath}`);
+	console.log(`Build log:  ${buildLogPath}`);
+	console.log(`Report:     ${reportPath}`);
+}
+
+function writeLine(record) {
+	try {
+		appendFileSync(pollLogPath, `${JSON.stringify(record)}\n`);
+	} catch (error) {
+		process.emitWarning(
+			`interference: could not append to ${pollLogPath}: ${error.message}`,
+		);
+	}
+}
+
+async function probeJson(url, what) {
+	let response;
+
+	try {
+		response = await fetch(url, { headers });
+	} catch (error) {
+		fail(`Could not reach the CMS for ${what}: ${error?.message ?? error}`);
+	}
+
+	if (!response.ok) {
+		fail(
+			`The CMS answered ${response.status} ${response.statusText} for ${what} (${pathAndQuery(url)}).`,
+		);
+	}
+
+	return await response.json();
+}
+
+function parseArgs(argv) {
+	const parsed = {
+		...DEFAULTS,
+		tag: null,
+		buildArgs: [],
+	};
+
+	for (let index = 0; index < argv.length; index += 1) {
+		const argument = argv[index];
+
+		if (argument === '--tag') {
+			parsed.tag = argv[++index] ?? null;
+		} else if (argument === '--collection') {
+			parsed.collection = argv[++index] ?? parsed.collection;
+		} else if (argument === '--idle') {
+			parsed.idleMs = seconds('--idle', argv[++index]);
+		} else if (argument === '--interval') {
+			parsed.intervalMs = seconds('--interval', argv[++index]);
+		} else if (argument === '--') {
+			parsed.buildArgs.push(...argv.slice(index + 1));
+			break;
+		} else if (argument === '--help' || argument === '-h') {
+			console.log(usage());
+			process.exit(0);
+		} else {
+			// A run costs three minutes, so a typo says so now rather than
+			// after the build has failed on an argument astro never wanted.
+			fail(
+				`Unknown argument \`${argument}\`. Build arguments go after \`--\`.\n\n${usage()}`,
+			);
+		}
+	}
+
+	return parsed;
+}
+
+function seconds(flag, value) {
+	const parsed = Number(value);
+
+	if (!Number.isFinite(parsed) || parsed <= 0) {
+		fail(`${flag} wants a number of seconds, not \`${value}\`.`);
+	}
+
+	return parsed * 1000;
+}
+
+function usage() {
+	return `Usage: cms-profile-interference [options] [-- astro build args]
+
+  --tag <label>        Recorded in the report, for runs that vary one thing
+  --collection <name>  The collection to poll (default ${DEFAULTS.collection})
+  --idle <seconds>     Each idle phase (default ${DEFAULTS.idleMs / 1000})
+  --interval <seconds> Poll interval (default ${DEFAULTS.intervalMs / 1000})`;
+}
+
+function relativeToCwd(path) {
+	const cwd = `${process.cwd()}/`;
+
+	return path.startsWith(cwd) ? path.slice(cwd.length) : path;
+}
+
+function round(value) {
+	return Math.round(value * 10) / 10;
+}
+
+function fail(message) {
+	console.error(message);
+	process.exit(1);
+}