@postedin/cms-client 0.1.0

package/bin/profile/interference-summary.mjs

@@ -0,0 +1,558 @@
+/**
+ * Turns an interference run into a report.
+ *
+ * Pure functions over the NDJSON written by `interference.mjs`, plus the build
+ * fetch log `fetch-log.mjs` wrote during the same run: no file system, no
+ * clock, no argv, so the whole thing can be checked against a fixture. The CLI
+ * that reads and writes files is `interference.mjs` itself.
+ *
+ * The question is one question, asked three times: what does a CMS request
+ * cost an editor before a build starts, while it runs, and after it finishes.
+ * Two things make the answer readable:
+ *
+ * 1. Percentiles per probe per phase, so "slower" is a number.
+ * 2. Every poll joined to how many of the build's CMS requests were in flight
+ *    at that moment, so latency can be read against what the build was doing
+ *    rather than against the wall clock. A build is not uniformly busy, and
+ *    the during-phase p50 hides that.
+ */
+
+import {
+	bar,
+	bullets,
+	count,
+	duration,
+	escapePipes,
+	offsetClock,
+	table,
+} from './format.mjs';
+import { group, parseNdjson, peakInFlight, percentile } from './measure.mjs';
+
+/** Phase order is the story the report tells; it is not alphabetical. */
+export const PHASES = ['before', 'during', 'after'];
+
+/**
+ * How many of the worst polls to name.
+ *
+ * The medians are the finding; the tail is the thing anyone reading the report
+ * actually wants to chase, and chasing it means one row per poll — when it
+ * went out, what the build was doing, and the CMS's own id for the request, if
+ * the deployment was profiling and gave one.
+ */
+const SLOWEST_POLLS = 10;
+
+const PHASE_LABELS = {
+	before: 'Before',
+	during: 'During the build',
+	after: 'After',
+};
+
+export function parsePollLog(text) {
+	const polls = [];
+	const meta = [];
+
+	for (const record of parseNdjson(text)) {
+		if (record.kind === 'poll') {
+			polls.push(record);
+		} else if (record.kind === 'meta') {
+			meta.push(record);
+		}
+	}
+
+	return { polls, meta };
+}
+
+export function summarizeInterference({ polls, meta, buildRequests = [] }) {
+	const start = meta.find((record) => record.event === 'start') ?? {};
+	const end = meta.find((record) => record.event === 'end') ?? {};
+	const build = meta.find((record) => record.event === 'build') ?? null;
+	const window = timeWindow(polls, start, end);
+	const probeNames = probeOrder(polls, start);
+	// A build fetches fonts and posts telemetry as well, and neither of those
+	// is something an editor is queueing behind. The overlay counts what
+	// reached the CMS, which is the same number the build report calls "peak
+	// CMS requests in flight".
+	const cmsRequests = buildRequests.filter((request) => request.cms);
+	const ticks = tickRows(polls, cmsRequests, window);
+
+	return {
+		tag: start.tag ?? null,
+		cmsOrigin: start.cmsOrigin ?? null,
+		project: start.project ?? null,
+		collection: start.collection ?? null,
+		documentId: start.documentId ?? null,
+		intervalMs: start.intervalMs ?? null,
+		idleMs: start.idleMs ?? null,
+		probes: start.probes ?? probeNames.map((name) => ({ name })),
+		window,
+		build: build && {
+			command: build.command ?? null,
+			logPath: build.logPath ?? null,
+			startedAt: build.startedAt ?? null,
+			endedAt: build.endedAt ?? null,
+			wallMs:
+				build.startedAt && build.endedAt
+					? build.endedAt - build.startedAt
+					: null,
+			exitCode: build.exitCode ?? null,
+			signal: build.signal ?? null,
+			// A signalled build has no exit code, and a build that never ran
+			// has neither. Only 0 is a build that finished.
+			ok: build.exitCode === 0 && !build.signal,
+			requests: buildRequests.length,
+			cmsRequests: cmsRequests.length,
+		},
+		phases: phaseRows(polls, probeNames),
+		comparison: comparisonRows(polls, probeNames),
+		ticks,
+		byInFlight: inFlightRows(ticks, probeNames),
+		slowest: slowestRows(ticks, probeNames),
+		errors: polls
+			.filter((poll) => !poll.ok)
+			.map((poll) => ({
+				atMs: poll.startedAt - window.startedAt,
+				phase: poll.phase,
+				probe: poll.probe,
+				status: poll.status ?? null,
+				error: poll.error ?? null,
+			})),
+		skipped: meta.filter((record) => record.event === 'skipped').length,
+		polls: polls.length,
+	};
+}
+
+/**
+ * The probes, in the order the run declared them, so every table reads the
+ * same way left to right. A log without a start record falls back to the order
+ * they were first polled in.
+ */
+function probeOrder(polls, start) {
+	if (start.probes?.length) {
+		return start.probes.map((probe) => probe.name);
+	}
+
+	const names = [];
+
+	for (const poll of polls) {
+		if (!names.includes(poll.probe)) {
+			names.push(poll.probe);
+		}
+	}
+
+	return names;
+}
+
+function timeWindow(polls, start, end) {
+	const starts = [start.startedAt, ...polls.map((poll) => poll.startedAt)];
+	const ends = [
+		end.endedAt,
+		...polls.map((poll) => poll.endedAt ?? poll.startedAt),
+	];
+
+	const startedAt = pick(starts, Math.min);
+	const endedAt = pick(ends, Math.max);
+
+	return { startedAt, endedAt: Math.max(endedAt, startedAt) };
+}
+
+function pick(values, choose) {
+	const numbers = values.filter((value) => typeof value === 'number');
+
+	return numbers.length ? choose(...numbers) : 0;
+}
+
+/** One row per phase, carrying one row per probe. */
+function phaseRows(polls, probeNames) {
+	return PHASES.filter((phase) =>
+		polls.some((poll) => poll.phase === phase),
+	).map((phase) => {
+		const members = polls.filter((poll) => poll.phase === phase);
+		const startedAt = pick(
+			members.map((poll) => poll.startedAt),
+			Math.min,
+		);
+		const endedAt = pick(
+			members.map((poll) => poll.endedAt ?? poll.startedAt),
+			Math.max,
+		);
+
+		return {
+			phase,
+			startedAt,
+			endedAt,
+			wallMs: endedAt - startedAt,
+			probes: probeNames.map((name) =>
+				statsFor(
+					name,
+					members.filter((poll) => poll.probe === name),
+				),
+			),
+		};
+	});
+}
+
+/**
+ * Latency for one probe over a set of polls.
+ *
+ * Percentiles cover the polls the CMS answered. A refused or failed poll has
+ * no latency worth ranking — it has a status — so it is counted beside them
+ * rather than mixed in.
+ */
+function statsFor(probe, polls) {
+	const answered = polls.filter((poll) => poll.ok);
+	const values = answered.map((poll) => poll.ms);
+
+	return {
+		probe,
+		polls: polls.length,
+		samples: answered.length,
+		errors: polls.length - answered.length,
+		p50: percentile(values, 0.5),
+		p95: percentile(values, 0.95),
+		min: values.length ? Math.min(...values) : null,
+		max: values.length ? Math.max(...values) : null,
+	};
+}
+
+/**
+ * The headline: each probe's p50 in each phase side by side, and what the
+ * build multiplied it by.
+ *
+ * `after` is in the table because it is the control. A during/before ratio on
+ * its own cannot tell a build's cost apart from a CMS that was simply cold at
+ * the start of the run, and the baseline showed how large that effect is.
+ */
+function comparisonRows(polls, probeNames) {
+	return probeNames.map((name) => {
+		const byPhase = {};
+
+		for (const phase of PHASES) {
+			byPhase[phase] = statsFor(
+				name,
+				polls.filter((poll) => poll.probe === name && poll.phase === phase),
+			);
+		}
+
+		return {
+			probe: name,
+			...byPhase,
+			ratio:
+				byPhase.before.p50 && byPhase.during.p50
+					? byPhase.during.p50 / byPhase.before.p50
+					: null,
+		};
+	});
+}
+
+/**
+ * One row per poll tick: when it went out, in which phase, what each probe
+ * cost, and how many of the build's CMS requests were in flight while it was
+ * open.
+ *
+ * The overlay is the peak concurrency over the tick's own span rather than the
+ * level at the instant it started. A poll that takes two seconds was not
+ * competing only with whatever happened to be open when it left.
+ */
+function tickRows(polls, cmsRequests, window) {
+	const ticks = group(polls, (poll) => poll.tick ?? poll.startedAt);
+
+	return [...ticks.values()]
+		.map(({ members }) => {
+			const startedAt = pick(
+				members.map((poll) => poll.startedAt),
+				Math.min,
+			);
+			const endedAt = pick(
+				members.map((poll) => poll.endedAt ?? poll.startedAt),
+				Math.max,
+			);
+			const samples = {};
+
+			for (const poll of members) {
+				samples[poll.probe] = poll;
+			}
+
+			return {
+				tick: members[0].tick ?? null,
+				phase: members[0].phase,
+				atMs: startedAt - window.startedAt,
+				startedAt,
+				endedAt,
+				inFlight: peakInFlight(cmsRequests, startedAt, endedAt),
+				samples,
+			};
+		})
+		.sort((a, b) => a.startedAt - b.startedAt);
+}
+
+/**
+ * The same latencies grouped by how busy the build was, which is the reading
+ * the wall clock cannot give: a build's during-phase p50 averages its quiet
+ * stretches together with its bursts.
+ */
+function inFlightRows(ticks, probeNames) {
+	const rows = group(ticks, (tick) => tick.inFlight);
+
+	return [...rows.values()]
+		.map(({ key, members }) => ({
+			inFlight: key,
+			ticks: members.length,
+			probes: probeNames.map((name) =>
+				statsFor(
+					name,
+					members.map((tick) => tick.samples[name]).filter(Boolean),
+				),
+			),
+		}))
+		.sort((a, b) => a.inFlight - b.inFlight);
+}
+
+/**
+ * The worst polls the CMS answered, each beside what the build was doing and
+ * the CMS's own id for it.
+ *
+ * Refused polls are left out: they are in the failed-poll table, and a 429's
+ * latency is not a measurement of a slow read. The ordering is by latency
+ * rather than by phase, because a stall in an idle phase is the most useful
+ * row in the table — it is the one that says the build is not the cause.
+ */
+function slowestRows(ticks, probeNames) {
+	const rows = [];
+
+	for (const tick of ticks) {
+		for (const name of probeNames) {
+			const poll = tick.samples[name];
+
+			if (!poll?.ok || typeof poll.ms !== 'number') {
+				continue;
+			}
+
+			rows.push({
+				atMs: tick.atMs,
+				phase: tick.phase,
+				probe: name,
+				ms: poll.ms,
+				inFlight: tick.inFlight,
+				profileId: poll.profileId ?? null,
+			});
+		}
+	}
+
+	return rows.sort((a, b) => b.ms - a.ms).slice(0, SLOWEST_POLLS);
+}
+
+export function renderInterferenceMarkdown(
+	summary,
+	{ logPath, generatedAt } = {},
+) {
+	const probeNames = summary.probes.map((probe) => probe.name);
+	const peak = Math.max(0, ...summary.ticks.map((tick) => tick.inFlight));
+
+	return `${[
+		'# Interference profile',
+		'',
+		...bullets([
+			['Log', logPath ? `\`${logPath}\`` : null],
+			['Generated', generatedAt ?? null],
+			['Tag', summary.tag ? `**${summary.tag}**` : null],
+			['CMS origin', summary.cmsOrigin ? `\`${summary.cmsOrigin}\`` : null],
+			['Project', summary.project ? `\`${summary.project}\`` : null],
+			[
+				'Poll interval',
+				summary.intervalMs ? `${summary.intervalMs / 1000}s` : null,
+			],
+			['Idle phases', summary.idleMs ? `${summary.idleMs / 1000}s each` : null],
+			[
+				'Polls',
+				`${count(summary.polls)} across ${count(summary.ticks.length)} tick${summary.ticks.length === 1 ? '' : 's'}`,
+			],
+			[
+				'Ticks skipped',
+				summary.skipped
+					? `${count(summary.skipped)} — the previous tick was still in flight`
+					: null,
+			],
+			...buildBullets(summary.build),
+		]),
+		'',
+		'## What was polled',
+		'',
+		...table(
+			['Probe', 'Request', 'Stands in for'],
+			summary.probes.map((probe) => [
+				`\`${probe.name}\``,
+				probe.request ? `\`${escapePipes(probe.request)}\`` : '-',
+				probe.description ?? '-',
+			]),
+		),
+		'',
+		'## Before, during, after',
+		'',
+		...table(
+			[
+				'Probe',
+				'Before p50',
+				'During p50',
+				'After p50',
+				'During / before',
+				'During p95',
+				'During max',
+			],
+			summary.comparison.map((row) => [
+				`\`${row.probe}\``,
+				duration(row.before.p50),
+				duration(row.during.p50),
+				duration(row.after.p50),
+				row.ratio ? `${row.ratio.toFixed(2)}×` : '-',
+				duration(row.during.p95),
+				duration(row.during.max),
+			]),
+		),
+		'',
+		'## Every phase in full',
+		'',
+		...table(
+			['Phase', 'Span', 'Probe', 'Polls', 'Errors', 'p50', 'p95', 'Max'],
+			summary.phases.flatMap((phase) =>
+				phase.probes.map((row, index) => [
+					index === 0 ? (PHASE_LABELS[phase.phase] ?? phase.phase) : '',
+					index === 0 ? duration(phase.wallMs) : '',
+					`\`${row.probe}\``,
+					count(row.polls),
+					count(row.errors),
+					duration(row.p50),
+					duration(row.p95),
+					duration(row.max),
+				]),
+			),
+		),
+		'',
+		'## Latency against build concurrency',
+		'',
+		...(summary.build
+			? [
+					`Polls grouped by how many of the build's CMS requests were open while they were, peaking at **${count(peak)}**.`,
+					'',
+					...table(
+						[
+							'CMS requests in flight',
+							'Ticks',
+							...probeNames.map((name) => `\`${name}\` p50`),
+							'Slowest poll',
+						],
+						summary.byInFlight.map((row) => [
+							count(row.inFlight),
+							count(row.ticks),
+							...row.probes.map((probe) => duration(probe.p50)),
+							duration(
+								Math.max(0, ...row.probes.map((probe) => probe.max ?? 0)),
+							),
+						]),
+					),
+				]
+			: ['No build ran in this log, so there is nothing to overlay.']),
+		'',
+		'## Slowest polls',
+		'',
+		...(summary.slowest.length
+			? [
+					...table(
+						[
+							'At',
+							'Phase',
+							'Probe',
+							'Latency',
+							'CMS requests in flight',
+							'CMS profile id',
+						],
+						summary.slowest.map((row) => [
+							offsetClock(row.atMs),
+							row.phase,
+							`\`${row.probe}\``,
+							duration(row.ms),
+							count(row.inFlight),
+							row.profileId ? `\`${row.profileId}\`` : '-',
+						]),
+					),
+					'',
+					...(summary.slowest.some((row) => row.profileId)
+						? [
+								'Each id is one `cms.profile.request` line on the CMS, so a row here can be opened into the commands and hooks that produced it.',
+							]
+						: [
+								'No response carried `x-cms-profile-id`, so these are latencies with nothing behind them. Run the target with `CMS_PROFILE=1` to join them to the CMS\u2019s own lines.',
+							]),
+				]
+			: ['No poll was answered, so there is nothing to rank.']),
+		'',
+		'## Poll by poll',
+		'',
+		...table(
+			[
+				'At',
+				'Phase',
+				'In flight',
+				'',
+				...probeNames.map((name) => `\`${name}\``),
+			],
+			summary.ticks.map((tick) => [
+				offsetClock(tick.atMs),
+				tick.phase,
+				count(tick.inFlight),
+				bar(tick.inFlight, peak, 8),
+				...probeNames.map((name) => cell(tick.samples[name])),
+			]),
+		),
+		'',
+		'## Failed polls',
+		'',
+		...(summary.errors.length
+			? table(
+					['At', 'Phase', 'Probe', 'Status', 'Error'],
+					summary.errors.map((row) => [
+						offsetClock(row.atMs),
+						row.phase,
+						`\`${row.probe}\``,
+						row.status === null ? '-' : count(row.status),
+						row.error ? escapePipes(row.error) : '-',
+					]),
+				)
+			: ['Every poll was answered.']),
+		'',
+	].join('\n')}\n`;
+}
+
+function buildBullets(build) {
+	if (!build) {
+		return [['Build', 'none — the harness polled and never built']];
+	}
+
+	return [
+		['Build command', build.command ? `\`${build.command}\`` : null],
+		['Build fetch log', build.logPath ? `\`${build.logPath}\`` : null],
+		['Build wall time', build.wallMs === null ? null : duration(build.wallMs)],
+		[
+			'Build outcome',
+			build.ok
+				? 'exit 0'
+				: `**failed** — ${build.signal ? `signal ${build.signal}` : `exit ${build.exitCode}`}`,
+		],
+		[
+			'Build requests',
+			`${count(build.requests)}, of which ${count(build.cmsRequests)} to the CMS`,
+		],
+	];
+}
+
+/** A poll's latency, marked if the CMS did not answer it. */
+function cell(poll) {
+	if (!poll) {
+		return '-';
+	}
+
+	if (!poll.ok) {
+		return `**${poll.status ?? 'error'}**`;
+	}
+
+	return duration(poll.ms);
+}