@fedify/cli 2.3.0-dev.994 → 2.3.1

package/dist/bench/metrics/stats-client.js

@@ -0,0 +1,216 @@
+import "@js-temporal/polyfill";
+import { STATS_PATH } from "../discovery/probe.js";
+//#region src/bench/metrics/stats-client.ts
+/**
+* Reading server-side metrics from the cooperative `stats` endpoint.
+*
+* The endpoint returns a JSON projection of the target's OpenTelemetry meters
+* (see *@fedify/fedify*'s benchmark module).  This module projects the relevant
+* instruments — signature verification latency and queue depth — into the
+* report's `server` section, marked distinct from client-measured numbers.
+*
+* The server reader is cumulative and has no reset, so a single snapshot covers
+* the target's whole lifetime.  To scope server numbers to one scenario's
+* measured window, callers take a {@link ServerSnapshot} baseline at the window
+* start and another at the end, {@link diffSnapshots} the two, and project the
+* difference with {@link snapshotToMetrics}.
+* @since 2.3.0
+* @module
+*/
+/**
+* Parses a `stats` snapshot into raw server instruments.  A successful parse
+* always yields a snapshot, even when it carries no relevant instruments (both
+* fields `null`); `null` is reserved for an unparseable snapshot, so callers can
+* tell "available but empty" apart from "unavailable".
+* @param snapshot The parsed `stats` JSON.
+* @returns The raw server snapshot, or `null` if it could not be parsed.
+*/
+function parseServerSnapshot(snapshot) {
+	try {
+		const metrics = flattenMetrics(snapshot);
+		const sig = metrics.find((m) => m.dataPointType === "histogram" && (m.name ?? "").includes("signature.verification"));
+		const signature = sig == null ? null : mergeHistogram(sig.dataPoints);
+		let queueDepthMax = null;
+		const depth = metrics.find((m) => m.name === "fedify.queue.depth");
+		if (depth != null && Array.isArray(depth.dataPoints)) {
+			const values = depth.dataPoints.map((p) => p.value).filter(isFiniteNumber);
+			if (values.length > 0) queueDepthMax = Math.max(...values);
+		}
+		const queueTasks = parseQueueTasks(metrics);
+		const deliveryPermanentFailures = sumMetric(metrics, "activitypub.delivery.permanent_failure");
+		return {
+			signature,
+			queueDepthMax,
+			...queueTasks == null ? {} : { queueTasks },
+			...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
+		};
+	} catch {
+		return null;
+	}
+}
+/**
+* Subtracts a baseline snapshot from an end snapshot, yielding the instruments
+* accumulated between the two (the measured window).  Signature histogram
+* counts are diffed bucket by bucket; the queue depth is a gauge, not a
+* cumulative count, so the end value is kept as-is.  Callers that cannot obtain
+* both snapshots should not call this (and should report no server metrics)
+* rather than passing a stand-in, since a missing baseline cannot be diffed.
+* @param baseline The snapshot taken at the measured-window start.
+* @param end The snapshot taken at the measured-window end.
+* @returns The windowed snapshot.
+*/
+function diffSnapshots(baseline, end) {
+	const queueTasks = diffQueueTasks(baseline.queueTasks ?? null, end.queueTasks ?? null);
+	const deliveryPermanentFailures = diffCounter(baseline.deliveryPermanentFailures ?? null, end.deliveryPermanentFailures ?? null);
+	return {
+		signature: diffHistogram(baseline.signature, end.signature),
+		queueDepthMax: end.queueDepthMax,
+		...queueTasks == null ? {} : { queueTasks },
+		...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
+	};
+}
+/**
+* Projects a raw server snapshot into the report's server metrics, or `null`
+* when it carries no usable measurement.
+* @param snapshot The raw (optionally diffed) server snapshot.
+* @returns The projected server metrics, or `null`.
+*/
+function snapshotToMetrics(snapshot) {
+	if (snapshot == null) return null;
+	const result = {};
+	if (snapshot.signature != null) {
+		if (snapshot.signature.counts.reduce((sum, n) => sum + n, 0) > 0) result.signatureVerificationMs = { overall: {
+			p50: histogramPercentile(snapshot.signature, 50),
+			p95: histogramPercentile(snapshot.signature, 95),
+			p99: histogramPercentile(snapshot.signature, 99)
+		} };
+	}
+	if (snapshot.queueDepthMax != null) result.queue = { depthMax: snapshot.queueDepthMax };
+	return Object.keys(result).length > 0 ? result : null;
+}
+/**
+* Fetches and parses the target's raw server snapshot.
+* @param target The target base URL.
+* @param fetchImpl The fetch implementation (overridable for tests).
+* @returns The raw server snapshot, or `null` if unavailable.
+*/
+async function fetchServerSnapshot(target, fetchImpl = fetch) {
+	try {
+		const response = await fetchImpl(new URL(STATS_PATH, target), { redirect: "manual" });
+		if (!response.ok) return null;
+		return parseServerSnapshot(await response.json());
+	} catch {
+		return null;
+	}
+}
+/**
+* Returns the remaining queue task backlog represented by a diffed snapshot.
+* @param snapshot The server snapshot to inspect, usually already diffed
+* against a baseline.
+* @param baselineRemaining Queue tasks that were already outstanding when the
+* diff baseline was taken.  These must drain before diffed completions can be
+* attributed to newly enqueued tasks.
+* @returns `Math.max(0, baselineRemaining + enqueued - completed - failed)`,
+* or `null` when the snapshot has no queue task counters.
+*/
+function queueTaskRemaining(snapshot, baselineRemaining = 0) {
+	if (snapshot?.queueTasks == null) return null;
+	const { enqueued, completed, failed } = snapshot.queueTasks;
+	return Math.max(0, baselineRemaining + enqueued - completed - failed);
+}
+function isFiniteNumber(value) {
+	return typeof value === "number" && Number.isFinite(value);
+}
+function flattenMetrics(snapshot) {
+	return (Array.isArray(snapshot?.scopeMetrics) ? snapshot.scopeMetrics : []).flatMap((scope) => {
+		const metrics = scope?.metrics;
+		return Array.isArray(metrics) ? metrics.filter((m) => m != null) : [];
+	});
+}
+function mergeHistogram(dataPoints) {
+	if (!Array.isArray(dataPoints)) return null;
+	let boundaries = null;
+	let counts = null;
+	for (const point of dataPoints) {
+		const value = point?.value;
+		if (typeof value !== "object" || value == null) continue;
+		const b = value.buckets?.boundaries;
+		const c = value.buckets?.counts;
+		if (!Array.isArray(b) || !Array.isArray(c)) continue;
+		if (!b.every(isFiniteNumber) || !c.every(isFiniteNumber)) continue;
+		if (boundaries == null) {
+			boundaries = [...b];
+			counts = [...c];
+		} else if (counts != null && counts.length === c.length && boundaries.length === b.length && boundaries.every((v, i) => v === b[i])) for (let i = 0; i < c.length; i++) counts[i] += c[i];
+	}
+	return boundaries != null && counts != null ? {
+		boundaries,
+		counts
+	} : null;
+}
+function parseQueueTasks(metrics) {
+	const enqueued = sumMetric(metrics, "fedify.queue.task.enqueued");
+	const completed = sumMetric(metrics, "fedify.queue.task.completed");
+	const failed = sumMetric(metrics, "fedify.queue.task.failed");
+	return enqueued == null && completed == null && failed == null ? null : {
+		enqueued: enqueued ?? 0,
+		completed: completed ?? 0,
+		failed: failed ?? 0
+	};
+}
+function sumMetric(metrics, name) {
+	let total = 0;
+	let found = false;
+	for (const metric of metrics) {
+		if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue;
+		for (const point of metric.dataPoints) if (isRecord(point) && isFiniteNumber(point["value"])) {
+			total += point["value"];
+			found = true;
+		}
+	}
+	return found ? total : null;
+}
+function diffHistogram(baseline, end) {
+	if (end == null) return null;
+	if (baseline == null) return end;
+	if (!histogramsCompatible(baseline, end)) return null;
+	const counts = end.counts.map((count, i) => Math.max(0, count - baseline.counts[i]));
+	return {
+		boundaries: end.boundaries,
+		counts
+	};
+}
+function diffQueueTasks(baseline, end) {
+	if (end == null) return null;
+	if (baseline == null) return end;
+	return {
+		enqueued: Math.max(0, end.enqueued - baseline.enqueued),
+		completed: Math.max(0, end.completed - baseline.completed),
+		failed: Math.max(0, end.failed - baseline.failed)
+	};
+}
+function diffCounter(baseline, end) {
+	if (end == null) return null;
+	if (baseline == null) return end;
+	return Math.max(0, end - baseline);
+}
+function histogramsCompatible(a, b) {
+	return a.boundaries.length === b.boundaries.length && a.counts.length === b.counts.length && a.boundaries.every((boundary, i) => boundary === b.boundaries[i]);
+}
+function isRecord(value) {
+	return value != null && typeof value === "object" && !Array.isArray(value);
+}
+function histogramPercentile(histogram, p) {
+	const { boundaries, counts } = histogram;
+	const total = counts.reduce((sum, n) => sum + n, 0);
+	if (total === 0) return 0;
+	const target = Math.ceil(p / 100 * total);
+	let accumulated = 0;
+	for (let i = 0; i < counts.length; i++) {
+		accumulated += counts[i];
+		if (accumulated >= target) return i < boundaries.length ? boundaries[i] : boundaries[boundaries.length - 1] ?? 0;
+	}
+	return boundaries[boundaries.length - 1] ?? 0;
+}
+//#endregion
+export { diffSnapshots, fetchServerSnapshot, queueTaskRemaining, snapshotToMetrics };

package/dist/bench/mod.js

@@ -0,0 +1,11 @@
+import "@js-temporal/polyfill";
+import "./command.js";
+import runBench$1 from "./action.js";
+import { runBenchCompare } from "./compare.js";
+//#region src/bench/mod.ts
+/** Runs a parsed benchmark command. */
+function runBench(command) {
+	return command.mode === "compare" ? runBenchCompare(command) : runBench$1(command);
+}
+//#endregion
+export { runBench };

package/dist/bench/render/format.js

@@ -0,0 +1,46 @@
+import "@js-temporal/polyfill";
+//#region src/bench/render/format.ts
+const OP_SYMBOLS = {
+	lt: "<",
+	lte: "<=",
+	gt: ">",
+	gte: ">=",
+	eq: "=="
+};
+/** Returns the symbolic form of a comparison operator. */
+function opSymbol(op) {
+	return OP_SYMBOLS[op];
+}
+/** Formats a number with grouping and at most three fractional digits. */
+function formatNumber(value) {
+	if (!Number.isFinite(value)) return String(value);
+	return (Math.round(value * 1e3) / 1e3).toLocaleString("en-US", { maximumFractionDigits: 3 });
+}
+/** Formats a ratio (0..1) as a percentage with at most two fractional digits. */
+function formatPercent(ratio) {
+	return `${(Math.round(ratio * 1e6) / 1e4).toLocaleString("en-US", { maximumFractionDigits: 2 })}%`;
+}
+/**
+* Formats a normalized threshold back into its human-friendly unit.
+* @param threshold The normalized numeric threshold.
+* @param unit The threshold's unit (`"ms"`, `"%"`, `"/s"`, or `null`).
+*/
+function formatThreshold(threshold, unit) {
+	switch (unit) {
+		case "%": return formatPercent(threshold);
+		case "ms": return `${formatNumber(threshold)}ms`;
+		case "/s": return `${formatNumber(threshold)}/s`;
+		default: return formatNumber(threshold);
+	}
+}
+/**
+* Formats a measured value using the unit of the assertion it is compared to.
+* @param actual The measured value, or `null` if unmeasured.
+* @param unit The assertion's unit.
+*/
+function formatActual(actual, unit) {
+	if (actual == null) return "n/a";
+	return formatThreshold(actual, unit);
+}
+//#endregion
+export { formatActual, formatNumber, formatPercent, formatThreshold, opSymbol };

package/dist/bench/render/index.js

@@ -0,0 +1,20 @@
+import "@js-temporal/polyfill";
+import { renderJson } from "./json.js";
+import { renderMarkdown } from "./markdown.js";
+import { renderText } from "./text.js";
+//#region src/bench/render/index.ts
+/**
+* Renders a report in the requested format.
+* @param report The report to render.
+* @param format The output format.
+* @returns The rendered text.
+*/
+function renderReport(report, format) {
+	switch (format) {
+		case "json": return renderJson(report);
+		case "markdown": return renderMarkdown(report);
+		case "text": return renderText(report);
+	}
+}
+//#endregion
+export { renderReport };

package/dist/bench/render/json.js

@@ -0,0 +1,12 @@
+import "@js-temporal/polyfill";
+//#region src/bench/render/json.ts
+/**
+* Renders a report as pretty-printed canonical JSON.
+* @param report The report to render.
+* @returns The JSON text, with a trailing newline.
+*/
+function renderJson(report) {
+	return `${JSON.stringify(report, null, 2)}\n`;
+}
+//#endregion
+export { renderJson };

package/dist/bench/render/markdown.js

@@ -0,0 +1,63 @@
+import "@js-temporal/polyfill";
+import { metricDisplayUnit } from "../result/expect/metrics.js";
+import { formatActual, formatNumber, formatPercent, formatThreshold, opSymbol } from "./format.js";
+//#region src/bench/render/markdown.ts
+/**
+* The Markdown renderer, suited to a GitHub Actions job summary or a PR
+* comment.  It is derived from the same report model as the text and JSON
+* forms.
+* @since 2.3.0
+* @module
+*/
+/**
+* Renders a report as Markdown.
+* @param report The report to render.
+* @returns The Markdown text.
+*/
+function renderMarkdown(report) {
+	const lines = [];
+	lines.push("# Fedify benchmark report", "");
+	lines.push(`**Result:** ${report.passed ? "✅ PASS" : "❌ FAIL"}`, "");
+	lines.push(`- **Target:** \`${report.target.url}\` (${report.target.statsAvailable ? "stats available" : "no stats"})`);
+	lines.push(`- **Environment:** ${report.environment.runtime} ${report.environment.runtimeVersion}, ${report.environment.os}, ${report.environment.cpuCount} CPUs`);
+	lines.push(`- **Config:** \`${report.suite.configHash}\``, "");
+	for (const scenario of report.scenarios) lines.push(...renderScenario(scenario), "");
+	return lines.join("\n");
+}
+function renderScenario(scenario) {
+	const lines = [];
+	lines.push(`## ${scenario.name} (${scenario.type}) ${scenario.passed ? "✅" : "❌"}`, "");
+	lines.push("| Metric | Value |", "| --- | --- |");
+	const r = scenario.requests;
+	lines.push(`| Requests | ${formatNumber(r.total)} |`);
+	lines.push(`| Success rate | ${formatPercent(r.successRate)} |`);
+	lines.push(`| Throughput | ${formatNumber(scenario.throughputPerSec)}/s |`);
+	if (scenario.deliveryThroughputPerSec != null) lines.push(`| Delivery throughput | ${formatNumber(scenario.deliveryThroughputPerSec)}/s |`);
+	const l = scenario.client.latencyMs;
+	lines.push(`| Latency p50 | ${formatNumber(l.p50)}ms |`);
+	lines.push(`| Latency p95 | ${formatNumber(l.p95)}ms |`);
+	lines.push(`| Latency p99 | ${formatNumber(l.p99)}ms |`);
+	const sig = scenario.server?.signatureVerificationMs?.overall;
+	if (sig?.p95 != null) lines.push(`| Signature verification p95 (server) | ${formatNumber(sig.p95)}ms |`);
+	const queue = scenario.server?.queue;
+	if (queue?.drainMs?.p95 != null) lines.push(`| Queue drain p95 (server) | ${formatNumber(queue.drainMs.p95)}ms |`);
+	if (queue?.depthMax != null) lines.push(`| Queue depth max (server) | ${formatNumber(queue.depthMax)} |`);
+	if (scenario.errors.length > 0) {
+		lines.push("", "| Error | Count |", "| --- | --- |");
+		for (const error of scenario.errors) {
+			const code = error.status == null ? error.kind : String(error.status);
+			lines.push(`| ${code} ${error.reason} | ${formatNumber(error.count)} |`);
+		}
+	}
+	if (scenario.expectations.length > 0) {
+		lines.push("", "| Expectation | Actual | Result |", "| --- | --- | --- |");
+		for (const e of scenario.expectations) {
+			const tag = e.pass ? "✅" : e.severity === "warn" ? "⚠️" : "❌";
+			const unit = metricDisplayUnit(e.metric);
+			lines.push(`| \`${e.metric} ${opSymbol(e.op)} ${formatThreshold(e.threshold, e.unit ?? unit)}\` | ${formatActual(e.actual, unit)} | ${tag} |`);
+		}
+	}
+	return lines;
+}
+//#endregion
+export { renderMarkdown };

package/dist/bench/render/text.js

@@ -0,0 +1,75 @@
+import "@js-temporal/polyfill";
+import { metricDisplayUnit } from "../result/expect/metrics.js";
+import { formatActual, formatNumber, formatPercent, formatThreshold, opSymbol } from "./format.js";
+//#region src/bench/render/text.ts
+/**
+* Renders a report as a plain-text terminal summary.
+* @param report The report to render.
+* @returns The summary text.
+*/
+function renderText(report) {
+	const lines = [];
+	lines.push("Fedify benchmark report", "");
+	const fedify = report.target.fedifyVersion == null ? "Fedify version unknown" : `Fedify ${report.target.fedifyVersion}`;
+	const stats = report.target.statsAvailable ? "stats available" : "stats unavailable";
+	lines.push(`Target: ${report.target.url}  (${fedify}, ${stats})`);
+	const env = report.environment;
+	lines.push(`Environment: ${env.runtime} ${env.runtimeVersion}, ${env.os}, ${env.cpuCount} CPUs`);
+	lines.push(`Started: ${report.startedAt}  Finished: ${report.finishedAt}`);
+	lines.push(`Config: ${report.suite.configHash}`, "");
+	for (const scenario of report.scenarios) lines.push(...renderScenario(scenario), "");
+	lines.push(`Overall: ${report.passed ? "PASS" : "FAIL"}`);
+	return lines.join("\n");
+}
+function renderScenario(scenario) {
+	const lines = [];
+	lines.push(`Scenario: ${scenario.name} (${scenario.type})  [${scenario.passed ? "PASS" : "FAIL"}]`);
+	lines.push(`  Load: ${describeLoad(scenario.load)}`);
+	const r = scenario.requests;
+	lines.push(`  Requests: ${formatNumber(r.total)}  (ok ${formatNumber(r.ok)}, failed ${formatNumber(r.failed)}, success ${formatPercent(r.successRate)})`);
+	lines.push(`  Throughput: ${formatNumber(scenario.throughputPerSec)} req/s`);
+	if (scenario.deliveryThroughputPerSec != null) lines.push(`  Delivery throughput: ${formatNumber(scenario.deliveryThroughputPerSec)} deliveries/s`);
+	const l = scenario.client.latencyMs;
+	lines.push(`  Client latency (ms): p50 ${formatNumber(l.p50)}  p95 ${formatNumber(l.p95)}  p99 ${formatNumber(l.p99)}  mean ${formatNumber(l.mean)}  max ${formatNumber(l.max)}`);
+	if (scenario.server?.signatureVerificationMs != null) lines.push(`  Server signature verification (ms): ${describePartial(scenario.server.signatureVerificationMs.overall)}`);
+	const queue = scenario.server?.queue;
+	if (queue?.drainMs != null && hasPartial(queue.drainMs)) {
+		const depth = queue.depthMax;
+		const suffix = depth == null ? "" : `  (depth max ${formatNumber(depth)})`;
+		lines.push(`  Server queue drain (ms): ${describePartial(queue.drainMs)}${suffix}`);
+	} else if (queue?.depthMax != null) lines.push(`  Server queue depth max: ${formatNumber(queue.depthMax)}`);
+	if (scenario.errors.length > 0) {
+		lines.push("  Errors:");
+		for (const error of scenario.errors) {
+			const code = error.status == null ? error.kind : String(error.status);
+			lines.push(`    ${code} ${error.reason}: ${formatNumber(error.count)}`);
+		}
+	}
+	if (scenario.expectations.length > 0) {
+		lines.push("  Expectations:");
+		for (const e of scenario.expectations) {
+			const tag = e.pass ? "PASS" : e.severity === "warn" ? "WARN" : "FAIL";
+			const unit = metricDisplayUnit(e.metric);
+			lines.push(`    [${tag}] ${e.metric} ${opSymbol(e.op)} ${formatThreshold(e.threshold, e.unit ?? unit)}  (actual ${formatActual(e.actual, unit)})`);
+		}
+	}
+	return lines;
+}
+function describeLoad(load) {
+	const tail = `duration ${formatNumber(load.durationMs)}ms, warmup ${formatNumber(load.warmupMs)}ms`;
+	if (load.model === "closed") return `closed, concurrency ${load.concurrency}, ${tail}`;
+	return `open, ${formatNumber(load.ratePerSec)}/s ${load.arrival}, ${tail}`;
+}
+function describePartial(latency) {
+	const parts = [];
+	if (latency.p50 != null) parts.push(`p50 ${formatNumber(latency.p50)}`);
+	if (latency.p95 != null) parts.push(`p95 ${formatNumber(latency.p95)}`);
+	if (latency.p99 != null) parts.push(`p99 ${formatNumber(latency.p99)}`);
+	return parts.join("  ");
+}
+/** Whether a partial latency carries at least one renderable percentile. */
+function hasPartial(latency) {
+	return latency.p50 != null || latency.p95 != null || latency.p99 != null;
+}
+//#endregion
+export { renderText };