@fedify/cli 2.3.0-dev.1361 → 2.3.0-dev.1367

package/dist/bench/load/generator.js

@@ -9,10 +9,10 @@ import { systemClock } from "./clock.js";
 * @param clock The clock (overridable for tests); defaults to the system clock.
 * @returns The recorded samples and run metadata.
 */
-function runLoad(plan, send, clock = systemClock()) {
-	return plan.load.kind === "open" ? runOpenLoop(plan, plan.load, send, clock) : runClosedLoop(plan, plan.load, send, clock);
+function runLoad(plan, send, clock = systemClock(), signal) {
+	return plan.load.kind === "open" ? runOpenLoop(plan, plan.load, send, clock, signal) : runClosedLoop(plan, plan.load, send, clock, signal);
 }
-async function runOpenLoop(plan, load, send, clock) {
+async function runOpenLoop(plan, load, send, clock, signal) {
 	const arrivals = scheduleArrivals({
 		ratePerSec: load.ratePerSec,
 		durationMs: plan.durationMs,
@@ -25,8 +25,13 @@ async function runOpenLoop(plan, load, send, clock) {
 	const start = clock.now();
 	const active = /* @__PURE__ */ new Set();
 	for (const offset of arrivals) {
-		await clock.sleepUntil(start + offset);
-		if (await slots.acquire()) saturated = true;
+		throwIfAborted(signal);
+		await clock.sleepUntil(start + offset, signal);
+		if (await slots.acquire(signal)) saturated = true;
+		if (signal?.aborted) {
+			slots.release();
+			throw abortReason(signal);
+		}
 		const dispatched = dispatch(send, offset, start, plan.warmupMs, clock, samples).finally(() => {
 			slots.release();
 			active.delete(dispatched);
@@ -40,7 +45,7 @@ async function runOpenLoop(plan, load, send, clock) {
 		wallDurationMs: clock.now() - start
 	};
 }
-async function runClosedLoop(plan, load, send, clock) {
+async function runClosedLoop(plan, load, send, clock, signal) {
 	const samples = [];
 	const slots = createSemaphore(load.maxInFlight);
 	let saturated = false;
@@ -48,7 +53,12 @@ async function runClosedLoop(plan, load, send, clock) {
 	const deadline = start + plan.durationMs;
 	async function worker() {
 		while (clock.now() < deadline) {
-			if (await slots.acquire()) saturated = true;
+			throwIfAborted(signal);
+			if (await slots.acquire(signal)) saturated = true;
+			if (signal?.aborted) {
+				slots.release();
+				throw abortReason(signal);
+			}
 			if (clock.now() >= deadline) {
 				slots.release();
 				break;
@@ -94,12 +104,29 @@ function createSemaphore(max) {
 	let count = 0;
 	const queue = [];
 	return {
-		acquire() {
+		acquire(signal) {
+			throwIfAborted(signal);
 			if (count < max) {
 				count++;
 				return Promise.resolve(false);
 			}
-			return new Promise((resolve) => queue.push(() => resolve(true)));
+			return new Promise((resolve, reject) => {
+				const waiter = () => {
+					cleanup();
+					resolve(true);
+				};
+				const onAbort = () => {
+					const index = queue.indexOf(waiter);
+					if (index >= 0) queue.splice(index, 1);
+					cleanup();
+					reject(abortReason(signal));
+				};
+				const cleanup = () => {
+					signal?.removeEventListener("abort", onAbort);
+				};
+				signal?.addEventListener("abort", onAbort, { once: true });
+				queue.push(waiter);
+			});
 		},
 		release() {
 			const next = queue.shift();
@@ -108,5 +135,11 @@ function createSemaphore(max) {
 		}
 	};
 }
+function throwIfAborted(signal) {
+	if (signal?.aborted) throw abortReason(signal);
+}
+function abortReason(signal) {
+	return signal.reason ?? /* @__PURE__ */ new Error("Benchmark load aborted.");
+}
 //#endregion
 export { runLoad };

package/dist/bench/metrics/stats-client.js

@@ -36,9 +36,13 @@ function parseServerSnapshot(snapshot) {
 			const values = depth.dataPoints.map((p) => p.value).filter(isFiniteNumber);
 			if (values.length > 0) queueDepthMax = Math.max(...values);
 		}
+		const queueTasks = parseQueueTasks(metrics);
+		const deliveryPermanentFailures = sumMetric(metrics, "activitypub.delivery.permanent_failure");
 		return {
 			signature,
-			queueDepthMax
+			queueDepthMax,
+			...queueTasks == null ? {} : { queueTasks },
+			...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
 		};
 	} catch {
 		return null;
@@ -56,9 +60,13 @@ function parseServerSnapshot(snapshot) {
 * @returns The windowed snapshot.
 */
 function diffSnapshots(baseline, end) {
+	const queueTasks = diffQueueTasks(baseline.queueTasks ?? null, end.queueTasks ?? null);
+	const deliveryPermanentFailures = diffCounter(baseline.deliveryPermanentFailures ?? null, end.deliveryPermanentFailures ?? null);
 	return {
 		signature: diffHistogram(baseline.signature, end.signature),
-		queueDepthMax: end.queueDepthMax
+		queueDepthMax: end.queueDepthMax,
+		...queueTasks == null ? {} : { queueTasks },
+		...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
 	};
 }
 /**
@@ -95,6 +103,21 @@ async function fetchServerSnapshot(target, fetchImpl = fetch) {
 		return null;
 	}
 }
+/**
+* Returns the remaining queue task backlog represented by a diffed snapshot.
+* @param snapshot The server snapshot to inspect, usually already diffed
+* against a baseline.
+* @param baselineRemaining Queue tasks that were already outstanding when the
+* diff baseline was taken.  These must drain before diffed completions can be
+* attributed to newly enqueued tasks.
+* @returns `Math.max(0, baselineRemaining + enqueued - completed - failed)`,
+* or `null` when the snapshot has no queue task counters.
+*/
+function queueTaskRemaining(snapshot, baselineRemaining = 0) {
+	if (snapshot?.queueTasks == null) return null;
+	const { enqueued, completed, failed } = snapshot.queueTasks;
+	return Math.max(0, baselineRemaining + enqueued - completed - failed);
+}
 function isFiniteNumber(value) {
 	return typeof value === "number" && Number.isFinite(value);
 }
@@ -125,6 +148,28 @@ function mergeHistogram(dataPoints) {
 		counts
 	} : null;
 }
+function parseQueueTasks(metrics) {
+	const enqueued = sumMetric(metrics, "fedify.queue.task.enqueued");
+	const completed = sumMetric(metrics, "fedify.queue.task.completed");
+	const failed = sumMetric(metrics, "fedify.queue.task.failed");
+	return enqueued == null && completed == null && failed == null ? null : {
+		enqueued: enqueued ?? 0,
+		completed: completed ?? 0,
+		failed: failed ?? 0
+	};
+}
+function sumMetric(metrics, name) {
+	let total = 0;
+	let found = false;
+	for (const metric of metrics) {
+		if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue;
+		for (const point of metric.dataPoints) if (isRecord(point) && isFiniteNumber(point["value"])) {
+			total += point["value"];
+			found = true;
+		}
+	}
+	return found ? total : null;
+}
 function diffHistogram(baseline, end) {
 	if (end == null) return null;
 	if (baseline == null) return end;
@@ -135,9 +180,26 @@ function diffHistogram(baseline, end) {
 		counts
 	};
 }
+function diffQueueTasks(baseline, end) {
+	if (end == null) return null;
+	if (baseline == null) return end;
+	return {
+		enqueued: Math.max(0, end.enqueued - baseline.enqueued),
+		completed: Math.max(0, end.completed - baseline.completed),
+		failed: Math.max(0, end.failed - baseline.failed)
+	};
+}
+function diffCounter(baseline, end) {
+	if (end == null) return null;
+	if (baseline == null) return end;
+	return Math.max(0, end - baseline);
+}
 function histogramsCompatible(a, b) {
 	return a.boundaries.length === b.boundaries.length && a.counts.length === b.counts.length && a.boundaries.every((boundary, i) => boundary === b.boundaries[i]);
 }
+function isRecord(value) {
+	return value != null && typeof value === "object" && !Array.isArray(value);
+}
 function histogramPercentile(histogram, p) {
 	const { boundaries, counts } = histogram;
 	const total = counts.reduce((sum, n) => sum + n, 0);
@@ -151,4 +213,4 @@ function histogramPercentile(histogram, p) {
 	return boundaries[boundaries.length - 1] ?? 0;
 }
 //#endregion
-export { diffSnapshots, fetchServerSnapshot, snapshotToMetrics };
+export { diffSnapshots, fetchServerSnapshot, queueTaskRemaining, snapshotToMetrics };

package/dist/bench/mod.js

@@ -1,4 +1,11 @@
 import "@js-temporal/polyfill";
-import "./action.js";
 import "./command.js";
-export {};
+import runBench$1 from "./action.js";
+import { runBenchCompare } from "./compare.js";
+//#region src/bench/mod.ts
+/** Runs a parsed benchmark command. */
+function runBench(command) {
+	return command.mode === "compare" ? runBenchCompare(command) : runBench$1(command);
+}
+//#endregion
+export { runBench };

package/dist/bench/render/markdown.js

@@ -32,6 +32,7 @@ function renderScenario(scenario) {
 	lines.push(`| Requests | ${formatNumber(r.total)} |`);
 	lines.push(`| Success rate | ${formatPercent(r.successRate)} |`);
 	lines.push(`| Throughput | ${formatNumber(scenario.throughputPerSec)}/s |`);
+	if (scenario.deliveryThroughputPerSec != null) lines.push(`| Delivery throughput | ${formatNumber(scenario.deliveryThroughputPerSec)}/s |`);
 	const l = scenario.client.latencyMs;
 	lines.push(`| Latency p50 | ${formatNumber(l.p50)}ms |`);
 	lines.push(`| Latency p95 | ${formatNumber(l.p95)}ms |`);

package/dist/bench/render/text.js

@@ -28,6 +28,7 @@ function renderScenario(scenario) {
 	const r = scenario.requests;
 	lines.push(`  Requests: ${formatNumber(r.total)}  (ok ${formatNumber(r.ok)}, failed ${formatNumber(r.failed)}, success ${formatPercent(r.successRate)})`);
 	lines.push(`  Throughput: ${formatNumber(scenario.throughputPerSec)} req/s`);
+	if (scenario.deliveryThroughputPerSec != null) lines.push(`  Delivery throughput: ${formatNumber(scenario.deliveryThroughputPerSec)} deliveries/s`);
 	const l = scenario.client.latencyMs;
 	lines.push(`  Client latency (ms): p50 ${formatNumber(l.p50)}  p95 ${formatNumber(l.p95)}  p99 ${formatNumber(l.p99)}  mean ${formatNumber(l.mean)}  max ${formatNumber(l.max)}`);
 	if (scenario.server?.signatureVerificationMs != null) lines.push(`  Server signature verification (ms): ${describePartial(scenario.server.signatureVerificationMs.overall)}`);

package/dist/bench/result/build.js

@@ -3,8 +3,8 @@ import { version } from "../../deno.js";
 import { evaluateExpect } from "./expect/evaluate.js";
 import { REPORT_SCHEMA_ID } from "./schema.js";
 import process from "node:process";
-import { createHash } from "node:crypto";
 import { cpus } from "node:os";
+import { createHash } from "node:crypto";
 //#region src/bench/result/build.ts
 /**
 * Assembly of the canonical benchmark report from measured scenario data.
@@ -23,19 +23,25 @@ import { cpus } from "node:os";
 * @returns The assembled scenario result.
 */
 function buildScenarioResult(scenario, measurement) {
-	const { results, passed } = evaluateExpect(scenario.expect, measurement);
+	const measurements = Array.isArray(measurement) ? measurement : [measurement];
+	if (measurements.length < 1) throw new RangeError("At least one scenario measurement is required.");
+	const aggregate = measurements.length === 1 ? measurements[0] : aggregateMeasurements(measurements);
+	const { results, passed } = evaluateExpect(scenario.expect, aggregate);
 	return {
 		name: scenario.name,
 		type: scenario.type,
 		load: loadSummary(scenario),
-		requests: measurement.requests,
-		throughputPerSec: measurement.throughputPerSec,
-		client: measurement.client,
-		server: measurement.server,
-		errors: measurement.errors,
+		requests: aggregate.requests,
+		throughputPerSec: aggregate.throughputPerSec,
+		...aggregate.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: aggregate.deliveryThroughputPerSec },
+		client: aggregate.client,
+		server: aggregate.server,
+		errors: aggregate.errors,
 		expectations: results,
-		passed: passed && measurement.requests.total > 0,
-		...measurement.histogram ? { histogram: measurement.histogram } : {}
+		passed: passed && measurements.every((m) => m.requests.total > 0),
+		runCount: measurements.length,
+		...measurements.length > 1 ? { runs: measurements.map((m, index) => runResult(index + 1, m)) } : {},
+		...aggregate.histogram ? { histogram: aggregate.histogram } : {}
 	};
 }
 /**
@@ -47,7 +53,7 @@ function buildScenarioResult(scenario, measurement) {
 function buildReport(input) {
 	return {
 		$schema: REPORT_SCHEMA_ID,
-		schemaVersion: 1,
+		schemaVersion: 3,
 		tool: {
 			name: "@fedify/cli",
 			version
@@ -61,6 +67,123 @@ function buildReport(input) {
 		scenarios: input.scenarios
 	};
 }
+function aggregateMeasurements(measurements) {
+	const errors = sumErrorBuckets(measurements.flatMap((m) => m.errors));
+	const total = sum(measurements.map((m) => m.requests.total));
+	const ok = sum(measurements.map((m) => m.requests.ok));
+	const failed = sum(measurements.map((m) => m.requests.failed));
+	const delivery = medianPresent(measurements.map((m) => m.deliveryThroughputPerSec));
+	return {
+		requests: {
+			total,
+			ok,
+			failed,
+			successRate: Math.min(...measurements.map((m) => m.requests.successRate))
+		},
+		throughputPerSec: median(measurements.map((m) => m.throughputPerSec)),
+		...delivery == null ? {} : { deliveryThroughputPerSec: delivery },
+		client: { latencyMs: {
+			p50: median(measurements.map((m) => m.client.latencyMs.p50)),
+			p95: median(measurements.map((m) => m.client.latencyMs.p95)),
+			p99: median(measurements.map((m) => m.client.latencyMs.p99)),
+			mean: median(measurements.map((m) => m.client.latencyMs.mean)),
+			max: median(measurements.map((m) => m.client.latencyMs.max))
+		} },
+		server: aggregateServer(measurements.map((m) => m.server)),
+		errors
+	};
+}
+function runResult(run, measurement) {
+	return {
+		run,
+		requests: measurement.requests,
+		throughputPerSec: measurement.throughputPerSec,
+		...measurement.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: measurement.deliveryThroughputPerSec },
+		client: measurement.client,
+		server: measurement.server,
+		errors: measurement.errors,
+		...measurement.histogram ? { histogram: measurement.histogram } : {}
+	};
+}
+function aggregateServer(servers) {
+	const present = servers.filter((s) => s != null);
+	if (present.length !== servers.length) return null;
+	const signature = aggregateSignatureVerification(present);
+	const queue = aggregateQueue(present);
+	return {
+		...signature == null ? {} : { signatureVerificationMs: signature },
+		...queue == null ? {} : { queue }
+	};
+}
+function aggregateSignatureVerification(servers) {
+	const values = servers.map((s) => s.signatureVerificationMs).filter((s) => s != null);
+	if (values.length !== servers.length) return null;
+	const standards = /* @__PURE__ */ new Set();
+	for (const value of values) for (const key of Object.keys(value.byStandard ?? {})) standards.add(key);
+	const byStandard = {};
+	for (const standard of standards) byStandard[standard] = aggregatePartial(values.map((v) => v.byStandard?.[standard]), "present");
+	return {
+		overall: aggregatePartial(values.map((v) => v.overall)),
+		...Object.keys(byStandard).length < 1 ? {} : { byStandard }
+	};
+}
+function aggregateQueue(servers) {
+	const values = servers.map((s) => s.queue).filter((q) => q != null);
+	if (values.length !== servers.length) return null;
+	const drainMs = aggregatePartial(values.map((v) => v.drainMs));
+	const depths = values.map((v) => v.depthMax);
+	return {
+		...hasPartial(drainMs) ? { drainMs } : {},
+		...depths.every(isNumber) ? { depthMax: Math.max(...depths) } : {}
+	};
+}
+function aggregatePartial(values, mode = "complete") {
+	return {
+		...partialField(values, "p50", mode),
+		...partialField(values, "p95", mode),
+		...partialField(values, "p99", mode)
+	};
+}
+function partialField(values, key, mode) {
+	const fieldValues = values.map((v) => v?.[key]);
+	if (mode === "present") {
+		const present = fieldValues.filter(isNumber);
+		return present.length < 1 ? {} : { [key]: median(present) };
+	}
+	return fieldValues.every(isNumber) ? { [key]: median(fieldValues) } : {};
+}
+function hasPartial(value) {
+	return value.p50 != null || value.p95 != null || value.p99 != null;
+}
+function sumErrorBuckets(errors) {
+	const buckets = /* @__PURE__ */ new Map();
+	for (const error of errors) {
+		const key = `${error.kind}|${error.status ?? ""}|${error.reason}`;
+		const previous = buckets.get(key);
+		buckets.set(key, {
+			...error,
+			count: (previous?.count ?? 0) + error.count
+		});
+	}
+	return [...buckets.values()].sort((a, b) => b.count - a.count);
+}
+function medianPresent(values) {
+	const present = values.filter(isNumber);
+	return present.length < 1 ? null : median(present);
+}
+function median(values) {
+	if (values.length < 1) return 0;
+	const sorted = [...values].sort((a, b) => a - b);
+	const middle = Math.floor(sorted.length / 2);
+	if (sorted.length % 2 === 1) return sorted[middle];
+	return (sorted[middle - 1] + sorted[middle]) / 2;
+}
+function sum(values) {
+	return values.reduce((a, b) => a + b, 0);
+}
+function isNumber(value) {
+	return typeof value === "number" && Number.isFinite(value);
+}
 /** Detects the current runtime environment for reproducibility metadata. */
 function detectEnvironment() {
 	const g = globalThis;

package/dist/bench/result/expect/evaluate.js

@@ -78,7 +78,7 @@ function lookupValue(metrics, metric) {
 	switch (metric) {
 		case "successRate": return metrics.requests.successRate;
 		case "throughputPerSec": return metrics.throughputPerSec;
-		case "deliveryThroughput": return null;
+		case "deliveryThroughput": return metrics.deliveryThroughputPerSec ?? null;
 		case "errors.total": return sumErrors(metrics.errors);
 		case "errors.4xx": return sumErrors(metrics.errors, {
 			min: 400,