@fedify/cli 2.3.0-dev.994 → 2.3.0

package/dist/bench/result/build.js

@@ -0,0 +1,252 @@
+import "@js-temporal/polyfill";
+import { version } from "../../deno.js";
+import { evaluateExpect } from "./expect/evaluate.js";
+import { REPORT_SCHEMA_ID } from "./schema.js";
+import process from "node:process";
+import { cpus } from "node:os";
+import { createHash } from "node:crypto";
+//#region src/bench/result/build.ts
+/**
+* Assembly of the canonical benchmark report from measured scenario data.
+*
+* The runners produce per-scenario measurements; this module turns each into a
+* {@link ScenarioResult} (evaluating its `expect` block) and assembles the
+* top-level {@link BenchReport} with reproducibility metadata.
+* @since 2.3.0
+* @module
+*/
+/**
+* Builds a scenario result from its resolved definition and measurement,
+* evaluating the `expect` block in the process.
+* @param scenario The resolved scenario.
+* @param measurement The measured client and server metrics.
+* @returns The assembled scenario result.
+*/
+function buildScenarioResult(scenario, measurement) {
+	const measurements = Array.isArray(measurement) ? measurement : [measurement];
+	if (measurements.length < 1) throw new RangeError("At least one scenario measurement is required.");
+	const aggregate = measurements.length === 1 ? measurements[0] : aggregateMeasurements(measurements);
+	const { results, passed } = evaluateExpect(scenario.expect, aggregate);
+	return {
+		name: scenario.name,
+		type: scenario.type,
+		load: loadSummary(scenario),
+		requests: aggregate.requests,
+		throughputPerSec: aggregate.throughputPerSec,
+		...aggregate.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: aggregate.deliveryThroughputPerSec },
+		client: aggregate.client,
+		server: aggregate.server,
+		errors: aggregate.errors,
+		expectations: results,
+		passed: passed && measurements.every((m) => m.requests.total > 0),
+		runCount: measurements.length,
+		...measurements.length > 1 ? { runs: measurements.map((m, index) => runResult(index + 1, m)) } : {},
+		...aggregate.histogram ? { histogram: aggregate.histogram } : {}
+	};
+}
+/**
+* Assembles the top-level report.  The gate passes only when every scenario
+* passes.
+* @param input The report inputs.
+* @returns The complete report.
+*/
+function buildReport(input) {
+	return {
+		$schema: REPORT_SCHEMA_ID,
+		schemaVersion: 3,
+		tool: {
+			name: "@fedify/cli",
+			version
+		},
+		environment: input.environment,
+		target: input.target,
+		startedAt: input.startedAt,
+		finishedAt: input.finishedAt,
+		suite: input.suite,
+		passed: input.scenarios.every((s) => s.passed),
+		scenarios: input.scenarios
+	};
+}
+function aggregateMeasurements(measurements) {
+	const errors = sumErrorBuckets(measurements.flatMap((m) => m.errors));
+	const total = sum(measurements.map((m) => m.requests.total));
+	const ok = sum(measurements.map((m) => m.requests.ok));
+	const failed = sum(measurements.map((m) => m.requests.failed));
+	const delivery = medianPresent(measurements.map((m) => m.deliveryThroughputPerSec));
+	return {
+		requests: {
+			total,
+			ok,
+			failed,
+			successRate: Math.min(...measurements.map((m) => m.requests.successRate))
+		},
+		throughputPerSec: median(measurements.map((m) => m.throughputPerSec)),
+		...delivery == null ? {} : { deliveryThroughputPerSec: delivery },
+		client: { latencyMs: {
+			p50: median(measurements.map((m) => m.client.latencyMs.p50)),
+			p95: median(measurements.map((m) => m.client.latencyMs.p95)),
+			p99: median(measurements.map((m) => m.client.latencyMs.p99)),
+			mean: median(measurements.map((m) => m.client.latencyMs.mean)),
+			max: median(measurements.map((m) => m.client.latencyMs.max))
+		} },
+		server: aggregateServer(measurements.map((m) => m.server)),
+		errors
+	};
+}
+function runResult(run, measurement) {
+	return {
+		run,
+		requests: measurement.requests,
+		throughputPerSec: measurement.throughputPerSec,
+		...measurement.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: measurement.deliveryThroughputPerSec },
+		client: measurement.client,
+		server: measurement.server,
+		errors: measurement.errors,
+		...measurement.histogram ? { histogram: measurement.histogram } : {}
+	};
+}
+function aggregateServer(servers) {
+	const present = servers.filter((s) => s != null);
+	if (present.length !== servers.length) return null;
+	const signature = aggregateSignatureVerification(present);
+	const queue = aggregateQueue(present);
+	return {
+		...signature == null ? {} : { signatureVerificationMs: signature },
+		...queue == null ? {} : { queue }
+	};
+}
+function aggregateSignatureVerification(servers) {
+	const values = servers.map((s) => s.signatureVerificationMs).filter((s) => s != null);
+	if (values.length !== servers.length) return null;
+	const standards = /* @__PURE__ */ new Set();
+	for (const value of values) for (const key of Object.keys(value.byStandard ?? {})) standards.add(key);
+	const byStandard = {};
+	for (const standard of standards) byStandard[standard] = aggregatePartial(values.map((v) => v.byStandard?.[standard]), "present");
+	return {
+		overall: aggregatePartial(values.map((v) => v.overall)),
+		...Object.keys(byStandard).length < 1 ? {} : { byStandard }
+	};
+}
+function aggregateQueue(servers) {
+	const values = servers.map((s) => s.queue).filter((q) => q != null);
+	if (values.length !== servers.length) return null;
+	const drainMs = aggregatePartial(values.map((v) => v.drainMs));
+	const depths = values.map((v) => v.depthMax);
+	return {
+		...hasPartial(drainMs) ? { drainMs } : {},
+		...depths.every(isNumber) ? { depthMax: Math.max(...depths) } : {}
+	};
+}
+function aggregatePartial(values, mode = "complete") {
+	return {
+		...partialField(values, "p50", mode),
+		...partialField(values, "p95", mode),
+		...partialField(values, "p99", mode)
+	};
+}
+function partialField(values, key, mode) {
+	const fieldValues = values.map((v) => v?.[key]);
+	if (mode === "present") {
+		const present = fieldValues.filter(isNumber);
+		return present.length < 1 ? {} : { [key]: median(present) };
+	}
+	return fieldValues.every(isNumber) ? { [key]: median(fieldValues) } : {};
+}
+function hasPartial(value) {
+	return value.p50 != null || value.p95 != null || value.p99 != null;
+}
+function sumErrorBuckets(errors) {
+	const buckets = /* @__PURE__ */ new Map();
+	for (const error of errors) {
+		const key = `${error.kind}|${error.status ?? ""}|${error.reason}`;
+		const previous = buckets.get(key);
+		buckets.set(key, {
+			...error,
+			count: (previous?.count ?? 0) + error.count
+		});
+	}
+	return [...buckets.values()].sort((a, b) => b.count - a.count);
+}
+function medianPresent(values) {
+	const present = values.filter(isNumber);
+	return present.length < 1 ? null : median(present);
+}
+function median(values) {
+	if (values.length < 1) return 0;
+	const sorted = [...values].sort((a, b) => a - b);
+	const middle = Math.floor(sorted.length / 2);
+	if (sorted.length % 2 === 1) return sorted[middle];
+	return (sorted[middle - 1] + sorted[middle]) / 2;
+}
+function sum(values) {
+	return values.reduce((a, b) => a + b, 0);
+}
+function isNumber(value) {
+	return typeof value === "number" && Number.isFinite(value);
+}
+/** Detects the current runtime environment for reproducibility metadata. */
+function detectEnvironment() {
+	const g = globalThis;
+	let runtime = "node";
+	let runtimeVersion = process.versions?.node ?? "unknown";
+	if (g.Deno?.version?.deno != null) {
+		runtime = "deno";
+		runtimeVersion = g.Deno.version.deno;
+	} else if (g.Bun?.version != null) {
+		runtime = "bun";
+		runtimeVersion = g.Bun.version;
+	}
+	let cpuCount = 0;
+	try {
+		cpuCount = cpus().length;
+	} catch {
+		cpuCount = 0;
+	}
+	return {
+		runtime,
+		runtimeVersion,
+		os: process.platform,
+		cpuCount
+	};
+}
+/**
+* Computes a stable `sha256:` hash of a resolved configuration, so CI only
+* compares runs from the same configuration.
+* @param config The configuration object to hash.
+* @returns A `sha256:`-prefixed hex digest.
+*/
+function configHash(config) {
+	return `sha256:${createHash("sha256").update(canonicalJson(config)).digest("hex")}`;
+}
+/** A guard against unbounded recursion on pathologically nested input. */
+const MAX_HASH_DEPTH = 100;
+function canonicalJson(value, depth = 0) {
+	if (depth > MAX_HASH_DEPTH) throw new RangeError("Maximum depth exceeded while hashing the config.");
+	if (value === void 0) return "null";
+	if (value === null || typeof value !== "object") return JSON.stringify(value);
+	const toJson = value.toJSON;
+	if (typeof toJson === "function") return canonicalJson(toJson.call(value), depth + 1);
+	if (Array.isArray(value)) return `[${value.map((v) => canonicalJson(v, depth + 1)).join(",")}]`;
+	return `{${Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v, depth + 1)}`).join(",")}}`;
+}
+function loadSummary(scenario) {
+	const { load, durationMs, warmupMs } = scenario;
+	const maxInFlight = load.maxInFlight == null ? {} : { maxInFlight: load.maxInFlight };
+	if (load.kind === "closed") return {
+		model: "closed",
+		concurrency: load.concurrency,
+		durationMs,
+		warmupMs,
+		...maxInFlight
+	};
+	return {
+		model: "open",
+		ratePerSec: load.ratePerSec,
+		arrival: load.arrival,
+		durationMs,
+		warmupMs,
+		...maxInFlight
+	};
+}
+//#endregion
+export { buildReport, buildScenarioResult, configHash, detectEnvironment };

package/dist/bench/result/expect/assert.js

@@ -0,0 +1,74 @@
+import "@js-temporal/polyfill";
+//#region src/bench/result/expect/assert.ts
+const ASSERT_RE = /^\s*(<=|>=|==|=|<|>)\s*(\d+(?:\.\d+)?)\s*(%|ms|s|\/s)?\s*$/;
+const OP_MAP = {
+	"<": "lt",
+	"<=": "lte",
+	">": "gt",
+	">=": "gte",
+	"==": "eq",
+	"=": "eq"
+};
+/** An error raised when an `expect` assertion cannot be parsed. */
+var AssertionParseError = class extends Error {};
+/**
+* Parses an `expect` assertion string.
+* @param text The assertion, e.g. `">= 99%"`.
+* @returns The parsed operator, normalized threshold, and unit.
+* @throws {AssertionParseError} If the assertion cannot be parsed.
+*/
+function parseAssertion(text) {
+	const match = text.match(ASSERT_RE);
+	if (match == null) throw new AssertionParseError(`Invalid expect assertion: ${JSON.stringify(text)}.`);
+	const op = OP_MAP[match[1]];
+	const value = Number.parseFloat(match[2]);
+	switch (match[3]) {
+		case "%": return {
+			op,
+			threshold: value / 100,
+			unit: "%"
+		};
+		case "ms": return {
+			op,
+			threshold: value,
+			unit: "ms"
+		};
+		case "s": return {
+			op,
+			threshold: value * 1e3,
+			unit: "ms"
+		};
+		case "/s": return {
+			op,
+			threshold: value,
+			unit: "/s"
+		};
+		default: return {
+			op,
+			threshold: value,
+			unit: null
+		};
+	}
+}
+/**
+* Compares a measured value against a threshold using a comparison operator.
+* @param actual The measured value.
+* @param op The comparison operator.
+* @param threshold The threshold.
+* @param tolerant Whether `eq` allows a small floating-point tolerance.  Pass
+*                 `false` for exact (count) metrics; defaults to `true` so
+*                 float-normalized thresholds (e.g. `"99.4%"` ->
+*                 `0.9940000000000001`) still match a measured `0.994`.
+* @returns Whether the comparison holds.
+*/
+function compare(actual, op, threshold, tolerant = true) {
+	switch (op) {
+		case "lt": return actual < threshold;
+		case "lte": return actual <= threshold;
+		case "gt": return actual > threshold;
+		case "gte": return actual >= threshold;
+		case "eq": return tolerant ? Math.abs(actual - threshold) <= 1e-9 + 1e-9 * Math.abs(threshold) : actual === threshold;
+	}
+}
+//#endregion
+export { AssertionParseError, compare, parseAssertion };

package/dist/bench/result/expect/evaluate.js

@@ -0,0 +1,128 @@
+import "@js-temporal/polyfill";
+import { AssertionParseError, compare, parseAssertion } from "./assert.js";
+import { metricUnit } from "./metrics.js";
+//#region src/bench/result/expect/evaluate.ts
+/**
+* Parses every assertion in an `expect` block, throwing on the first malformed
+* one.  Run during preflight so that a typo in a CI gate is reported as a
+* configuration error before any load is sent, instead of crashing the run with
+* an uncaught {@link AssertionParseError} after the traffic has already gone out.
+* @param expect The scenario's `expect` block.
+* @throws {AssertionParseError} If an entry has no assertion string or its
+*         assertion cannot be parsed.
+*/
+function validateExpectBlock(expect) {
+	for (const [metric, value] of Object.entries(expect)) {
+		const assertion = typeof value === "string" ? value : value.assert;
+		if (typeof assertion !== "string") throw new AssertionParseError(`The \`expect\` entry for "${metric}" has no assertion string.`);
+		try {
+			parseAssertion(assertion);
+		} catch (error) {
+			if (!(error instanceof AssertionParseError)) throw error;
+			throw new AssertionParseError(`Invalid \`expect\` assertion for "${metric}": ${JSON.stringify(assertion)}.`);
+		}
+	}
+}
+/**
+* Evaluates an `expect` block against measured metrics.
+* @param expect The scenario's `expect` block.
+* @param metrics The measured metrics to evaluate against.
+* @returns The evaluated assertions and whether the gate passed.
+*/
+function evaluateExpect(expect, metrics) {
+	const results = [];
+	for (const [metric, value] of Object.entries(expect)) {
+		const assertion = typeof value === "string" ? value : value.assert;
+		const severity = typeof value === "string" ? "fail" : value.severity ?? "fail";
+		const { op, threshold, unit } = parseAssertion(assertion);
+		const lookup = lookupMetric(metrics, metric);
+		const actual = lookup?.value ?? null;
+		const pass = lookup != null && actual != null && unitCompatible(unit, lookup.unit) && compare(actual, op, threshold, lookup.unit !== "count");
+		results.push({
+			metric,
+			op,
+			threshold,
+			unit,
+			actual,
+			severity,
+			pass
+		});
+	}
+	return {
+		results,
+		passed: results.every((r) => r.severity === "warn" || r.pass)
+	};
+}
+/**
+* Whether an assertion's (normalized) unit is compatible with a metric's
+* natural unit.  A unitless assertion is always compatible.
+*/
+function unitCompatible(assertionUnit, unit) {
+	if (assertionUnit == null) return true;
+	switch (unit) {
+		case "ratio": return assertionUnit === "%";
+		case "ms": return assertionUnit === "ms";
+		case "rate": return assertionUnit === "/s";
+		case "count": return false;
+	}
+}
+function lookupMetric(metrics, metric) {
+	const unit = metricUnit(metric);
+	if (unit == null) return null;
+	return {
+		value: lookupValue(metrics, metric),
+		unit
+	};
+}
+function lookupValue(metrics, metric) {
+	switch (metric) {
+		case "successRate": return metrics.requests.successRate;
+		case "throughputPerSec": return metrics.throughputPerSec;
+		case "deliveryThroughput": return metrics.deliveryThroughputPerSec ?? null;
+		case "errors.total": return sumErrors(metrics.errors);
+		case "errors.4xx": return sumErrors(metrics.errors, {
+			min: 400,
+			max: 500
+		});
+		case "errors.5xx": return sumErrors(metrics.errors, {
+			min: 500,
+			max: 600
+		});
+	}
+	if (metric.startsWith("latency.")) return latencyField(metrics.client.latencyMs, metric.slice(8));
+	if (metric.startsWith("signatureVerification.")) return partialField(metrics.server?.signatureVerificationMs?.overall, metric.slice(22));
+	if (metric.startsWith("queueDrain.")) return partialField(metrics.server?.queue?.drainMs, metric.slice(11));
+	return null;
+}
+function latencyField(latency, key) {
+	switch (key) {
+		case "p50": return latency.p50;
+		case "p95": return latency.p95;
+		case "p99": return latency.p99;
+		case "mean": return latency.mean;
+		case "max": return latency.max;
+		default: return null;
+	}
+}
+function partialField(source, key) {
+	if (source == null) return null;
+	switch (key) {
+		case "p50": return source.p50 ?? null;
+		case "p95": return source.p95 ?? null;
+		case "p99": return source.p99 ?? null;
+		default: return null;
+	}
+}
+/**
+* Sums error counts, optionally restricted to a half-open HTTP status range.
+* The bounds are a single coupled argument so a caller cannot pass one without
+* the other.
+*/
+function sumErrors(errors, range) {
+	let total = 0;
+	for (const error of errors) if (range == null) total += error.count;
+	else if (error.status != null && error.status >= range.min && error.status < range.max) total += error.count;
+	return total;
+}
+//#endregion
+export { evaluateExpect, validateExpectBlock };

package/dist/bench/result/expect/metrics.js

@@ -0,0 +1,34 @@
+import "@js-temporal/polyfill";
+//#region src/bench/result/expect/metrics.ts
+/**
+* Returns the natural unit class of a metric, or `null` if the metric name is
+* not recognized.
+* @param metric The metric name, e.g. `"latency.p95"`.
+*/
+function metricUnit(metric) {
+	switch (metric) {
+		case "successRate": return "ratio";
+		case "throughputPerSec":
+		case "deliveryThroughput": return "rate";
+		case "errors.total":
+		case "errors.4xx":
+		case "errors.5xx": return "count";
+	}
+	if (metric.startsWith("latency.") || metric.startsWith("signatureVerification.") || metric.startsWith("queueDrain.")) return "ms";
+	return null;
+}
+/**
+* Returns the human display unit for a metric (`"%"`, `"ms"`, `"/s"`), or
+* `null` for counts and unknown metrics.
+* @param metric The metric name.
+*/
+function metricDisplayUnit(metric) {
+	switch (metricUnit(metric)) {
+		case "ratio": return "%";
+		case "ms": return "ms";
+		case "rate": return "/s";
+		default: return null;
+	}
+}
+//#endregion
+export { metricDisplayUnit, metricUnit };