@fedify/cli 2.3.0-dev.1214 → 2.3.0-dev.1258

package/dist/bench/result/build.js

@@ -0,0 +1,129 @@
+import "@js-temporal/polyfill";
+import { version } from "../../deno.js";
+import { evaluateExpect } from "./expect/evaluate.js";
+import { REPORT_SCHEMA_ID } from "./schema.js";
+import process from "node:process";
+import { createHash } from "node:crypto";
+import { cpus } from "node:os";
+//#region src/bench/result/build.ts
+/**
+* Assembly of the canonical benchmark report from measured scenario data.
+*
+* The runners produce per-scenario measurements; this module turns each into a
+* {@link ScenarioResult} (evaluating its `expect` block) and assembles the
+* top-level {@link BenchReport} with reproducibility metadata.
+* @since 2.3.0
+* @module
+*/
+/**
+* Builds a scenario result from its resolved definition and measurement,
+* evaluating the `expect` block in the process.
+* @param scenario The resolved scenario.
+* @param measurement The measured client and server metrics.
+* @returns The assembled scenario result.
+*/
+function buildScenarioResult(scenario, measurement) {
+	const { results, passed } = evaluateExpect(scenario.expect, measurement);
+	return {
+		name: scenario.name,
+		type: scenario.type,
+		load: loadSummary(scenario),
+		requests: measurement.requests,
+		throughputPerSec: measurement.throughputPerSec,
+		client: measurement.client,
+		server: measurement.server,
+		errors: measurement.errors,
+		expectations: results,
+		passed: passed && measurement.requests.total > 0,
+		...measurement.histogram ? { histogram: measurement.histogram } : {}
+	};
+}
+/**
+* Assembles the top-level report.  The gate passes only when every scenario
+* passes.
+* @param input The report inputs.
+* @returns The complete report.
+*/
+function buildReport(input) {
+	return {
+		$schema: REPORT_SCHEMA_ID,
+		schemaVersion: 1,
+		tool: {
+			name: "@fedify/cli",
+			version
+		},
+		environment: input.environment,
+		target: input.target,
+		startedAt: input.startedAt,
+		finishedAt: input.finishedAt,
+		suite: input.suite,
+		passed: input.scenarios.every((s) => s.passed),
+		scenarios: input.scenarios
+	};
+}
+/** Detects the current runtime environment for reproducibility metadata. */
+function detectEnvironment() {
+	const g = globalThis;
+	let runtime = "node";
+	let runtimeVersion = process.versions?.node ?? "unknown";
+	if (g.Deno?.version?.deno != null) {
+		runtime = "deno";
+		runtimeVersion = g.Deno.version.deno;
+	} else if (g.Bun?.version != null) {
+		runtime = "bun";
+		runtimeVersion = g.Bun.version;
+	}
+	let cpuCount = 0;
+	try {
+		cpuCount = cpus().length;
+	} catch {
+		cpuCount = 0;
+	}
+	return {
+		runtime,
+		runtimeVersion,
+		os: process.platform,
+		cpuCount
+	};
+}
+/**
+* Computes a stable `sha256:` hash of a resolved configuration, so CI only
+* compares runs from the same configuration.
+* @param config The configuration object to hash.
+* @returns A `sha256:`-prefixed hex digest.
+*/
+function configHash(config) {
+	return `sha256:${createHash("sha256").update(canonicalJson(config)).digest("hex")}`;
+}
+/** A guard against unbounded recursion on pathologically nested input. */
+const MAX_HASH_DEPTH = 100;
+function canonicalJson(value, depth = 0) {
+	if (depth > MAX_HASH_DEPTH) throw new RangeError("Maximum depth exceeded while hashing the config.");
+	if (value === void 0) return "null";
+	if (value === null || typeof value !== "object") return JSON.stringify(value);
+	const toJson = value.toJSON;
+	if (typeof toJson === "function") return canonicalJson(toJson.call(value), depth + 1);
+	if (Array.isArray(value)) return `[${value.map((v) => canonicalJson(v, depth + 1)).join(",")}]`;
+	return `{${Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v, depth + 1)}`).join(",")}}`;
+}
+function loadSummary(scenario) {
+	const { load, durationMs, warmupMs } = scenario;
+	const maxInFlight = load.maxInFlight == null ? {} : { maxInFlight: load.maxInFlight };
+	if (load.kind === "closed") return {
+		model: "closed",
+		concurrency: load.concurrency,
+		durationMs,
+		warmupMs,
+		...maxInFlight
+	};
+	return {
+		model: "open",
+		ratePerSec: load.ratePerSec,
+		arrival: load.arrival,
+		durationMs,
+		warmupMs,
+		...maxInFlight
+	};
+}
+//#endregion
+export { buildReport, buildScenarioResult, configHash, detectEnvironment };

package/dist/bench/result/expect/assert.js

@@ -0,0 +1,74 @@
+import "@js-temporal/polyfill";
+//#region src/bench/result/expect/assert.ts
+const ASSERT_RE = /^\s*(<=|>=|==|=|<|>)\s*(\d+(?:\.\d+)?)\s*(%|ms|s|\/s)?\s*$/;
+const OP_MAP = {
+	"<": "lt",
+	"<=": "lte",
+	">": "gt",
+	">=": "gte",
+	"==": "eq",
+	"=": "eq"
+};
+/** An error raised when an `expect` assertion cannot be parsed. */
+var AssertionParseError = class extends Error {};
+/**
+* Parses an `expect` assertion string.
+* @param text The assertion, e.g. `">= 99%"`.
+* @returns The parsed operator, normalized threshold, and unit.
+* @throws {AssertionParseError} If the assertion cannot be parsed.
+*/
+function parseAssertion(text) {
+	const match = text.match(ASSERT_RE);
+	if (match == null) throw new AssertionParseError(`Invalid expect assertion: ${JSON.stringify(text)}.`);
+	const op = OP_MAP[match[1]];
+	const value = Number.parseFloat(match[2]);
+	switch (match[3]) {
+		case "%": return {
+			op,
+			threshold: value / 100,
+			unit: "%"
+		};
+		case "ms": return {
+			op,
+			threshold: value,
+			unit: "ms"
+		};
+		case "s": return {
+			op,
+			threshold: value * 1e3,
+			unit: "ms"
+		};
+		case "/s": return {
+			op,
+			threshold: value,
+			unit: "/s"
+		};
+		default: return {
+			op,
+			threshold: value,
+			unit: null
+		};
+	}
+}
+/**
+* Compares a measured value against a threshold using a comparison operator.
+* @param actual The measured value.
+* @param op The comparison operator.
+* @param threshold The threshold.
+* @param tolerant Whether `eq` allows a small floating-point tolerance.  Pass
+*                 `false` for exact (count) metrics; defaults to `true` so
+*                 float-normalized thresholds (e.g. `"99.4%"` ->
+*                 `0.9940000000000001`) still match a measured `0.994`.
+* @returns Whether the comparison holds.
+*/
+function compare(actual, op, threshold, tolerant = true) {
+	switch (op) {
+		case "lt": return actual < threshold;
+		case "lte": return actual <= threshold;
+		case "gt": return actual > threshold;
+		case "gte": return actual >= threshold;
+		case "eq": return tolerant ? Math.abs(actual - threshold) <= 1e-9 + 1e-9 * Math.abs(threshold) : actual === threshold;
+	}
+}
+//#endregion
+export { AssertionParseError, compare, parseAssertion };

package/dist/bench/result/expect/evaluate.js

@@ -0,0 +1,128 @@
+import "@js-temporal/polyfill";
+import { AssertionParseError, compare, parseAssertion } from "./assert.js";
+import { metricUnit } from "./metrics.js";
+//#region src/bench/result/expect/evaluate.ts
+/**
+* Parses every assertion in an `expect` block, throwing on the first malformed
+* one.  Run during preflight so that a typo in a CI gate is reported as a
+* configuration error before any load is sent, instead of crashing the run with
+* an uncaught {@link AssertionParseError} after the traffic has already gone out.
+* @param expect The scenario's `expect` block.
+* @throws {AssertionParseError} If an entry has no assertion string or its
+*         assertion cannot be parsed.
+*/
+function validateExpectBlock(expect) {
+	for (const [metric, value] of Object.entries(expect)) {
+		const assertion = typeof value === "string" ? value : value.assert;
+		if (typeof assertion !== "string") throw new AssertionParseError(`The \`expect\` entry for "${metric}" has no assertion string.`);
+		try {
+			parseAssertion(assertion);
+		} catch (error) {
+			if (!(error instanceof AssertionParseError)) throw error;
+			throw new AssertionParseError(`Invalid \`expect\` assertion for "${metric}": ${JSON.stringify(assertion)}.`);
+		}
+	}
+}
+/**
+* Evaluates an `expect` block against measured metrics.
+* @param expect The scenario's `expect` block.
+* @param metrics The measured metrics to evaluate against.
+* @returns The evaluated assertions and whether the gate passed.
+*/
+function evaluateExpect(expect, metrics) {
+	const results = [];
+	for (const [metric, value] of Object.entries(expect)) {
+		const assertion = typeof value === "string" ? value : value.assert;
+		const severity = typeof value === "string" ? "fail" : value.severity ?? "fail";
+		const { op, threshold, unit } = parseAssertion(assertion);
+		const lookup = lookupMetric(metrics, metric);
+		const actual = lookup?.value ?? null;
+		const pass = lookup != null && actual != null && unitCompatible(unit, lookup.unit) && compare(actual, op, threshold, lookup.unit !== "count");
+		results.push({
+			metric,
+			op,
+			threshold,
+			unit,
+			actual,
+			severity,
+			pass
+		});
+	}
+	return {
+		results,
+		passed: results.every((r) => r.severity === "warn" || r.pass)
+	};
+}
+/**
+* Whether an assertion's (normalized) unit is compatible with a metric's
+* natural unit.  A unitless assertion is always compatible.
+*/
+function unitCompatible(assertionUnit, unit) {
+	if (assertionUnit == null) return true;
+	switch (unit) {
+		case "ratio": return assertionUnit === "%";
+		case "ms": return assertionUnit === "ms";
+		case "rate": return assertionUnit === "/s";
+		case "count": return false;
+	}
+}
+function lookupMetric(metrics, metric) {
+	const unit = metricUnit(metric);
+	if (unit == null) return null;
+	return {
+		value: lookupValue(metrics, metric),
+		unit
+	};
+}
+function lookupValue(metrics, metric) {
+	switch (metric) {
+		case "successRate": return metrics.requests.successRate;
+		case "throughputPerSec": return metrics.throughputPerSec;
+		case "deliveryThroughput": return null;
+		case "errors.total": return sumErrors(metrics.errors);
+		case "errors.4xx": return sumErrors(metrics.errors, {
+			min: 400,
+			max: 500
+		});
+		case "errors.5xx": return sumErrors(metrics.errors, {
+			min: 500,
+			max: 600
+		});
+	}
+	if (metric.startsWith("latency.")) return latencyField(metrics.client.latencyMs, metric.slice(8));
+	if (metric.startsWith("signatureVerification.")) return partialField(metrics.server?.signatureVerificationMs?.overall, metric.slice(22));
+	if (metric.startsWith("queueDrain.")) return partialField(metrics.server?.queue?.drainMs, metric.slice(11));
+	return null;
+}
+function latencyField(latency, key) {
+	switch (key) {
+		case "p50": return latency.p50;
+		case "p95": return latency.p95;
+		case "p99": return latency.p99;
+		case "mean": return latency.mean;
+		case "max": return latency.max;
+		default: return null;
+	}
+}
+function partialField(source, key) {
+	if (source == null) return null;
+	switch (key) {
+		case "p50": return source.p50 ?? null;
+		case "p95": return source.p95 ?? null;
+		case "p99": return source.p99 ?? null;
+		default: return null;
+	}
+}
+/**
+* Sums error counts, optionally restricted to a half-open HTTP status range.
+* The bounds are a single coupled argument so a caller cannot pass one without
+* the other.
+*/
+function sumErrors(errors, range) {
+	let total = 0;
+	for (const error of errors) if (range == null) total += error.count;
+	else if (error.status != null && error.status >= range.min && error.status < range.max) total += error.count;
+	return total;
+}
+//#endregion
+export { evaluateExpect, validateExpectBlock };

package/dist/bench/result/expect/metrics.js

@@ -0,0 +1,34 @@
+import "@js-temporal/polyfill";
+//#region src/bench/result/expect/metrics.ts
+/**
+* Returns the natural unit class of a metric, or `null` if the metric name is
+* not recognized.
+* @param metric The metric name, e.g. `"latency.p95"`.
+*/
+function metricUnit(metric) {
+	switch (metric) {
+		case "successRate": return "ratio";
+		case "throughputPerSec":
+		case "deliveryThroughput": return "rate";
+		case "errors.total":
+		case "errors.4xx":
+		case "errors.5xx": return "count";
+	}
+	if (metric.startsWith("latency.") || metric.startsWith("signatureVerification.") || metric.startsWith("queueDrain.")) return "ms";
+	return null;
+}
+/**
+* Returns the human display unit for a metric (`"%"`, `"ms"`, `"/s"`), or
+* `null` for counts and unknown metrics.
+* @param metric The metric name.
+*/
+function metricDisplayUnit(metric) {
+	switch (metricUnit(metric)) {
+		case "ratio": return "%";
+		case "ms": return "ms";
+		case "rate": return "/s";
+		default: return null;
+	}
+}
+//#endregion
+export { metricDisplayUnit, metricUnit };

package/dist/bench/result/schema.js

@@ -0,0 +1,15 @@
+import "@js-temporal/polyfill";
+//#region src/bench/result/schema.ts
+/**
+* The embedded JSON Schema (draft 2020-12) for benchmark report output.
+*
+* Like the scenario schema, this object is the runtime copy and is published,
+* byte-for-byte, as *schema/bench/report-v1.json*; a drift guard keeps the two
+* in sync.  The matching TypeScript types live in {@link ./model.ts}.
+* @since 2.3.0
+* @module
+*/
+/** The hosted URL that serves the report schema. */
+const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v1.json";
+//#endregion
+export { REPORT_SCHEMA_ID };

package/dist/bench/safety/gate.js

@@ -0,0 +1,54 @@
+import "@js-temporal/polyfill";
+import { classifyTarget } from "./tiers.js";
+//#region src/bench/safety/gate.ts
+/**
+* The client-side safety gate.
+*
+* A run is allowed without friction when the target is loopback/private or
+* advertises benchmark mode (the operator's "not production" assertion).  Only
+* a public target that does not advertise benchmark mode is gated, behind an
+* explicit `--allow-unsafe-target`.  There is no interactive prompt, so the
+* flag is mandatory in CI and any non-TTY context.  An inspection-only run
+* (the `dryRun` flag) sends no load, so it bypasses the gate.
+* @since 2.3.0
+* @module
+*/
+/** An error raised when a target is refused by the safety gate. */
+var UnsafeTargetError = class extends Error {};
+/**
+* Asserts that a target may be benchmarked, throwing otherwise.
+* @param context The gate decision inputs.
+* @throws {UnsafeTargetError} If the target is public, does not advertise
+*         benchmark mode, and `--allow-unsafe-target` was not given.
+*/
+function assertTargetAllowed(context) {
+	if (context.dryRun) return;
+	if (context.tier !== "public") return;
+	if (context.benchmarkMode) return;
+	if (context.allowUnsafe) return;
+	throw new UnsafeTargetError("Refusing to benchmark a public target that does not advertise benchmark mode.  If you control this target, pass --allow-unsafe-target (mandatory in CI and any non-interactive context).");
+}
+/**
+* Asserts that a resolved inbox URL — the actual destination of signed
+* benchmark load — may be sent to.  The suite's `target` is gated separately by
+* {@link assertTargetAllowed}; this catches a destination that differs from it
+* (a public `recipient`, or an explicit `inbox:` URL), so production cannot be
+* benchmarked through the back door.
+*
+* A destination is allowed when it is loopback or private, or shares the gated
+* target's host while the target advertises benchmark mode (inheriting its
+* gate), or `--allow-unsafe-target` is given.  Because the destination's server
+* dereferences the synthetic actor while verifying signatures, a non-loopback
+* destination additionally requires an advertised, reachable synthetic host.
+* @param url The resolved inbox URL.
+* @param context The destination gate inputs.
+* @throws {UnsafeTargetError} If the destination is refused.
+*/
+function assertInboxDestinationAllowed(url, context) {
+	const tier = classifyTarget(url);
+	const inheritsTargetGate = url.origin === context.targetOrigin && context.targetBenchmarkMode;
+	if (tier === "public" && !inheritsTargetGate && !context.allowUnsafe) throw new UnsafeTargetError(`Refusing to send benchmark load to ${url.href}: it is a public inbox that is neither part of the benchmarked target nor covered by benchmark mode.  Pass --allow-unsafe-target to override.`);
+	if (tier !== "loopback" && !context.advertised) throw new UnsafeTargetError(`Refusing to send signed benchmark load to ${url.href}: the synthetic actor server is unreachable from a non-loopback inbox.  Pass --advertise-host with an address it can reach.`);
+}
+//#endregion
+export { UnsafeTargetError, assertInboxDestinationAllowed, assertTargetAllowed };

package/dist/bench/safety/tiers.js

@@ -0,0 +1,41 @@
+import "@js-temporal/polyfill";
+//#region src/bench/safety/tiers.ts
+/**
+* Classifies a target URL into a risk tier from its host.
+* @param target The target URL.
+* @returns The risk tier.
+*/
+function classifyTarget(target) {
+	let host = target.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
+	if (host.endsWith(".")) host = host.slice(0, -1);
+	if (host === "localhost" || host.endsWith(".localhost")) return "loopback";
+	if (host.endsWith(".local")) return "private";
+	if (isIpv4(host)) return classifyIpv4(host);
+	if (host.includes(":")) return classifyIpv6(host);
+	return "public";
+}
+function isIpv4(host) {
+	const match = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+	return match != null && match.slice(1).every((octet) => Number(octet) <= 255);
+}
+function classifyIpv4(host) {
+	if (host === "0.0.0.0" || /^127\./.test(host)) return "loopback";
+	if (/^10\./.test(host) || /^192\.168\./.test(host) || /^172\.(1[6-9]|2\d|3[01])\./.test(host) || /^169\.254\./.test(host)) return "private";
+	return "public";
+}
+function classifyIpv6(host) {
+	if (host === "::1") return "loopback";
+	const dotted = host.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
+	if (dotted != null && isIpv4(dotted[1])) return classifyIpv4(dotted[1]);
+	const hex = host.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
+	if (hex != null) {
+		const hi = Number.parseInt(hex[1], 16);
+		const lo = Number.parseInt(hex[2], 16);
+		return classifyIpv4(`${hi >> 8 & 255}.${hi & 255}.${lo >> 8 & 255}.${lo & 255}`);
+	}
+	if (/^f[cd][0-9a-f]*:/.test(host)) return "private";
+	if (/^fe[89ab][0-9a-f]*:/.test(host)) return "private";
+	return "public";
+}
+//#endregion
+export { classifyTarget };

package/dist/bench/scenario/coerce.js

@@ -0,0 +1,24 @@
+import "@js-temporal/polyfill";
+//#region src/bench/scenario/coerce.ts
+/**
+* Scalar-or-list coercion used throughout the scenario format, where many
+* fields (`recipient`, `seed`, `collection`, `type`, and so on) accept either a
+* single value or a list of values so the common single-value case stays terse.
+* @since 2.3.0
+* @module
+*/
+/**
+* Normalizes a scalar-or-list value into an array.
+*
+* A single value becomes a one-element array, an array is shallow-copied, and
+* `null`/`undefined` becomes an empty array.
+* @typeParam T The element type.
+* @param value A single value, a list of values, or nothing.
+* @returns A new array of values.
+*/
+function asList(value) {
+	if (value == null) return [];
+	return Array.isArray(value) ? [...value] : [value];
+}
+//#endregion
+export { asList };

package/dist/bench/scenario/errors.js

@@ -0,0 +1,36 @@
+import "@js-temporal/polyfill";
+//#region src/bench/scenario/errors.ts
+/** An error raised when a scenario suite fails schema validation. */
+var SuiteValidationError = class extends Error {
+	/** The individual validation problems, most specific first. */
+	problems;
+	constructor(problems, source) {
+		super(formatMessage(problems, source));
+		this.name = "SuiteValidationError";
+		this.problems = problems;
+	}
+};
+function formatMessage(problems, source) {
+	const where = source == null ? "scenario suite" : source;
+	if (problems.length === 0) return `Invalid ${where}.`;
+	return `Invalid ${where}:\n${dedupe(problems).map((problem) => {
+		return `  - ${problem.instanceLocation === "#" || problem.instanceLocation === "" ? "(root)" : problem.instanceLocation.replace(/^#/, "")}: ${problem.error}`;
+	}).join("\n")}`;
+}
+function dedupe(problems) {
+	const seen = /* @__PURE__ */ new Set();
+	const result = [];
+	const sorted = [...problems].sort((a, b) => depth(b.instanceLocation) - depth(a.instanceLocation));
+	for (const problem of sorted) {
+		const key = JSON.stringify([problem.instanceLocation, problem.error]);
+		if (seen.has(key)) continue;
+		seen.add(key);
+		result.push(problem);
+	}
+	return result;
+}
+function depth(instanceLocation) {
+	return (instanceLocation.match(/\//g) ?? []).length;
+}
+//#endregion
+export { SuiteValidationError };

package/dist/bench/scenario/load.js

@@ -0,0 +1,69 @@
+import "@js-temporal/polyfill";
+import { defaultHelpers } from "../template/helpers.js";
+import { renderTemplates } from "../template/template.js";
+import { readFile } from "node:fs/promises";
+import { parse } from "yaml";
+//#region src/bench/scenario/load.ts
+/**
+* Reading and parsing scenario suite files.
+*
+* Files may be written in YAML or JSON; because YAML is a superset of JSON, a
+* single YAML parser handles both, and YAML anchors/aliases are available for
+* in-document reuse.
+* @since 2.3.0
+* @module
+*/
+/**
+* Parses scenario suite text (YAML or JSON) into an untyped value.
+* @param text The file contents.
+* @returns The parsed value, to be validated with `validateSuite()`.
+*/
+function parseSuiteText(text) {
+	return parse(text);
+}
+/**
+* Reads and parses a scenario suite file.
+* @param path The path to the suite file.
+* @returns The parsed value, to be validated with `validateSuite()`.
+*/
+async function loadSuiteFile(path) {
+	return parseSuiteText(await readFile(path, { encoding: "utf-8" }));
+}
+/**
+* Expands `${{ ... }}` templates in a parsed suite.
+*
+* The context exposes `target` (its `host`, `hostname`, `port`, `origin`,
+* `href`, and `protocol`) plus the default helpers.  The target comes from the
+* `--target` override or the suite's own `target`, neither of which is
+* templated.
+* @param raw The parsed suite value.
+* @param targetOverride A target URL from `--target`, if any.
+* @returns The suite with templates expanded.
+*/
+function renderSuiteTemplates(raw, targetOverride) {
+	const target = targetOverride ?? suiteTarget(raw);
+	const values = {};
+	if (target != null) try {
+		const url = new URL(target);
+		values.target = {
+			host: url.host,
+			hostname: url.hostname,
+			port: url.port,
+			origin: url.origin,
+			href: url.href,
+			protocol: url.protocol.replace(/:$/, "")
+		};
+	} catch {}
+	return renderTemplates(raw, {
+		values,
+		helpers: defaultHelpers()
+	});
+}
+function suiteTarget(raw) {
+	if (raw != null && typeof raw === "object" && "target" in raw) {
+		const target = raw.target;
+		return typeof target === "string" ? target : void 0;
+	}
+}
+//#endregion
+export { loadSuiteFile, renderSuiteTemplates };