@fedify/cli 2.3.0-dev.1347 → 2.3.0-dev.1361

package/dist/bench/mod.js

@@ -1,11 +1,4 @@
 import "@js-temporal/polyfill";
+import "./action.js";
 import "./command.js";
-import runBench$1 from "./action.js";
-import { runBenchCompare } from "./compare.js";
-//#region src/bench/mod.ts
-/** Runs a parsed benchmark command. */
-function runBench(command) {
-	return command.mode === "compare" ? runBenchCompare(command) : runBench$1(command);
-}
-//#endregion
-export { runBench };
+export {};

package/dist/bench/render/markdown.js

@@ -32,7 +32,6 @@ function renderScenario(scenario) {
 	lines.push(`| Requests | ${formatNumber(r.total)} |`);
 	lines.push(`| Success rate | ${formatPercent(r.successRate)} |`);
 	lines.push(`| Throughput | ${formatNumber(scenario.throughputPerSec)}/s |`);
-	if (scenario.deliveryThroughputPerSec != null) lines.push(`| Delivery throughput | ${formatNumber(scenario.deliveryThroughputPerSec)}/s |`);
 	const l = scenario.client.latencyMs;
 	lines.push(`| Latency p50 | ${formatNumber(l.p50)}ms |`);
 	lines.push(`| Latency p95 | ${formatNumber(l.p95)}ms |`);

package/dist/bench/render/text.js

@@ -28,7 +28,6 @@ function renderScenario(scenario) {
 	const r = scenario.requests;
 	lines.push(`  Requests: ${formatNumber(r.total)}  (ok ${formatNumber(r.ok)}, failed ${formatNumber(r.failed)}, success ${formatPercent(r.successRate)})`);
 	lines.push(`  Throughput: ${formatNumber(scenario.throughputPerSec)} req/s`);
-	if (scenario.deliveryThroughputPerSec != null) lines.push(`  Delivery throughput: ${formatNumber(scenario.deliveryThroughputPerSec)} deliveries/s`);
 	const l = scenario.client.latencyMs;
 	lines.push(`  Client latency (ms): p50 ${formatNumber(l.p50)}  p95 ${formatNumber(l.p95)}  p99 ${formatNumber(l.p99)}  mean ${formatNumber(l.mean)}  max ${formatNumber(l.max)}`);
 	if (scenario.server?.signatureVerificationMs != null) lines.push(`  Server signature verification (ms): ${describePartial(scenario.server.signatureVerificationMs.overall)}`);

package/dist/bench/result/build.js

@@ -3,8 +3,8 @@ import { version } from "../../deno.js";
 import { evaluateExpect } from "./expect/evaluate.js";
 import { REPORT_SCHEMA_ID } from "./schema.js";
 import process from "node:process";
-import { cpus } from "node:os";
 import { createHash } from "node:crypto";
+import { cpus } from "node:os";
 //#region src/bench/result/build.ts
 /**
 * Assembly of the canonical benchmark report from measured scenario data.
@@ -23,25 +23,19 @@ import { createHash } from "node:crypto";
 * @returns The assembled scenario result.
 */
 function buildScenarioResult(scenario, measurement) {
-	const measurements = Array.isArray(measurement) ? measurement : [measurement];
-	if (measurements.length < 1) throw new RangeError("At least one scenario measurement is required.");
-	const aggregate = measurements.length === 1 ? measurements[0] : aggregateMeasurements(measurements);
-	const { results, passed } = evaluateExpect(scenario.expect, aggregate);
+	const { results, passed } = evaluateExpect(scenario.expect, measurement);
 	return {
 		name: scenario.name,
 		type: scenario.type,
 		load: loadSummary(scenario),
-		requests: aggregate.requests,
-		throughputPerSec: aggregate.throughputPerSec,
-		...aggregate.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: aggregate.deliveryThroughputPerSec },
-		client: aggregate.client,
-		server: aggregate.server,
-		errors: aggregate.errors,
+		requests: measurement.requests,
+		throughputPerSec: measurement.throughputPerSec,
+		client: measurement.client,
+		server: measurement.server,
+		errors: measurement.errors,
 		expectations: results,
-		passed: passed && measurements.every((m) => m.requests.total > 0),
-		runCount: measurements.length,
-		...measurements.length > 1 ? { runs: measurements.map((m, index) => runResult(index + 1, m)) } : {},
-		...aggregate.histogram ? { histogram: aggregate.histogram } : {}
+		passed: passed && measurement.requests.total > 0,
+		...measurement.histogram ? { histogram: measurement.histogram } : {}
 	};
 }
 /**
@@ -53,7 +47,7 @@ function buildScenarioResult(scenario, measurement) {
 function buildReport(input) {
 	return {
 		$schema: REPORT_SCHEMA_ID,
-		schemaVersion: 3,
+		schemaVersion: 1,
 		tool: {
 			name: "@fedify/cli",
 			version
@@ -67,123 +61,6 @@ function buildReport(input) {
 		scenarios: input.scenarios
 	};
 }
-function aggregateMeasurements(measurements) {
-	const errors = sumErrorBuckets(measurements.flatMap((m) => m.errors));
-	const total = sum(measurements.map((m) => m.requests.total));
-	const ok = sum(measurements.map((m) => m.requests.ok));
-	const failed = sum(measurements.map((m) => m.requests.failed));
-	const delivery = medianPresent(measurements.map((m) => m.deliveryThroughputPerSec));
-	return {
-		requests: {
-			total,
-			ok,
-			failed,
-			successRate: Math.min(...measurements.map((m) => m.requests.successRate))
-		},
-		throughputPerSec: median(measurements.map((m) => m.throughputPerSec)),
-		...delivery == null ? {} : { deliveryThroughputPerSec: delivery },
-		client: { latencyMs: {
-			p50: median(measurements.map((m) => m.client.latencyMs.p50)),
-			p95: median(measurements.map((m) => m.client.latencyMs.p95)),
-			p99: median(measurements.map((m) => m.client.latencyMs.p99)),
-			mean: median(measurements.map((m) => m.client.latencyMs.mean)),
-			max: median(measurements.map((m) => m.client.latencyMs.max))
-		} },
-		server: aggregateServer(measurements.map((m) => m.server)),
-		errors
-	};
-}
-function runResult(run, measurement) {
-	return {
-		run,
-		requests: measurement.requests,
-		throughputPerSec: measurement.throughputPerSec,
-		...measurement.deliveryThroughputPerSec == null ? {} : { deliveryThroughputPerSec: measurement.deliveryThroughputPerSec },
-		client: measurement.client,
-		server: measurement.server,
-		errors: measurement.errors,
-		...measurement.histogram ? { histogram: measurement.histogram } : {}
-	};
-}
-function aggregateServer(servers) {
-	const present = servers.filter((s) => s != null);
-	if (present.length !== servers.length) return null;
-	const signature = aggregateSignatureVerification(present);
-	const queue = aggregateQueue(present);
-	return {
-		...signature == null ? {} : { signatureVerificationMs: signature },
-		...queue == null ? {} : { queue }
-	};
-}
-function aggregateSignatureVerification(servers) {
-	const values = servers.map((s) => s.signatureVerificationMs).filter((s) => s != null);
-	if (values.length !== servers.length) return null;
-	const standards = /* @__PURE__ */ new Set();
-	for (const value of values) for (const key of Object.keys(value.byStandard ?? {})) standards.add(key);
-	const byStandard = {};
-	for (const standard of standards) byStandard[standard] = aggregatePartial(values.map((v) => v.byStandard?.[standard]), "present");
-	return {
-		overall: aggregatePartial(values.map((v) => v.overall)),
-		...Object.keys(byStandard).length < 1 ? {} : { byStandard }
-	};
-}
-function aggregateQueue(servers) {
-	const values = servers.map((s) => s.queue).filter((q) => q != null);
-	if (values.length !== servers.length) return null;
-	const drainMs = aggregatePartial(values.map((v) => v.drainMs));
-	const depths = values.map((v) => v.depthMax);
-	return {
-		...hasPartial(drainMs) ? { drainMs } : {},
-		...depths.every(isNumber) ? { depthMax: Math.max(...depths) } : {}
-	};
-}
-function aggregatePartial(values, mode = "complete") {
-	return {
-		...partialField(values, "p50", mode),
-		...partialField(values, "p95", mode),
-		...partialField(values, "p99", mode)
-	};
-}
-function partialField(values, key, mode) {
-	const fieldValues = values.map((v) => v?.[key]);
-	if (mode === "present") {
-		const present = fieldValues.filter(isNumber);
-		return present.length < 1 ? {} : { [key]: median(present) };
-	}
-	return fieldValues.every(isNumber) ? { [key]: median(fieldValues) } : {};
-}
-function hasPartial(value) {
-	return value.p50 != null || value.p95 != null || value.p99 != null;
-}
-function sumErrorBuckets(errors) {
-	const buckets = /* @__PURE__ */ new Map();
-	for (const error of errors) {
-		const key = `${error.kind}|${error.status ?? ""}|${error.reason}`;
-		const previous = buckets.get(key);
-		buckets.set(key, {
-			...error,
-			count: (previous?.count ?? 0) + error.count
-		});
-	}
-	return [...buckets.values()].sort((a, b) => b.count - a.count);
-}
-function medianPresent(values) {
-	const present = values.filter(isNumber);
-	return present.length < 1 ? null : median(present);
-}
-function median(values) {
-	if (values.length < 1) return 0;
-	const sorted = [...values].sort((a, b) => a - b);
-	const middle = Math.floor(sorted.length / 2);
-	if (sorted.length % 2 === 1) return sorted[middle];
-	return (sorted[middle - 1] + sorted[middle]) / 2;
-}
-function sum(values) {
-	return values.reduce((a, b) => a + b, 0);
-}
-function isNumber(value) {
-	return typeof value === "number" && Number.isFinite(value);
-}
 /** Detects the current runtime environment for reproducibility metadata. */
 function detectEnvironment() {
 	const g = globalThis;

package/dist/bench/result/expect/evaluate.js

@@ -78,7 +78,7 @@ function lookupValue(metrics, metric) {
 	switch (metric) {
 		case "successRate": return metrics.requests.successRate;
 		case "throughputPerSec": return metrics.throughputPerSec;
-		case "deliveryThroughput": return metrics.deliveryThroughputPerSec ?? null;
+		case "deliveryThroughput": return null;
 		case "errors.total": return sumErrors(metrics.errors);
 		case "errors.4xx": return sumErrors(metrics.errors, {
 			min: 400,

package/dist/bench/result/schema.js

@@ -4,362 +4,12 @@ import "@js-temporal/polyfill";
 * The embedded JSON Schema (draft 2020-12) for benchmark report output.
 *
 * Like the scenario schema, this object is the runtime copy and is published,
-* byte-for-byte, as files under *schema/bench/*; a drift guard keeps the two in
-* sync.  The matching TypeScript types live in {@link ./model.ts}.
+* byte-for-byte, as *schema/bench/report-v1.json*; a drift guard keeps the two
+* in sync.  The matching TypeScript types live in {@link ./model.ts}.
 * @since 2.3.0
 * @module
 */
 /** The hosted URL that serves the report schema. */
-const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v3.json";
-/** The hosted URL for the version 2 report schema. */
-const REPORT_SCHEMA_V2_ID = "https://json-schema.fedify.dev/bench/report-v2.json";
-/** The benchmark report JSON Schema (draft 2020-12). */
-const reportSchemaV1 = {
-	$schema: "https://json-schema.org/draft/2020-12/schema",
-	$id: "https://json-schema.fedify.dev/bench/report-v1.json",
-	title: "Fedify benchmark report",
-	type: "object",
-	additionalProperties: false,
-	required: [
-		"schemaVersion",
-		"tool",
-		"environment",
-		"target",
-		"startedAt",
-		"finishedAt",
-		"suite",
-		"passed",
-		"scenarios"
-	],
-	properties: {
-		$schema: { type: "string" },
-		schemaVersion: { const: 1 },
-		tool: {
-			type: "object",
-			additionalProperties: false,
-			required: ["name", "version"],
-			properties: {
-				name: { type: "string" },
-				version: { type: "string" }
-			}
-		},
-		environment: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"runtime",
-				"runtimeVersion",
-				"os",
-				"cpuCount"
-			],
-			properties: {
-				runtime: { type: "string" },
-				runtimeVersion: { type: "string" },
-				os: { type: "string" },
-				cpuCount: {
-					type: "integer",
-					minimum: 0
-				}
-			}
-		},
-		target: {
-			type: "object",
-			additionalProperties: false,
-			required: ["url", "statsAvailable"],
-			properties: {
-				url: { type: "string" },
-				fedifyVersion: { type: ["string", "null"] },
-				statsAvailable: { type: "boolean" }
-			}
-		},
-		startedAt: { type: "string" },
-		finishedAt: { type: "string" },
-		suite: {
-			type: "object",
-			additionalProperties: false,
-			required: ["configHash"],
-			properties: {
-				name: { type: "string" },
-				configHash: { type: "string" }
-			}
-		},
-		passed: { type: "boolean" },
-		scenarios: {
-			type: "array",
-			items: { $ref: "#/$defs/scenarioResult" }
-		}
-	},
-	$defs: {
-		latencyMs: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"p50",
-				"p95",
-				"p99",
-				"mean",
-				"max"
-			],
-			properties: {
-				p50: { type: "number" },
-				p95: { type: "number" },
-				p99: { type: "number" },
-				mean: { type: "number" },
-				max: { type: "number" }
-			}
-		},
-		partialLatencyMs: {
-			type: "object",
-			additionalProperties: false,
-			properties: {
-				p50: { type: "number" },
-				p95: { type: "number" },
-				p99: { type: "number" }
-			}
-		},
-		loadSummary: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"model",
-				"durationMs",
-				"warmupMs"
-			],
-			properties: {
-				model: { enum: ["open", "closed"] },
-				ratePerSec: { type: "number" },
-				arrival: { type: "string" },
-				concurrency: { type: "integer" },
-				durationMs: { type: "number" },
-				warmupMs: { type: "number" },
-				maxInFlight: { type: "integer" }
-			},
-			oneOf: [{
-				properties: { model: { const: "open" } },
-				required: ["ratePerSec", "arrival"],
-				not: { required: ["concurrency"] }
-			}, {
-				properties: { model: { const: "closed" } },
-				required: ["concurrency"],
-				not: { anyOf: [{ required: ["ratePerSec"] }, { required: ["arrival"] }] }
-			}]
-		},
-		requestSummary: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"total",
-				"ok",
-				"failed",
-				"successRate"
-			],
-			properties: {
-				total: {
-					type: "integer",
-					minimum: 0
-				},
-				ok: {
-					type: "integer",
-					minimum: 0
-				},
-				failed: {
-					type: "integer",
-					minimum: 0
-				},
-				successRate: {
-					type: "number",
-					minimum: 0,
-					maximum: 1
-				}
-			}
-		},
-		clientMetrics: {
-			type: "object",
-			additionalProperties: false,
-			required: ["latencyMs"],
-			properties: { latencyMs: { $ref: "#/$defs/latencyMs" } }
-		},
-		serverMetrics: {
-			type: "object",
-			additionalProperties: false,
-			properties: {
-				signatureVerificationMs: {
-					type: "object",
-					additionalProperties: false,
-					required: ["overall"],
-					properties: {
-						overall: { $ref: "#/$defs/partialLatencyMs" },
-						byStandard: {
-							type: "object",
-							additionalProperties: { $ref: "#/$defs/partialLatencyMs" }
-						}
-					}
-				},
-				queue: {
-					type: "object",
-					additionalProperties: false,
-					properties: {
-						drainMs: { $ref: "#/$defs/partialLatencyMs" },
-						depthMax: { type: "number" }
-					}
-				}
-			}
-		},
-		errorBucket: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"kind",
-				"reason",
-				"count"
-			],
-			properties: {
-				kind: { type: "string" },
-				status: { type: "integer" },
-				reason: { type: "string" },
-				count: {
-					type: "integer",
-					minimum: 0
-				}
-			}
-		},
-		expectResult: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"metric",
-				"op",
-				"threshold",
-				"unit",
-				"actual",
-				"severity",
-				"pass"
-			],
-			properties: {
-				metric: { type: "string" },
-				op: { enum: [
-					"lt",
-					"lte",
-					"gt",
-					"gte",
-					"eq"
-				] },
-				threshold: { type: "number" },
-				unit: { type: ["string", "null"] },
-				actual: { type: ["number", "null"] },
-				severity: { enum: ["warn", "fail"] },
-				pass: { type: "boolean" }
-			}
-		},
-		scenarioResult: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"name",
-				"type",
-				"load",
-				"requests",
-				"throughputPerSec",
-				"client",
-				"server",
-				"errors",
-				"expectations",
-				"passed"
-			],
-			properties: {
-				name: { type: "string" },
-				type: { enum: [
-					"inbox",
-					"webfinger",
-					"actor",
-					"object",
-					"fanout",
-					"collection",
-					"failure",
-					"mixed"
-				] },
-				load: { $ref: "#/$defs/loadSummary" },
-				requests: { $ref: "#/$defs/requestSummary" },
-				throughputPerSec: { type: "number" },
-				client: { $ref: "#/$defs/clientMetrics" },
-				server: { anyOf: [{ $ref: "#/$defs/serverMetrics" }, { type: "null" }] },
-				errors: {
-					type: "array",
-					items: { $ref: "#/$defs/errorBucket" }
-				},
-				expectations: {
-					type: "array",
-					items: { $ref: "#/$defs/expectResult" }
-				},
-				passed: { type: "boolean" },
-				histogram: { $ref: "#/$defs/serializedHistogram" }
-			}
-		},
-		serializedHistogram: {
-			type: "object",
-			additionalProperties: false,
-			required: [
-				"version",
-				"subBucketCount",
-				"count",
-				"zeroCount",
-				"min",
-				"max",
-				"sum",
-				"indices",
-				"counts"
-			],
-			properties: {
-				version: { const: 1 },
-				subBucketCount: {
-					type: "integer",
-					minimum: 1
-				},
-				count: {
-					type: "integer",
-					minimum: 0
-				},
-				zeroCount: {
-					type: "integer",
-					minimum: 0
-				},
-				min: { type: "number" },
-				max: { type: "number" },
-				sum: { type: "number" },
-				indices: {
-					type: "array",
-					items: { type: "integer" }
-				},
-				counts: {
-					type: "array",
-					items: {
-						type: "integer",
-						minimum: 0
-					}
-				}
-			}
-		}
-	}
-};
-/** The benchmark report JSON Schema (draft 2020-12). */
-const reportSchemaV2 = {
-	...reportSchemaV1,
-	$id: REPORT_SCHEMA_V2_ID,
-	properties: {
-		...reportSchemaV1.properties,
-		schemaVersion: { const: 2 }
-	},
-	$defs: {
-		...reportSchemaV1.$defs,
-		scenarioResult: {
-			...reportSchemaV1.$defs.scenarioResult,
-			properties: {
-				...reportSchemaV1.$defs.scenarioResult.properties,
-				deliveryThroughputPerSec: { type: "number" }
-			}
-		}
-	}
-};
-({ ...reportSchemaV2 }), { ...reportSchemaV2.properties }, { ...reportSchemaV2.$defs }, { ...reportSchemaV2.$defs.scenarioResult }, [...reportSchemaV2.$defs.scenarioResult.required], { ...reportSchemaV2.$defs.scenarioResult.properties };
+const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v1.json";
 //#endregion
 export { REPORT_SCHEMA_ID };

package/dist/bench/safety/gate.js

@@ -20,9 +20,8 @@ function assertTargetAllowed(context) {
 *
 * The override is only meaningful for a public target that does not advertise
 * benchmark mode.  In that caution tier, the operator must name the target on
-* the command line for this run and must explicitly set load, duration, and
-* runs, so the built-in defaults cannot accidentally create a long public
-* benchmark.
+* the command line for this run and must explicitly set load and duration, so
+* the built-in defaults cannot accidentally create a long public benchmark.
 * @param context The unsafe override decision inputs.
 * @throws {UnsafeTargetError} If the unsafe override is too broad.
 */
@@ -32,7 +31,6 @@ function assertUnsafeOverrideAllowed(context) {
 	for (const scenario of context.scenarios) {
 		if (!scenario.explicitLoad) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark load default.  Set rate or concurrency explicitly before using --allow-unsafe-target against a public target.`);
 		if (!scenario.explicitDuration) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark duration default.  Set duration explicitly before using --allow-unsafe-target against a public target.`);
-		if (!scenario.explicitRuns) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark runs default.  Set runs explicitly before using --allow-unsafe-target against a public target.`);
 	}
 }
 /**

package/dist/bench/scenario/normalize.js

@@ -19,7 +19,7 @@ const DEFAULT_DURATION_MS = 6e4;
 const DEFAULT_WARMUP_MS = 0;
 const DEFAULT_RATE_PER_SEC = 50;
 const DEFAULT_SIGNING = "pipeline";
-const DEFAULT_RUNS = 3;
+const DEFAULT_RUNS = 1;
 /** An error raised when a suite cannot be normalized. */
 var SuiteNormalizeError = class extends Error {};
 /**
@@ -57,6 +57,7 @@ function resolveScenario(scenario, suite) {
 	const warmupMs = resolveDuration(scenario.warmup ?? defaults.warmup, DEFAULT_WARMUP_MS);
 	if (warmupMs >= durationMs) throw new SuiteNormalizeError(`Scenario "${scenario.name}": warmup (${warmupMs}ms) must be shorter than duration (${durationMs}ms); otherwise no requests are measured.`);
 	const runs = scenario.runs ?? defaults.runs ?? DEFAULT_RUNS;
+	if (runs > 1) throw new SuiteNormalizeError(`Scenario "${scenario.name}": multiple runs (runs > 1) are not yet implemented in fedify bench; set runs to 1.`);
 	return {
 		name: scenario.name,
 		type: scenario.type,