@fedify/cli 2.3.0-dev.1281 → 2.3.0-dev.1336

package/dist/bench/action.js

@@ -4,6 +4,7 @@ import { describeError } from "../utils.js";
 import { buildFleet } from "./actor/fleet.js";
 import { convertUrlIfHandle } from "../webfinger/lib.js";
 import { discoverInbox, selectInbox } from "./discovery/discover.js";
+import { actorUrlsFromRecipients, objectUrlsFromSource } from "./scenarios/object-discovery.js";
 import { validateExpectBlock } from "./result/expect/evaluate.js";
 import { buildReport, buildScenarioResult, configHash, detectEnvironment } from "./result/build.js";
 import { probeBenchmarkMode } from "./discovery/probe.js";
@@ -13,13 +14,11 @@ import { normalizeSuite } from "./scenario/normalize.js";
 import { validateSuite } from "./scenario/validate.js";
 import { UnsafeTargetError, assertInboxDestinationAllowed, assertTargetAllowed, assertUnsafeOverrideAllowed } from "./safety/gate.js";
 import { classifyResolvedTarget } from "./safety/tiers.js";
-import { runnerFor } from "./scenarios/registry.js";
 import { resolveAdvertiseHost, spawnSyntheticServer } from "./server/synthetic.js";
+import { runnerFor } from "./scenarios/registry.js";
 import { writeFile } from "node:fs/promises";
 import process from "node:process";
 //#region src/bench/action.ts
-/** The scenario types that need the synthetic actor/key server. */
-const SIGNED_TYPES = new Set(["inbox"]);
 /**
 * Runs the `fedify bench` command: load and validate the suite, gate the
 * target, run each scenario, and render the report.  The process exits 0 when
@@ -34,7 +33,10 @@ async function runBench(command, deps = {}) {
 	});
 	const writeOutput = deps.writeOutput ?? defaultWriteOutput;
 	const log = deps.log ?? ((message) => process.stderr.write(`${message}\n`));
-	const fetchImpl = withUserAgent(deps.fetch ?? fetch, command.userAgent);
+	const signal = deps.signal;
+	const fetchImpl = withUserAgent(withAbortSignal(deps.fetch ?? fetch, signal), command.userAgent);
+	const explicitCliTarget = command.explicitCliTarget ?? command.target != null;
+	throwIfAborted(signal);
 	let validated;
 	let suite;
 	try {
@@ -45,11 +47,12 @@ async function runBench(command, deps = {}) {
 		exit(2);
 		return;
 	}
+	throwIfAborted(signal);
 	let runners;
 	try {
 		runners = suite.scenarios.map((scenario) => {
 			const runner = runnerFor(scenario.type);
-			runner.validate?.(scenario);
+			runner.validate?.(scenario, { scenarios: suite.scenarios });
 			validateExpectBlock(scenario.expect);
 			return runner;
 		});
@@ -59,14 +62,17 @@ async function runBench(command, deps = {}) {
 		exit(2);
 		return;
 	}
+	throwIfAborted(signal);
 	const tier = await classifyResolvedTarget(suite.target, deps.resolveTargetAddresses);
+	throwIfAborted(signal);
 	const probe = await probeBenchmarkMode(suite.target, fetchImpl);
+	throwIfAborted(signal);
 	try {
 		if (!command.dryRun) assertUnsafeOverrideAllowed({
 			tier,
 			benchmarkMode: probe.benchmarkMode,
 			allowUnsafe: command.allowUnsafeTarget,
-			explicitCliTarget: command.target != null,
+			explicitCliTarget,
 			scenarios: unsafeOverrideScenarios(validated)
 		});
 		assertTargetAllowed({
@@ -106,17 +112,35 @@ async function runBench(command, deps = {}) {
 			targetOrigin: suite.target.origin,
 			targetBenchmarkMode: probe.benchmarkMode,
 			allowUnsafe: command.allowUnsafeTarget,
-			explicitCliTarget: command.target != null,
+			explicitCliTarget,
+			destinationTier,
+			defaults: validated.defaults
+		});
+	};
+	const assertDestinationWithoutSyntheticServerAllowed = async (url, scenario, loadDescription) => {
+		const sameOrigin = url.origin === suite.target.origin;
+		const destinationTier = sameOrigin ? tier : await classifyResolvedTarget(url, deps.resolveTargetAddresses);
+		const inheritsTargetGate = sameOrigin && probe.benchmarkMode;
+		if (destinationTier === "public" && !inheritsTargetGate && !command.allowUnsafeTarget) throw new UnsafeTargetError(`Refusing to send ${loadDescription} to ${url.href}: it is public and not part of the benchmarked target.  Pass --allow-unsafe-target to override.`);
+		assertPublicDestinationOverrideAllowed(url, scenario, {
+			targetOrigin: suite.target.origin,
+			targetBenchmarkMode: probe.benchmarkMode,
+			allowUnsafe: command.allowUnsafeTarget,
+			explicitCliTarget,
 			destinationTier,
 			defaults: validated.defaults
 		});
 	};
+	const assertReadDestinationAllowed = (url, scenario) => assertDestinationWithoutSyntheticServerAllowed(url, scenario, "benchmark read load");
+	const assertActorlessDestinationAllowed = (url, scenario) => assertDestinationWithoutSyntheticServerAllowed(url, scenario, "benchmark load");
 	if (command.dryRun) try {
 		await writeOutput(await renderPlan(suite, {
 			documentLoader,
 			contextLoader,
 			allowPrivateAddress,
-			assertDestinationAllowed
+			fetch: fetchImpl,
+			assertDestinationAllowed,
+			assertReadDestinationAllowed
 		}), command.output);
 		exit(0);
 		return;
@@ -125,30 +149,42 @@ async function runBench(command, deps = {}) {
 		exit(2);
 		return;
 	}
-	if (tier !== "loopback" && command.advertiseHost == null && suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))) {
-		log("Signed scenarios (inbox) need the benchmark's synthetic actor server to be reachable from the target.  A loopback target reaches it automatically; for a non-loopback target, pass --advertise-host with an address the target can reach (the synthetic server then binds all interfaces), or use a read scenario such as webfinger.");
+	if (tier !== "loopback" && command.advertiseHost == null && suite.scenarios.some((scenario) => scenarioNeedsReachableLocalServer(scenario, suite.scenarios))) {
+		log("Some scenarios need benchmark-owned local servers to be reachable from the target.  A loopback target reaches them automatically; for a non-loopback target, pass --advertise-host with an address the target can reach, or use a scenario that does not need local benchmark servers such as webfinger.");
 		exit(2);
 		return;
 	}
 	let fleet;
 	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
 	try {
-		if (suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))) fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost });
+		throwIfAborted(signal);
+		if (suite.scenarios.some((scenario) => scenarioNeedsSyntheticServer(scenario, suite.scenarios))) fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost });
 		const results = [];
 		for (let i = 0; i < suite.scenarios.length; i++) {
 			const scenario = suite.scenarios[i];
-			log(`Running scenario "${scenario.name}" (${scenario.type})…`);
-			const measurement = await runners[i].run({
-				scenario,
-				target: suite.target,
-				documentLoader,
-				contextLoader,
-				allowPrivateAddress,
-				fleet: fleet ?? null,
-				fetch: fetchImpl,
-				assertDestinationAllowed: (url) => assertDestinationAllowed(url, scenario)
-			});
-			results.push(buildScenarioResult(scenario, measurement));
+			const measurements = [];
+			for (let run = 1; run <= scenario.runs; run++) {
+				throwIfAborted(signal);
+				const suffix = scenario.runs === 1 ? "" : ` run ${run}/${scenario.runs}`;
+				log(`Running scenario "${scenario.name}" (${scenario.type})${suffix}…`);
+				measurements.push(await runners[i].run({
+					scenario,
+					scenarios: suite.scenarios,
+					target: suite.target,
+					documentLoader,
+					contextLoader,
+					allowPrivateAddress,
+					fleet: fleet ?? null,
+					advertiseHost: command.advertiseHost,
+					fetch: fetchImpl,
+					assertDestinationAllowed: (url, gateScenario) => assertDestinationAllowed(url, gateScenario ?? scenario),
+					assertReadDestinationAllowed: (url, gateScenario) => assertReadDestinationAllowed(url, gateScenario ?? scenario),
+					assertActorlessDestinationAllowed: (url, gateScenario) => assertActorlessDestinationAllowed(url, gateScenario ?? scenario),
+					signal
+				}));
+				throwIfAborted(signal);
+			}
+			results.push(buildScenarioResult(scenario, measurements));
 		}
 		const report = buildReport({
 			scenarios: results,
@@ -210,6 +246,22 @@ function withUserAgent(fetchImpl, userAgent) {
 		});
 	});
 }
+function withAbortSignal(fetchImpl, signal) {
+	if (signal == null) return fetchImpl;
+	return ((input, init) => {
+		if (signal.aborted) return Promise.reject(abortReason(signal));
+		return fetchImpl(input, {
+			...init,
+			signal
+		});
+	});
+}
+function throwIfAborted(signal) {
+	if (signal?.aborted) throw abortReason(signal);
+}
+function abortReason(signal) {
+	return signal.reason ?? /* @__PURE__ */ new Error("Benchmark run aborted.");
+}
 async function defaultWriteOutput(content, outputPath) {
 	if (outputPath == null) {
 		process.stdout.write(content.endsWith("\n") ? content : `${content}\n`);
@@ -232,12 +284,34 @@ async function renderPlan(suite, context) {
 	return `${lines.join("\n")}\n`;
 }
 function describePlan(scenario) {
-	return `${scenario.load.kind === "open" ? `open-loop ${scenario.load.ratePerSec}/s ${scenario.load.arrival}` : `closed-loop concurrency ${scenario.load.concurrency}`}, duration ${scenario.durationMs}ms, signing ${scenario.signing}`;
+	const load = scenario.load.kind === "open" ? `open-loop ${scenario.load.ratePerSec}/s ${scenario.load.arrival}` : `closed-loop concurrency ${scenario.load.concurrency}`;
+	const totalDurationMs = scenario.durationMs * scenario.runs;
+	const volume = describePlannedRequestVolume(scenario);
+	return [
+		load,
+		`duration ${scenario.durationMs}ms`,
+		`runs ${scenario.runs}`,
+		`total duration ${totalDurationMs}ms`,
+		...volume == null ? [] : [volume],
+		`signing ${scenario.signing}`
+	].join(", ");
+}
+function describePlannedRequestVolume(scenario) {
+	if (scenario.load.kind !== "open") return null;
+	return `estimated scheduled requests ${formatPlanNumber(scenario.load.ratePerSec * (scenario.durationMs / 1e3) * scenario.runs)}`;
+}
+function formatPlanNumber(value) {
+	if (Number.isInteger(value)) return String(value);
+	const formatted = value.toFixed(2).replace(/\.?0+$/, "");
+	return formatted === "" ? "0" : formatted;
 }
 async function describeDiscoveryPlan(scenario, suite, context) {
 	switch (scenario.type) {
 		case "inbox": return await describeInboxDiscoveryPlan(scenario, context);
 		case "webfinger": return describeWebFingerPlan(scenario, suite.target);
+		case "actor": return await describeActorPlan(scenario, suite, context);
+		case "object": return await describeObjectPlan(scenario, suite, context);
+		case "mixed": return describeMixedPlan(scenario);
 		default: return ["  discovery: not available for this scenario type"];
 	}
 }
@@ -269,15 +343,59 @@ function describeWebFingerPlan(scenario, target) {
 		return `  webfinger ${resource}: GET ${url.href}`;
 	});
 }
-async function describeDestinationSafety(inbox, scenario, context) {
+async function describeActorPlan(scenario, suite, context) {
 	try {
-		await context.assertDestinationAllowed(inbox, scenario);
+		const urls = await actorUrlsFromRecipients(scenario.recipients, {
+			target: suite.target,
+			fetch: context.fetch
+		});
+		const lines = [];
+		for (const url of urls) {
+			lines.push(`  actor: GET ${url.href}`);
+			lines.push(`  destination safety: ${await describeDestinationSafety(url, scenario, context)}`);
+		}
+		return lines;
+	} catch (error) {
+		return [`  actor discovery failed (${describeError(error)})`];
+	}
+}
+async function describeObjectPlan(scenario, suite, context) {
+	try {
+		const urls = await objectUrlsFromSource({
+			source: scenario.source,
+			target: suite.target,
+			fetch: context.fetch,
+			assertReadDestinationAllowed: (url) => context.assertReadDestinationAllowed(url, scenario)
+		});
+		const lines = [`  objects: ${urls.length} URL(s) resolved`];
+		for (const url of urls.slice(0, 10)) {
+			lines.push(`  object: GET ${url.href}`);
+			lines.push(`  destination safety: ${await describeDestinationSafety(url, scenario, context)}`);
+		}
+		if (urls.length > 10) lines.push(`  ... ${urls.length - 10} more`);
+		return lines;
+	} catch (error) {
+		return [`  object discovery failed (${describeError(error)})`];
+	}
+}
+function describeMixedPlan(scenario) {
+	const entries = scenario.raw.mix ?? [];
+	if (entries.length < 1) return ["  mix: no child scenarios"];
+	return entries.map((entry) => `  mix: ${entry.scenario} weight ${entry.weight}`);
+}
+async function describeDestinationSafety(url, scenario, context) {
+	try {
+		if (usesReadDestinationGate(scenario)) await context.assertReadDestinationAllowed(url, scenario);
+		else await context.assertDestinationAllowed(url, scenario);
 		return "allowed";
 	} catch (error) {
 		if (error instanceof UnsafeTargetError) return `would be refused: ${error.message}`;
 		throw error;
 	}
 }
+function usesReadDestinationGate(scenario) {
+	return (scenario.type === "actor" || scenario.type === "object") && !scenario.authenticated;
+}
 function assertPublicDestinationOverrideAllowed(url, scenario, context) {
 	const inheritsTargetGate = url.origin === context.targetOrigin && context.targetBenchmarkMode;
 	if (context.destinationTier !== "public" || inheritsTargetGate || !context.allowUnsafe) return;
@@ -295,15 +413,57 @@ function unsafeOverrideScenarios(suite) {
 function unsafeOverrideScenario(scenario, defaults) {
 	const defaultDuration = defaults?.duration != null;
 	const defaultLoad = hasExplicitLoad(defaults?.load);
+	const defaultRuns = defaults?.runs != null;
 	const raw = "raw" in scenario ? scenario.raw : scenario;
 	return {
 		name: scenario.name,
 		explicitDuration: raw.duration != null || defaultDuration,
-		explicitLoad: hasExplicitLoad(raw.load) || defaultLoad
+		explicitLoad: hasExplicitLoad(raw.load) || defaultLoad,
+		explicitRuns: raw.runs != null || defaultRuns
 	};
 }
 function hasExplicitLoad(load) {
 	return load != null && typeof load === "object" && ("rate" in load && load.rate != null || "concurrency" in load && load.concurrency != null);
 }
+function scenarioNeedsSyntheticServer(scenario, scenarios, seen = /* @__PURE__ */ new Set()) {
+	if (seen.has(scenario.name)) return false;
+	const nextSeen = new Set(seen).add(scenario.name);
+	switch (scenario.type) {
+		case "inbox": return true;
+		case "actor":
+		case "object": return scenario.authenticated;
+		case "failure": return failureFaultsOf(scenario).some(isInboundFailureFault);
+		case "mixed": return mixedChildrenOf(scenario, scenarios).some((child) => scenarioNeedsSyntheticServer(child, scenarios, nextSeen));
+		default: return false;
+	}
+}
+function scenarioNeedsReachableLocalServer(scenario, scenarios, seen = /* @__PURE__ */ new Set()) {
+	if (scenario.type === "fanout") return scenario.raw.sinkBase == null;
+	if (scenario.type === "failure") {
+		const faults = failureFaultsOf(scenario);
+		return faults.includes("invalid-signature") || scenario.raw.sinkBase == null && faults.some(isRemoteFailureFault);
+	}
+	if (scenario.type === "mixed") {
+		if (seen.has(scenario.name)) return false;
+		const nextSeen = new Set(seen).add(scenario.name);
+		return mixedChildrenOf(scenario, scenarios).some((child) => scenarioNeedsReachableLocalServer(child, scenarios, nextSeen));
+	}
+	return scenarioNeedsSyntheticServer(scenario, scenarios, seen);
+}
+function failureFaultsOf(scenario) {
+	return scenario.faults.length < 1 ? ["remote-404"] : scenario.faults;
+}
+function mixedChildrenOf(scenario, scenarios) {
+	return (scenario.raw.mix ?? []).flatMap((entry) => {
+		const child = scenarios.find((candidate) => candidate.name === entry.scenario);
+		return child == null ? [] : [child];
+	});
+}
+function isInboundFailureFault(fault) {
+	return fault === "invalid-signature" || fault === "missing-actor";
+}
+function isRemoteFailureFault(fault) {
+	return fault === "remote-404" || fault === "remote-410" || fault === "slow-inbox" || fault === "network-error";
+}
 //#endregion
 export { runBench as default };

package/dist/bench/command.js

@@ -1,7 +1,7 @@
 import "@js-temporal/polyfill";
 import { configContext } from "../config.js";
 import { userAgentOption } from "../options.js";
-import { argument, choice, command, constant, flag, group, merge, message, object, option, optional, string, withDefault } from "@optique/core";
+import { argument, choice, command, constant, flag, group, merge, message, object, option, optional, or, string, withDefault } from "@optique/core";
 import { bindConfig } from "@optique/config";
 //#region src/bench/command.ts
 const formatOption = bindConfig(option("-f", "--format", choice([
@@ -16,27 +16,55 @@ const formatOption = bindConfig(option("-f", "--format", choice([
 const allowUnsafeTarget = withDefault(flag("--allow-unsafe-target", { description: message`Allow benchmarking a public target that does not advertise \
 benchmark mode.  Must be given on the command line for each run; it cannot be \
 set in a configuration file.` }), false);
-const benchCommand = command("bench", merge("Benchmark options", object({
+const outputOption = optional(option("-o", "--output", string({ metavar: "OUTPUT_PATH" }), { description: message`Write the report to a file instead of standard output.` }));
+const targetOption = optional(option("-t", "--target", string({ metavar: "URL" }), { description: message`Override the target URL declared in the suite.` }));
+const advertiseHostOption = optional(option("--advertise-host", string({ metavar: "HOST" }), { description: message`Host (name or IP) a non-loopback target can reach the \
+benchmark's synthetic actor server at.  Required for signed scenarios against a \
+non-loopback target; binds the synthetic server on all interfaces and uses this \
+host in the actor and key URLs the target dereferences.` }));
+const runParser = merge("Benchmark options", object({
 	command: constant("bench"),
+	mode: constant("run"),
 	scenario: group("Arguments", argument(string({ metavar: "SCENARIO_FILE" }), { description: message`Path to the benchmark suite file (YAML or JSON).` })),
-	target: optional(option("-t", "--target", string({ metavar: "URL" }), { description: message`Override the target URL declared in the suite.` })),
+	target: targetOption,
 	format: formatOption,
-	output: optional(option("-o", "--output", string({ metavar: "OUTPUT_PATH" }), { description: message`Write the report to a file instead of standard output.` })),
+	output: outputOption,
 	dryRun: withDefault(flag("--dry-run", { description: message`Resolve discovery and print the benchmark plan without \
 sending load.` }), false),
-	advertiseHost: optional(option("--advertise-host", string({ metavar: "HOST" }), { description: message`Host (name or IP) a non-loopback target can reach the \
-benchmark's synthetic actor server at.  Required for signed scenarios against a \
-non-loopback target; binds the synthetic server on all interfaces and uses this \
-host in the actor and key URLs the target dereferences.` })),
+	advertiseHost: advertiseHostOption,
+	allowUnsafeTarget
+}), userAgentOption);
+const benchCommand = command("bench", or(command("compare", merge("Compare options", object({
+	command: constant("bench"),
+	mode: constant("compare"),
+	base: option("--base", string({ metavar: "REF" }), { description: message`The base git ref to benchmark.` }),
+	head: option("--head", string({ metavar: "REF" }), { description: message`The head git ref to benchmark.` }),
+	file: option("--file", string({ metavar: "SCENARIO_FILE" }), { description: message`Path to the benchmark suite file (YAML or JSON).` }),
+	startCommand: option("--start-command", string({ metavar: "COMMAND" }), { description: message`Shell command that starts the target application in each \
+checked-out worktree.` }),
+	readyUrl: option("--ready-url", string({ metavar: "URL" }), { description: message`URL that returns success when the started target is ready.` }),
+	readyTimeout: withDefault(option("--ready-timeout", string({ metavar: "DURATION" }), { description: message`How long to wait for --ready-url.` }), "30s"),
+	maxRegression: option("--max-regression", string({ metavar: "PERCENT" }), { description: message`Maximum regression tolerated after the measured noise band.` }),
+	target: targetOption,
+	format: formatOption,
+	output: outputOption,
+	dryRun: constant(false),
+	advertiseHost: advertiseHostOption,
 	allowUnsafeTarget
 }), userAgentOption), {
+	brief: message`Compare base and head benchmark runs.`,
+	description: message`Run the same benchmark suite against two git revisions on the \
+same runner, then fail when the head revision regresses beyond the configured \
+tolerance and measured noise band.`
+}), runParser), {
 	brief: message`Benchmark a Fedify federation workload.`,
 	description: message`Run an ActivityPub-specific load benchmark against a \
 cooperative Fedify target running in benchmark mode.
 
-The suite file declares the target, actors, and scenarios.  Only the \`inbox\` \
-and \`webfinger\` scenario types are executed in this version; the format \
-itself can express every scenario type.`
+The suite file declares the target, actors, and scenarios.  This version \
+executes the \`inbox\`, \`webfinger\`, \`actor\`, \`object\`, \`fanout\`, \
+\`failure\`, and \`mixed\` scenario types; \`collection\` remains reserved by \
+the suite format.`
 });
 //#endregion
 export { benchCommand };

package/dist/bench/compare/schema.js

@@ -0,0 +1,16 @@
+import "@js-temporal/polyfill";
+//#region src/bench/compare/schema.ts
+/**
+* The embedded JSON Schema (draft 2020-12) for benchmark comparison output.
+*
+* The comparison report embeds the two benchmark reports it compares; this
+* schema validates the comparison envelope and checks that the embedded reports
+* look like current benchmark reports without duplicating the complete report
+* schema in two published files.
+* @since 2.3.0
+* @module
+*/
+/** The hosted URL that serves the comparison report schema. */
+const COMPARE_REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/compare-report-v1.json";
+//#endregion
+export { COMPARE_REPORT_SCHEMA_ID };