@fedify/cli 2.3.0-dev.1299 → 2.3.0-dev.1344

package/dist/bench/result/schema.js

@@ -4,12 +4,362 @@ import "@js-temporal/polyfill";
 * The embedded JSON Schema (draft 2020-12) for benchmark report output.
 *
 * Like the scenario schema, this object is the runtime copy and is published,
-* byte-for-byte, as *schema/bench/report-v1.json*; a drift guard keeps the two
-* in sync.  The matching TypeScript types live in {@link ./model.ts}.
+* byte-for-byte, as files under *schema/bench/*; a drift guard keeps the two in
+* sync.  The matching TypeScript types live in {@link ./model.ts}.
 * @since 2.3.0
 * @module
 */
 /** The hosted URL that serves the report schema. */
-const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v1.json";
+const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v3.json";
+/** The hosted URL for the version 2 report schema. */
+const REPORT_SCHEMA_V2_ID = "https://json-schema.fedify.dev/bench/report-v2.json";
+/** The benchmark report JSON Schema (draft 2020-12). */
+const reportSchemaV1 = {
+	$schema: "https://json-schema.org/draft/2020-12/schema",
+	$id: "https://json-schema.fedify.dev/bench/report-v1.json",
+	title: "Fedify benchmark report",
+	type: "object",
+	additionalProperties: false,
+	required: [
+		"schemaVersion",
+		"tool",
+		"environment",
+		"target",
+		"startedAt",
+		"finishedAt",
+		"suite",
+		"passed",
+		"scenarios"
+	],
+	properties: {
+		$schema: { type: "string" },
+		schemaVersion: { const: 1 },
+		tool: {
+			type: "object",
+			additionalProperties: false,
+			required: ["name", "version"],
+			properties: {
+				name: { type: "string" },
+				version: { type: "string" }
+			}
+		},
+		environment: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"runtime",
+				"runtimeVersion",
+				"os",
+				"cpuCount"
+			],
+			properties: {
+				runtime: { type: "string" },
+				runtimeVersion: { type: "string" },
+				os: { type: "string" },
+				cpuCount: {
+					type: "integer",
+					minimum: 0
+				}
+			}
+		},
+		target: {
+			type: "object",
+			additionalProperties: false,
+			required: ["url", "statsAvailable"],
+			properties: {
+				url: { type: "string" },
+				fedifyVersion: { type: ["string", "null"] },
+				statsAvailable: { type: "boolean" }
+			}
+		},
+		startedAt: { type: "string" },
+		finishedAt: { type: "string" },
+		suite: {
+			type: "object",
+			additionalProperties: false,
+			required: ["configHash"],
+			properties: {
+				name: { type: "string" },
+				configHash: { type: "string" }
+			}
+		},
+		passed: { type: "boolean" },
+		scenarios: {
+			type: "array",
+			items: { $ref: "#/$defs/scenarioResult" }
+		}
+	},
+	$defs: {
+		latencyMs: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"p50",
+				"p95",
+				"p99",
+				"mean",
+				"max"
+			],
+			properties: {
+				p50: { type: "number" },
+				p95: { type: "number" },
+				p99: { type: "number" },
+				mean: { type: "number" },
+				max: { type: "number" }
+			}
+		},
+		partialLatencyMs: {
+			type: "object",
+			additionalProperties: false,
+			properties: {
+				p50: { type: "number" },
+				p95: { type: "number" },
+				p99: { type: "number" }
+			}
+		},
+		loadSummary: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"model",
+				"durationMs",
+				"warmupMs"
+			],
+			properties: {
+				model: { enum: ["open", "closed"] },
+				ratePerSec: { type: "number" },
+				arrival: { type: "string" },
+				concurrency: { type: "integer" },
+				durationMs: { type: "number" },
+				warmupMs: { type: "number" },
+				maxInFlight: { type: "integer" }
+			},
+			oneOf: [{
+				properties: { model: { const: "open" } },
+				required: ["ratePerSec", "arrival"],
+				not: { required: ["concurrency"] }
+			}, {
+				properties: { model: { const: "closed" } },
+				required: ["concurrency"],
+				not: { anyOf: [{ required: ["ratePerSec"] }, { required: ["arrival"] }] }
+			}]
+		},
+		requestSummary: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"total",
+				"ok",
+				"failed",
+				"successRate"
+			],
+			properties: {
+				total: {
+					type: "integer",
+					minimum: 0
+				},
+				ok: {
+					type: "integer",
+					minimum: 0
+				},
+				failed: {
+					type: "integer",
+					minimum: 0
+				},
+				successRate: {
+					type: "number",
+					minimum: 0,
+					maximum: 1
+				}
+			}
+		},
+		clientMetrics: {
+			type: "object",
+			additionalProperties: false,
+			required: ["latencyMs"],
+			properties: { latencyMs: { $ref: "#/$defs/latencyMs" } }
+		},
+		serverMetrics: {
+			type: "object",
+			additionalProperties: false,
+			properties: {
+				signatureVerificationMs: {
+					type: "object",
+					additionalProperties: false,
+					required: ["overall"],
+					properties: {
+						overall: { $ref: "#/$defs/partialLatencyMs" },
+						byStandard: {
+							type: "object",
+							additionalProperties: { $ref: "#/$defs/partialLatencyMs" }
+						}
+					}
+				},
+				queue: {
+					type: "object",
+					additionalProperties: false,
+					properties: {
+						drainMs: { $ref: "#/$defs/partialLatencyMs" },
+						depthMax: { type: "number" }
+					}
+				}
+			}
+		},
+		errorBucket: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"kind",
+				"reason",
+				"count"
+			],
+			properties: {
+				kind: { type: "string" },
+				status: { type: "integer" },
+				reason: { type: "string" },
+				count: {
+					type: "integer",
+					minimum: 0
+				}
+			}
+		},
+		expectResult: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"metric",
+				"op",
+				"threshold",
+				"unit",
+				"actual",
+				"severity",
+				"pass"
+			],
+			properties: {
+				metric: { type: "string" },
+				op: { enum: [
+					"lt",
+					"lte",
+					"gt",
+					"gte",
+					"eq"
+				] },
+				threshold: { type: "number" },
+				unit: { type: ["string", "null"] },
+				actual: { type: ["number", "null"] },
+				severity: { enum: ["warn", "fail"] },
+				pass: { type: "boolean" }
+			}
+		},
+		scenarioResult: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"name",
+				"type",
+				"load",
+				"requests",
+				"throughputPerSec",
+				"client",
+				"server",
+				"errors",
+				"expectations",
+				"passed"
+			],
+			properties: {
+				name: { type: "string" },
+				type: { enum: [
+					"inbox",
+					"webfinger",
+					"actor",
+					"object",
+					"fanout",
+					"collection",
+					"failure",
+					"mixed"
+				] },
+				load: { $ref: "#/$defs/loadSummary" },
+				requests: { $ref: "#/$defs/requestSummary" },
+				throughputPerSec: { type: "number" },
+				client: { $ref: "#/$defs/clientMetrics" },
+				server: { anyOf: [{ $ref: "#/$defs/serverMetrics" }, { type: "null" }] },
+				errors: {
+					type: "array",
+					items: { $ref: "#/$defs/errorBucket" }
+				},
+				expectations: {
+					type: "array",
+					items: { $ref: "#/$defs/expectResult" }
+				},
+				passed: { type: "boolean" },
+				histogram: { $ref: "#/$defs/serializedHistogram" }
+			}
+		},
+		serializedHistogram: {
+			type: "object",
+			additionalProperties: false,
+			required: [
+				"version",
+				"subBucketCount",
+				"count",
+				"zeroCount",
+				"min",
+				"max",
+				"sum",
+				"indices",
+				"counts"
+			],
+			properties: {
+				version: { const: 1 },
+				subBucketCount: {
+					type: "integer",
+					minimum: 1
+				},
+				count: {
+					type: "integer",
+					minimum: 0
+				},
+				zeroCount: {
+					type: "integer",
+					minimum: 0
+				},
+				min: { type: "number" },
+				max: { type: "number" },
+				sum: { type: "number" },
+				indices: {
+					type: "array",
+					items: { type: "integer" }
+				},
+				counts: {
+					type: "array",
+					items: {
+						type: "integer",
+						minimum: 0
+					}
+				}
+			}
+		}
+	}
+};
+/** The benchmark report JSON Schema (draft 2020-12). */
+const reportSchemaV2 = {
+	...reportSchemaV1,
+	$id: REPORT_SCHEMA_V2_ID,
+	properties: {
+		...reportSchemaV1.properties,
+		schemaVersion: { const: 2 }
+	},
+	$defs: {
+		...reportSchemaV1.$defs,
+		scenarioResult: {
+			...reportSchemaV1.$defs.scenarioResult,
+			properties: {
+				...reportSchemaV1.$defs.scenarioResult.properties,
+				deliveryThroughputPerSec: { type: "number" }
+			}
+		}
+	}
+};
+({ ...reportSchemaV2 }), { ...reportSchemaV2.properties }, { ...reportSchemaV2.$defs }, { ...reportSchemaV2.$defs.scenarioResult }, [...reportSchemaV2.$defs.scenarioResult.required], { ...reportSchemaV2.$defs.scenarioResult.properties };
 //#endregion
 export { REPORT_SCHEMA_ID };

package/dist/bench/safety/gate.js

@@ -20,8 +20,9 @@ function assertTargetAllowed(context) {
 *
 * The override is only meaningful for a public target that does not advertise
 * benchmark mode.  In that caution tier, the operator must name the target on
-* the command line for this run and must explicitly set load and duration, so
-* the built-in defaults cannot accidentally create a long public benchmark.
+* the command line for this run and must explicitly set load, duration, and
+* runs, so the built-in defaults cannot accidentally create a long public
+* benchmark.
 * @param context The unsafe override decision inputs.
 * @throws {UnsafeTargetError} If the unsafe override is too broad.
 */
@@ -31,6 +32,7 @@ function assertUnsafeOverrideAllowed(context) {
 	for (const scenario of context.scenarios) {
 		if (!scenario.explicitLoad) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark load default.  Set rate or concurrency explicitly before using --allow-unsafe-target against a public target.`);
 		if (!scenario.explicitDuration) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark duration default.  Set duration explicitly before using --allow-unsafe-target against a public target.`);
+		if (!scenario.explicitRuns) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark runs default.  Set runs explicitly before using --allow-unsafe-target against a public target.`);
 	}
 }
 /**

package/dist/bench/scenario/normalize.js

@@ -19,7 +19,7 @@ const DEFAULT_DURATION_MS = 6e4;
 const DEFAULT_WARMUP_MS = 0;
 const DEFAULT_RATE_PER_SEC = 50;
 const DEFAULT_SIGNING = "pipeline";
-const DEFAULT_RUNS = 1;
+const DEFAULT_RUNS = 3;
 /** An error raised when a suite cannot be normalized. */
 var SuiteNormalizeError = class extends Error {};
 /**
@@ -57,7 +57,6 @@ function resolveScenario(scenario, suite) {
 	const warmupMs = resolveDuration(scenario.warmup ?? defaults.warmup, DEFAULT_WARMUP_MS);
 	if (warmupMs >= durationMs) throw new SuiteNormalizeError(`Scenario "${scenario.name}": warmup (${warmupMs}ms) must be shorter than duration (${durationMs}ms); otherwise no requests are measured.`);
 	const runs = scenario.runs ?? defaults.runs ?? DEFAULT_RUNS;
-	if (runs > 1) throw new SuiteNormalizeError(`Scenario "${scenario.name}": multiple runs (runs > 1) are not yet implemented in fedify bench; set runs to 1.`);
 	return {
 		name: scenario.name,
 		type: scenario.type,

package/dist/bench/scenario/schema.js

@@ -3,14 +3,14 @@ import "@js-temporal/polyfill";
 /**
 * The embedded JSON Schema (draft 2020-12) for benchmark scenario suite files.
 *
-* This object is the runtime copy used by the validator; it is published,
-* byte-for-byte, as *schema/bench/scenario-v1.json* and a drift guard keeps the
-* two in sync.  The matching TypeScript types live in {@link ./types.ts}.
+* These objects are the runtime copies used by the validator; they are
+* published, byte-for-byte, under *schema/bench/* and a drift guard keeps them
+* in sync.  The matching TypeScript types live in {@link ./types.ts}.
 *
 * The schema expresses every scenario type discussed for `fedify bench`
 * (`inbox`, `webfinger`, `actor`, `object`, `fanout`, `collection`, `failure`,
-* `mixed`), even though only `inbox` and `webfinger` have runners in this
-* version.  Three cross-field rules are enforced here rather than in code:
+* `mixed`).  All but `collection` have runners in this version.  Three
+* cross-field rules are enforced here rather than in code:
 *
 *  -  exactly one HTTP request signature scheme per actor group
 *     (`contains` + `minContains`/`maxContains`);
@@ -20,8 +20,10 @@ import "@js-temporal/polyfill";
 * @since 2.3.0
 * @module
 */
-/** The hosted URL that serves the scenario schema. */
-const SCENARIO_SCHEMA_ID = "https://json-schema.fedify.dev/bench/scenario-v1.json";
+/** The hosted URL that serves the current scenario schema. */
+const SCENARIO_SCHEMA_ID = "https://json-schema.fedify.dev/bench/scenario-v2.json";
+/** The hosted URL that serves the version 1 scenario schema. */
+const SCENARIO_SCHEMA_ID_V1 = "https://json-schema.fedify.dev/bench/scenario-v1.json";
 const READ_METRICS = [
 	"successRate",
 	"throughputPerSec",
@@ -51,10 +53,11 @@ const FANOUT_METRICS = [
 	"queueDrain.p99"
 ];
 const MIXED_METRICS = [...new Set([...INBOX_METRICS, ...FANOUT_METRICS])];
+const MIXED_V2_METRICS = [...READ_METRICS, "deliveryThroughput"];
 /** The benchmark scenario suite JSON Schema (draft 2020-12). */
 const scenarioSchemaV1 = {
 	$schema: "https://json-schema.org/draft/2020-12/schema",
-	$id: SCENARIO_SCHEMA_ID,
+	$id: SCENARIO_SCHEMA_ID_V1,
 	title: "Fedify benchmark scenario suite",
 	type: "object",
 	required: ["version", "scenarios"],
@@ -354,5 +357,43 @@ const scenarioSchemaV1 = {
 		}
 	}
 };
+/** The current benchmark scenario suite JSON Schema (draft 2020-12). */
+const scenarioSchemaV2 = {
+	...scenarioSchemaV1,
+	$id: SCENARIO_SCHEMA_ID,
+	$defs: {
+		...scenarioSchemaV1.$defs,
+		scenario: {
+			...scenarioSchemaV1.$defs.scenario,
+			properties: {
+				...scenarioSchemaV1.$defs.scenario.properties,
+				sinkBase: { type: "string" }
+			},
+			allOf: scenarioSchemaV1.$defs.scenario.allOf.map((condition) => condition.if.properties.type.const === "failure" ? {
+				if: condition.if,
+				then: { properties: condition.then.properties }
+			} : condition.if.properties.type.const === "actor" || condition.if.properties.type.const === "object" ? {
+				if: condition.if,
+				then: {
+					required: condition.then.required,
+					allOf: [{
+						if: {
+							required: ["authenticated"],
+							properties: { authenticated: { const: true } }
+						},
+						then: { properties: { expect: { propertyNames: { enum: INBOX_METRICS } } } },
+						else: { properties: { expect: { propertyNames: { enum: READ_METRICS } } } }
+					}]
+				}
+			} : condition.if.properties.type.const === "mixed" ? {
+				if: condition.if,
+				then: {
+					required: condition.then.required,
+					properties: { expect: { propertyNames: { enum: MIXED_V2_METRICS } } }
+				}
+			} : condition)
+		}
+	}
+};
 //#endregion
-export { scenarioSchemaV1 };
+export { scenarioSchemaV2 };

package/dist/bench/scenario/validate.js

@@ -1,5 +1,5 @@
 import "@js-temporal/polyfill";
-import { scenarioSchemaV1 } from "./schema.js";
+import { scenarioSchemaV2 } from "./schema.js";
 import { SuiteValidationError } from "./errors.js";
 import { Validator } from "@cfworker/json-schema";
 //#region src/bench/scenario/validate.ts
@@ -10,7 +10,7 @@ import { Validator } from "@cfworker/json-schema";
 */
 let validator;
 function getValidator() {
-	validator ??= new Validator(scenarioSchemaV1, "2020-12");
+	validator ??= new Validator(scenarioSchemaV2, "2020-12");
 	return validator;
 }
 /**

package/dist/bench/scenarios/actor.js

@@ -0,0 +1,38 @@
+import "@js-temporal/polyfill";
+import { convertUrlIfHandle } from "../../webfinger/lib.js";
+import { actorUrlsFromRecipients } from "./object-discovery.js";
+import { isBareHttpUrl } from "./runner.js";
+import { runReadLoad } from "./read.js";
+//#region src/bench/scenarios/actor.ts
+/**
+* The `actor` scenario runner.
+* @since 2.3.0
+* @module
+*/
+/** The `actor` scenario runner. */
+const actorRunner = {
+	validate(scenario) {
+		if (scenario.recipients.length < 1) throw new Error("The actor scenario requires a recipient.");
+		for (const recipient of scenario.recipients) {
+			let url;
+			try {
+				url = convertUrlIfHandle(recipient);
+			} catch {
+				throw new Error(`Scenario "${scenario.name}": invalid actor recipient ${JSON.stringify(recipient)}.`);
+			}
+			if (url.protocol !== "acct:" && !isBareHttpUrl(url)) throw new Error(`Scenario "${scenario.name}": actor recipient must be an acct: handle or a bare http(s) URL with a host and no credentials; got ${JSON.stringify(url.href)}.`);
+		}
+	},
+	async run(context) {
+		this.validate?.(context.scenario);
+		return await runReadLoad(context, {
+			urls: await actorUrlsFromRecipients(context.scenario.recipients, {
+				target: context.target,
+				fetch: context.fetch
+			}),
+			authenticated: context.scenario.authenticated
+		});
+	}
+};
+//#endregion
+export { actorRunner };