@fedify/cli 2.3.0-dev.1219 → 2.3.0-dev.1273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bench/action.js +309 -0
- package/dist/bench/actor/documents.js +39 -0
- package/dist/bench/actor/fleet.js +39 -0
- package/dist/bench/actor/keys.js +35 -0
- package/dist/bench/command.js +42 -0
- package/dist/bench/discovery/discover.js +67 -0
- package/dist/bench/discovery/probe.js +50 -0
- package/dist/bench/load/arrival.js +27 -0
- package/dist/bench/load/clock.js +15 -0
- package/dist/bench/load/generator.js +112 -0
- package/dist/bench/metrics/aggregate.js +64 -0
- package/dist/bench/metrics/histogram.js +141 -0
- package/dist/bench/metrics/stats-client.js +154 -0
- package/dist/bench/mod.js +4 -0
- package/dist/bench/render/format.js +46 -0
- package/dist/bench/render/index.js +20 -0
- package/dist/bench/render/json.js +12 -0
- package/dist/bench/render/markdown.js +62 -0
- package/dist/bench/render/text.js +74 -0
- package/dist/bench/result/build.js +129 -0
- package/dist/bench/result/expect/assert.js +74 -0
- package/dist/bench/result/expect/evaluate.js +128 -0
- package/dist/bench/result/expect/metrics.js +34 -0
- package/dist/bench/result/schema.js +15 -0
- package/dist/bench/safety/gate.js +60 -0
- package/dist/bench/safety/tiers.js +97 -0
- package/dist/bench/scenario/coerce.js +24 -0
- package/dist/bench/scenario/errors.js +36 -0
- package/dist/bench/scenario/load.js +69 -0
- package/dist/bench/scenario/normalize.js +126 -0
- package/dist/bench/scenario/schema.js +358 -0
- package/dist/bench/scenario/units.js +56 -0
- package/dist/bench/scenario/validate.js +29 -0
- package/dist/bench/scenarios/inbox.js +155 -0
- package/dist/bench/scenarios/registry.js +21 -0
- package/dist/bench/scenarios/runner.js +76 -0
- package/dist/bench/scenarios/webfinger.js +44 -0
- package/dist/bench/server/synthetic.js +118 -0
- package/dist/bench/signing/activity-id.js +18 -0
- package/dist/bench/signing/pipeline.js +134 -0
- package/dist/bench/signing/signer.js +39 -0
- package/dist/bench/template/generate.js +90 -0
- package/dist/bench/template/helpers.js +19 -0
- package/dist/bench/template/template.js +132 -0
- package/dist/cache.js +1 -1
- package/dist/config.js +14 -2
- package/dist/deno.js +1 -1
- package/dist/generate-vocab/action.js +3 -3
- package/dist/generate-vocab/command.js +1 -1
- package/dist/imagerenderer.js +1 -1
- package/dist/inbox/view.js +1 -1
- package/dist/inbox.js +2 -2
- package/dist/lookup.js +37 -37
- package/dist/mod.js +3 -0
- package/dist/nodeinfo.js +5 -5
- package/dist/options.js +1 -1
- package/dist/relay.js +1 -1
- package/dist/runner.js +10 -8
- package/dist/tempserver.js +1 -1
- package/dist/tunnel.js +2 -2
- package/dist/utils.js +8 -4
- package/dist/webfinger/action.js +1 -1
- package/package.json +12 -10
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
import { metricDisplayUnit } from "../result/expect/metrics.js";
|
|
3
|
+
import { formatActual, formatNumber, formatPercent, formatThreshold, opSymbol } from "./format.js";
|
|
4
|
+
//#region src/bench/render/text.ts
|
|
5
|
+
/**
|
|
6
|
+
* Renders a report as a plain-text terminal summary.
|
|
7
|
+
* @param report The report to render.
|
|
8
|
+
* @returns The summary text.
|
|
9
|
+
*/
|
|
10
|
+
function renderText(report) {
|
|
11
|
+
const lines = [];
|
|
12
|
+
lines.push("Fedify benchmark report", "");
|
|
13
|
+
const fedify = report.target.fedifyVersion == null ? "Fedify version unknown" : `Fedify ${report.target.fedifyVersion}`;
|
|
14
|
+
const stats = report.target.statsAvailable ? "stats available" : "stats unavailable";
|
|
15
|
+
lines.push(`Target: ${report.target.url} (${fedify}, ${stats})`);
|
|
16
|
+
const env = report.environment;
|
|
17
|
+
lines.push(`Environment: ${env.runtime} ${env.runtimeVersion}, ${env.os}, ${env.cpuCount} CPUs`);
|
|
18
|
+
lines.push(`Started: ${report.startedAt} Finished: ${report.finishedAt}`);
|
|
19
|
+
lines.push(`Config: ${report.suite.configHash}`, "");
|
|
20
|
+
for (const scenario of report.scenarios) lines.push(...renderScenario(scenario), "");
|
|
21
|
+
lines.push(`Overall: ${report.passed ? "PASS" : "FAIL"}`);
|
|
22
|
+
return lines.join("\n");
|
|
23
|
+
}
|
|
24
|
+
function renderScenario(scenario) {
|
|
25
|
+
const lines = [];
|
|
26
|
+
lines.push(`Scenario: ${scenario.name} (${scenario.type}) [${scenario.passed ? "PASS" : "FAIL"}]`);
|
|
27
|
+
lines.push(` Load: ${describeLoad(scenario.load)}`);
|
|
28
|
+
const r = scenario.requests;
|
|
29
|
+
lines.push(` Requests: ${formatNumber(r.total)} (ok ${formatNumber(r.ok)}, failed ${formatNumber(r.failed)}, success ${formatPercent(r.successRate)})`);
|
|
30
|
+
lines.push(` Throughput: ${formatNumber(scenario.throughputPerSec)} req/s`);
|
|
31
|
+
const l = scenario.client.latencyMs;
|
|
32
|
+
lines.push(` Client latency (ms): p50 ${formatNumber(l.p50)} p95 ${formatNumber(l.p95)} p99 ${formatNumber(l.p99)} mean ${formatNumber(l.mean)} max ${formatNumber(l.max)}`);
|
|
33
|
+
if (scenario.server?.signatureVerificationMs != null) lines.push(` Server signature verification (ms): ${describePartial(scenario.server.signatureVerificationMs.overall)}`);
|
|
34
|
+
const queue = scenario.server?.queue;
|
|
35
|
+
if (queue?.drainMs != null && hasPartial(queue.drainMs)) {
|
|
36
|
+
const depth = queue.depthMax;
|
|
37
|
+
const suffix = depth == null ? "" : ` (depth max ${formatNumber(depth)})`;
|
|
38
|
+
lines.push(` Server queue drain (ms): ${describePartial(queue.drainMs)}${suffix}`);
|
|
39
|
+
} else if (queue?.depthMax != null) lines.push(` Server queue depth max: ${formatNumber(queue.depthMax)}`);
|
|
40
|
+
if (scenario.errors.length > 0) {
|
|
41
|
+
lines.push(" Errors:");
|
|
42
|
+
for (const error of scenario.errors) {
|
|
43
|
+
const code = error.status == null ? error.kind : String(error.status);
|
|
44
|
+
lines.push(` ${code} ${error.reason}: ${formatNumber(error.count)}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (scenario.expectations.length > 0) {
|
|
48
|
+
lines.push(" Expectations:");
|
|
49
|
+
for (const e of scenario.expectations) {
|
|
50
|
+
const tag = e.pass ? "PASS" : e.severity === "warn" ? "WARN" : "FAIL";
|
|
51
|
+
const unit = metricDisplayUnit(e.metric);
|
|
52
|
+
lines.push(` [${tag}] ${e.metric} ${opSymbol(e.op)} ${formatThreshold(e.threshold, e.unit ?? unit)} (actual ${formatActual(e.actual, unit)})`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return lines;
|
|
56
|
+
}
|
|
57
|
+
function describeLoad(load) {
|
|
58
|
+
const tail = `duration ${formatNumber(load.durationMs)}ms, warmup ${formatNumber(load.warmupMs)}ms`;
|
|
59
|
+
if (load.model === "closed") return `closed, concurrency ${load.concurrency}, ${tail}`;
|
|
60
|
+
return `open, ${formatNumber(load.ratePerSec)}/s ${load.arrival}, ${tail}`;
|
|
61
|
+
}
|
|
62
|
+
function describePartial(latency) {
|
|
63
|
+
const parts = [];
|
|
64
|
+
if (latency.p50 != null) parts.push(`p50 ${formatNumber(latency.p50)}`);
|
|
65
|
+
if (latency.p95 != null) parts.push(`p95 ${formatNumber(latency.p95)}`);
|
|
66
|
+
if (latency.p99 != null) parts.push(`p99 ${formatNumber(latency.p99)}`);
|
|
67
|
+
return parts.join(" ");
|
|
68
|
+
}
|
|
69
|
+
/** Whether a partial latency carries at least one renderable percentile. */
|
|
70
|
+
function hasPartial(latency) {
|
|
71
|
+
return latency.p50 != null || latency.p95 != null || latency.p99 != null;
|
|
72
|
+
}
|
|
73
|
+
//#endregion
|
|
74
|
+
export { renderText };
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
import { version } from "../../deno.js";
|
|
3
|
+
import { evaluateExpect } from "./expect/evaluate.js";
|
|
4
|
+
import { REPORT_SCHEMA_ID } from "./schema.js";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { cpus } from "node:os";
|
|
8
|
+
//#region src/bench/result/build.ts
|
|
9
|
+
/**
|
|
10
|
+
* Assembly of the canonical benchmark report from measured scenario data.
|
|
11
|
+
*
|
|
12
|
+
* The runners produce per-scenario measurements; this module turns each into a
|
|
13
|
+
* {@link ScenarioResult} (evaluating its `expect` block) and assembles the
|
|
14
|
+
* top-level {@link BenchReport} with reproducibility metadata.
|
|
15
|
+
* @since 2.3.0
|
|
16
|
+
* @module
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Builds a scenario result from its resolved definition and measurement,
|
|
20
|
+
* evaluating the `expect` block in the process.
|
|
21
|
+
* @param scenario The resolved scenario.
|
|
22
|
+
* @param measurement The measured client and server metrics.
|
|
23
|
+
* @returns The assembled scenario result.
|
|
24
|
+
*/
|
|
25
|
+
function buildScenarioResult(scenario, measurement) {
|
|
26
|
+
const { results, passed } = evaluateExpect(scenario.expect, measurement);
|
|
27
|
+
return {
|
|
28
|
+
name: scenario.name,
|
|
29
|
+
type: scenario.type,
|
|
30
|
+
load: loadSummary(scenario),
|
|
31
|
+
requests: measurement.requests,
|
|
32
|
+
throughputPerSec: measurement.throughputPerSec,
|
|
33
|
+
client: measurement.client,
|
|
34
|
+
server: measurement.server,
|
|
35
|
+
errors: measurement.errors,
|
|
36
|
+
expectations: results,
|
|
37
|
+
passed: passed && measurement.requests.total > 0,
|
|
38
|
+
...measurement.histogram ? { histogram: measurement.histogram } : {}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Assembles the top-level report. The gate passes only when every scenario
|
|
43
|
+
* passes.
|
|
44
|
+
* @param input The report inputs.
|
|
45
|
+
* @returns The complete report.
|
|
46
|
+
*/
|
|
47
|
+
function buildReport(input) {
|
|
48
|
+
return {
|
|
49
|
+
$schema: REPORT_SCHEMA_ID,
|
|
50
|
+
schemaVersion: 1,
|
|
51
|
+
tool: {
|
|
52
|
+
name: "@fedify/cli",
|
|
53
|
+
version
|
|
54
|
+
},
|
|
55
|
+
environment: input.environment,
|
|
56
|
+
target: input.target,
|
|
57
|
+
startedAt: input.startedAt,
|
|
58
|
+
finishedAt: input.finishedAt,
|
|
59
|
+
suite: input.suite,
|
|
60
|
+
passed: input.scenarios.every((s) => s.passed),
|
|
61
|
+
scenarios: input.scenarios
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** Detects the current runtime environment for reproducibility metadata. */
|
|
65
|
+
function detectEnvironment() {
|
|
66
|
+
const g = globalThis;
|
|
67
|
+
let runtime = "node";
|
|
68
|
+
let runtimeVersion = process.versions?.node ?? "unknown";
|
|
69
|
+
if (g.Deno?.version?.deno != null) {
|
|
70
|
+
runtime = "deno";
|
|
71
|
+
runtimeVersion = g.Deno.version.deno;
|
|
72
|
+
} else if (g.Bun?.version != null) {
|
|
73
|
+
runtime = "bun";
|
|
74
|
+
runtimeVersion = g.Bun.version;
|
|
75
|
+
}
|
|
76
|
+
let cpuCount = 0;
|
|
77
|
+
try {
|
|
78
|
+
cpuCount = cpus().length;
|
|
79
|
+
} catch {
|
|
80
|
+
cpuCount = 0;
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
runtime,
|
|
84
|
+
runtimeVersion,
|
|
85
|
+
os: process.platform,
|
|
86
|
+
cpuCount
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Computes a stable `sha256:` hash of a resolved configuration, so CI only
|
|
91
|
+
* compares runs from the same configuration.
|
|
92
|
+
* @param config The configuration object to hash.
|
|
93
|
+
* @returns A `sha256:`-prefixed hex digest.
|
|
94
|
+
*/
|
|
95
|
+
function configHash(config) {
|
|
96
|
+
return `sha256:${createHash("sha256").update(canonicalJson(config)).digest("hex")}`;
|
|
97
|
+
}
|
|
98
|
+
/** A guard against unbounded recursion on pathologically nested input. */
|
|
99
|
+
const MAX_HASH_DEPTH = 100;
|
|
100
|
+
function canonicalJson(value, depth = 0) {
|
|
101
|
+
if (depth > MAX_HASH_DEPTH) throw new RangeError("Maximum depth exceeded while hashing the config.");
|
|
102
|
+
if (value === void 0) return "null";
|
|
103
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
104
|
+
const toJson = value.toJSON;
|
|
105
|
+
if (typeof toJson === "function") return canonicalJson(toJson.call(value), depth + 1);
|
|
106
|
+
if (Array.isArray(value)) return `[${value.map((v) => canonicalJson(v, depth + 1)).join(",")}]`;
|
|
107
|
+
return `{${Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v, depth + 1)}`).join(",")}}`;
|
|
108
|
+
}
|
|
109
|
+
function loadSummary(scenario) {
|
|
110
|
+
const { load, durationMs, warmupMs } = scenario;
|
|
111
|
+
const maxInFlight = load.maxInFlight == null ? {} : { maxInFlight: load.maxInFlight };
|
|
112
|
+
if (load.kind === "closed") return {
|
|
113
|
+
model: "closed",
|
|
114
|
+
concurrency: load.concurrency,
|
|
115
|
+
durationMs,
|
|
116
|
+
warmupMs,
|
|
117
|
+
...maxInFlight
|
|
118
|
+
};
|
|
119
|
+
return {
|
|
120
|
+
model: "open",
|
|
121
|
+
ratePerSec: load.ratePerSec,
|
|
122
|
+
arrival: load.arrival,
|
|
123
|
+
durationMs,
|
|
124
|
+
warmupMs,
|
|
125
|
+
...maxInFlight
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
//#endregion
|
|
129
|
+
export { buildReport, buildScenarioResult, configHash, detectEnvironment };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/result/expect/assert.ts
|
|
3
|
+
const ASSERT_RE = /^\s*(<=|>=|==|=|<|>)\s*(\d+(?:\.\d+)?)\s*(%|ms|s|\/s)?\s*$/;
|
|
4
|
+
const OP_MAP = {
|
|
5
|
+
"<": "lt",
|
|
6
|
+
"<=": "lte",
|
|
7
|
+
">": "gt",
|
|
8
|
+
">=": "gte",
|
|
9
|
+
"==": "eq",
|
|
10
|
+
"=": "eq"
|
|
11
|
+
};
|
|
12
|
+
/** An error raised when an `expect` assertion cannot be parsed. */
|
|
13
|
+
var AssertionParseError = class extends Error {};
|
|
14
|
+
/**
|
|
15
|
+
* Parses an `expect` assertion string.
|
|
16
|
+
* @param text The assertion, e.g. `">= 99%"`.
|
|
17
|
+
* @returns The parsed operator, normalized threshold, and unit.
|
|
18
|
+
* @throws {AssertionParseError} If the assertion cannot be parsed.
|
|
19
|
+
*/
|
|
20
|
+
function parseAssertion(text) {
|
|
21
|
+
const match = text.match(ASSERT_RE);
|
|
22
|
+
if (match == null) throw new AssertionParseError(`Invalid expect assertion: ${JSON.stringify(text)}.`);
|
|
23
|
+
const op = OP_MAP[match[1]];
|
|
24
|
+
const value = Number.parseFloat(match[2]);
|
|
25
|
+
switch (match[3]) {
|
|
26
|
+
case "%": return {
|
|
27
|
+
op,
|
|
28
|
+
threshold: value / 100,
|
|
29
|
+
unit: "%"
|
|
30
|
+
};
|
|
31
|
+
case "ms": return {
|
|
32
|
+
op,
|
|
33
|
+
threshold: value,
|
|
34
|
+
unit: "ms"
|
|
35
|
+
};
|
|
36
|
+
case "s": return {
|
|
37
|
+
op,
|
|
38
|
+
threshold: value * 1e3,
|
|
39
|
+
unit: "ms"
|
|
40
|
+
};
|
|
41
|
+
case "/s": return {
|
|
42
|
+
op,
|
|
43
|
+
threshold: value,
|
|
44
|
+
unit: "/s"
|
|
45
|
+
};
|
|
46
|
+
default: return {
|
|
47
|
+
op,
|
|
48
|
+
threshold: value,
|
|
49
|
+
unit: null
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Compares a measured value against a threshold using a comparison operator.
|
|
55
|
+
* @param actual The measured value.
|
|
56
|
+
* @param op The comparison operator.
|
|
57
|
+
* @param threshold The threshold.
|
|
58
|
+
* @param tolerant Whether `eq` allows a small floating-point tolerance. Pass
|
|
59
|
+
* `false` for exact (count) metrics; defaults to `true` so
|
|
60
|
+
* float-normalized thresholds (e.g. `"99.4%"` ->
|
|
61
|
+
* `0.9940000000000001`) still match a measured `0.994`.
|
|
62
|
+
* @returns Whether the comparison holds.
|
|
63
|
+
*/
|
|
64
|
+
function compare(actual, op, threshold, tolerant = true) {
|
|
65
|
+
switch (op) {
|
|
66
|
+
case "lt": return actual < threshold;
|
|
67
|
+
case "lte": return actual <= threshold;
|
|
68
|
+
case "gt": return actual > threshold;
|
|
69
|
+
case "gte": return actual >= threshold;
|
|
70
|
+
case "eq": return tolerant ? Math.abs(actual - threshold) <= 1e-9 + 1e-9 * Math.abs(threshold) : actual === threshold;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
//#endregion
|
|
74
|
+
export { AssertionParseError, compare, parseAssertion };
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
import { AssertionParseError, compare, parseAssertion } from "./assert.js";
|
|
3
|
+
import { metricUnit } from "./metrics.js";
|
|
4
|
+
//#region src/bench/result/expect/evaluate.ts
|
|
5
|
+
/**
|
|
6
|
+
* Parses every assertion in an `expect` block, throwing on the first malformed
|
|
7
|
+
* one. Run during preflight so that a typo in a CI gate is reported as a
|
|
8
|
+
* configuration error before any load is sent, instead of crashing the run with
|
|
9
|
+
* an uncaught {@link AssertionParseError} after the traffic has already gone out.
|
|
10
|
+
* @param expect The scenario's `expect` block.
|
|
11
|
+
* @throws {AssertionParseError} If an entry has no assertion string or its
|
|
12
|
+
* assertion cannot be parsed.
|
|
13
|
+
*/
|
|
14
|
+
function validateExpectBlock(expect) {
|
|
15
|
+
for (const [metric, value] of Object.entries(expect)) {
|
|
16
|
+
const assertion = typeof value === "string" ? value : value.assert;
|
|
17
|
+
if (typeof assertion !== "string") throw new AssertionParseError(`The \`expect\` entry for "${metric}" has no assertion string.`);
|
|
18
|
+
try {
|
|
19
|
+
parseAssertion(assertion);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
if (!(error instanceof AssertionParseError)) throw error;
|
|
22
|
+
throw new AssertionParseError(`Invalid \`expect\` assertion for "${metric}": ${JSON.stringify(assertion)}.`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Evaluates an `expect` block against measured metrics.
|
|
28
|
+
* @param expect The scenario's `expect` block.
|
|
29
|
+
* @param metrics The measured metrics to evaluate against.
|
|
30
|
+
* @returns The evaluated assertions and whether the gate passed.
|
|
31
|
+
*/
|
|
32
|
+
function evaluateExpect(expect, metrics) {
|
|
33
|
+
const results = [];
|
|
34
|
+
for (const [metric, value] of Object.entries(expect)) {
|
|
35
|
+
const assertion = typeof value === "string" ? value : value.assert;
|
|
36
|
+
const severity = typeof value === "string" ? "fail" : value.severity ?? "fail";
|
|
37
|
+
const { op, threshold, unit } = parseAssertion(assertion);
|
|
38
|
+
const lookup = lookupMetric(metrics, metric);
|
|
39
|
+
const actual = lookup?.value ?? null;
|
|
40
|
+
const pass = lookup != null && actual != null && unitCompatible(unit, lookup.unit) && compare(actual, op, threshold, lookup.unit !== "count");
|
|
41
|
+
results.push({
|
|
42
|
+
metric,
|
|
43
|
+
op,
|
|
44
|
+
threshold,
|
|
45
|
+
unit,
|
|
46
|
+
actual,
|
|
47
|
+
severity,
|
|
48
|
+
pass
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
results,
|
|
53
|
+
passed: results.every((r) => r.severity === "warn" || r.pass)
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Whether an assertion's (normalized) unit is compatible with a metric's
|
|
58
|
+
* natural unit. A unitless assertion is always compatible.
|
|
59
|
+
*/
|
|
60
|
+
function unitCompatible(assertionUnit, unit) {
|
|
61
|
+
if (assertionUnit == null) return true;
|
|
62
|
+
switch (unit) {
|
|
63
|
+
case "ratio": return assertionUnit === "%";
|
|
64
|
+
case "ms": return assertionUnit === "ms";
|
|
65
|
+
case "rate": return assertionUnit === "/s";
|
|
66
|
+
case "count": return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function lookupMetric(metrics, metric) {
|
|
70
|
+
const unit = metricUnit(metric);
|
|
71
|
+
if (unit == null) return null;
|
|
72
|
+
return {
|
|
73
|
+
value: lookupValue(metrics, metric),
|
|
74
|
+
unit
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function lookupValue(metrics, metric) {
|
|
78
|
+
switch (metric) {
|
|
79
|
+
case "successRate": return metrics.requests.successRate;
|
|
80
|
+
case "throughputPerSec": return metrics.throughputPerSec;
|
|
81
|
+
case "deliveryThroughput": return null;
|
|
82
|
+
case "errors.total": return sumErrors(metrics.errors);
|
|
83
|
+
case "errors.4xx": return sumErrors(metrics.errors, {
|
|
84
|
+
min: 400,
|
|
85
|
+
max: 500
|
|
86
|
+
});
|
|
87
|
+
case "errors.5xx": return sumErrors(metrics.errors, {
|
|
88
|
+
min: 500,
|
|
89
|
+
max: 600
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
if (metric.startsWith("latency.")) return latencyField(metrics.client.latencyMs, metric.slice(8));
|
|
93
|
+
if (metric.startsWith("signatureVerification.")) return partialField(metrics.server?.signatureVerificationMs?.overall, metric.slice(22));
|
|
94
|
+
if (metric.startsWith("queueDrain.")) return partialField(metrics.server?.queue?.drainMs, metric.slice(11));
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
function latencyField(latency, key) {
|
|
98
|
+
switch (key) {
|
|
99
|
+
case "p50": return latency.p50;
|
|
100
|
+
case "p95": return latency.p95;
|
|
101
|
+
case "p99": return latency.p99;
|
|
102
|
+
case "mean": return latency.mean;
|
|
103
|
+
case "max": return latency.max;
|
|
104
|
+
default: return null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function partialField(source, key) {
|
|
108
|
+
if (source == null) return null;
|
|
109
|
+
switch (key) {
|
|
110
|
+
case "p50": return source.p50 ?? null;
|
|
111
|
+
case "p95": return source.p95 ?? null;
|
|
112
|
+
case "p99": return source.p99 ?? null;
|
|
113
|
+
default: return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Sums error counts, optionally restricted to a half-open HTTP status range.
|
|
118
|
+
* The bounds are a single coupled argument so a caller cannot pass one without
|
|
119
|
+
* the other.
|
|
120
|
+
*/
|
|
121
|
+
function sumErrors(errors, range) {
|
|
122
|
+
let total = 0;
|
|
123
|
+
for (const error of errors) if (range == null) total += error.count;
|
|
124
|
+
else if (error.status != null && error.status >= range.min && error.status < range.max) total += error.count;
|
|
125
|
+
return total;
|
|
126
|
+
}
|
|
127
|
+
//#endregion
|
|
128
|
+
export { evaluateExpect, validateExpectBlock };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/result/expect/metrics.ts
|
|
3
|
+
/**
|
|
4
|
+
* Returns the natural unit class of a metric, or `null` if the metric name is
|
|
5
|
+
* not recognized.
|
|
6
|
+
* @param metric The metric name, e.g. `"latency.p95"`.
|
|
7
|
+
*/
|
|
8
|
+
function metricUnit(metric) {
|
|
9
|
+
switch (metric) {
|
|
10
|
+
case "successRate": return "ratio";
|
|
11
|
+
case "throughputPerSec":
|
|
12
|
+
case "deliveryThroughput": return "rate";
|
|
13
|
+
case "errors.total":
|
|
14
|
+
case "errors.4xx":
|
|
15
|
+
case "errors.5xx": return "count";
|
|
16
|
+
}
|
|
17
|
+
if (metric.startsWith("latency.") || metric.startsWith("signatureVerification.") || metric.startsWith("queueDrain.")) return "ms";
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Returns the human display unit for a metric (`"%"`, `"ms"`, `"/s"`), or
|
|
22
|
+
* `null` for counts and unknown metrics.
|
|
23
|
+
* @param metric The metric name.
|
|
24
|
+
*/
|
|
25
|
+
function metricDisplayUnit(metric) {
|
|
26
|
+
switch (metricUnit(metric)) {
|
|
27
|
+
case "ratio": return "%";
|
|
28
|
+
case "ms": return "ms";
|
|
29
|
+
case "rate": return "/s";
|
|
30
|
+
default: return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
34
|
+
export { metricDisplayUnit, metricUnit };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/result/schema.ts
|
|
3
|
+
/**
|
|
4
|
+
* The embedded JSON Schema (draft 2020-12) for benchmark report output.
|
|
5
|
+
*
|
|
6
|
+
* Like the scenario schema, this object is the runtime copy and is published,
|
|
7
|
+
* byte-for-byte, as *schema/bench/report-v1.json*; a drift guard keeps the two
|
|
8
|
+
* in sync. The matching TypeScript types live in {@link ./model.ts}.
|
|
9
|
+
* @since 2.3.0
|
|
10
|
+
* @module
|
|
11
|
+
*/
|
|
12
|
+
/** The hosted URL that serves the report schema. */
|
|
13
|
+
const REPORT_SCHEMA_ID = "https://json-schema.fedify.dev/bench/report-v1.json";
|
|
14
|
+
//#endregion
|
|
15
|
+
export { REPORT_SCHEMA_ID };
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/safety/gate.ts
|
|
3
|
+
/** An error raised when a target is refused by the safety gate. */
|
|
4
|
+
var UnsafeTargetError = class extends Error {};
|
|
5
|
+
/**
|
|
6
|
+
* Asserts that a target may be benchmarked, throwing otherwise.
|
|
7
|
+
* @param context The gate decision inputs.
|
|
8
|
+
* @throws {UnsafeTargetError} If the target is public, does not advertise
|
|
9
|
+
* benchmark mode, and `--allow-unsafe-target` was not given.
|
|
10
|
+
*/
|
|
11
|
+
function assertTargetAllowed(context) {
|
|
12
|
+
if (context.dryRun) return;
|
|
13
|
+
if (context.tier !== "public") return;
|
|
14
|
+
if (context.benchmarkMode) return;
|
|
15
|
+
if (context.allowUnsafe) return;
|
|
16
|
+
throw new UnsafeTargetError("Refusing to benchmark a public target that does not advertise benchmark mode. If you control this target, pass --allow-unsafe-target (mandatory in CI and any non-interactive context).");
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Asserts that an unsafe public-target override is specific and bounded.
|
|
20
|
+
*
|
|
21
|
+
* The override is only meaningful for a public target that does not advertise
|
|
22
|
+
* benchmark mode. In that caution tier, the operator must name the target on
|
|
23
|
+
* the command line for this run and must explicitly set load and duration, so
|
|
24
|
+
* the built-in defaults cannot accidentally create a long public benchmark.
|
|
25
|
+
* @param context The unsafe override decision inputs.
|
|
26
|
+
* @throws {UnsafeTargetError} If the unsafe override is too broad.
|
|
27
|
+
*/
|
|
28
|
+
function assertUnsafeOverrideAllowed(context) {
|
|
29
|
+
if (context.tier !== "public" || context.benchmarkMode || !context.allowUnsafe) return;
|
|
30
|
+
if (!context.explicitCliTarget) throw new UnsafeTargetError("The --allow-unsafe-target override must be paired with an explicit --target for this run.");
|
|
31
|
+
for (const scenario of context.scenarios) {
|
|
32
|
+
if (!scenario.explicitLoad) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark load default. Set rate or concurrency explicitly before using --allow-unsafe-target against a public target.`);
|
|
33
|
+
if (!scenario.explicitDuration) throw new UnsafeTargetError(`Scenario "${scenario.name}" uses the built-in benchmark duration default. Set duration explicitly before using --allow-unsafe-target against a public target.`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Asserts that a resolved inbox URL — the actual destination of signed
|
|
38
|
+
* benchmark load — may be sent to. The suite's `target` is gated separately by
|
|
39
|
+
* {@link assertTargetAllowed}; this catches a destination that differs from it
|
|
40
|
+
* (a public `recipient`, or an explicit `inbox:` URL), so production cannot be
|
|
41
|
+
* benchmarked through the back door.
|
|
42
|
+
*
|
|
43
|
+
* A destination is allowed when it is loopback or private, or shares the gated
|
|
44
|
+
* target's host while the target advertises benchmark mode (inheriting its
|
|
45
|
+
* gate), or `--allow-unsafe-target` is given. Because the destination's server
|
|
46
|
+
* dereferences the synthetic actor while verifying signatures, a non-loopback
|
|
47
|
+
* destination additionally requires an advertised, reachable synthetic host.
|
|
48
|
+
* @param url The resolved inbox URL.
|
|
49
|
+
* @param context The destination gate inputs.
|
|
50
|
+
* @throws {UnsafeTargetError} If the destination is refused.
|
|
51
|
+
*/
|
|
52
|
+
function assertInboxDestinationAllowed(url, context) {
|
|
53
|
+
const sameOrigin = url.origin === context.targetOrigin;
|
|
54
|
+
const tier = sameOrigin ? context.targetTier : context.destinationTier;
|
|
55
|
+
const inheritsTargetGate = sameOrigin && context.targetBenchmarkMode;
|
|
56
|
+
if (tier === "public" && !inheritsTargetGate && !context.allowUnsafe) throw new UnsafeTargetError(`Refusing to send benchmark load to ${url.href}: it is a public inbox that is neither part of the benchmarked target nor covered by benchmark mode. Pass --allow-unsafe-target to override.`);
|
|
57
|
+
if (tier !== "loopback" && !context.advertised) throw new UnsafeTargetError(`Refusing to send signed benchmark load to ${url.href}: the synthetic actor server is unreachable from a non-loopback inbox. Pass --advertise-host with an address it can reach.`);
|
|
58
|
+
}
|
|
59
|
+
//#endregion
|
|
60
|
+
export { UnsafeTargetError, assertInboxDestinationAllowed, assertTargetAllowed, assertUnsafeOverrideAllowed };
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
import { lookup } from "node:dns/promises";
|
|
3
|
+
//#region src/bench/safety/tiers.ts
|
|
4
|
+
/**
|
|
5
|
+
* Target risk classification.
|
|
6
|
+
*
|
|
7
|
+
* A target is `loopback` or `private` when it is clearly one of the operator's
|
|
8
|
+
* own boxes, and `public` otherwise. Classification is conservative: a host
|
|
9
|
+
* that is not obviously loopback or private is treated as `public` (the gated
|
|
10
|
+
* tier), since the tool cannot tell staging from production without resolving
|
|
11
|
+
* and trusting DNS.
|
|
12
|
+
* @since 2.3.0
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Classifies a target URL into a risk tier from its host.
|
|
17
|
+
* @param target The target URL.
|
|
18
|
+
* @returns The risk tier.
|
|
19
|
+
*/
|
|
20
|
+
function classifyTarget(target) {
|
|
21
|
+
let host = target.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
|
|
22
|
+
if (host.endsWith(".")) host = host.slice(0, -1);
|
|
23
|
+
if (host === "localhost" || host.endsWith(".localhost")) return "loopback";
|
|
24
|
+
if (host.endsWith(".local")) return "private";
|
|
25
|
+
if (isIpv4(host)) return classifyIpv4(host);
|
|
26
|
+
if (host.includes(":")) return classifyIpv6(host);
|
|
27
|
+
return "public";
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Classifies a target URL, resolving DNS for public-looking hostnames.
|
|
31
|
+
*
|
|
32
|
+
* Literal addresses and known local hostname suffixes are classified directly.
|
|
33
|
+
* Other hostnames are resolved and classified from their addresses; any public
|
|
34
|
+
* address in the answer keeps the target public, and DNS failure is treated as
|
|
35
|
+
* public so the safety gate remains conservative.
|
|
36
|
+
* @param target The target URL.
|
|
37
|
+
* @param resolveAddresses Hostname resolver, overridable for tests.
|
|
38
|
+
* @returns The resolved target tier.
|
|
39
|
+
*/
|
|
40
|
+
async function classifyResolvedTarget(target, resolveAddresses = defaultResolveTargetAddresses) {
|
|
41
|
+
const host = normalizedHost(target);
|
|
42
|
+
const direct = classifyTarget(target);
|
|
43
|
+
if (direct !== "public" || isIpLiteral(host)) return direct;
|
|
44
|
+
try {
|
|
45
|
+
const resolved = await resolveAddresses(host);
|
|
46
|
+
if (!Array.isArray(resolved) || resolved.length < 1) return "public";
|
|
47
|
+
let aggregate = "loopback";
|
|
48
|
+
for (const address of resolved) {
|
|
49
|
+
const tier = classifyTarget(new URL(`http://${hostForAddress(address)}/`));
|
|
50
|
+
if (tier === "public") return "public";
|
|
51
|
+
if (tier === "private") aggregate = "private";
|
|
52
|
+
}
|
|
53
|
+
return aggregate;
|
|
54
|
+
} catch {
|
|
55
|
+
return "public";
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/** Resolves a hostname with the platform DNS resolver. */
|
|
59
|
+
async function defaultResolveTargetAddresses(hostname) {
|
|
60
|
+
return (await lookup(hostname, { all: true })).map((entry) => entry.address);
|
|
61
|
+
}
|
|
62
|
+
function normalizedHost(target) {
|
|
63
|
+
let host = target.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
|
|
64
|
+
if (host.endsWith(".")) host = host.slice(0, -1);
|
|
65
|
+
return host;
|
|
66
|
+
}
|
|
67
|
+
function isIpLiteral(host) {
|
|
68
|
+
return isIpv4(host) || host.includes(":");
|
|
69
|
+
}
|
|
70
|
+
function hostForAddress(address) {
|
|
71
|
+
return address.includes(":") ? `[${address}]` : address;
|
|
72
|
+
}
|
|
73
|
+
function isIpv4(host) {
|
|
74
|
+
const match = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
75
|
+
return match != null && match.slice(1).every((octet) => Number(octet) <= 255);
|
|
76
|
+
}
|
|
77
|
+
function classifyIpv4(host) {
|
|
78
|
+
if (host === "0.0.0.0" || /^127\./.test(host)) return "loopback";
|
|
79
|
+
if (/^10\./.test(host) || /^192\.168\./.test(host) || /^172\.(1[6-9]|2\d|3[01])\./.test(host) || /^169\.254\./.test(host)) return "private";
|
|
80
|
+
return "public";
|
|
81
|
+
}
|
|
82
|
+
function classifyIpv6(host) {
|
|
83
|
+
if (host === "::1") return "loopback";
|
|
84
|
+
const dotted = host.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
|
|
85
|
+
if (dotted != null && isIpv4(dotted[1])) return classifyIpv4(dotted[1]);
|
|
86
|
+
const hex = host.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
|
87
|
+
if (hex != null) {
|
|
88
|
+
const hi = Number.parseInt(hex[1], 16);
|
|
89
|
+
const lo = Number.parseInt(hex[2], 16);
|
|
90
|
+
return classifyIpv4(`${hi >> 8 & 255}.${hi & 255}.${lo >> 8 & 255}.${lo & 255}`);
|
|
91
|
+
}
|
|
92
|
+
if (/^f[cd][0-9a-f]*:/.test(host)) return "private";
|
|
93
|
+
if (/^fe[89ab][0-9a-f]*:/.test(host)) return "private";
|
|
94
|
+
return "public";
|
|
95
|
+
}
|
|
96
|
+
//#endregion
|
|
97
|
+
export { classifyResolvedTarget };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/scenario/coerce.ts
|
|
3
|
+
/**
|
|
4
|
+
* Scalar-or-list coercion used throughout the scenario format, where many
|
|
5
|
+
* fields (`recipient`, `seed`, `collection`, `type`, and so on) accept either a
|
|
6
|
+
* single value or a list of values so the common single-value case stays terse.
|
|
7
|
+
* @since 2.3.0
|
|
8
|
+
* @module
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Normalizes a scalar-or-list value into an array.
|
|
12
|
+
*
|
|
13
|
+
* A single value becomes a one-element array, an array is shallow-copied, and
|
|
14
|
+
* `null`/`undefined` becomes an empty array.
|
|
15
|
+
* @typeParam T The element type.
|
|
16
|
+
* @param value A single value, a list of values, or nothing.
|
|
17
|
+
* @returns A new array of values.
|
|
18
|
+
*/
|
|
19
|
+
function asList(value) {
|
|
20
|
+
if (value == null) return [];
|
|
21
|
+
return Array.isArray(value) ? [...value] : [value];
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
export { asList };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import "@js-temporal/polyfill";
|
|
2
|
+
//#region src/bench/scenario/errors.ts
|
|
3
|
+
/** An error raised when a scenario suite fails schema validation. */
|
|
4
|
+
var SuiteValidationError = class extends Error {
|
|
5
|
+
/** The individual validation problems, most specific first. */
|
|
6
|
+
problems;
|
|
7
|
+
constructor(problems, source) {
|
|
8
|
+
super(formatMessage(problems, source));
|
|
9
|
+
this.name = "SuiteValidationError";
|
|
10
|
+
this.problems = problems;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function formatMessage(problems, source) {
|
|
14
|
+
const where = source == null ? "scenario suite" : source;
|
|
15
|
+
if (problems.length === 0) return `Invalid ${where}.`;
|
|
16
|
+
return `Invalid ${where}:\n${dedupe(problems).map((problem) => {
|
|
17
|
+
return ` - ${problem.instanceLocation === "#" || problem.instanceLocation === "" ? "(root)" : problem.instanceLocation.replace(/^#/, "")}: ${problem.error}`;
|
|
18
|
+
}).join("\n")}`;
|
|
19
|
+
}
|
|
20
|
+
function dedupe(problems) {
|
|
21
|
+
const seen = /* @__PURE__ */ new Set();
|
|
22
|
+
const result = [];
|
|
23
|
+
const sorted = [...problems].sort((a, b) => depth(b.instanceLocation) - depth(a.instanceLocation));
|
|
24
|
+
for (const problem of sorted) {
|
|
25
|
+
const key = JSON.stringify([problem.instanceLocation, problem.error]);
|
|
26
|
+
if (seen.has(key)) continue;
|
|
27
|
+
seen.add(key);
|
|
28
|
+
result.push(problem);
|
|
29
|
+
}
|
|
30
|
+
return result;
|
|
31
|
+
}
|
|
32
|
+
function depth(instanceLocation) {
|
|
33
|
+
return (instanceLocation.match(/\//g) ?? []).length;
|
|
34
|
+
}
|
|
35
|
+
//#endregion
|
|
36
|
+
export { SuiteValidationError };
|