@fedify/cli 2.3.0-dev.1347 → 2.3.0-dev.1361
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bench/action.js +29 -189
- package/dist/bench/command.js +13 -43
- package/dist/bench/load/clock.js +2 -20
- package/dist/bench/load/generator.js +9 -42
- package/dist/bench/metrics/stats-client.js +3 -65
- package/dist/bench/mod.js +2 -9
- package/dist/bench/render/markdown.js +0 -1
- package/dist/bench/render/text.js +0 -1
- package/dist/bench/result/build.js +10 -133
- package/dist/bench/result/expect/evaluate.js +1 -1
- package/dist/bench/result/schema.js +3 -353
- package/dist/bench/safety/gate.js +2 -4
- package/dist/bench/scenario/normalize.js +2 -1
- package/dist/bench/scenario/schema.js +9 -50
- package/dist/bench/scenario/validate.js +2 -2
- package/dist/bench/scenarios/inbox.js +12 -4
- package/dist/bench/scenarios/registry.js +1 -19
- package/dist/bench/scenarios/runner.js +1 -21
- package/dist/bench/scenarios/webfinger.js +1 -1
- package/dist/cache.js +1 -1
- package/dist/config.js +1 -1
- package/dist/deno.js +1 -1
- package/dist/docloader.js +1 -1
- package/dist/generate-vocab/action.js +1 -1
- package/dist/generate-vocab/command.js +3 -5
- package/dist/generate-vocab/mod.js +4 -0
- package/dist/imagerenderer.js +2 -2
- package/dist/inbox/command.js +4 -6
- package/dist/inbox.js +4 -4
- package/dist/init/mod.js +3 -0
- package/dist/log.js +2 -2
- package/dist/lookup.js +123 -12
- package/dist/mod.js +23 -2
- package/dist/nodeinfo.js +9 -11
- package/dist/options.js +1 -1
- package/dist/relay/command.js +4 -6
- package/dist/relay.js +2 -2
- package/dist/runner.js +46 -69
- package/dist/tempserver.js +1 -1
- package/dist/tunnel.js +4 -6
- package/dist/utils.js +4 -5
- package/dist/webfinger/action.js +1 -1
- package/dist/webfinger/command.js +4 -6
- package/dist/webfinger/lib.js +1 -1
- package/dist/webfinger/mod.js +4 -0
- package/package.json +12 -13
- package/dist/bench/compare/schema.js +0 -16
- package/dist/bench/compare.js +0 -667
- package/dist/bench/scenarios/actor.js +0 -38
- package/dist/bench/scenarios/failure.js +0 -363
- package/dist/bench/scenarios/fanout.js +0 -261
- package/dist/bench/scenarios/mixed.js +0 -244
- package/dist/bench/scenarios/object-discovery.js +0 -211
- package/dist/bench/scenarios/object.js +0 -54
- package/dist/bench/scenarios/read.js +0 -108
- package/dist/commands.js +0 -110
- package/dist/lookup/command.js +0 -121
package/dist/bench/action.js
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import "@js-temporal/polyfill";
|
|
2
|
-
import { describeError } from "../utils.js";
|
|
3
2
|
import { getContextLoader, getDocumentLoader } from "../docloader.js";
|
|
3
|
+
import { describeError } from "../utils.js";
|
|
4
4
|
import { buildFleet } from "./actor/fleet.js";
|
|
5
5
|
import { convertUrlIfHandle } from "../webfinger/lib.js";
|
|
6
6
|
import { discoverInbox, selectInbox } from "./discovery/discover.js";
|
|
7
|
-
import { actorUrlsFromRecipients, objectUrlsFromSource } from "./scenarios/object-discovery.js";
|
|
8
7
|
import { validateExpectBlock } from "./result/expect/evaluate.js";
|
|
9
8
|
import { buildReport, buildScenarioResult, configHash, detectEnvironment } from "./result/build.js";
|
|
10
9
|
import { probeBenchmarkMode } from "./discovery/probe.js";
|
|
@@ -14,11 +13,13 @@ import { normalizeSuite } from "./scenario/normalize.js";
|
|
|
14
13
|
import { validateSuite } from "./scenario/validate.js";
|
|
15
14
|
import { UnsafeTargetError, assertInboxDestinationAllowed, assertTargetAllowed, assertUnsafeOverrideAllowed } from "./safety/gate.js";
|
|
16
15
|
import { classifyResolvedTarget } from "./safety/tiers.js";
|
|
17
|
-
import { resolveAdvertiseHost, spawnSyntheticServer } from "./server/synthetic.js";
|
|
18
16
|
import { runnerFor } from "./scenarios/registry.js";
|
|
19
|
-
import
|
|
17
|
+
import { resolveAdvertiseHost, spawnSyntheticServer } from "./server/synthetic.js";
|
|
20
18
|
import { writeFile } from "node:fs/promises";
|
|
19
|
+
import process from "node:process";
|
|
21
20
|
//#region src/bench/action.ts
|
|
21
|
+
/** The scenario types that need the synthetic actor/key server. */
|
|
22
|
+
const SIGNED_TYPES = new Set(["inbox"]);
|
|
22
23
|
/**
|
|
23
24
|
* Runs the `fedify bench` command: load and validate the suite, gate the
|
|
24
25
|
* target, run each scenario, and render the report. The process exits 0 when
|
|
@@ -33,10 +34,7 @@ async function runBench(command, deps = {}) {
|
|
|
33
34
|
});
|
|
34
35
|
const writeOutput = deps.writeOutput ?? defaultWriteOutput;
|
|
35
36
|
const log = deps.log ?? ((message) => process.stderr.write(`${message}\n`));
|
|
36
|
-
const
|
|
37
|
-
const fetchImpl = withUserAgent(withAbortSignal(deps.fetch ?? fetch, signal), command.userAgent);
|
|
38
|
-
const explicitCliTarget = command.explicitCliTarget ?? command.target != null;
|
|
39
|
-
throwIfAborted(signal);
|
|
37
|
+
const fetchImpl = withUserAgent(deps.fetch ?? fetch, command.userAgent);
|
|
40
38
|
let validated;
|
|
41
39
|
let suite;
|
|
42
40
|
try {
|
|
@@ -47,12 +45,11 @@ async function runBench(command, deps = {}) {
|
|
|
47
45
|
exit(2);
|
|
48
46
|
return;
|
|
49
47
|
}
|
|
50
|
-
throwIfAborted(signal);
|
|
51
48
|
let runners;
|
|
52
49
|
try {
|
|
53
50
|
runners = suite.scenarios.map((scenario) => {
|
|
54
51
|
const runner = runnerFor(scenario.type);
|
|
55
|
-
runner.validate?.(scenario
|
|
52
|
+
runner.validate?.(scenario);
|
|
56
53
|
validateExpectBlock(scenario.expect);
|
|
57
54
|
return runner;
|
|
58
55
|
});
|
|
@@ -62,17 +59,14 @@ async function runBench(command, deps = {}) {
|
|
|
62
59
|
exit(2);
|
|
63
60
|
return;
|
|
64
61
|
}
|
|
65
|
-
throwIfAborted(signal);
|
|
66
62
|
const tier = await classifyResolvedTarget(suite.target, deps.resolveTargetAddresses);
|
|
67
|
-
throwIfAborted(signal);
|
|
68
63
|
const probe = await probeBenchmarkMode(suite.target, fetchImpl);
|
|
69
|
-
throwIfAborted(signal);
|
|
70
64
|
try {
|
|
71
65
|
if (!command.dryRun) assertUnsafeOverrideAllowed({
|
|
72
66
|
tier,
|
|
73
67
|
benchmarkMode: probe.benchmarkMode,
|
|
74
68
|
allowUnsafe: command.allowUnsafeTarget,
|
|
75
|
-
explicitCliTarget,
|
|
69
|
+
explicitCliTarget: command.target != null,
|
|
76
70
|
scenarios: unsafeOverrideScenarios(validated)
|
|
77
71
|
});
|
|
78
72
|
assertTargetAllowed({
|
|
@@ -112,35 +106,17 @@ async function runBench(command, deps = {}) {
|
|
|
112
106
|
targetOrigin: suite.target.origin,
|
|
113
107
|
targetBenchmarkMode: probe.benchmarkMode,
|
|
114
108
|
allowUnsafe: command.allowUnsafeTarget,
|
|
115
|
-
explicitCliTarget,
|
|
116
|
-
destinationTier,
|
|
117
|
-
defaults: validated.defaults
|
|
118
|
-
});
|
|
119
|
-
};
|
|
120
|
-
const assertDestinationWithoutSyntheticServerAllowed = async (url, scenario, loadDescription) => {
|
|
121
|
-
const sameOrigin = url.origin === suite.target.origin;
|
|
122
|
-
const destinationTier = sameOrigin ? tier : await classifyResolvedTarget(url, deps.resolveTargetAddresses);
|
|
123
|
-
const inheritsTargetGate = sameOrigin && probe.benchmarkMode;
|
|
124
|
-
if (destinationTier === "public" && !inheritsTargetGate && !command.allowUnsafeTarget) throw new UnsafeTargetError(`Refusing to send ${loadDescription} to ${url.href}: it is public and not part of the benchmarked target. Pass --allow-unsafe-target to override.`);
|
|
125
|
-
assertPublicDestinationOverrideAllowed(url, scenario, {
|
|
126
|
-
targetOrigin: suite.target.origin,
|
|
127
|
-
targetBenchmarkMode: probe.benchmarkMode,
|
|
128
|
-
allowUnsafe: command.allowUnsafeTarget,
|
|
129
|
-
explicitCliTarget,
|
|
109
|
+
explicitCliTarget: command.target != null,
|
|
130
110
|
destinationTier,
|
|
131
111
|
defaults: validated.defaults
|
|
132
112
|
});
|
|
133
113
|
};
|
|
134
|
-
const assertReadDestinationAllowed = (url, scenario) => assertDestinationWithoutSyntheticServerAllowed(url, scenario, "benchmark read load");
|
|
135
|
-
const assertActorlessDestinationAllowed = (url, scenario) => assertDestinationWithoutSyntheticServerAllowed(url, scenario, "benchmark load");
|
|
136
114
|
if (command.dryRun) try {
|
|
137
115
|
await writeOutput(await renderPlan(suite, {
|
|
138
116
|
documentLoader,
|
|
139
117
|
contextLoader,
|
|
140
118
|
allowPrivateAddress,
|
|
141
|
-
|
|
142
|
-
assertDestinationAllowed,
|
|
143
|
-
assertReadDestinationAllowed
|
|
119
|
+
assertDestinationAllowed
|
|
144
120
|
}), command.output);
|
|
145
121
|
exit(0);
|
|
146
122
|
return;
|
|
@@ -149,42 +125,30 @@ async function runBench(command, deps = {}) {
|
|
|
149
125
|
exit(2);
|
|
150
126
|
return;
|
|
151
127
|
}
|
|
152
|
-
if (tier !== "loopback" && command.advertiseHost == null && suite.scenarios.some((
|
|
153
|
-
log("
|
|
128
|
+
if (tier !== "loopback" && command.advertiseHost == null && suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))) {
|
|
129
|
+
log("Signed scenarios (inbox) need the benchmark's synthetic actor server to be reachable from the target. A loopback target reaches it automatically; for a non-loopback target, pass --advertise-host with an address the target can reach (the synthetic server then binds all interfaces), or use a read scenario such as webfinger.");
|
|
154
130
|
exit(2);
|
|
155
131
|
return;
|
|
156
132
|
}
|
|
157
133
|
let fleet;
|
|
158
134
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
159
135
|
try {
|
|
160
|
-
|
|
161
|
-
if (suite.scenarios.some((scenario) => scenarioNeedsSyntheticServer(scenario, suite.scenarios))) fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost });
|
|
136
|
+
if (suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))) fleet = await spawnSyntheticServer(await buildFleet(suite.actors), { advertiseHost: command.advertiseHost });
|
|
162
137
|
const results = [];
|
|
163
138
|
for (let i = 0; i < suite.scenarios.length; i++) {
|
|
164
139
|
const scenario = suite.scenarios[i];
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
fleet: fleet ?? null,
|
|
178
|
-
advertiseHost: command.advertiseHost,
|
|
179
|
-
fetch: fetchImpl,
|
|
180
|
-
assertDestinationAllowed: (url, gateScenario) => assertDestinationAllowed(url, gateScenario ?? scenario),
|
|
181
|
-
assertReadDestinationAllowed: (url, gateScenario) => assertReadDestinationAllowed(url, gateScenario ?? scenario),
|
|
182
|
-
assertActorlessDestinationAllowed: (url, gateScenario) => assertActorlessDestinationAllowed(url, gateScenario ?? scenario),
|
|
183
|
-
signal
|
|
184
|
-
}));
|
|
185
|
-
throwIfAborted(signal);
|
|
186
|
-
}
|
|
187
|
-
results.push(buildScenarioResult(scenario, measurements));
|
|
140
|
+
log(`Running scenario "${scenario.name}" (${scenario.type})…`);
|
|
141
|
+
const measurement = await runners[i].run({
|
|
142
|
+
scenario,
|
|
143
|
+
target: suite.target,
|
|
144
|
+
documentLoader,
|
|
145
|
+
contextLoader,
|
|
146
|
+
allowPrivateAddress,
|
|
147
|
+
fleet: fleet ?? null,
|
|
148
|
+
fetch: fetchImpl,
|
|
149
|
+
assertDestinationAllowed: (url) => assertDestinationAllowed(url, scenario)
|
|
150
|
+
});
|
|
151
|
+
results.push(buildScenarioResult(scenario, measurement));
|
|
188
152
|
}
|
|
189
153
|
const report = buildReport({
|
|
190
154
|
scenarios: results,
|
|
@@ -246,22 +210,6 @@ function withUserAgent(fetchImpl, userAgent) {
|
|
|
246
210
|
});
|
|
247
211
|
});
|
|
248
212
|
}
|
|
249
|
-
function withAbortSignal(fetchImpl, signal) {
|
|
250
|
-
if (signal == null) return fetchImpl;
|
|
251
|
-
return ((input, init) => {
|
|
252
|
-
if (signal.aborted) return Promise.reject(abortReason(signal));
|
|
253
|
-
return fetchImpl(input, {
|
|
254
|
-
...init,
|
|
255
|
-
signal
|
|
256
|
-
});
|
|
257
|
-
});
|
|
258
|
-
}
|
|
259
|
-
function throwIfAborted(signal) {
|
|
260
|
-
if (signal?.aborted) throw abortReason(signal);
|
|
261
|
-
}
|
|
262
|
-
function abortReason(signal) {
|
|
263
|
-
return signal.reason ?? /* @__PURE__ */ new Error("Benchmark run aborted.");
|
|
264
|
-
}
|
|
265
213
|
async function defaultWriteOutput(content, outputPath) {
|
|
266
214
|
if (outputPath == null) {
|
|
267
215
|
process.stdout.write(content.endsWith("\n") ? content : `${content}\n`);
|
|
@@ -284,34 +232,12 @@ async function renderPlan(suite, context) {
|
|
|
284
232
|
return `${lines.join("\n")}\n`;
|
|
285
233
|
}
|
|
286
234
|
function describePlan(scenario) {
|
|
287
|
-
|
|
288
|
-
const totalDurationMs = scenario.durationMs * scenario.runs;
|
|
289
|
-
const volume = describePlannedRequestVolume(scenario);
|
|
290
|
-
return [
|
|
291
|
-
load,
|
|
292
|
-
`duration ${scenario.durationMs}ms`,
|
|
293
|
-
`runs ${scenario.runs}`,
|
|
294
|
-
`total duration ${totalDurationMs}ms`,
|
|
295
|
-
...volume == null ? [] : [volume],
|
|
296
|
-
`signing ${scenario.signing}`
|
|
297
|
-
].join(", ");
|
|
298
|
-
}
|
|
299
|
-
function describePlannedRequestVolume(scenario) {
|
|
300
|
-
if (scenario.load.kind !== "open") return null;
|
|
301
|
-
return `estimated scheduled requests ${formatPlanNumber(scenario.load.ratePerSec * (scenario.durationMs / 1e3) * scenario.runs)}`;
|
|
302
|
-
}
|
|
303
|
-
function formatPlanNumber(value) {
|
|
304
|
-
if (Number.isInteger(value)) return String(value);
|
|
305
|
-
const formatted = value.toFixed(2).replace(/\.?0+$/, "");
|
|
306
|
-
return formatted === "" ? "0" : formatted;
|
|
235
|
+
return `${scenario.load.kind === "open" ? `open-loop ${scenario.load.ratePerSec}/s ${scenario.load.arrival}` : `closed-loop concurrency ${scenario.load.concurrency}`}, duration ${scenario.durationMs}ms, signing ${scenario.signing}`;
|
|
307
236
|
}
|
|
308
237
|
async function describeDiscoveryPlan(scenario, suite, context) {
|
|
309
238
|
switch (scenario.type) {
|
|
310
239
|
case "inbox": return await describeInboxDiscoveryPlan(scenario, context);
|
|
311
240
|
case "webfinger": return describeWebFingerPlan(scenario, suite.target);
|
|
312
|
-
case "actor": return await describeActorPlan(scenario, suite, context);
|
|
313
|
-
case "object": return await describeObjectPlan(scenario, suite, context);
|
|
314
|
-
case "mixed": return describeMixedPlan(scenario);
|
|
315
241
|
default: return [" discovery: not available for this scenario type"];
|
|
316
242
|
}
|
|
317
243
|
}
|
|
@@ -343,59 +269,15 @@ function describeWebFingerPlan(scenario, target) {
|
|
|
343
269
|
return ` webfinger ${resource}: GET ${url.href}`;
|
|
344
270
|
});
|
|
345
271
|
}
|
|
346
|
-
async function
|
|
272
|
+
async function describeDestinationSafety(inbox, scenario, context) {
|
|
347
273
|
try {
|
|
348
|
-
|
|
349
|
-
target: suite.target,
|
|
350
|
-
fetch: context.fetch
|
|
351
|
-
});
|
|
352
|
-
const lines = [];
|
|
353
|
-
for (const url of urls) {
|
|
354
|
-
lines.push(` actor: GET ${url.href}`);
|
|
355
|
-
lines.push(` destination safety: ${await describeDestinationSafety(url, scenario, context)}`);
|
|
356
|
-
}
|
|
357
|
-
return lines;
|
|
358
|
-
} catch (error) {
|
|
359
|
-
return [` actor discovery failed (${describeError(error)})`];
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
async function describeObjectPlan(scenario, suite, context) {
|
|
363
|
-
try {
|
|
364
|
-
const urls = await objectUrlsFromSource({
|
|
365
|
-
source: scenario.source,
|
|
366
|
-
target: suite.target,
|
|
367
|
-
fetch: context.fetch,
|
|
368
|
-
assertReadDestinationAllowed: (url) => context.assertReadDestinationAllowed(url, scenario)
|
|
369
|
-
});
|
|
370
|
-
const lines = [` objects: ${urls.length} URL(s) resolved`];
|
|
371
|
-
for (const url of urls.slice(0, 10)) {
|
|
372
|
-
lines.push(` object: GET ${url.href}`);
|
|
373
|
-
lines.push(` destination safety: ${await describeDestinationSafety(url, scenario, context)}`);
|
|
374
|
-
}
|
|
375
|
-
if (urls.length > 10) lines.push(` ... ${urls.length - 10} more`);
|
|
376
|
-
return lines;
|
|
377
|
-
} catch (error) {
|
|
378
|
-
return [` object discovery failed (${describeError(error)})`];
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
function describeMixedPlan(scenario) {
|
|
382
|
-
const entries = scenario.raw.mix ?? [];
|
|
383
|
-
if (entries.length < 1) return [" mix: no child scenarios"];
|
|
384
|
-
return entries.map((entry) => ` mix: ${entry.scenario} weight ${entry.weight}`);
|
|
385
|
-
}
|
|
386
|
-
async function describeDestinationSafety(url, scenario, context) {
|
|
387
|
-
try {
|
|
388
|
-
if (usesReadDestinationGate(scenario)) await context.assertReadDestinationAllowed(url, scenario);
|
|
389
|
-
else await context.assertDestinationAllowed(url, scenario);
|
|
274
|
+
await context.assertDestinationAllowed(inbox, scenario);
|
|
390
275
|
return "allowed";
|
|
391
276
|
} catch (error) {
|
|
392
277
|
if (error instanceof UnsafeTargetError) return `would be refused: ${error.message}`;
|
|
393
278
|
throw error;
|
|
394
279
|
}
|
|
395
280
|
}
|
|
396
|
-
function usesReadDestinationGate(scenario) {
|
|
397
|
-
return (scenario.type === "actor" || scenario.type === "object") && !scenario.authenticated;
|
|
398
|
-
}
|
|
399
281
|
function assertPublicDestinationOverrideAllowed(url, scenario, context) {
|
|
400
282
|
const inheritsTargetGate = url.origin === context.targetOrigin && context.targetBenchmarkMode;
|
|
401
283
|
if (context.destinationTier !== "public" || inheritsTargetGate || !context.allowUnsafe) return;
|
|
@@ -413,57 +295,15 @@ function unsafeOverrideScenarios(suite) {
|
|
|
413
295
|
function unsafeOverrideScenario(scenario, defaults) {
|
|
414
296
|
const defaultDuration = defaults?.duration != null;
|
|
415
297
|
const defaultLoad = hasExplicitLoad(defaults?.load);
|
|
416
|
-
const defaultRuns = defaults?.runs != null;
|
|
417
298
|
const raw = "raw" in scenario ? scenario.raw : scenario;
|
|
418
299
|
return {
|
|
419
300
|
name: scenario.name,
|
|
420
301
|
explicitDuration: raw.duration != null || defaultDuration,
|
|
421
|
-
explicitLoad: hasExplicitLoad(raw.load) || defaultLoad
|
|
422
|
-
explicitRuns: raw.runs != null || defaultRuns
|
|
302
|
+
explicitLoad: hasExplicitLoad(raw.load) || defaultLoad
|
|
423
303
|
};
|
|
424
304
|
}
|
|
425
305
|
function hasExplicitLoad(load) {
|
|
426
306
|
return load != null && typeof load === "object" && ("rate" in load && load.rate != null || "concurrency" in load && load.concurrency != null);
|
|
427
307
|
}
|
|
428
|
-
function scenarioNeedsSyntheticServer(scenario, scenarios, seen = /* @__PURE__ */ new Set()) {
|
|
429
|
-
if (seen.has(scenario.name)) return false;
|
|
430
|
-
const nextSeen = new Set(seen).add(scenario.name);
|
|
431
|
-
switch (scenario.type) {
|
|
432
|
-
case "inbox": return true;
|
|
433
|
-
case "actor":
|
|
434
|
-
case "object": return scenario.authenticated;
|
|
435
|
-
case "failure": return failureFaultsOf(scenario).some(isInboundFailureFault);
|
|
436
|
-
case "mixed": return mixedChildrenOf(scenario, scenarios).some((child) => scenarioNeedsSyntheticServer(child, scenarios, nextSeen));
|
|
437
|
-
default: return false;
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
function scenarioNeedsReachableLocalServer(scenario, scenarios, seen = /* @__PURE__ */ new Set()) {
|
|
441
|
-
if (scenario.type === "fanout") return scenario.raw.sinkBase == null;
|
|
442
|
-
if (scenario.type === "failure") {
|
|
443
|
-
const faults = failureFaultsOf(scenario);
|
|
444
|
-
return faults.includes("invalid-signature") || scenario.raw.sinkBase == null && faults.some(isRemoteFailureFault);
|
|
445
|
-
}
|
|
446
|
-
if (scenario.type === "mixed") {
|
|
447
|
-
if (seen.has(scenario.name)) return false;
|
|
448
|
-
const nextSeen = new Set(seen).add(scenario.name);
|
|
449
|
-
return mixedChildrenOf(scenario, scenarios).some((child) => scenarioNeedsReachableLocalServer(child, scenarios, nextSeen));
|
|
450
|
-
}
|
|
451
|
-
return scenarioNeedsSyntheticServer(scenario, scenarios, seen);
|
|
452
|
-
}
|
|
453
|
-
function failureFaultsOf(scenario) {
|
|
454
|
-
return scenario.faults.length < 1 ? ["remote-404"] : scenario.faults;
|
|
455
|
-
}
|
|
456
|
-
function mixedChildrenOf(scenario, scenarios) {
|
|
457
|
-
return (scenario.raw.mix ?? []).flatMap((entry) => {
|
|
458
|
-
const child = scenarios.find((candidate) => candidate.name === entry.scenario);
|
|
459
|
-
return child == null ? [] : [child];
|
|
460
|
-
});
|
|
461
|
-
}
|
|
462
|
-
function isInboundFailureFault(fault) {
|
|
463
|
-
return fault === "invalid-signature" || fault === "missing-actor";
|
|
464
|
-
}
|
|
465
|
-
function isRemoteFailureFault(fault) {
|
|
466
|
-
return fault === "remote-404" || fault === "remote-410" || fault === "slow-inbox" || fault === "network-error";
|
|
467
|
-
}
|
|
468
308
|
//#endregion
|
|
469
309
|
export { runBench as default };
|
package/dist/bench/command.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import "@js-temporal/polyfill";
|
|
2
2
|
import { configContext } from "../config.js";
|
|
3
3
|
import { userAgentOption } from "../options.js";
|
|
4
|
-
import { argument, choice, command, constant, flag, group, merge, message, object, option, optional,
|
|
4
|
+
import { argument, choice, command, constant, flag, group, merge, message, object, option, optional, string, withDefault } from "@optique/core";
|
|
5
5
|
import { bindConfig } from "@optique/config";
|
|
6
6
|
//#region src/bench/command.ts
|
|
7
7
|
const formatOption = bindConfig(option("-f", "--format", choice([
|
|
@@ -16,57 +16,27 @@ const formatOption = bindConfig(option("-f", "--format", choice([
|
|
|
16
16
|
const allowUnsafeTarget = withDefault(flag("--allow-unsafe-target", { description: message`Allow benchmarking a public target that does not advertise \
|
|
17
17
|
benchmark mode. Must be given on the command line for each run; it cannot be \
|
|
18
18
|
set in a configuration file.` }), false);
|
|
19
|
-
const
|
|
20
|
-
const targetOption = optional(option("-t", "--target", string({ metavar: "URL" }), { description: message`Override the target URL declared in the suite.` }));
|
|
21
|
-
const advertiseHostOption = optional(option("--advertise-host", string({ metavar: "HOST" }), { description: message`Host (name or IP) a non-loopback target can reach the \
|
|
22
|
-
benchmark's synthetic actor server at. Required for signed scenarios against a \
|
|
23
|
-
non-loopback target; binds the synthetic server on all interfaces and uses this \
|
|
24
|
-
host in the actor and key URLs the target dereferences.` }));
|
|
25
|
-
const runParser = merge("Benchmark options", object({
|
|
19
|
+
const benchCommand = command("bench", merge("Benchmark options", object({
|
|
26
20
|
command: constant("bench"),
|
|
27
|
-
mode: constant("run"),
|
|
28
21
|
scenario: group("Arguments", argument(string({ metavar: "SCENARIO_FILE" }), { description: message`Path to the benchmark suite file (YAML or JSON).` })),
|
|
29
|
-
target:
|
|
22
|
+
target: optional(option("-t", "--target", string({ metavar: "URL" }), { description: message`Override the target URL declared in the suite.` })),
|
|
30
23
|
format: formatOption,
|
|
31
|
-
output:
|
|
24
|
+
output: optional(option("-o", "--output", string({ metavar: "OUTPUT_PATH" }), { description: message`Write the report to a file instead of standard output.` })),
|
|
32
25
|
dryRun: withDefault(flag("--dry-run", { description: message`Resolve discovery and print the benchmark plan without \
|
|
33
26
|
sending load.` }), false),
|
|
34
|
-
advertiseHost:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
command: constant("bench"),
|
|
39
|
-
mode: constant("compare"),
|
|
40
|
-
base: option("--base", string({ metavar: "REF" }), { description: message`The base git ref to benchmark.` }),
|
|
41
|
-
head: option("--head", string({ metavar: "REF" }), { description: message`The head git ref to benchmark.` }),
|
|
42
|
-
file: option("--file", string({ metavar: "SCENARIO_FILE" }), { description: message`Path to the benchmark suite file (YAML or JSON).` }),
|
|
43
|
-
startCommand: option("--start-command", string({ metavar: "COMMAND" }), { description: message`Shell command that starts the target application in each \
|
|
44
|
-
checked-out worktree.` }),
|
|
45
|
-
readyUrl: option("--ready-url", string({ metavar: "URL" }), { description: message`URL that returns success when the started target is ready.` }),
|
|
46
|
-
readyTimeout: withDefault(option("--ready-timeout", string({ metavar: "DURATION" }), { description: message`How long to wait for --ready-url.` }), "30s"),
|
|
47
|
-
maxRegression: option("--max-regression", string({ metavar: "PERCENT" }), { description: message`Maximum regression tolerated after the measured noise band.` }),
|
|
48
|
-
target: targetOption,
|
|
49
|
-
format: formatOption,
|
|
50
|
-
output: outputOption,
|
|
51
|
-
dryRun: constant(false),
|
|
52
|
-
advertiseHost: advertiseHostOption,
|
|
27
|
+
advertiseHost: optional(option("--advertise-host", string({ metavar: "HOST" }), { description: message`Host (name or IP) a non-loopback target can reach the \
|
|
28
|
+
benchmark's synthetic actor server at. Required for signed scenarios against a \
|
|
29
|
+
non-loopback target; binds the synthetic server on all interfaces and uses this \
|
|
30
|
+
host in the actor and key URLs the target dereferences.` })),
|
|
53
31
|
allowUnsafeTarget
|
|
54
32
|
}), userAgentOption), {
|
|
55
|
-
brief: message`Compare base and head benchmark runs.`,
|
|
56
|
-
description: message`Run the same benchmark suite against two git revisions on the \
|
|
57
|
-
same runner, then fail when the head revision regresses beyond the configured \
|
|
58
|
-
tolerance and measured noise band.`
|
|
59
|
-
}), runParser);
|
|
60
|
-
const benchMetadata = {
|
|
61
33
|
brief: message`Benchmark a Fedify federation workload.`,
|
|
62
34
|
description: message`Run an ActivityPub-specific load benchmark against a \
|
|
63
35
|
cooperative Fedify target running in benchmark mode.
|
|
64
36
|
|
|
65
|
-
The suite file declares the target, actors, and scenarios.
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
};
|
|
70
|
-
command("bench", benchOptions, benchMetadata);
|
|
37
|
+
The suite file declares the target, actors, and scenarios. Only the \`inbox\` \
|
|
38
|
+
and \`webfinger\` scenario types are executed in this version; the format \
|
|
39
|
+
itself can express every scenario type.`
|
|
40
|
+
});
|
|
71
41
|
//#endregion
|
|
72
|
-
export {
|
|
42
|
+
export { benchCommand };
|
package/dist/bench/load/clock.js
CHANGED
|
@@ -4,30 +4,12 @@ import "@js-temporal/polyfill";
|
|
|
4
4
|
function systemClock() {
|
|
5
5
|
return {
|
|
6
6
|
now: () => performance.now(),
|
|
7
|
-
sleepUntil(timeMs
|
|
8
|
-
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
7
|
+
sleepUntil(timeMs) {
|
|
9
8
|
const remaining = timeMs - performance.now();
|
|
10
9
|
if (remaining <= 0) return Promise.resolve();
|
|
11
|
-
return new Promise((resolve
|
|
12
|
-
const timer = setTimeout(() => {
|
|
13
|
-
cleanup();
|
|
14
|
-
resolve();
|
|
15
|
-
}, remaining);
|
|
16
|
-
const onAbort = () => {
|
|
17
|
-
clearTimeout(timer);
|
|
18
|
-
cleanup();
|
|
19
|
-
reject(abortReason(signal));
|
|
20
|
-
};
|
|
21
|
-
const cleanup = () => {
|
|
22
|
-
signal?.removeEventListener("abort", onAbort);
|
|
23
|
-
};
|
|
24
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
25
|
-
});
|
|
10
|
+
return new Promise((resolve) => setTimeout(resolve, remaining));
|
|
26
11
|
}
|
|
27
12
|
};
|
|
28
13
|
}
|
|
29
|
-
function abortReason(signal) {
|
|
30
|
-
return signal.reason ?? /* @__PURE__ */ new Error("Operation aborted.");
|
|
31
|
-
}
|
|
32
14
|
//#endregion
|
|
33
15
|
export { systemClock };
|
|
@@ -9,10 +9,10 @@ import { systemClock } from "./clock.js";
|
|
|
9
9
|
* @param clock The clock (overridable for tests); defaults to the system clock.
|
|
10
10
|
* @returns The recorded samples and run metadata.
|
|
11
11
|
*/
|
|
12
|
-
function runLoad(plan, send, clock = systemClock()
|
|
13
|
-
return plan.load.kind === "open" ? runOpenLoop(plan, plan.load, send, clock
|
|
12
|
+
function runLoad(plan, send, clock = systemClock()) {
|
|
13
|
+
return plan.load.kind === "open" ? runOpenLoop(plan, plan.load, send, clock) : runClosedLoop(plan, plan.load, send, clock);
|
|
14
14
|
}
|
|
15
|
-
async function runOpenLoop(plan, load, send, clock
|
|
15
|
+
async function runOpenLoop(plan, load, send, clock) {
|
|
16
16
|
const arrivals = scheduleArrivals({
|
|
17
17
|
ratePerSec: load.ratePerSec,
|
|
18
18
|
durationMs: plan.durationMs,
|
|
@@ -25,13 +25,8 @@ async function runOpenLoop(plan, load, send, clock, signal) {
|
|
|
25
25
|
const start = clock.now();
|
|
26
26
|
const active = /* @__PURE__ */ new Set();
|
|
27
27
|
for (const offset of arrivals) {
|
|
28
|
-
|
|
29
|
-
await
|
|
30
|
-
if (await slots.acquire(signal)) saturated = true;
|
|
31
|
-
if (signal?.aborted) {
|
|
32
|
-
slots.release();
|
|
33
|
-
throw abortReason(signal);
|
|
34
|
-
}
|
|
28
|
+
await clock.sleepUntil(start + offset);
|
|
29
|
+
if (await slots.acquire()) saturated = true;
|
|
35
30
|
const dispatched = dispatch(send, offset, start, plan.warmupMs, clock, samples).finally(() => {
|
|
36
31
|
slots.release();
|
|
37
32
|
active.delete(dispatched);
|
|
@@ -45,7 +40,7 @@ async function runOpenLoop(plan, load, send, clock, signal) {
|
|
|
45
40
|
wallDurationMs: clock.now() - start
|
|
46
41
|
};
|
|
47
42
|
}
|
|
48
|
-
async function runClosedLoop(plan, load, send, clock
|
|
43
|
+
async function runClosedLoop(plan, load, send, clock) {
|
|
49
44
|
const samples = [];
|
|
50
45
|
const slots = createSemaphore(load.maxInFlight);
|
|
51
46
|
let saturated = false;
|
|
@@ -53,12 +48,7 @@ async function runClosedLoop(plan, load, send, clock, signal) {
|
|
|
53
48
|
const deadline = start + plan.durationMs;
|
|
54
49
|
async function worker() {
|
|
55
50
|
while (clock.now() < deadline) {
|
|
56
|
-
|
|
57
|
-
if (await slots.acquire(signal)) saturated = true;
|
|
58
|
-
if (signal?.aborted) {
|
|
59
|
-
slots.release();
|
|
60
|
-
throw abortReason(signal);
|
|
61
|
-
}
|
|
51
|
+
if (await slots.acquire()) saturated = true;
|
|
62
52
|
if (clock.now() >= deadline) {
|
|
63
53
|
slots.release();
|
|
64
54
|
break;
|
|
@@ -104,29 +94,12 @@ function createSemaphore(max) {
|
|
|
104
94
|
let count = 0;
|
|
105
95
|
const queue = [];
|
|
106
96
|
return {
|
|
107
|
-
acquire(
|
|
108
|
-
throwIfAborted(signal);
|
|
97
|
+
acquire() {
|
|
109
98
|
if (count < max) {
|
|
110
99
|
count++;
|
|
111
100
|
return Promise.resolve(false);
|
|
112
101
|
}
|
|
113
|
-
return new Promise((resolve
|
|
114
|
-
const waiter = () => {
|
|
115
|
-
cleanup();
|
|
116
|
-
resolve(true);
|
|
117
|
-
};
|
|
118
|
-
const onAbort = () => {
|
|
119
|
-
const index = queue.indexOf(waiter);
|
|
120
|
-
if (index >= 0) queue.splice(index, 1);
|
|
121
|
-
cleanup();
|
|
122
|
-
reject(abortReason(signal));
|
|
123
|
-
};
|
|
124
|
-
const cleanup = () => {
|
|
125
|
-
signal?.removeEventListener("abort", onAbort);
|
|
126
|
-
};
|
|
127
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
128
|
-
queue.push(waiter);
|
|
129
|
-
});
|
|
102
|
+
return new Promise((resolve) => queue.push(() => resolve(true)));
|
|
130
103
|
},
|
|
131
104
|
release() {
|
|
132
105
|
const next = queue.shift();
|
|
@@ -135,11 +108,5 @@ function createSemaphore(max) {
|
|
|
135
108
|
}
|
|
136
109
|
};
|
|
137
110
|
}
|
|
138
|
-
function throwIfAborted(signal) {
|
|
139
|
-
if (signal?.aborted) throw abortReason(signal);
|
|
140
|
-
}
|
|
141
|
-
function abortReason(signal) {
|
|
142
|
-
return signal.reason ?? /* @__PURE__ */ new Error("Benchmark load aborted.");
|
|
143
|
-
}
|
|
144
111
|
//#endregion
|
|
145
112
|
export { runLoad };
|
|
@@ -36,13 +36,9 @@ function parseServerSnapshot(snapshot) {
|
|
|
36
36
|
const values = depth.dataPoints.map((p) => p.value).filter(isFiniteNumber);
|
|
37
37
|
if (values.length > 0) queueDepthMax = Math.max(...values);
|
|
38
38
|
}
|
|
39
|
-
const queueTasks = parseQueueTasks(metrics);
|
|
40
|
-
const deliveryPermanentFailures = sumMetric(metrics, "activitypub.delivery.permanent_failure");
|
|
41
39
|
return {
|
|
42
40
|
signature,
|
|
43
|
-
queueDepthMax
|
|
44
|
-
...queueTasks == null ? {} : { queueTasks },
|
|
45
|
-
...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
|
|
41
|
+
queueDepthMax
|
|
46
42
|
};
|
|
47
43
|
} catch {
|
|
48
44
|
return null;
|
|
@@ -60,13 +56,9 @@ function parseServerSnapshot(snapshot) {
|
|
|
60
56
|
* @returns The windowed snapshot.
|
|
61
57
|
*/
|
|
62
58
|
function diffSnapshots(baseline, end) {
|
|
63
|
-
const queueTasks = diffQueueTasks(baseline.queueTasks ?? null, end.queueTasks ?? null);
|
|
64
|
-
const deliveryPermanentFailures = diffCounter(baseline.deliveryPermanentFailures ?? null, end.deliveryPermanentFailures ?? null);
|
|
65
59
|
return {
|
|
66
60
|
signature: diffHistogram(baseline.signature, end.signature),
|
|
67
|
-
queueDepthMax: end.queueDepthMax
|
|
68
|
-
...queueTasks == null ? {} : { queueTasks },
|
|
69
|
-
...deliveryPermanentFailures == null ? {} : { deliveryPermanentFailures }
|
|
61
|
+
queueDepthMax: end.queueDepthMax
|
|
70
62
|
};
|
|
71
63
|
}
|
|
72
64
|
/**
|
|
@@ -103,21 +95,6 @@ async function fetchServerSnapshot(target, fetchImpl = fetch) {
|
|
|
103
95
|
return null;
|
|
104
96
|
}
|
|
105
97
|
}
|
|
106
|
-
/**
|
|
107
|
-
* Returns the remaining queue task backlog represented by a diffed snapshot.
|
|
108
|
-
* @param snapshot The server snapshot to inspect, usually already diffed
|
|
109
|
-
* against a baseline.
|
|
110
|
-
* @param baselineRemaining Queue tasks that were already outstanding when the
|
|
111
|
-
* diff baseline was taken. These must drain before diffed completions can be
|
|
112
|
-
* attributed to newly enqueued tasks.
|
|
113
|
-
* @returns `Math.max(0, baselineRemaining + enqueued - completed - failed)`,
|
|
114
|
-
* or `null` when the snapshot has no queue task counters.
|
|
115
|
-
*/
|
|
116
|
-
function queueTaskRemaining(snapshot, baselineRemaining = 0) {
|
|
117
|
-
if (snapshot?.queueTasks == null) return null;
|
|
118
|
-
const { enqueued, completed, failed } = snapshot.queueTasks;
|
|
119
|
-
return Math.max(0, baselineRemaining + enqueued - completed - failed);
|
|
120
|
-
}
|
|
121
98
|
function isFiniteNumber(value) {
|
|
122
99
|
return typeof value === "number" && Number.isFinite(value);
|
|
123
100
|
}
|
|
@@ -148,28 +125,6 @@ function mergeHistogram(dataPoints) {
|
|
|
148
125
|
counts
|
|
149
126
|
} : null;
|
|
150
127
|
}
|
|
151
|
-
function parseQueueTasks(metrics) {
|
|
152
|
-
const enqueued = sumMetric(metrics, "fedify.queue.task.enqueued");
|
|
153
|
-
const completed = sumMetric(metrics, "fedify.queue.task.completed");
|
|
154
|
-
const failed = sumMetric(metrics, "fedify.queue.task.failed");
|
|
155
|
-
return enqueued == null && completed == null && failed == null ? null : {
|
|
156
|
-
enqueued: enqueued ?? 0,
|
|
157
|
-
completed: completed ?? 0,
|
|
158
|
-
failed: failed ?? 0
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
function sumMetric(metrics, name) {
|
|
162
|
-
let total = 0;
|
|
163
|
-
let found = false;
|
|
164
|
-
for (const metric of metrics) {
|
|
165
|
-
if (metric.name !== name || !Array.isArray(metric.dataPoints)) continue;
|
|
166
|
-
for (const point of metric.dataPoints) if (isRecord(point) && isFiniteNumber(point["value"])) {
|
|
167
|
-
total += point["value"];
|
|
168
|
-
found = true;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
return found ? total : null;
|
|
172
|
-
}
|
|
173
128
|
function diffHistogram(baseline, end) {
|
|
174
129
|
if (end == null) return null;
|
|
175
130
|
if (baseline == null) return end;
|
|
@@ -180,26 +135,9 @@ function diffHistogram(baseline, end) {
|
|
|
180
135
|
counts
|
|
181
136
|
};
|
|
182
137
|
}
|
|
183
|
-
function diffQueueTasks(baseline, end) {
|
|
184
|
-
if (end == null) return null;
|
|
185
|
-
if (baseline == null) return end;
|
|
186
|
-
return {
|
|
187
|
-
enqueued: Math.max(0, end.enqueued - baseline.enqueued),
|
|
188
|
-
completed: Math.max(0, end.completed - baseline.completed),
|
|
189
|
-
failed: Math.max(0, end.failed - baseline.failed)
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
function diffCounter(baseline, end) {
|
|
193
|
-
if (end == null) return null;
|
|
194
|
-
if (baseline == null) return end;
|
|
195
|
-
return Math.max(0, end - baseline);
|
|
196
|
-
}
|
|
197
138
|
function histogramsCompatible(a, b) {
|
|
198
139
|
return a.boundaries.length === b.boundaries.length && a.counts.length === b.counts.length && a.boundaries.every((boundary, i) => boundary === b.boundaries[i]);
|
|
199
140
|
}
|
|
200
|
-
function isRecord(value) {
|
|
201
|
-
return value != null && typeof value === "object" && !Array.isArray(value);
|
|
202
|
-
}
|
|
203
141
|
function histogramPercentile(histogram, p) {
|
|
204
142
|
const { boundaries, counts } = histogram;
|
|
205
143
|
const total = counts.reduce((sum, n) => sum + n, 0);
|
|
@@ -213,4 +151,4 @@ function histogramPercentile(histogram, p) {
|
|
|
213
151
|
return boundaries[boundaries.length - 1] ?? 0;
|
|
214
152
|
}
|
|
215
153
|
//#endregion
|
|
216
|
-
export { diffSnapshots, fetchServerSnapshot,
|
|
154
|
+
export { diffSnapshots, fetchServerSnapshot, snapshotToMetrics };
|