@sdods/core 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/a11y/audit.d.ts +88 -0
- package/dist/a11y/audit.js +134 -0
- package/dist/config/defaults.js +2 -0
- package/dist/config/resolve.js +1 -1
- package/dist/data/index.d.ts +1 -0
- package/dist/data/index.js +1 -0
- package/dist/data/pool-capacity.d.ts +58 -0
- package/dist/data/pool-capacity.js +138 -0
- package/dist/data/user-pool.d.ts +43 -2
- package/dist/data/user-pool.js +170 -40
- package/dist/errors.d.ts +1 -1
- package/dist/fixtures/scenario.d.ts +11 -0
- package/dist/fixtures/scenario.js +15 -0
- package/dist/fixtures/test.js +24 -3
- package/dist/fixtures/types.d.ts +2 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/matrix/index.d.ts +13 -0
- package/dist/matrix/index.js +24 -3
- package/dist/perf/vitals.d.ts +118 -0
- package/dist/perf/vitals.js +236 -0
- package/dist/quality/a11y-scenario.d.ts +68 -0
- package/dist/quality/a11y-scenario.js +63 -0
- package/dist/quality/gates.d.ts +54 -0
- package/dist/quality/gates.js +144 -0
- package/dist/quality/hooks.d.ts +2 -0
- package/dist/quality/hooks.js +38 -0
- package/dist/quality/index.d.ts +4 -0
- package/dist/quality/index.js +2 -0
- package/dist/quality/perf-scenario.d.ts +132 -0
- package/dist/quality/perf-scenario.js +241 -0
- package/dist/shots/baselines.d.ts +72 -0
- package/dist/shots/baselines.js +201 -0
- package/dist/shots/hooks.js +2 -2
- package/dist/shots/index.d.ts +1 -0
- package/dist/shots/index.js +1 -0
- package/dist/shots/narrator.d.ts +10 -2
- package/dist/shots/narrator.js +106 -10
- package/dist/shots/policy.d.ts +5 -1
- package/dist/shots/policy.js +2 -0
- package/dist/steps/a11y.steps.d.ts +18 -51
- package/dist/steps/a11y.steps.js +6 -107
- package/dist/steps/data.steps.js +17 -5
- package/dist/steps/hooks.steps.d.ts +1 -0
- package/dist/steps/hooks.steps.js +3 -1
- package/dist/steps/index.d.ts +1 -0
- package/dist/steps/index.js +1 -0
- package/dist/steps/perf.steps.d.ts +3 -129
- package/dist/steps/perf.steps.js +4 -227
- package/dist/steps/ui.steps.js +5 -0
- package/package.json +3 -3
package/dist/data/user-pool.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, statSync, unlinkSync, writeSync, } from 'node:fs';
|
|
1
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync, } from 'node:fs';
|
|
2
|
+
import { cpus } from 'node:os';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
import { SdodsError } from '../errors.js';
|
|
4
5
|
import { Logger } from '../logger.js';
|
|
@@ -82,9 +83,46 @@ export class FileLeaseStore {
|
|
|
82
83
|
return out;
|
|
83
84
|
}
|
|
84
85
|
}
|
|
86
|
+
/** `data.userPool.waitMs` when unset and leases are held per worker. */
|
|
87
|
+
export const DEFAULT_LEASE_WAIT_MS = 30_000;
|
|
88
|
+
const POLL_MS = 500;
|
|
89
|
+
/** A waiter refreshes its ticket every poll; one untouched this long belongs to a waiter that is gone. */
|
|
90
|
+
const TICKET_STALE_MS = 10_000;
|
|
91
|
+
/**
|
|
92
|
+
* The worker count Playwright will actually use: `--workers` / `SDODS_WORKERS` when set (both land
|
|
93
|
+
* in `runtime.workers`), else Playwright's own default of half the logical cores.
|
|
94
|
+
*
|
|
95
|
+
* Every browser and layer of one `sdods run` shares this pool: the CLI starts ONE `playwright test`
|
|
96
|
+
* with a `--project` per run target, and Playwright's `workers` caps the whole invocation.
|
|
97
|
+
*/
|
|
98
|
+
export function effectiveWorkers(workers) {
|
|
99
|
+
return workers ?? Math.max(1, Math.floor(cpus().length / 2));
|
|
100
|
+
}
|
|
101
|
+
/** How long a lease request waits for a free account (see `data.userPool.waitMs`). */
|
|
102
|
+
export function leaseWaitMs(pool) {
|
|
103
|
+
if (pool.waitMs !== undefined)
|
|
104
|
+
return pool.waitMs;
|
|
105
|
+
return pool.leaseScope === 'scenario' ? pool.leaseTtlMs : DEFAULT_LEASE_WAIT_MS;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* With `leaseScope: scenario`, waiting for an account is how one account serialises the scenarios
|
|
109
|
+
* that need it, so the wait must not count against the scenario's own timeout: a scenario third in
|
|
110
|
+
* line would otherwise fail having done nothing. The timeout is raised by the longest possible wait
|
|
111
|
+
* before leasing, then settled to what was actually waited.
|
|
112
|
+
*/
|
|
113
|
+
export function scenarioLeaseClock(pool, testInfo) {
|
|
114
|
+
const base = testInfo.timeout;
|
|
115
|
+
if (pool?.leaseScope !== 'scenario' || pool.mode === 'shared' || base <= 0) {
|
|
116
|
+
return { settle: () => undefined };
|
|
117
|
+
}
|
|
118
|
+
const started = Date.now();
|
|
119
|
+
testInfo.setTimeout(base + leaseWaitMs(pool));
|
|
120
|
+
return { settle: () => testInfo.setTimeout(base + (Date.now() - started)) };
|
|
121
|
+
}
|
|
85
122
|
/**
|
|
86
123
|
* Pool of accounts from the configured dataset, partitioned by role and capped by env.users.poolSize.
|
|
87
|
-
* Leases are keyed by owner (`runId:shardOffset+parallelIndex`) and released on worker teardown
|
|
124
|
+
* Leases are keyed by owner (`runId:shardOffset+parallelIndex`) and released on worker teardown, or
|
|
125
|
+
* at scenario teardown with `leaseScope: scenario`.
|
|
88
126
|
*/
|
|
89
127
|
export class FileUserPool {
|
|
90
128
|
config;
|
|
@@ -93,18 +131,28 @@ export class FileUserPool {
|
|
|
93
131
|
log = new Logger('pool');
|
|
94
132
|
leased = new Map(); // by role
|
|
95
133
|
rows;
|
|
96
|
-
|
|
134
|
+
storeInstance;
|
|
135
|
+
leaseDir;
|
|
97
136
|
constructor(config, data, opts) {
|
|
98
137
|
this.config = config;
|
|
99
138
|
this.data = data;
|
|
100
139
|
this.opts = opts;
|
|
101
|
-
this.
|
|
102
|
-
|
|
103
|
-
|
|
140
|
+
this.leaseDir = join(config.runtime.artifactsDir, '..', 'leases', config.project.slug, config.env.name);
|
|
141
|
+
this.storeInstance = opts.store;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Created on first use. Every scenario depends on the pool (the scenario-scope release fixture is
|
|
145
|
+
* automatic), and a project that never leases must not grow a lease directory for it.
|
|
146
|
+
*/
|
|
147
|
+
get store() {
|
|
148
|
+
return (this.storeInstance ??= new FileLeaseStore(this.leaseDir));
|
|
104
149
|
}
|
|
105
150
|
static ownerFor(config, parallelIndex) {
|
|
106
151
|
const shard = config.runtime.shard;
|
|
107
|
-
|
|
152
|
+
// Shards each number their workers from 0, so the offset must span a whole shard's workers.
|
|
153
|
+
// It used `workers ?? 1`, which let shard 2's worker 0 share an owner with shard 1's worker 1
|
|
154
|
+
// whenever --workers was left to Playwright's default.
|
|
155
|
+
const offset = shard ? (shard.current - 1) * effectiveWorkers(config.runtime.workers) : 0;
|
|
108
156
|
return `${config.runtime.runId}:${offset + parallelIndex}`;
|
|
109
157
|
}
|
|
110
158
|
async users() {
|
|
@@ -161,42 +209,122 @@ export class FileUserPool {
|
|
|
161
209
|
return user;
|
|
162
210
|
}
|
|
163
211
|
const ttl = pool.leaseTtlMs;
|
|
164
|
-
const waitMs = this.opts.waitMs ?? pool
|
|
212
|
+
const waitMs = this.opts.waitMs ?? leaseWaitMs(pool);
|
|
165
213
|
const started = Date.now();
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
index
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
214
|
+
const ticket = this.queueTicket(role, started);
|
|
215
|
+
try {
|
|
216
|
+
while (true) {
|
|
217
|
+
// First come, first served. Without a queue the worker that just released an account won
|
|
218
|
+
// it straight back: its next scenario asks within milliseconds while every waiter polls
|
|
219
|
+
// every 500ms, so under `leaseScope: scenario` the same waiters lost round after round and
|
|
220
|
+
// failed on the timeout while the account changed hands the whole time.
|
|
221
|
+
if (ticket.ahead() < candidates.length) {
|
|
222
|
+
for (const { r, index } of candidates) {
|
|
223
|
+
const id = String(r.id ?? r.username ?? index);
|
|
224
|
+
if (await this.store.tryAcquire(id, this.opts.owner, ttl)) {
|
|
225
|
+
const user = {
|
|
226
|
+
id,
|
|
227
|
+
username: String(r.username ?? id),
|
|
228
|
+
password: String(r.password ?? ''),
|
|
229
|
+
role,
|
|
230
|
+
index,
|
|
231
|
+
extra: r,
|
|
232
|
+
leaseKey: id,
|
|
233
|
+
owner: this.opts.owner,
|
|
234
|
+
};
|
|
235
|
+
this.leased.set(role, user);
|
|
236
|
+
this.log.debug(`leased ${user.username} (${role}) for ${this.opts.owner}`);
|
|
237
|
+
return user;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
183
240
|
}
|
|
241
|
+
if (Date.now() - started > waitMs) {
|
|
242
|
+
const owners = await this.store.owners();
|
|
243
|
+
throw new SdodsError('USER_POOL_EXHAUSTED', `All ${candidates.length} user(s) with role "${role}" are leased.`, {
|
|
244
|
+
hint: `Owners: ${JSON.stringify(owners)}. Waited ${Math.round(waitMs / 1000)}s. ` +
|
|
245
|
+
`Four ways out, in the order worth trying: (1) if these scenarios do not ` +
|
|
246
|
+
`mutate user-scoped state, set data.userPool.mode: shared — one account ` +
|
|
247
|
+
`then serves every worker, which is what a read-only suite needs; ` +
|
|
248
|
+
`(2) add more accounts with role "${role}" to dataset "${pool.dataset}"; ` +
|
|
249
|
+
`(3) set data.userPool.leaseScope: scenario, so an account is released when ` +
|
|
250
|
+
`each scenario ends rather than when its worker exits, or raise ` +
|
|
251
|
+
`data.userPool.waitMs (currently ${waitMs}ms); (4) lower --workers. ` +
|
|
252
|
+
`Note env.users.poolSize slices the dataset BEFORE role filtering, so a ` +
|
|
253
|
+
`small value can starve a role on its own.`,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
await new Promise((r) => setTimeout(r, POLL_MS));
|
|
257
|
+
ticket.touch();
|
|
184
258
|
}
|
|
185
|
-
if (Date.now() - started > waitMs) {
|
|
186
|
-
const owners = await this.store.owners();
|
|
187
|
-
throw new SdodsError('USER_POOL_EXHAUSTED', `All ${candidates.length} user(s) with role "${role}" are leased.`, {
|
|
188
|
-
hint: `Owners: ${JSON.stringify(owners)}. Waited ${Math.round(waitMs / 1000)}s. ` +
|
|
189
|
-
`Four ways out, in the order worth trying: (1) if these scenarios do not ` +
|
|
190
|
-
`mutate user-scoped state, set data.userPool.mode: shared — one account ` +
|
|
191
|
-
`then serves every worker, which is what a read-only suite needs; ` +
|
|
192
|
-
`(2) add more accounts with role "${role}" to dataset "${pool.dataset}"; ` +
|
|
193
|
-
`(3) raise data.userPool.waitMs (currently ${waitMs}ms); (4) lower --workers. ` +
|
|
194
|
-
`Note env.users.poolSize slices the dataset BEFORE role filtering, so a ` +
|
|
195
|
-
`small value can starve a role on its own.`,
|
|
196
|
-
});
|
|
197
|
-
}
|
|
198
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
199
259
|
}
|
|
260
|
+
finally {
|
|
261
|
+
ticket.drop();
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* A place in line for `role`: an empty file named by arrival time and owner, refreshed while its
|
|
266
|
+
* waiter polls. `ahead()` counts live tickets that arrived earlier, and a waiter only tries to
|
|
267
|
+
* acquire while fewer are ahead of it than the role has accounts. A ticket nobody refreshes (its
|
|
268
|
+
* worker crashed) goes stale and stops holding the line.
|
|
269
|
+
*/
|
|
270
|
+
queueTicket(role, arrivedAt) {
|
|
271
|
+
const dir = join(this.opts.queueDir ?? join(this.leaseDir, '.queue'), encodeURIComponent(role));
|
|
272
|
+
const name = `${String(arrivedAt).padStart(15, '0')}-${encodeURIComponent(this.opts.owner)}`;
|
|
273
|
+
const file = join(dir, name);
|
|
274
|
+
const write = () => {
|
|
275
|
+
try {
|
|
276
|
+
mkdirSync(dir, { recursive: true });
|
|
277
|
+
writeFileSync(file, '');
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
/* a queue that cannot be written degrades to unordered polling */
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
write();
|
|
284
|
+
return {
|
|
285
|
+
ahead: () => {
|
|
286
|
+
let names;
|
|
287
|
+
try {
|
|
288
|
+
names = readdirSync(dir);
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return 0;
|
|
292
|
+
}
|
|
293
|
+
let ahead = 0;
|
|
294
|
+
for (const other of names) {
|
|
295
|
+
if (other >= name)
|
|
296
|
+
continue;
|
|
297
|
+
try {
|
|
298
|
+
if (Date.now() - statSync(join(dir, other)).mtimeMs > TICKET_STALE_MS) {
|
|
299
|
+
unlinkSync(join(dir, other));
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
continue; // dropped by its owner between readdir and stat
|
|
305
|
+
}
|
|
306
|
+
ahead++;
|
|
307
|
+
}
|
|
308
|
+
return ahead;
|
|
309
|
+
},
|
|
310
|
+
touch: () => {
|
|
311
|
+
try {
|
|
312
|
+
const now = new Date();
|
|
313
|
+
utimesSync(file, now, now);
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
write(); // removed as stale while this worker was blocked: back in, same place
|
|
317
|
+
}
|
|
318
|
+
},
|
|
319
|
+
drop: () => {
|
|
320
|
+
try {
|
|
321
|
+
unlinkSync(file);
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
/* already gone */
|
|
325
|
+
}
|
|
326
|
+
},
|
|
327
|
+
};
|
|
200
328
|
}
|
|
201
329
|
async release(user) {
|
|
202
330
|
// A shared user holds no lease (leaseKey is empty). Releasing it would be at
|
|
@@ -206,7 +334,9 @@ export class FileUserPool {
|
|
|
206
334
|
this.leased.delete(user.role);
|
|
207
335
|
}
|
|
208
336
|
async releaseAll() {
|
|
209
|
-
|
|
337
|
+
// A store that was never created holds nothing to release.
|
|
338
|
+
if (this.storeInstance)
|
|
339
|
+
await this.storeInstance.releaseAll(this.opts.owner);
|
|
210
340
|
this.leased.clear();
|
|
211
341
|
}
|
|
212
342
|
async status() {
|
package/dist/errors.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** Structured errors: every CLI/MCP/server surface renders `{ code, message, hint, docsUrl }`. */
|
|
2
|
-
export type SdodsErrorCode = 'CONFIG_INVALID' | 'CONFIG_NOT_FOUND' | 'CONFIG_SECRET_LITERAL' | 'CONFIG_UNRESOLVED_VAR' | 'PROJECT_NOT_FOUND' | 'ENV_NOT_FOUND' | 'DATASET_NOT_FOUND' | 'DATASET_ROW_NOT_FOUND' | 'USER_POOL_EXHAUSTED' | 'HEAL_FAILED' | 'LINT_FAILED' | 'RUN_FAILED' | 'HAR_MISS' | 'DB_REQUIRED' | 'MAIL_REQUIRED' | 'MAIL_NOT_RECEIVED' | 'AUTH_FAILED' | 'NOT_SUPPORTED' | 'INTERNAL';
|
|
2
|
+
export type SdodsErrorCode = 'CONFIG_INVALID' | 'CONFIG_NOT_FOUND' | 'CONFIG_SECRET_LITERAL' | 'CONFIG_UNRESOLVED_VAR' | 'PROJECT_NOT_FOUND' | 'ENV_NOT_FOUND' | 'DATASET_NOT_FOUND' | 'DATASET_ROW_NOT_FOUND' | 'USER_POOL_EXHAUSTED' | 'USER_POOL_TOO_SMALL' | 'HEAL_FAILED' | 'LINT_FAILED' | 'RUN_FAILED' | 'GATE_FAILED' | 'HAR_MISS' | 'DB_REQUIRED' | 'MAIL_REQUIRED' | 'MAIL_NOT_RECEIVED' | 'AUTH_FAILED' | 'NOT_SUPPORTED' | 'INTERNAL';
|
|
3
3
|
export declare const DOCS_BASE_URL = "https://docs.sdods.com";
|
|
4
4
|
export interface SdodsErrorOptions {
|
|
5
5
|
hint?: string;
|
|
@@ -17,9 +17,20 @@ export interface ScenarioInit {
|
|
|
17
17
|
export declare class ScenarioMeta {
|
|
18
18
|
readonly data: ScenarioMetaDto;
|
|
19
19
|
readonly dir: string;
|
|
20
|
+
/** the run directory the scenario directory lives in */
|
|
21
|
+
readonly runDir: string;
|
|
22
|
+
/** root of the SDODS project (`projects/<slug>`) */
|
|
23
|
+
readonly projectRoot: string;
|
|
20
24
|
readonly suiteTag?: string;
|
|
21
25
|
constructor(init: ScenarioInit);
|
|
26
|
+
/**
|
|
27
|
+
* The first failure an after-scenario hook (`@a11y`, `@perf`) raised. Playwright marks the test
|
|
28
|
+
* failed only once every after-hook has returned, so `sdods:finalize` would otherwise record a
|
|
29
|
+
* scenario those hooks failed as passed.
|
|
30
|
+
*/
|
|
31
|
+
hookFailure?: string;
|
|
22
32
|
get fingerprint(): string;
|
|
33
|
+
noteFailure(message: string): void;
|
|
23
34
|
file(rel: string): string;
|
|
24
35
|
appendJsonl(rel: string, obj: unknown): void;
|
|
25
36
|
write(): void;
|
|
@@ -7,6 +7,10 @@ import { moduleForFeature } from '../config/workspace.js';
|
|
|
7
7
|
export class ScenarioMeta {
|
|
8
8
|
data;
|
|
9
9
|
dir;
|
|
10
|
+
/** the run directory the scenario directory lives in */
|
|
11
|
+
runDir;
|
|
12
|
+
/** root of the SDODS project (`projects/<slug>`) */
|
|
13
|
+
projectRoot;
|
|
10
14
|
suiteTag;
|
|
11
15
|
constructor(init) {
|
|
12
16
|
const { config, testInfo, tags } = init;
|
|
@@ -21,6 +25,8 @@ export class ScenarioMeta {
|
|
|
21
25
|
exampleIndex: null,
|
|
22
26
|
layer,
|
|
23
27
|
});
|
|
28
|
+
this.runDir = config.runtime.runDir;
|
|
29
|
+
this.projectRoot = config.project.root;
|
|
24
30
|
this.dir = join(config.runtime.runDir, config.project.slug, fp, `r${testInfo.retry}`);
|
|
25
31
|
this.suiteTag = suiteOfTags(tags, config.project.tags.suites.map((s) => `@${s}`));
|
|
26
32
|
this.data = {
|
|
@@ -48,9 +54,18 @@ export class ScenarioMeta {
|
|
|
48
54
|
mkdirSync(this.dir, { recursive: true });
|
|
49
55
|
this.write();
|
|
50
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The first failure an after-scenario hook (`@a11y`, `@perf`) raised. Playwright marks the test
|
|
59
|
+
* failed only once every after-hook has returned, so `sdods:finalize` would otherwise record a
|
|
60
|
+
* scenario those hooks failed as passed.
|
|
61
|
+
*/
|
|
62
|
+
hookFailure;
|
|
51
63
|
get fingerprint() {
|
|
52
64
|
return this.data.fingerprint;
|
|
53
65
|
}
|
|
66
|
+
noteFailure(message) {
|
|
67
|
+
this.hookFailure ??= message;
|
|
68
|
+
}
|
|
54
69
|
file(rel) {
|
|
55
70
|
const f = join(this.dir, rel);
|
|
56
71
|
const parent = join(f, '..');
|
package/dist/fixtures/test.js
CHANGED
|
@@ -6,7 +6,7 @@ import { applyEmulation, emulationFromTags } from '../config/emulation.js';
|
|
|
6
6
|
import { noopAuth } from '../auth/index.js';
|
|
7
7
|
import { ApiClient, isolatedRequestFactory } from '../api/client.js';
|
|
8
8
|
import { CompositeDataProvider } from '../data/provider.js';
|
|
9
|
-
import { FileUserPool } from '../data/user-pool.js';
|
|
9
|
+
import { FileUserPool, scenarioLeaseClock } from '../data/user-pool.js';
|
|
10
10
|
import { apiHarForScenario } from '../har/hooks.js';
|
|
11
11
|
import { Healer } from '../heal/healer.js';
|
|
12
12
|
import { HealHistory } from '../heal/history.js';
|
|
@@ -188,11 +188,32 @@ export const test = base.extend({
|
|
|
188
188
|
// `$sdodsTagGate` first, and it is not unused: it is here to force the ordering.
|
|
189
189
|
// Leasing an account for a scenario that the tag gate is about to skip burns a
|
|
190
190
|
// pool slot, and on a role with no rows it throws before the skip can happen.
|
|
191
|
-
user: async ({ $sdodsTagGate, userPool, $tags }, use, testInfo) => {
|
|
191
|
+
user: async ({ $sdodsTagGate, userPool, config, $tags }, use, testInfo) => {
|
|
192
192
|
void $sdodsTagGate;
|
|
193
193
|
const role = parseTagValue($tags, 'user');
|
|
194
|
-
|
|
194
|
+
if (!role)
|
|
195
|
+
return use(undefined);
|
|
196
|
+
const clock = scenarioLeaseClock(config.project.data.userPool, testInfo);
|
|
197
|
+
const leased = await userPool.lease(role, testInfo.parallelIndex);
|
|
198
|
+
clock.settle();
|
|
199
|
+
await use(leased);
|
|
195
200
|
},
|
|
201
|
+
/**
|
|
202
|
+
* `data.userPool.leaseScope: scenario` (#153): hand back every account this worker leased once
|
|
203
|
+
* the scenario is over — the `@user:` lease above and the `I use a leased user …` steps alike.
|
|
204
|
+
* A worker runs one scenario at a time, so what it holds at teardown is exactly this scenario's.
|
|
205
|
+
*
|
|
206
|
+
* Automatic and dependent only on worker fixtures, so it is set up before and torn down after
|
|
207
|
+
* everything that could still be signed in as the account (page, context, storageState).
|
|
208
|
+
*/
|
|
209
|
+
$sdodsScenarioLeases: [
|
|
210
|
+
async ({ config, userPool }, use) => {
|
|
211
|
+
await use();
|
|
212
|
+
if (config.project.data.userPool?.leaseScope === 'scenario')
|
|
213
|
+
await userPool.releaseAll();
|
|
214
|
+
},
|
|
215
|
+
{ auto: true },
|
|
216
|
+
],
|
|
196
217
|
storageState: async ({ user, authCache, auth, sdods, browser, config }, use) => {
|
|
197
218
|
if (!user || sdods.layer === 'api' || !config.project.auth.storageState)
|
|
198
219
|
return use(undefined);
|
package/dist/fixtures/types.d.ts
CHANGED
|
@@ -49,6 +49,8 @@ export interface TestFixtures {
|
|
|
49
49
|
$sdodsAnnotations: void;
|
|
50
50
|
/** auto fixture: applies @env:, @skip:<browser>, @quarantine and @flag: */
|
|
51
51
|
$sdodsTagGate: void;
|
|
52
|
+
/** auto fixture: releases the scenario's pool leases when `leaseScope: scenario` */
|
|
53
|
+
$sdodsScenarioLeases: void;
|
|
52
54
|
/** What the scenario's @locale: @timezone: @theme: @viewport: @device: tags ask for. */
|
|
53
55
|
$sdodsEmulation: Emulation;
|
|
54
56
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -18,4 +18,5 @@ export type { TestFixtures, WorkerFixtures, SdodsOption } from './fixtures/types
|
|
|
18
18
|
export { coreStepsGlob, coreStepsPatterns, coreStepNames, coreStepsDir } from './steps/glob.js';
|
|
19
19
|
export { VERSION } from './version.js';
|
|
20
20
|
export * from './matrix/index.js';
|
|
21
|
+
export * from './quality/index.js';
|
|
21
22
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -17,4 +17,5 @@ export { HeaderMap } from './fixtures/api-context.js';
|
|
|
17
17
|
export { coreStepsGlob, coreStepsPatterns, coreStepNames, coreStepsDir } from './steps/glob.js';
|
|
18
18
|
export { VERSION } from './version.js';
|
|
19
19
|
export * from './matrix/index.js';
|
|
20
|
+
export * from './quality/index.js';
|
|
20
21
|
//# sourceMappingURL=index.js.map
|
package/dist/matrix/index.d.ts
CHANGED
|
@@ -101,4 +101,17 @@ export declare function checkMatrixRoles(project: ProjectConfig & {
|
|
|
101
101
|
export declare function poolRolesByEnv(project: ProjectConfig & {
|
|
102
102
|
root: string;
|
|
103
103
|
}): Map<string, Set<string>> | undefined;
|
|
104
|
+
/**
|
|
105
|
+
* Accounts per role in the user-pool dataset, per available environment. Undefined if not
|
|
106
|
+
* file-backed (a db or openapi dataset is not knowable without connecting).
|
|
107
|
+
*
|
|
108
|
+
* `poolSize` is the env's `users.poolSize`, applied as the lease applies it: the dataset is sliced
|
|
109
|
+
* BEFORE rows are split by role, so a small value can leave a role with no account at all.
|
|
110
|
+
*/
|
|
111
|
+
export declare function poolAccountsByEnv(project: ProjectConfig & {
|
|
112
|
+
root: string;
|
|
113
|
+
}, opts?: {
|
|
114
|
+
envs?: readonly string[];
|
|
115
|
+
poolSize?: (env: string) => number | undefined;
|
|
116
|
+
}): Map<string, Map<string, number>> | undefined;
|
|
104
117
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/matrix/index.js
CHANGED
|
@@ -451,18 +451,31 @@ export function checkMatrixRoles(project, matrices) {
|
|
|
451
451
|
}
|
|
452
452
|
/** Roles present in the user-pool dataset, per available environment. Undefined if not file-backed. */
|
|
453
453
|
export function poolRolesByEnv(project) {
|
|
454
|
+
const counts = poolAccountsByEnv(project);
|
|
455
|
+
if (!counts)
|
|
456
|
+
return undefined;
|
|
457
|
+
return new Map([...counts].map(([env, roles]) => [env, new Set(roles.keys())]));
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Accounts per role in the user-pool dataset, per available environment. Undefined if not
|
|
461
|
+
* file-backed (a db or openapi dataset is not knowable without connecting).
|
|
462
|
+
*
|
|
463
|
+
* `poolSize` is the env's `users.poolSize`, applied as the lease applies it: the dataset is sliced
|
|
464
|
+
* BEFORE rows are split by role, so a small value can leave a role with no account at all.
|
|
465
|
+
*/
|
|
466
|
+
export function poolAccountsByEnv(project, opts = {}) {
|
|
454
467
|
const pool = project.data.userPool;
|
|
455
468
|
const spec = pool ? project.data.sources[pool.dataset] : undefined;
|
|
456
469
|
if (!pool || !spec || !('path' in spec))
|
|
457
470
|
return undefined;
|
|
458
471
|
const out = new Map();
|
|
459
|
-
for (const env of project.envs.available) {
|
|
472
|
+
for (const env of opts.envs ?? project.envs.available) {
|
|
460
473
|
let file;
|
|
461
474
|
try {
|
|
462
475
|
file = resolveDataPath(project.root, spec, env);
|
|
463
476
|
}
|
|
464
477
|
catch {
|
|
465
|
-
out.set(env, new
|
|
478
|
+
out.set(env, new Map());
|
|
466
479
|
continue;
|
|
467
480
|
}
|
|
468
481
|
let rows = [];
|
|
@@ -478,8 +491,16 @@ export function poolRolesByEnv(project) {
|
|
|
478
491
|
catch {
|
|
479
492
|
rows = [];
|
|
480
493
|
}
|
|
494
|
+
const size = opts.poolSize?.(env);
|
|
495
|
+
if (size)
|
|
496
|
+
rows = rows.slice(0, size);
|
|
497
|
+
const roles = new Map();
|
|
481
498
|
// Same fallback as the lease: a row without a role column is a "standard" account.
|
|
482
|
-
|
|
499
|
+
for (const r of rows) {
|
|
500
|
+
const role = String(r[pool.roleColumn] ?? 'standard');
|
|
501
|
+
roles.set(role, (roles.get(role) ?? 0) + 1);
|
|
502
|
+
}
|
|
503
|
+
out.set(env, roles);
|
|
483
504
|
}
|
|
484
505
|
return out;
|
|
485
506
|
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import type { PerformanceMetrics } from '@sdods/contracts';
|
|
2
|
+
/**
|
|
3
|
+
* Page vitals, latency percentiles and the budgets they are judged against, shared by the explicit
|
|
4
|
+
* perf steps (`steps/perf.steps.ts`) and the end-of-scenario check every `@perf` scenario gets
|
|
5
|
+
* (`quality/hooks.ts`). Outside the step file for the reason `a11y/audit.ts` gives: importing a step
|
|
6
|
+
* file registers its phrases, and `steps.core.exclude: [perf]` must stay able to switch them off.
|
|
7
|
+
* The rules these functions keep are written up at the top of `steps/perf.steps.ts`.
|
|
8
|
+
*/
|
|
9
|
+
/** The page vitals SDODS records. Each name is also its key in `perf.budgets`. */
|
|
10
|
+
export declare const VITAL_KEYS: readonly ["pageLoadMs", "lcpMs", "fcpMs", "ttfbMs"];
|
|
11
|
+
export type VitalKey = (typeof VITAL_KEYS)[number];
|
|
12
|
+
/** The `perf.budgets` key a sampled API latency distribution is judged against. */
|
|
13
|
+
export declare const API_P95_BUDGET_KEY = "apiP95Ms";
|
|
14
|
+
type BudgetLayer = {
|
|
15
|
+
budgets?: Record<string, number | undefined>;
|
|
16
|
+
} | undefined;
|
|
17
|
+
/** The slice of a resolved config these steps read. Nothing here writes config. */
|
|
18
|
+
export interface BudgetConfig {
|
|
19
|
+
project: {
|
|
20
|
+
perf?: BudgetLayer;
|
|
21
|
+
};
|
|
22
|
+
env: {
|
|
23
|
+
name?: string;
|
|
24
|
+
perf?: BudgetLayer;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Effective budgets, env overriding project per key.
|
|
29
|
+
*
|
|
30
|
+
* `resolveConfig` already deep-merges `env.perf` into `config.project.perf`, so on a resolved
|
|
31
|
+
* config the project side is the whole answer. Merging the env side again is idempotent and keeps
|
|
32
|
+
* this correct for a config assembled by hand. Non-numeric values are dropped rather than spread —
|
|
33
|
+
* an explicit `undefined` in the env layer would otherwise erase a real project budget and turn a
|
|
34
|
+
* configured assertion into an unconfigured one.
|
|
35
|
+
*/
|
|
36
|
+
export declare function resolvedBudgets(config: BudgetConfig): Record<string, number>;
|
|
37
|
+
/**
|
|
38
|
+
* The configured budget for one key, or a hard error naming it.
|
|
39
|
+
*
|
|
40
|
+
* PROVES the assertion that follows is gated on something real. A missing budget must never
|
|
41
|
+
* degrade to a skip or to `Infinity`: an unconfigured budget is the single failure mode that lets
|
|
42
|
+
* an entire perf suite report success while asserting nothing at all.
|
|
43
|
+
*/
|
|
44
|
+
export declare function requireBudget(config: BudgetConfig, key: string): number;
|
|
45
|
+
export declare function requireVitalKey(name: string): VitalKey;
|
|
46
|
+
/** One `When I record the page vitals`. `null` in `vitals` means NOT MEASURED, never "0 ms". */
|
|
47
|
+
export interface VitalsRecording {
|
|
48
|
+
url: string;
|
|
49
|
+
vitals: Record<VitalKey, number | null>;
|
|
50
|
+
/** Why a vital is null, keyed by vital. Surfaced verbatim in the failure message. */
|
|
51
|
+
unmeasured: Partial<Record<VitalKey, string>>;
|
|
52
|
+
/** The contract shape written to `sdods/perf/<step>` and ingested into `steps.perf_json`. */
|
|
53
|
+
metrics: PerformanceMetrics;
|
|
54
|
+
}
|
|
55
|
+
export interface LatencySample {
|
|
56
|
+
status: number;
|
|
57
|
+
ms: number;
|
|
58
|
+
}
|
|
59
|
+
export interface LatencySampleSet {
|
|
60
|
+
method: string;
|
|
61
|
+
path: string;
|
|
62
|
+
samples: LatencySample[];
|
|
63
|
+
p95Ms: number;
|
|
64
|
+
}
|
|
65
|
+
interface PerfStore {
|
|
66
|
+
recordings: VitalsRecording[];
|
|
67
|
+
sampleSets: LatencySampleSet[];
|
|
68
|
+
}
|
|
69
|
+
export declare function perfStore(key: object): PerfStore;
|
|
70
|
+
export declare function lastRecording(key: object): VitalsRecording;
|
|
71
|
+
export declare function lastSampleSet(key: object): LatencySampleSet;
|
|
72
|
+
/**
|
|
73
|
+
* Nearest-rank p95 — deterministic for a given set of samples, with no interpolation to argue
|
|
74
|
+
* about. Below 20 samples the nearest rank IS the maximum; the sample count travels in the
|
|
75
|
+
* attachment and in every failure message so a reader can see when "p95" means "slowest of five".
|
|
76
|
+
*
|
|
77
|
+
* Throws rather than returning 0 on an empty sample: a p95 of 0 slides under every budget.
|
|
78
|
+
*/
|
|
79
|
+
export declare function p95Of(msValues: readonly number[]): number;
|
|
80
|
+
/**
|
|
81
|
+
* How long the LCP observer is given to settle. LCP arrives as a stream of ever-larger candidates;
|
|
82
|
+
* `buffered: true` replays the ones already emitted and this window catches any still in flight.
|
|
83
|
+
* A fixed constant rather than a computed wait, so two runs of one scenario observe the same
|
|
84
|
+
* window and a baseline stays comparable.
|
|
85
|
+
*/
|
|
86
|
+
export declare const LCP_SETTLE_MS = 500;
|
|
87
|
+
/** Bounded wait for the load event before reading `loadEventEnd`. */
|
|
88
|
+
export declare const LOAD_WAIT_MS = 5000;
|
|
89
|
+
/** What the page hands back. Every timing is `number | null`; null always means "did not fire". */
|
|
90
|
+
export interface RawVitals {
|
|
91
|
+
hasNavigationEntry: boolean;
|
|
92
|
+
navigationStartEpochMs: number | null;
|
|
93
|
+
pageLoadMs: number | null;
|
|
94
|
+
ttfbMs: number | null;
|
|
95
|
+
fcpMs: number | null;
|
|
96
|
+
lcpMs: number | null;
|
|
97
|
+
domContentLoadedMs: number | null;
|
|
98
|
+
totalResources: number;
|
|
99
|
+
totalResourceSizeKB: number;
|
|
100
|
+
lcpSupported: boolean;
|
|
101
|
+
fcpSupported: boolean;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Runs inside the page. Serialised by Playwright, so it closes over nothing but its argument — the
|
|
105
|
+
* type aliases above are erased at compile time and are safe to reference; the constants are not,
|
|
106
|
+
* which is why the settle window is passed in.
|
|
107
|
+
*/
|
|
108
|
+
export declare function collectRawVitals(settleMs: number): Promise<RawVitals>;
|
|
109
|
+
/**
|
|
110
|
+
* Turns the page's answer into a recording, capturing WHY a vital is null while the browser's
|
|
111
|
+
* capabilities are still in hand. By the time a `Then` step fails, "lcpMs was not measured" on its
|
|
112
|
+
* own sends the reader hunting for a bug in a page that is merely running in a browser that does
|
|
113
|
+
* not implement LCP (WebKit). Support is read from `PerformanceObserver.supportedEntryTypes`, never
|
|
114
|
+
* assumed from the browser name.
|
|
115
|
+
*/
|
|
116
|
+
export declare function toRecording(raw: RawVitals, url: string, browserName?: string): VitalsRecording;
|
|
117
|
+
export {};
|
|
118
|
+
//# sourceMappingURL=vitals.d.ts.map
|