@sdods/cli 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ import type { Command } from 'commander';
2
+ export declare function register(program: Command): void;
3
+ //# sourceMappingURL=baselines.d.ts.map
@@ -0,0 +1,111 @@
1
+ import { relative, resolve as resolvePath } from 'node:path';
2
+ import pc from 'picocolors';
3
+ import { DEFAULT_ARTIFACTS_DIR, SdodsError, acceptVisualFailures, listVisualFailures, selectVisualFailures, } from '@sdods/core';
4
+ import { createContext } from '../context.js';
5
+ import { heading, json, ok, out, table } from '../ui.js';
6
+ import { listRuns } from './trace.js';
7
+ function artifactsDirOf(ctx, flag) {
8
+ return resolvePath(ctx.rootDir, flag ?? process.env.SDODS_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
9
+ }
10
+ function findRun(ctx, opts) {
11
+ const artifactsDir = artifactsDirOf(ctx, opts.artifactsDir);
12
+ const runs = listRuns(artifactsDir);
13
+ const run = opts.run ? runs.find((r) => r.runId === opts.run) : runs[0];
14
+ if (!run)
15
+ throw new SdodsError('CONFIG_NOT_FOUND', opts.run
16
+ ? `Run ${opts.run} not found under ${artifactsDir}.`
17
+ : `No runs under ${artifactsDir}.`, {
18
+ hint: 'Run `sdods run -t @visual …` first, or download the CI run into .sdods/runs/<id>.',
19
+ exitCode: 2,
20
+ });
21
+ return run;
22
+ }
23
+ const pct = (r) => (r === undefined ? '' : `${(r * 100).toFixed(2)}%`);
24
+ export function register(program) {
25
+ const baselines = program
26
+ .command('baselines')
27
+ .description('Review failed visual baseline checks of a run and accept the new screenshots');
28
+ baselines
29
+ .command('diff')
30
+ .description('List the visual checks that failed in a run: expected, actual and diff images')
31
+ .option('--run <id>', 'run id (default: the most recent run)')
32
+ .option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
33
+ .action(async (opts, cmd) => {
34
+ const ctx = createContext(cmd);
35
+ const run = findRun(ctx, opts);
36
+ const failures = listVisualFailures(run.dir);
37
+ const abs = (p) => (p ? resolvePath(run.dir, p) : undefined);
38
+ if (ctx.opts.json)
39
+ return json({
40
+ runId: run.runId,
41
+ failures: failures.map((f) => ({
42
+ ...f,
43
+ actual: abs(f.actual),
44
+ expected: abs(f.expected),
45
+ diff: abs(f.diff),
46
+ })),
47
+ });
48
+ if (!failures.length) {
49
+ ok(`No failed visual checks in run ${run.runId}.`);
50
+ return;
51
+ }
52
+ heading(`${failures.length} failed visual check(s) in run ${run.runId}`);
53
+ const rel = (p) => (p ? relative(ctx.rootDir, p) : '—');
54
+ for (const f of failures) {
55
+ out('');
56
+ out(`${pc.bold(`${f.runnerProject}/${f.name}`)} ${pc.dim(f.platform)} ${reasonLabel(f)}`);
57
+ out(` scenario ${f.scenarioName} ${pc.dim(`(${f.featureUri})`)}`);
58
+ out(` baseline ${f.baseline}`);
59
+ table([
60
+ { image: 'expected', path: rel(abs(f.expected)) },
61
+ { image: 'actual', path: rel(abs(f.actual)) },
62
+ { image: 'diff', path: rel(abs(f.diff)) },
63
+ ]);
64
+ }
65
+ out('');
66
+ out(pc.dim(`Accept after reviewing the diff: sdods baselines accept <name…> --run ${run.runId} (or --all)`));
67
+ });
68
+ baselines
69
+ .command('accept [names...]')
70
+ .description('Copy the actual screenshot of failed visual checks over their baselines (writes to the project tree)')
71
+ .requiredOption('--run <id>', 'run id the screenshots come from (required)')
72
+ .option('--all', 'accept every failed visual check of the run')
73
+ .option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
74
+ .action(async (names, opts, cmd) => {
75
+ const ctx = createContext(cmd);
76
+ if (opts.all && names.length)
77
+ throw new SdodsError('CONFIG_INVALID', 'Pass baseline names or --all, not both.', {
78
+ exitCode: 2,
79
+ });
80
+ const run = findRun(ctx, opts);
81
+ const failures = listVisualFailures(run.dir);
82
+ if (!failures.length)
83
+ throw new SdodsError('CONFIG_NOT_FOUND', `No failed visual checks in run ${run.runId}.`, {
84
+ hint: 'Nothing to accept. `sdods baselines diff --run <id>` lists what failed.',
85
+ exitCode: 2,
86
+ });
87
+ const picked = selectVisualFailures(failures, { names, all: opts.all });
88
+ const accepted = acceptVisualFailures({
89
+ runDir: run.dir,
90
+ failures: picked,
91
+ projectRoot: (slug) => ctx.registry.rootOf(slug),
92
+ });
93
+ if (ctx.opts.json)
94
+ return json({ runId: run.runId, accepted });
95
+ for (const a of accepted) {
96
+ ok(`${a.created ? 'created' : 'updated'} ${relative(ctx.rootDir, a.to)} ${pc.dim(`${a.runnerProject}/${a.name} · ${a.platform} · ${reasonLabel(a)}`)}`);
97
+ out(pc.dim(` from ${relative(ctx.rootDir, a.from)}`));
98
+ if (a.diff)
99
+ out(pc.dim(` diff ${relative(ctx.rootDir, a.diff)}`));
100
+ }
101
+ out(pc.dim('Review the PNGs with `git diff --stat` and commit them with the change that moved the pixels.'));
102
+ });
103
+ }
104
+ function reasonLabel(f) {
105
+ if (f.reason === 'missing')
106
+ return pc.cyan('new baseline');
107
+ if (f.reason === 'size')
108
+ return pc.yellow('size changed');
109
+ return pc.yellow(`changed ${pct(f.diffRatio)}`.trim());
110
+ }
111
+ //# sourceMappingURL=baselines.js.map
@@ -75,7 +75,7 @@ export async function browserStatuses(names = DOWNLOADED_BROWSERS) {
75
75
  return names.map((name) => {
76
76
  const engine = ENGINE_OF[name] ?? 'chromium';
77
77
  const channel = CHANNEL_OF[name] ?? null;
78
- let executable = null;
78
+ let executable;
79
79
  try {
80
80
  executable = channel ? channelExecutable(channel) : pw[engine].executablePath();
81
81
  }
@@ -2,17 +2,18 @@ import { existsSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
3
  import { execa } from 'execa';
4
4
  import pc from 'picocolors';
5
- import { collectVarRefs, loadDotEnvLayer, loadEnvFile } from '@sdods/core';
5
+ import { collectVarRefs, describeWorkers, effectiveWorkers, loadDotEnvLayer, loadEnvFile, poolAccountsByEnv, } from '@sdods/core';
6
6
  import { createContext } from '../context.js';
7
7
  import { browserStatuses } from './browsers.js';
8
8
  import { MIN_PLAYWRIGHT_FOR_STEP_RESULTS, installedPlaywrightVersion, stepResultsWarning, } from '../runner-compat.js';
9
- import { json, out } from '../ui.js';
9
+ import { json, out, parseIntFlag } from '../ui.js';
10
10
  export function registerDoctorCommand(program) {
11
11
  program
12
12
  .command('doctor')
13
13
  .description('Check Node, Bun, browsers, projects, env vars and database reachability')
14
14
  .option('-p, --project <slug>', 'limit env-var checks to one project')
15
15
  .option('--fix', 'install missing browsers')
16
+ .option('-w, --workers <n>', 'worker count to check user pools against (default: SDODS_WORKERS, else Playwright default)', parseIntFlag('workers'))
16
17
  .action(async (opts, cmd) => {
17
18
  const ctx = createContext(cmd);
18
19
  const checks = [];
@@ -45,6 +46,8 @@ export function registerDoctorCommand(program) {
45
46
  cwd: ctx.rootDir,
46
47
  }).catch(() => undefined);
47
48
  }
49
+ const envWorkers = Number(process.env.SDODS_WORKERS);
50
+ const workers = opts.workers ?? (Number.isInteger(envWorkers) && envWorkers > 0 ? envWorkers : undefined);
48
51
  const entries = opts.project
49
52
  ? [ctx.registry.entry(opts.project)]
50
53
  : ctx.registry.entriesList();
@@ -80,6 +83,9 @@ export function registerDoctorCommand(program) {
80
83
  ? `Add ${missing.join(', ')} to ${join(e.root, `.env.${envName}`)}`
81
84
  : undefined,
82
85
  });
86
+ const pool = poolCheck(e.slug, envName, { ...e.config, root: e.root }, envCfg.users.poolSize, workers);
87
+ if (pool)
88
+ checks.push(pool);
83
89
  }
84
90
  }
85
91
  const driver = process.env.DB_DRIVER ?? 'sqlite';
@@ -121,6 +127,42 @@ export function registerDoctorCommand(program) {
121
127
  process.exitCode = 1;
122
128
  });
123
129
  }
130
+ /**
131
+ * #153: accounts per role in an exclusive, worker-held pool against the worker count. Advisory —
132
+ * doctor does not know which roles a run will select; `sdods run` makes the real decision and
133
+ * fails with USER_POOL_TOO_SMALL. Undefined when there is nothing to compare.
134
+ */
135
+ function poolCheck(slug, envName, project, poolSize, workers) {
136
+ const pool = project.data.userPool;
137
+ if (!pool)
138
+ return undefined;
139
+ const name = `${slug}/${envName} user pool`;
140
+ if (pool.mode === 'shared' || pool.leaseScope === 'scenario') {
141
+ return {
142
+ name,
143
+ ok: true,
144
+ detail: pool.mode === 'shared' ? 'mode: shared (no leases)' : 'leaseScope: scenario (queues)',
145
+ };
146
+ }
147
+ const accounts = poolAccountsByEnv(project, {
148
+ envs: [envName],
149
+ poolSize: () => poolSize,
150
+ })?.get(envName);
151
+ if (!accounts)
152
+ return undefined; // not file-backed
153
+ const n = effectiveWorkers(workers);
154
+ const short = [...accounts].filter(([, count]) => count < n);
155
+ return {
156
+ name,
157
+ ok: short.length === 0,
158
+ optional: true,
159
+ detail: `${[...accounts].map(([role, count]) => `${role} ${count}`).join(', ') || 'no accounts'} ` +
160
+ `vs ${describeWorkers(workers)}`,
161
+ fix: short.length
162
+ ? `fewer accounts than workers for ${short.map(([r]) => r).join(', ')}: sdods run stops with USER_POOL_TOO_SMALL when more of their @user: scenarios could run at once than there are accounts. Set data.userPool.leaseScope: scenario, add accounts, or lower --workers.`
163
+ : undefined,
164
+ };
165
+ }
124
166
  /**
125
167
  * The token matrix. "one-of" rows form a group: agents need ANY one of them.
126
168
  * Platform features never need an external token.
@@ -140,12 +140,12 @@ export async function initWorkspace(target, flags) {
140
140
  // 1.63 is the floor, not a preference: below it an npm-installed workspace records every
141
141
  // Gherkin step SKIPPED in messages.ndjson (see runner-compat.ts).
142
142
  '@playwright/test': flags.link && flags.pm === 'bun' ? 'link:@playwright/test' : '^1.63.0',
143
- 'playwright-bdd': flags.link && flags.pm === 'bun' ? 'link:playwright-bdd' : '^9.2.0',
143
+ 'playwright-bdd': flags.link && flags.pm === 'bun' ? 'link:playwright-bdd' : '^9.2.1',
144
144
  },
145
145
  devDependencies: {
146
- '@types/node': '^22.15.0',
147
- typescript: '~5.9.3',
148
- tsx: '^4.23.0',
146
+ '@types/node': '^22.20.2',
147
+ typescript: '~6.0.3',
148
+ tsx: '^4.23.13',
149
149
  },
150
150
  trustedDependencies: [
151
151
  '@playwright/test',
@@ -1,10 +1,12 @@
1
- import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
2
- import { dirname, extname, join, resolve } from 'node:path';
1
+ import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync, } from 'node:fs';
2
+ import { tmpdir } from 'node:os';
3
+ import { basename, dirname, extname, join, resolve } from 'node:path';
3
4
  import { execa } from 'execa';
4
5
  import pc from 'picocolors';
5
- import { runFiles } from '@sdods/contracts';
6
- import { SdodsError } from '@sdods/core';
6
+ import { legacyRunFiles, runFiles, } from '@sdods/contracts';
7
+ import { SdodsError, collectGateEvidence, evaluateGates, gateReportsFromRunnerJson, gateTotalsFromRunnerStats, hasGates, ingestGateJudge, } from '@sdods/core';
7
8
  import { createContext } from '../context.js';
9
+ import { gateFailedError, printGates } from '../gates.js';
8
10
  import { json, ok, out, table, warn } from '../ui.js';
9
11
  function artifactsRoot(rootDir) {
10
12
  return resolve(rootDir, process.env.SDODS_ARTIFACTS_DIR ?? '.sdods/runs');
@@ -35,6 +37,82 @@ async function openPath(p) {
35
37
  const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
36
38
  await execa(cmd, [p], { shell: process.platform === 'win32' }).catch(() => undefined);
37
39
  }
40
+ /**
41
+ * The process `report merge` judges: `--process` (with `-p`, or the project in a run.json), or the
42
+ * process a run.json names when it declares gates. `undefined` when there is nothing to judge.
43
+ */
44
+ function resolveMergeProcess(ctx, input) {
45
+ const manifest = input.manifestDirs
46
+ .map((d) => readJsonFile(join(d, runFiles.manifest)))
47
+ .find(Boolean);
48
+ const name = input.process ?? manifest?.process;
49
+ if (!name)
50
+ return undefined;
51
+ const slug = input.project ?? manifest?.projectSlug;
52
+ if (!slug)
53
+ throw new SdodsError('CONFIG_INVALID', `--process ${name} needs the project that defines it, and no run.json names one.`, {
54
+ hint: `Pass -p <slug>: sdods report merge --process ${name} -p <slug> <dirs...>`,
55
+ exitCode: 2,
56
+ });
57
+ let proc;
58
+ try {
59
+ proc = ctx.registry.processOf(slug, name);
60
+ }
61
+ catch (e) {
62
+ // Named explicitly, an unknown process is an error. Inferred from a run.json this checkout
63
+ // cannot resolve, it only means there is nothing here to judge it by; the merge still runs.
64
+ if (input.process)
65
+ throw e;
66
+ warn(`run.json names process "${name}" of ${slug}, which this checkout does not define: gates not judged.`);
67
+ return undefined;
68
+ }
69
+ if (!hasGates(proc.gates)) {
70
+ // Named explicitly, a process without gates is a mistake worth saying; inferred, it is not.
71
+ if (input.process)
72
+ warn(`process "${name}" of ${slug} declares no gates, so there is nothing to judge.`);
73
+ return undefined;
74
+ }
75
+ return { slug, process: proc };
76
+ }
77
+ /**
78
+ * A merge workspace for `playwright merge-reports`: the blob reports of every directory copied into
79
+ * one (it reads exactly one directory; prefixed, because two runs' shard 1 carry the same file
80
+ * name), and a merge config that pins the root the report paths are relative to. Blobs recorded
81
+ * under different checkouts, such as CI's Playwright container (`/__w/…`) and its bare runner
82
+ * (`/home/runner/work/…`), otherwise refuse to merge ("recorded with different test directories").
83
+ */
84
+ function mergeWorkspace(dirs, rootDir) {
85
+ const dir = mkdtempSync(join(tmpdir(), 'sdods-merge-'));
86
+ const reports = join(dir, 'reports');
87
+ mkdirSync(reports);
88
+ dirs.forEach((from, i) => {
89
+ for (const name of readdirSync(from))
90
+ if (name.endsWith('.zip'))
91
+ copyFileSync(join(from, name), join(reports, `${i + 1}-${name}`));
92
+ });
93
+ const config = join(dir, 'merge.config.mjs');
94
+ writeFileSync(config, `export default ${JSON.stringify({ testDir: rootDir })};\n`);
95
+ return { reports, config, dir };
96
+ }
97
+ /** The error Playwright printed, rather than the last lines of its stack. */
98
+ function mergeFailureHint(stderr) {
99
+ const lines = (stderr ?? '').split('\n');
100
+ const at = lines.findIndex((l) => /^\s*Error\b/.test(l));
101
+ const picked = at >= 0 ? lines.slice(at, at + 12).filter((l) => !/^\s+at /.test(l)) : lines.slice(-3);
102
+ return picked.join(' ').replace(/\s+/g, ' ').trim() || undefined;
103
+ }
104
+ /** Writes the merged verdict where `sdods run --process` writes one, so ingest records it the same way. */
105
+ function recordMergedGates(runDir, gates) {
106
+ writeFileSync(join(runDir, runFiles.gates), JSON.stringify(gates, null, 2));
107
+ const manifestFile = join(runDir, runFiles.manifest);
108
+ const manifest = readJsonFile(manifestFile);
109
+ if (manifest)
110
+ writeFileSync(manifestFile, JSON.stringify({ ...manifest, gatesPassed: gates.passed }, null, 2));
111
+ const summaryFile = join(runDir, runFiles.summary);
112
+ const summary = readJsonFile(summaryFile);
113
+ if (summary)
114
+ writeFileSync(summaryFile, JSON.stringify({ ...summary, gates }, null, 2));
115
+ }
38
116
  export function register(program) {
39
117
  const report = program
40
118
  .command('report')
@@ -172,12 +250,25 @@ export function register(program) {
172
250
  artifactsRoot: root,
173
251
  replace: Boolean(opts.replace),
174
252
  env: opts.env,
253
+ // A run with no gates.json whose process declares gates is judged here, as the run would have been.
254
+ gates: ingestGateJudge({
255
+ manifest: manifest ?? m.readManifest(join(root, opts.runId)),
256
+ runDir: join(root, opts.runId),
257
+ processOf: (slug, name) => ctx.registry.processOf(slug, name),
258
+ }),
175
259
  });
176
260
  if (ctx.opts.json)
177
261
  return json(res);
178
262
  ok(`Ingested run ${res.runId} (${res.projectSlug}): ${res.scenarios} scenario(s), ${res.attempts} attempt(s), ${res.steps} step(s), ${res.artifacts} artifact(s), ${res.healEvents} heal event(s)`);
179
263
  const t = res.totals;
180
264
  out(`${pc.green(`${t.passed} passed`)} ${pc.red(`${t.failed} failed`)} ${pc.yellow(`${t.flaky} flaky`)} ${pc.dim(`${t.skipped} skipped`)} → ${res.status}`);
265
+ if (res.gates)
266
+ out(`gates · process ${res.gates.process}: ${res.gates.passed
267
+ ? pc.green('passed')
268
+ : pc.red(`FAILED (${res.gates.rows
269
+ .filter((r) => !r.passed)
270
+ .map((r) => r.gate)
271
+ .join(', ')})`)}`);
181
272
  if (res.filesSkipped.length)
182
273
  out(pc.dim(`skipped (already ingested): ${res.filesSkipped.join(', ')}`));
183
274
  if (res.parseErrors)
@@ -189,9 +280,11 @@ export function register(program) {
189
280
  });
190
281
  report
191
282
  .command('merge <dirs...>')
192
- .description('Merge sharded run reports into one HTML report and JUnit file')
283
+ .description('Merge sharded run reports into one HTML report and JUnit file, and judge process gates on the whole run')
193
284
  .option('--run <id>', 'run id to write the merged report into (default: latest)')
194
285
  .option('--reporter <list>', 'reporters for the merged output', 'html,junit')
286
+ .option('--process <name>', "evaluate this process's gates over the merged shards; exit 1 when one is not met (default: the process in run.json, when it declares gates)")
287
+ .option('-p, --project <slug>', 'project that defines --process (default: from run.json)')
195
288
  .action(async (dirs, opts, cmd) => {
196
289
  const ctx = createContext(cmd);
197
290
  const root = artifactsRoot(ctx.rootDir);
@@ -212,28 +305,82 @@ export function register(program) {
212
305
  exitCode: 2,
213
306
  });
214
307
  }
308
+ const shardDirs = dirs.map((d) => resolve(ctx.rootDir, d));
309
+ // The run directory each shard report sits in (`<run>/shard-reports`) holds that shard's
310
+ // @a11y and @perf scenario reports, which is the gate evidence the merged report lacks.
311
+ const shardRunDirs = shardDirs
312
+ .filter((d) => [runFiles.shardReports, legacyRunFiles.shardReports].includes(basename(d)))
313
+ .map((d) => dirname(d));
314
+ const gated = resolveMergeProcess(ctx, {
315
+ process: opts.process,
316
+ project: opts.project,
317
+ manifestDirs: [runDir, ...shardRunDirs],
318
+ });
319
+ const reporters = String(opts.reporter)
320
+ .split(',')
321
+ .map((r) => r.trim())
322
+ .filter(Boolean);
323
+ const mergedJson = join(runDir, runFiles.mergedResults);
324
+ if (gated && !reporters.includes('json'))
325
+ reporters.push('json');
326
+ if (gated)
327
+ mkdirSync(runDir, { recursive: true });
215
328
  // Shard reports are produced by `sdods run --reporter blob --shard i/n`; merging them
216
329
  // rebuilds one HTML report and JUnit file for the whole matrix.
330
+ const workspace = mergeWorkspace(shardDirs, ctx.rootDir);
217
331
  const args = [
218
332
  'playwright',
219
333
  'merge-reports',
334
+ '--config',
335
+ workspace.config,
220
336
  '--reporter',
221
- opts.reporter,
222
- ...dirs.map((d) => resolve(ctx.rootDir, d)),
337
+ reporters.join(','),
338
+ workspace.reports,
223
339
  ];
224
340
  const res = await execa('npx', args, {
225
341
  cwd: ctx.rootDir,
226
342
  reject: false,
227
- env: { ...process.env, PLAYWRIGHT_HTML_OUTPUT_DIR: join(runDir, runFiles.htmlReport) },
343
+ env: {
344
+ ...process.env,
345
+ PLAYWRIGHT_HTML_OUTPUT_DIR: join(runDir, runFiles.htmlReport),
346
+ ...(gated ? { PLAYWRIGHT_JSON_OUTPUT_FILE: mergedJson } : {}),
347
+ },
228
348
  });
229
- if (res.exitCode !== 0) {
230
- throw new SdodsError('RUN_FAILED', `Merging shard reports failed (exit ${res.exitCode}).`, {
231
- hint: res.stderr?.split('\n').slice(-3).join(' ') || undefined,
232
- });
349
+ let gates;
350
+ try {
351
+ if (res.exitCode !== 0) {
352
+ throw new SdodsError('RUN_FAILED', `Merging shard reports failed (exit ${res.exitCode}).`, { hint: mergeFailureHint(res.stderr) });
353
+ }
354
+ if (gated) {
355
+ const merged = readJsonFile(mergedJson);
356
+ gates = evaluateGates({
357
+ process: gated.process.name,
358
+ gates: gated.process.gates,
359
+ totals: gateTotalsFromRunnerStats(merged?.stats),
360
+ // The shards' run directories, and the reports attached in the blobs themselves: a
361
+ // download of the shard reports alone still carries its evidence.
362
+ evidence: collectGateEvidence([runDir, ...shardRunDirs], gateReportsFromRunnerJson(merged)),
363
+ });
364
+ recordMergedGates(runDir, gates);
365
+ }
366
+ }
367
+ finally {
368
+ rmSync(workspace.dir, { recursive: true, force: true });
233
369
  }
234
370
  if (ctx.opts.json)
235
- return json({ runId, merged: dirs.length, htmlReport: join(runDir, runFiles.htmlReport) });
236
- ok(`Merged ${dirs.length} shard report dir(s) into ${join(runDir, runFiles.htmlReport)}`);
371
+ json({
372
+ runId,
373
+ merged: dirs.length,
374
+ htmlReport: join(runDir, runFiles.htmlReport),
375
+ ...(gates ? { gates } : {}),
376
+ });
377
+ else {
378
+ ok(`Merged ${dirs.length} shard report dir(s) into ${join(runDir, runFiles.htmlReport)}`);
379
+ if (gates)
380
+ printGates(gates);
381
+ }
382
+ if (gates && !gates.passed)
383
+ throw gateFailedError(gates, join(runDir, runFiles.gates));
237
384
  });
238
385
  report
239
386
  .command('traceability')
@@ -40,6 +40,7 @@ export interface RunFlags {
40
40
  trigger?: string;
41
41
  allowEmpty?: boolean;
42
42
  notify?: boolean;
43
+ allowPoolContention?: boolean;
43
44
  }
44
45
  export declare function register(program: Command): void;
45
46
  export declare function runCommand(flags: RunFlags, cmd: Command): Promise<number>;
@@ -3,13 +3,14 @@ import { isAbsolute, join, relative, resolve as resolvePath } from 'node:path';
3
3
  import { execa } from 'execa';
4
4
  import pc from 'picocolors';
5
5
  import { RecordingModeSchema, newRunId, runFiles, } from '@sdods/contracts';
6
- import { SdodsError, CLI_OVERRIDES_ENV, VERSION, formatFindings, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
6
+ import { SdodsError, CLI_OVERRIDES_ENV, VERSION, checkPoolCapacity, collectGateEvidence, effectiveWorkers, evaluateGates, formatFindings, hasGates, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, poolAccountsFor, poolDemandByRole, poolTooSmallError, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
7
7
  import { analyzeChangeImpact } from '@sdods/mcp';
8
8
  import { createContext } from '../context.js';
9
9
  import { browserStatuses } from './browsers.js';
10
+ import { gateFailedError, printGates } from '../gates.js';
10
11
  import { maybeNotify, notifyRun } from '../notify.js';
11
12
  import { installedPlaywrightVersion, stepResultsWarning } from '../runner-compat.js';
12
- import { collect, json, out, parseIntFlag, table, warn } from '../ui.js';
13
+ import { collect, json, out, parseIntFlag, renderError, table, warn } from '../ui.js';
13
14
  const RECORDING_MODES = RecordingModeSchema.options.join(' | ');
14
15
  /** `--trace`/`--video` take Playwright's modes; refuse anything else before specs are generated. */
15
16
  function recordingMode(flag, value) {
@@ -37,6 +38,7 @@ function addRunOptions(cmd) {
37
38
  .option('--device <name>', 'device name for mobile emulation, e.g. "iPhone 15"')
38
39
  .option('--headed', 'run headed')
39
40
  .option('-w, --workers <n>', 'parallel workers', parseIntFlag('workers'))
41
+ .option('--allow-pool-contention', 'run even when a @user: role has fewer pool accounts than the workers that need it')
40
42
  .option('--shard <i/n>', 'shard, e.g. 1/3')
41
43
  .option('--retries <n>', 'retries per test', parseIntFlag('retries'))
42
44
  .option('--trace <mode>', `Playwright trace: ${RECORDING_MODES} (default: evidence.trace)`)
@@ -54,7 +56,7 @@ function addRunOptions(cmd) {
54
56
  .option('--har-update', 'record/refresh HAR files for @har scenarios')
55
57
  .option('--har-replay', 'replay HAR files for @har scenarios')
56
58
  .option('--strict', 'with --har-replay: abort on any request not in the HAR (offline)')
57
- .option('--update-snapshots', 'update visual baselines')
59
+ .option('--update-snapshots', 'overwrite visual baselines without review (prefer `sdods baselines diff` / `accept`)')
58
60
  .option('--ui', 'interactive UI mode')
59
61
  .option('--debug', 'step debugger')
60
62
  .option('--list', 'list the run targets and tests that would run')
@@ -150,6 +152,9 @@ export async function runCommand(flags, cmd) {
150
152
  if (stepResults)
151
153
  warn(stepResults);
152
154
  const runId = flags.runId ?? newRunId();
155
+ if (flags.updateSnapshots)
156
+ warn('--update-snapshots overwrites every compared baseline without showing the diff. ' +
157
+ `Prefer a normal run, then \`sdods baselines diff --run ${runId}\` and \`sdods baselines accept <name> --run ${runId}\`.`);
153
158
  const cli = {
154
159
  env: envName,
155
160
  headed: flags.headed,
@@ -206,6 +211,59 @@ export async function runCommand(flags, cmd) {
206
211
  if (result.warnings.length && ctx.opts.verbose)
207
212
  process.stderr.write(formatFindings(result) + '\n');
208
213
  }
214
+ // --since selects features by git range; resolved here so the pool check counts the same selection.
215
+ const impact = flags.since
216
+ ? analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since)
217
+ : undefined;
218
+ // #153: refuse, before generating anything, a run whose user pool cannot cover its workers.
219
+ // Listing runs nothing, and --ui / --debug run one scenario at a time by hand.
220
+ if (!flags.list && !flags.ui && !flags.debug) {
221
+ const poolCfg = cfg.project.data.userPool;
222
+ const accounts = poolCfg && poolCfg.mode !== 'shared' && poolCfg.leaseScope !== 'scenario'
223
+ ? poolAccountsFor(cfg)
224
+ : undefined;
225
+ if (accounts) {
226
+ const moduleRels = modules.map((m) => relative(cfg.project.root, moduleDir(cfg.project.root, moduleByName(projectCfg, m)))
227
+ .replace(/\\/g, '/')
228
+ .concat('/'));
229
+ const impactedRels = impact ? new Set(impact.impacted.map((r) => r.feature)) : undefined;
230
+ const grep = flags.grep ? safeRegExp(flags.grep) : undefined;
231
+ const features = cfg.env.vars?.features;
232
+ const demand = poolDemandByRole(cfg.project, cfg.env.name, {
233
+ layers: layers.length ? layers : projectCfg.layers,
234
+ browsers: browsers.length ? browsers : projectCfg.browsers,
235
+ tags,
236
+ setupTags: setup?.tags,
237
+ flags: typeof features === 'string'
238
+ ? features
239
+ .split(',')
240
+ .map((f) => f.trim())
241
+ .filter(Boolean)
242
+ : undefined,
243
+ quarantine: process.env.SDODS_QUARANTINE === 'run' ? 'run' : 'skip',
244
+ fullyParallel: proc?.fullyParallel ?? projectCfg.fullyParallel,
245
+ includeFeature: (rel) => (!featureRel || rel === featureRel) &&
246
+ (!moduleRels.length || moduleRels.some((m) => rel.startsWith(m))) &&
247
+ (!impactedRels || impactedRels.has(rel)),
248
+ includeScenario: (name) => (!flags.scenario || name.includes(flags.scenario)) &&
249
+ (!flags.grep || (grep?.test(name) ?? false)),
250
+ repeatEach: flags.repeatEach,
251
+ shardTotal: cfg.runtime.shard?.total,
252
+ });
253
+ const shortfalls = checkPoolCapacity({
254
+ pool: poolCfg,
255
+ accounts,
256
+ demand,
257
+ workers: effectiveWorkers(cfg.runtime.workers),
258
+ });
259
+ if (shortfalls.length) {
260
+ const error = poolTooSmallError(cfg, shortfalls, cfg.runtime.workers);
261
+ if (!flags.allowPoolContention)
262
+ throw error;
263
+ warn(`${error.message} Running anyway (--allow-pool-contention).`);
264
+ }
265
+ }
266
+ }
209
267
  const selection = {
210
268
  project: entry.slug,
211
269
  env: cfg.env.name,
@@ -318,8 +376,7 @@ export async function runCommand(flags, cmd) {
318
376
  // `analyzeChangeImpact` has existed for a while and was reachable only
319
377
  // through MCP, so it could advise a human and could not select a run. This is
320
378
  // the seam that was missing; the mapping itself is unchanged.
321
- if (flags.since) {
322
- const impact = analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since);
379
+ if (flags.since && impact) {
323
380
  if (impact.impacted.length === 0) {
324
381
  // Deliberately NOT "run everything" and deliberately not a silent empty
325
382
  // run. A run that registered zero scenarios exits 0 and looks identical
@@ -389,6 +446,30 @@ export async function runCommand(flags, cmd) {
389
446
  warn(`trace redaction failed for ${relative(ctx.rootDir, e.file)}: ${e.error}`);
390
447
  }
391
448
  const summary = readSummary(runDir, runId, exitCode, Date.now() - started);
449
+ // Process gates are judged here, from the run that just finished, before anything publishes it.
450
+ // One shard of several holds only part of the run, so its pass rate and its a11y/perf evidence
451
+ // are not the run's; a sharded process is gated where the shards are merged
452
+ // (`sdods report merge --process`), not per shard.
453
+ const shardTotal = cfg.runtime.shard?.total ?? 1;
454
+ let gates;
455
+ if (proc && hasGates(proc.gates) && exitCode !== 130) {
456
+ if (shardTotal > 1) {
457
+ warn(`process "${proc.name}" gates are not evaluated on shard ${cfg.runtime.shard?.current}/${shardTotal}: one shard is not the whole run. Judge them on the merged shards with \`sdods report merge --process ${proc.name}\`.`);
458
+ }
459
+ else {
460
+ gates = evaluateGates({
461
+ process: proc.name,
462
+ gates: proc.gates,
463
+ totals: summary?.totals,
464
+ evidence: collectGateEvidence(runDir),
465
+ });
466
+ writeFileSync(join(runDir, runFiles.gates), JSON.stringify(gates, null, 2));
467
+ if (summary)
468
+ summary.gates = gates;
469
+ manifest.gatesPassed = gates.passed;
470
+ writeManifest();
471
+ }
472
+ }
392
473
  if (summary)
393
474
  writeFileSync(join(runDir, runFiles.summary), JSON.stringify(summary, null, 2));
394
475
  const dbConfigured = process.env.DB_DRIVER === 'postgres'
@@ -428,6 +509,13 @@ export async function runCommand(flags, cmd) {
428
509
  writeManifest();
429
510
  }
430
511
  }
512
+ // A breached gate fails a run whose scenarios passed. Exit 1, as the exit-code reference says:
513
+ // a gate not met is a failure, not a configuration error.
514
+ if (gates && !gates.passed && finalExit === 0) {
515
+ finalExit = 1;
516
+ manifest.exitCode = finalExit;
517
+ writeManifest();
518
+ }
431
519
  // Enabled integrations (check runs, PR comment, issues) act on the run here, the same way
432
520
  // `sdods integrations notify --run-id` does. Failures are reported and never change the exit code.
433
521
  const notify = await maybeNotify({
@@ -461,9 +549,11 @@ export async function runCommand(flags, cmd) {
461
549
  out(pc.dim(`integrations: not notified (${notify.reason})`));
462
550
  }
463
551
  if (ctx.opts.json) {
464
- json({ runId, runDir, exitCode: finalExit, summary, manifest, notify });
552
+ json({ runId, runDir, exitCode: finalExit, summary, manifest, notify, gates });
465
553
  }
466
554
  else {
555
+ if (gates)
556
+ printGates(gates);
467
557
  const t = summary?.totals;
468
558
  out('');
469
559
  out(`${finalExit === 0 ? pc.green('✔ passed') : finalExit === 130 ? pc.yellow('■ cancelled') : finalExit === 2 ? pc.yellow('■ nothing ran') : pc.red('✖ failed')} ${t ? `${t.passed} passed, ${t.failed} failed, ${t.skipped} skipped, ${t.flaky} flaky` : ''} ${pc.dim(`(${Math.round((Date.now() - started) / 1000)}s)`)}`);
@@ -471,6 +561,8 @@ export async function runCommand(flags, cmd) {
471
561
  out(pc.dim(`html report: ${join(runDir, runFiles.htmlReport, 'index.html')}`));
472
562
  out(pc.dim(`dashboard: ${join(runDir, runFiles.dashboard, 'index.html')}`));
473
563
  }
564
+ if (gates && !gates.passed)
565
+ renderError(gateFailedError(gates, join(runDir, runFiles.gates)), Boolean(ctx.opts.json));
474
566
  return finalExit;
475
567
  }
476
568
  /**
@@ -496,6 +588,15 @@ function projectFeaturePath(projectRoot, repoRoot, input) {
496
588
  });
497
589
  return rel;
498
590
  }
591
+ /** `--grep` as the pool check reads it; a pattern Playwright would reject matches no scenario. */
592
+ function safeRegExp(pattern) {
593
+ try {
594
+ return new RegExp(pattern);
595
+ }
596
+ catch {
597
+ return undefined;
598
+ }
599
+ }
499
600
  function escapeRe(s) {
500
601
  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
501
602
  }