@sdods/cli 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/commands/baselines.d.ts +3 -0
- package/dist/commands/baselines.js +111 -0
- package/dist/commands/browsers.js +1 -1
- package/dist/commands/doctor.js +44 -2
- package/dist/commands/init.js +4 -4
- package/dist/commands/report.js +161 -14
- package/dist/commands/run.d.ts +1 -0
- package/dist/commands/run.js +107 -6
- package/dist/commands/schedule.js +1 -1
- package/dist/commands/trace.js +2 -2
- package/dist/gates.d.ts +11 -0
- package/dist/gates.js +29 -0
- package/dist/program.js +2 -0
- package/package.json +10 -10
- package/skills/sdods-start-ui/SKILL.md +5 -4
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--chromium/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--firefox/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--webkit/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/auth/user-pool.feature +1 -1
- package/templates/projects/demo-shop/features/cart/performance.feature +14 -0
- package/templates/projects/demo-shop/features/inventory/accessibility.feature +12 -0
- package/templates/projects/demo-shop/features/inventory/visual.feature +2 -1
- package/templates/projects/demo-shop/sdods.project.yaml +3 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { relative, resolve as resolvePath } from 'node:path';
|
|
2
|
+
import pc from 'picocolors';
|
|
3
|
+
import { DEFAULT_ARTIFACTS_DIR, SdodsError, acceptVisualFailures, listVisualFailures, selectVisualFailures, } from '@sdods/core';
|
|
4
|
+
import { createContext } from '../context.js';
|
|
5
|
+
import { heading, json, ok, out, table } from '../ui.js';
|
|
6
|
+
import { listRuns } from './trace.js';
|
|
7
|
+
function artifactsDirOf(ctx, flag) {
|
|
8
|
+
return resolvePath(ctx.rootDir, flag ?? process.env.SDODS_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
|
|
9
|
+
}
|
|
10
|
+
function findRun(ctx, opts) {
|
|
11
|
+
const artifactsDir = artifactsDirOf(ctx, opts.artifactsDir);
|
|
12
|
+
const runs = listRuns(artifactsDir);
|
|
13
|
+
const run = opts.run ? runs.find((r) => r.runId === opts.run) : runs[0];
|
|
14
|
+
if (!run)
|
|
15
|
+
throw new SdodsError('CONFIG_NOT_FOUND', opts.run
|
|
16
|
+
? `Run ${opts.run} not found under ${artifactsDir}.`
|
|
17
|
+
: `No runs under ${artifactsDir}.`, {
|
|
18
|
+
hint: 'Run `sdods run -t @visual …` first, or download the CI run into .sdods/runs/<id>.',
|
|
19
|
+
exitCode: 2,
|
|
20
|
+
});
|
|
21
|
+
return run;
|
|
22
|
+
}
|
|
23
|
+
const pct = (r) => (r === undefined ? '' : `${(r * 100).toFixed(2)}%`);
|
|
24
|
+
export function register(program) {
|
|
25
|
+
const baselines = program
|
|
26
|
+
.command('baselines')
|
|
27
|
+
.description('Review failed visual baseline checks of a run and accept the new screenshots');
|
|
28
|
+
baselines
|
|
29
|
+
.command('diff')
|
|
30
|
+
.description('List the visual checks that failed in a run: expected, actual and diff images')
|
|
31
|
+
.option('--run <id>', 'run id (default: the most recent run)')
|
|
32
|
+
.option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
|
|
33
|
+
.action(async (opts, cmd) => {
|
|
34
|
+
const ctx = createContext(cmd);
|
|
35
|
+
const run = findRun(ctx, opts);
|
|
36
|
+
const failures = listVisualFailures(run.dir);
|
|
37
|
+
const abs = (p) => (p ? resolvePath(run.dir, p) : undefined);
|
|
38
|
+
if (ctx.opts.json)
|
|
39
|
+
return json({
|
|
40
|
+
runId: run.runId,
|
|
41
|
+
failures: failures.map((f) => ({
|
|
42
|
+
...f,
|
|
43
|
+
actual: abs(f.actual),
|
|
44
|
+
expected: abs(f.expected),
|
|
45
|
+
diff: abs(f.diff),
|
|
46
|
+
})),
|
|
47
|
+
});
|
|
48
|
+
if (!failures.length) {
|
|
49
|
+
ok(`No failed visual checks in run ${run.runId}.`);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
heading(`${failures.length} failed visual check(s) in run ${run.runId}`);
|
|
53
|
+
const rel = (p) => (p ? relative(ctx.rootDir, p) : '—');
|
|
54
|
+
for (const f of failures) {
|
|
55
|
+
out('');
|
|
56
|
+
out(`${pc.bold(`${f.runnerProject}/${f.name}`)} ${pc.dim(f.platform)} ${reasonLabel(f)}`);
|
|
57
|
+
out(` scenario ${f.scenarioName} ${pc.dim(`(${f.featureUri})`)}`);
|
|
58
|
+
out(` baseline ${f.baseline}`);
|
|
59
|
+
table([
|
|
60
|
+
{ image: 'expected', path: rel(abs(f.expected)) },
|
|
61
|
+
{ image: 'actual', path: rel(abs(f.actual)) },
|
|
62
|
+
{ image: 'diff', path: rel(abs(f.diff)) },
|
|
63
|
+
]);
|
|
64
|
+
}
|
|
65
|
+
out('');
|
|
66
|
+
out(pc.dim(`Accept after reviewing the diff: sdods baselines accept <name…> --run ${run.runId} (or --all)`));
|
|
67
|
+
});
|
|
68
|
+
baselines
|
|
69
|
+
.command('accept [names...]')
|
|
70
|
+
.description('Copy the actual screenshot of failed visual checks over their baselines (writes to the project tree)')
|
|
71
|
+
.requiredOption('--run <id>', 'run id the screenshots come from (required)')
|
|
72
|
+
.option('--all', 'accept every failed visual check of the run')
|
|
73
|
+
.option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
|
|
74
|
+
.action(async (names, opts, cmd) => {
|
|
75
|
+
const ctx = createContext(cmd);
|
|
76
|
+
if (opts.all && names.length)
|
|
77
|
+
throw new SdodsError('CONFIG_INVALID', 'Pass baseline names or --all, not both.', {
|
|
78
|
+
exitCode: 2,
|
|
79
|
+
});
|
|
80
|
+
const run = findRun(ctx, opts);
|
|
81
|
+
const failures = listVisualFailures(run.dir);
|
|
82
|
+
if (!failures.length)
|
|
83
|
+
throw new SdodsError('CONFIG_NOT_FOUND', `No failed visual checks in run ${run.runId}.`, {
|
|
84
|
+
hint: 'Nothing to accept. `sdods baselines diff --run <id>` lists what failed.',
|
|
85
|
+
exitCode: 2,
|
|
86
|
+
});
|
|
87
|
+
const picked = selectVisualFailures(failures, { names, all: opts.all });
|
|
88
|
+
const accepted = acceptVisualFailures({
|
|
89
|
+
runDir: run.dir,
|
|
90
|
+
failures: picked,
|
|
91
|
+
projectRoot: (slug) => ctx.registry.rootOf(slug),
|
|
92
|
+
});
|
|
93
|
+
if (ctx.opts.json)
|
|
94
|
+
return json({ runId: run.runId, accepted });
|
|
95
|
+
for (const a of accepted) {
|
|
96
|
+
ok(`${a.created ? 'created' : 'updated'} ${relative(ctx.rootDir, a.to)} ${pc.dim(`${a.runnerProject}/${a.name} · ${a.platform} · ${reasonLabel(a)}`)}`);
|
|
97
|
+
out(pc.dim(` from ${relative(ctx.rootDir, a.from)}`));
|
|
98
|
+
if (a.diff)
|
|
99
|
+
out(pc.dim(` diff ${relative(ctx.rootDir, a.diff)}`));
|
|
100
|
+
}
|
|
101
|
+
out(pc.dim('Review the PNGs with `git diff --stat` and commit them with the change that moved the pixels.'));
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function reasonLabel(f) {
|
|
105
|
+
if (f.reason === 'missing')
|
|
106
|
+
return pc.cyan('new baseline');
|
|
107
|
+
if (f.reason === 'size')
|
|
108
|
+
return pc.yellow('size changed');
|
|
109
|
+
return pc.yellow(`changed ${pct(f.diffRatio)}`.trim());
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=baselines.js.map
|
|
@@ -75,7 +75,7 @@ export async function browserStatuses(names = DOWNLOADED_BROWSERS) {
|
|
|
75
75
|
return names.map((name) => {
|
|
76
76
|
const engine = ENGINE_OF[name] ?? 'chromium';
|
|
77
77
|
const channel = CHANNEL_OF[name] ?? null;
|
|
78
|
-
let executable
|
|
78
|
+
let executable;
|
|
79
79
|
try {
|
|
80
80
|
executable = channel ? channelExecutable(channel) : pw[engine].executablePath();
|
|
81
81
|
}
|
package/dist/commands/doctor.js
CHANGED
|
@@ -2,17 +2,18 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { execa } from 'execa';
|
|
4
4
|
import pc from 'picocolors';
|
|
5
|
-
import { collectVarRefs, loadDotEnvLayer, loadEnvFile } from '@sdods/core';
|
|
5
|
+
import { collectVarRefs, describeWorkers, effectiveWorkers, loadDotEnvLayer, loadEnvFile, poolAccountsByEnv, } from '@sdods/core';
|
|
6
6
|
import { createContext } from '../context.js';
|
|
7
7
|
import { browserStatuses } from './browsers.js';
|
|
8
8
|
import { MIN_PLAYWRIGHT_FOR_STEP_RESULTS, installedPlaywrightVersion, stepResultsWarning, } from '../runner-compat.js';
|
|
9
|
-
import { json, out } from '../ui.js';
|
|
9
|
+
import { json, out, parseIntFlag } from '../ui.js';
|
|
10
10
|
export function registerDoctorCommand(program) {
|
|
11
11
|
program
|
|
12
12
|
.command('doctor')
|
|
13
13
|
.description('Check Node, Bun, browsers, projects, env vars and database reachability')
|
|
14
14
|
.option('-p, --project <slug>', 'limit env-var checks to one project')
|
|
15
15
|
.option('--fix', 'install missing browsers')
|
|
16
|
+
.option('-w, --workers <n>', 'worker count to check user pools against (default: SDODS_WORKERS, else Playwright default)', parseIntFlag('workers'))
|
|
16
17
|
.action(async (opts, cmd) => {
|
|
17
18
|
const ctx = createContext(cmd);
|
|
18
19
|
const checks = [];
|
|
@@ -45,6 +46,8 @@ export function registerDoctorCommand(program) {
|
|
|
45
46
|
cwd: ctx.rootDir,
|
|
46
47
|
}).catch(() => undefined);
|
|
47
48
|
}
|
|
49
|
+
const envWorkers = Number(process.env.SDODS_WORKERS);
|
|
50
|
+
const workers = opts.workers ?? (Number.isInteger(envWorkers) && envWorkers > 0 ? envWorkers : undefined);
|
|
48
51
|
const entries = opts.project
|
|
49
52
|
? [ctx.registry.entry(opts.project)]
|
|
50
53
|
: ctx.registry.entriesList();
|
|
@@ -80,6 +83,9 @@ export function registerDoctorCommand(program) {
|
|
|
80
83
|
? `Add ${missing.join(', ')} to ${join(e.root, `.env.${envName}`)}`
|
|
81
84
|
: undefined,
|
|
82
85
|
});
|
|
86
|
+
const pool = poolCheck(e.slug, envName, { ...e.config, root: e.root }, envCfg.users.poolSize, workers);
|
|
87
|
+
if (pool)
|
|
88
|
+
checks.push(pool);
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
91
|
const driver = process.env.DB_DRIVER ?? 'sqlite';
|
|
@@ -121,6 +127,42 @@ export function registerDoctorCommand(program) {
|
|
|
121
127
|
process.exitCode = 1;
|
|
122
128
|
});
|
|
123
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* #153: accounts per role in an exclusive, worker-held pool against the worker count. Advisory —
|
|
132
|
+
* doctor does not know which roles a run will select; `sdods run` makes the real decision and
|
|
133
|
+
* fails with USER_POOL_TOO_SMALL. Undefined when there is nothing to compare.
|
|
134
|
+
*/
|
|
135
|
+
function poolCheck(slug, envName, project, poolSize, workers) {
|
|
136
|
+
const pool = project.data.userPool;
|
|
137
|
+
if (!pool)
|
|
138
|
+
return undefined;
|
|
139
|
+
const name = `${slug}/${envName} user pool`;
|
|
140
|
+
if (pool.mode === 'shared' || pool.leaseScope === 'scenario') {
|
|
141
|
+
return {
|
|
142
|
+
name,
|
|
143
|
+
ok: true,
|
|
144
|
+
detail: pool.mode === 'shared' ? 'mode: shared (no leases)' : 'leaseScope: scenario (queues)',
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const accounts = poolAccountsByEnv(project, {
|
|
148
|
+
envs: [envName],
|
|
149
|
+
poolSize: () => poolSize,
|
|
150
|
+
})?.get(envName);
|
|
151
|
+
if (!accounts)
|
|
152
|
+
return undefined; // not file-backed
|
|
153
|
+
const n = effectiveWorkers(workers);
|
|
154
|
+
const short = [...accounts].filter(([, count]) => count < n);
|
|
155
|
+
return {
|
|
156
|
+
name,
|
|
157
|
+
ok: short.length === 0,
|
|
158
|
+
optional: true,
|
|
159
|
+
detail: `${[...accounts].map(([role, count]) => `${role} ${count}`).join(', ') || 'no accounts'} ` +
|
|
160
|
+
`vs ${describeWorkers(workers)}`,
|
|
161
|
+
fix: short.length
|
|
162
|
+
? `fewer accounts than workers for ${short.map(([r]) => r).join(', ')}: sdods run stops with USER_POOL_TOO_SMALL when more of their @user: scenarios could run at once than there are accounts. Set data.userPool.leaseScope: scenario, add accounts, or lower --workers.`
|
|
163
|
+
: undefined,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
124
166
|
/**
|
|
125
167
|
* The token matrix. "one-of" rows form a group: agents need ANY one of them.
|
|
126
168
|
* Platform features never need an external token.
|
package/dist/commands/init.js
CHANGED
|
@@ -140,12 +140,12 @@ export async function initWorkspace(target, flags) {
|
|
|
140
140
|
// 1.63 is the floor, not a preference: below it an npm-installed workspace records every
|
|
141
141
|
// Gherkin step SKIPPED in messages.ndjson (see runner-compat.ts).
|
|
142
142
|
'@playwright/test': flags.link && flags.pm === 'bun' ? 'link:@playwright/test' : '^1.63.0',
|
|
143
|
-
'playwright-bdd': flags.link && flags.pm === 'bun' ? 'link:playwright-bdd' : '^9.2.
|
|
143
|
+
'playwright-bdd': flags.link && flags.pm === 'bun' ? 'link:playwright-bdd' : '^9.2.1',
|
|
144
144
|
},
|
|
145
145
|
devDependencies: {
|
|
146
|
-
'@types/node': '^22.
|
|
147
|
-
typescript: '~
|
|
148
|
-
tsx: '^4.23.
|
|
146
|
+
'@types/node': '^22.20.2',
|
|
147
|
+
typescript: '~6.0.3',
|
|
148
|
+
tsx: '^4.23.13',
|
|
149
149
|
},
|
|
150
150
|
trustedDependencies: [
|
|
151
151
|
'@playwright/test',
|
package/dist/commands/report.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import {
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync, } from 'node:fs';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { basename, dirname, extname, join, resolve } from 'node:path';
|
|
3
4
|
import { execa } from 'execa';
|
|
4
5
|
import pc from 'picocolors';
|
|
5
|
-
import { runFiles } from '@sdods/contracts';
|
|
6
|
-
import { SdodsError } from '@sdods/core';
|
|
6
|
+
import { legacyRunFiles, runFiles, } from '@sdods/contracts';
|
|
7
|
+
import { SdodsError, collectGateEvidence, evaluateGates, gateReportsFromRunnerJson, gateTotalsFromRunnerStats, hasGates, ingestGateJudge, } from '@sdods/core';
|
|
7
8
|
import { createContext } from '../context.js';
|
|
9
|
+
import { gateFailedError, printGates } from '../gates.js';
|
|
8
10
|
import { json, ok, out, table, warn } from '../ui.js';
|
|
9
11
|
function artifactsRoot(rootDir) {
|
|
10
12
|
return resolve(rootDir, process.env.SDODS_ARTIFACTS_DIR ?? '.sdods/runs');
|
|
@@ -35,6 +37,82 @@ async function openPath(p) {
|
|
|
35
37
|
const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'start' : 'xdg-open';
|
|
36
38
|
await execa(cmd, [p], { shell: process.platform === 'win32' }).catch(() => undefined);
|
|
37
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* The process `report merge` judges: `--process` (with `-p`, or the project in a run.json), or the
|
|
42
|
+
* process a run.json names when it declares gates. `undefined` when there is nothing to judge.
|
|
43
|
+
*/
|
|
44
|
+
function resolveMergeProcess(ctx, input) {
|
|
45
|
+
const manifest = input.manifestDirs
|
|
46
|
+
.map((d) => readJsonFile(join(d, runFiles.manifest)))
|
|
47
|
+
.find(Boolean);
|
|
48
|
+
const name = input.process ?? manifest?.process;
|
|
49
|
+
if (!name)
|
|
50
|
+
return undefined;
|
|
51
|
+
const slug = input.project ?? manifest?.projectSlug;
|
|
52
|
+
if (!slug)
|
|
53
|
+
throw new SdodsError('CONFIG_INVALID', `--process ${name} needs the project that defines it, and no run.json names one.`, {
|
|
54
|
+
hint: `Pass -p <slug>: sdods report merge --process ${name} -p <slug> <dirs...>`,
|
|
55
|
+
exitCode: 2,
|
|
56
|
+
});
|
|
57
|
+
let proc;
|
|
58
|
+
try {
|
|
59
|
+
proc = ctx.registry.processOf(slug, name);
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
// Named explicitly, an unknown process is an error. Inferred from a run.json this checkout
|
|
63
|
+
// cannot resolve, it only means there is nothing here to judge it by; the merge still runs.
|
|
64
|
+
if (input.process)
|
|
65
|
+
throw e;
|
|
66
|
+
warn(`run.json names process "${name}" of ${slug}, which this checkout does not define: gates not judged.`);
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
if (!hasGates(proc.gates)) {
|
|
70
|
+
// Named explicitly, a process without gates is a mistake worth saying; inferred, it is not.
|
|
71
|
+
if (input.process)
|
|
72
|
+
warn(`process "${name}" of ${slug} declares no gates, so there is nothing to judge.`);
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
return { slug, process: proc };
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* A merge workspace for `playwright merge-reports`: the blob reports of every directory copied into
|
|
79
|
+
* one (it reads exactly one directory; prefixed, because two runs' shard 1 carry the same file
|
|
80
|
+
* name), and a merge config that pins the root the report paths are relative to. Blobs recorded
|
|
81
|
+
* under different checkouts, such as CI's Playwright container (`/__w/…`) and its bare runner
|
|
82
|
+
* (`/home/runner/work/…`), otherwise refuse to merge ("recorded with different test directories").
|
|
83
|
+
*/
|
|
84
|
+
function mergeWorkspace(dirs, rootDir) {
|
|
85
|
+
const dir = mkdtempSync(join(tmpdir(), 'sdods-merge-'));
|
|
86
|
+
const reports = join(dir, 'reports');
|
|
87
|
+
mkdirSync(reports);
|
|
88
|
+
dirs.forEach((from, i) => {
|
|
89
|
+
for (const name of readdirSync(from))
|
|
90
|
+
if (name.endsWith('.zip'))
|
|
91
|
+
copyFileSync(join(from, name), join(reports, `${i + 1}-${name}`));
|
|
92
|
+
});
|
|
93
|
+
const config = join(dir, 'merge.config.mjs');
|
|
94
|
+
writeFileSync(config, `export default ${JSON.stringify({ testDir: rootDir })};\n`);
|
|
95
|
+
return { reports, config, dir };
|
|
96
|
+
}
|
|
97
|
+
/** The error Playwright printed, rather than the last lines of its stack. */
|
|
98
|
+
function mergeFailureHint(stderr) {
|
|
99
|
+
const lines = (stderr ?? '').split('\n');
|
|
100
|
+
const at = lines.findIndex((l) => /^\s*Error\b/.test(l));
|
|
101
|
+
const picked = at >= 0 ? lines.slice(at, at + 12).filter((l) => !/^\s+at /.test(l)) : lines.slice(-3);
|
|
102
|
+
return picked.join(' ').replace(/\s+/g, ' ').trim() || undefined;
|
|
103
|
+
}
|
|
104
|
+
/** Writes the merged verdict where `sdods run --process` writes one, so ingest records it the same way. */
|
|
105
|
+
function recordMergedGates(runDir, gates) {
|
|
106
|
+
writeFileSync(join(runDir, runFiles.gates), JSON.stringify(gates, null, 2));
|
|
107
|
+
const manifestFile = join(runDir, runFiles.manifest);
|
|
108
|
+
const manifest = readJsonFile(manifestFile);
|
|
109
|
+
if (manifest)
|
|
110
|
+
writeFileSync(manifestFile, JSON.stringify({ ...manifest, gatesPassed: gates.passed }, null, 2));
|
|
111
|
+
const summaryFile = join(runDir, runFiles.summary);
|
|
112
|
+
const summary = readJsonFile(summaryFile);
|
|
113
|
+
if (summary)
|
|
114
|
+
writeFileSync(summaryFile, JSON.stringify({ ...summary, gates }, null, 2));
|
|
115
|
+
}
|
|
38
116
|
export function register(program) {
|
|
39
117
|
const report = program
|
|
40
118
|
.command('report')
|
|
@@ -172,12 +250,25 @@ export function register(program) {
|
|
|
172
250
|
artifactsRoot: root,
|
|
173
251
|
replace: Boolean(opts.replace),
|
|
174
252
|
env: opts.env,
|
|
253
|
+
// A run with no gates.json whose process declares gates is judged here, as the run would have been.
|
|
254
|
+
gates: ingestGateJudge({
|
|
255
|
+
manifest: manifest ?? m.readManifest(join(root, opts.runId)),
|
|
256
|
+
runDir: join(root, opts.runId),
|
|
257
|
+
processOf: (slug, name) => ctx.registry.processOf(slug, name),
|
|
258
|
+
}),
|
|
175
259
|
});
|
|
176
260
|
if (ctx.opts.json)
|
|
177
261
|
return json(res);
|
|
178
262
|
ok(`Ingested run ${res.runId} (${res.projectSlug}): ${res.scenarios} scenario(s), ${res.attempts} attempt(s), ${res.steps} step(s), ${res.artifacts} artifact(s), ${res.healEvents} heal event(s)`);
|
|
179
263
|
const t = res.totals;
|
|
180
264
|
out(`${pc.green(`${t.passed} passed`)} ${pc.red(`${t.failed} failed`)} ${pc.yellow(`${t.flaky} flaky`)} ${pc.dim(`${t.skipped} skipped`)} → ${res.status}`);
|
|
265
|
+
if (res.gates)
|
|
266
|
+
out(`gates · process ${res.gates.process}: ${res.gates.passed
|
|
267
|
+
? pc.green('passed')
|
|
268
|
+
: pc.red(`FAILED (${res.gates.rows
|
|
269
|
+
.filter((r) => !r.passed)
|
|
270
|
+
.map((r) => r.gate)
|
|
271
|
+
.join(', ')})`)}`);
|
|
181
272
|
if (res.filesSkipped.length)
|
|
182
273
|
out(pc.dim(`skipped (already ingested): ${res.filesSkipped.join(', ')}`));
|
|
183
274
|
if (res.parseErrors)
|
|
@@ -189,9 +280,11 @@ export function register(program) {
|
|
|
189
280
|
});
|
|
190
281
|
report
|
|
191
282
|
.command('merge <dirs...>')
|
|
192
|
-
.description('Merge sharded run reports into one HTML report and JUnit file')
|
|
283
|
+
.description('Merge sharded run reports into one HTML report and JUnit file, and judge process gates on the whole run')
|
|
193
284
|
.option('--run <id>', 'run id to write the merged report into (default: latest)')
|
|
194
285
|
.option('--reporter <list>', 'reporters for the merged output', 'html,junit')
|
|
286
|
+
.option('--process <name>', "evaluate this process's gates over the merged shards; exit 1 when one is not met (default: the process in run.json, when it declares gates)")
|
|
287
|
+
.option('-p, --project <slug>', 'project that defines --process (default: from run.json)')
|
|
195
288
|
.action(async (dirs, opts, cmd) => {
|
|
196
289
|
const ctx = createContext(cmd);
|
|
197
290
|
const root = artifactsRoot(ctx.rootDir);
|
|
@@ -212,28 +305,82 @@ export function register(program) {
|
|
|
212
305
|
exitCode: 2,
|
|
213
306
|
});
|
|
214
307
|
}
|
|
308
|
+
const shardDirs = dirs.map((d) => resolve(ctx.rootDir, d));
|
|
309
|
+
// The run directory each shard report sits in (`<run>/shard-reports`) holds that shard's
|
|
310
|
+
// @a11y and @perf scenario reports, which is the gate evidence the merged report lacks.
|
|
311
|
+
const shardRunDirs = shardDirs
|
|
312
|
+
.filter((d) => [runFiles.shardReports, legacyRunFiles.shardReports].includes(basename(d)))
|
|
313
|
+
.map((d) => dirname(d));
|
|
314
|
+
const gated = resolveMergeProcess(ctx, {
|
|
315
|
+
process: opts.process,
|
|
316
|
+
project: opts.project,
|
|
317
|
+
manifestDirs: [runDir, ...shardRunDirs],
|
|
318
|
+
});
|
|
319
|
+
const reporters = String(opts.reporter)
|
|
320
|
+
.split(',')
|
|
321
|
+
.map((r) => r.trim())
|
|
322
|
+
.filter(Boolean);
|
|
323
|
+
const mergedJson = join(runDir, runFiles.mergedResults);
|
|
324
|
+
if (gated && !reporters.includes('json'))
|
|
325
|
+
reporters.push('json');
|
|
326
|
+
if (gated)
|
|
327
|
+
mkdirSync(runDir, { recursive: true });
|
|
215
328
|
// Shard reports are produced by `sdods run --reporter blob --shard i/n`; merging them
|
|
216
329
|
// rebuilds one HTML report and JUnit file for the whole matrix.
|
|
330
|
+
const workspace = mergeWorkspace(shardDirs, ctx.rootDir);
|
|
217
331
|
const args = [
|
|
218
332
|
'playwright',
|
|
219
333
|
'merge-reports',
|
|
334
|
+
'--config',
|
|
335
|
+
workspace.config,
|
|
220
336
|
'--reporter',
|
|
221
|
-
|
|
222
|
-
|
|
337
|
+
reporters.join(','),
|
|
338
|
+
workspace.reports,
|
|
223
339
|
];
|
|
224
340
|
const res = await execa('npx', args, {
|
|
225
341
|
cwd: ctx.rootDir,
|
|
226
342
|
reject: false,
|
|
227
|
-
env: {
|
|
343
|
+
env: {
|
|
344
|
+
...process.env,
|
|
345
|
+
PLAYWRIGHT_HTML_OUTPUT_DIR: join(runDir, runFiles.htmlReport),
|
|
346
|
+
...(gated ? { PLAYWRIGHT_JSON_OUTPUT_FILE: mergedJson } : {}),
|
|
347
|
+
},
|
|
228
348
|
});
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
349
|
+
let gates;
|
|
350
|
+
try {
|
|
351
|
+
if (res.exitCode !== 0) {
|
|
352
|
+
throw new SdodsError('RUN_FAILED', `Merging shard reports failed (exit ${res.exitCode}).`, { hint: mergeFailureHint(res.stderr) });
|
|
353
|
+
}
|
|
354
|
+
if (gated) {
|
|
355
|
+
const merged = readJsonFile(mergedJson);
|
|
356
|
+
gates = evaluateGates({
|
|
357
|
+
process: gated.process.name,
|
|
358
|
+
gates: gated.process.gates,
|
|
359
|
+
totals: gateTotalsFromRunnerStats(merged?.stats),
|
|
360
|
+
// The shards' run directories, and the reports attached in the blobs themselves: a
|
|
361
|
+
// download of the shard reports alone still carries its evidence.
|
|
362
|
+
evidence: collectGateEvidence([runDir, ...shardRunDirs], gateReportsFromRunnerJson(merged)),
|
|
363
|
+
});
|
|
364
|
+
recordMergedGates(runDir, gates);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
finally {
|
|
368
|
+
rmSync(workspace.dir, { recursive: true, force: true });
|
|
233
369
|
}
|
|
234
370
|
if (ctx.opts.json)
|
|
235
|
-
|
|
236
|
-
|
|
371
|
+
json({
|
|
372
|
+
runId,
|
|
373
|
+
merged: dirs.length,
|
|
374
|
+
htmlReport: join(runDir, runFiles.htmlReport),
|
|
375
|
+
...(gates ? { gates } : {}),
|
|
376
|
+
});
|
|
377
|
+
else {
|
|
378
|
+
ok(`Merged ${dirs.length} shard report dir(s) into ${join(runDir, runFiles.htmlReport)}`);
|
|
379
|
+
if (gates)
|
|
380
|
+
printGates(gates);
|
|
381
|
+
}
|
|
382
|
+
if (gates && !gates.passed)
|
|
383
|
+
throw gateFailedError(gates, join(runDir, runFiles.gates));
|
|
237
384
|
});
|
|
238
385
|
report
|
|
239
386
|
.command('traceability')
|
package/dist/commands/run.d.ts
CHANGED
|
@@ -40,6 +40,7 @@ export interface RunFlags {
|
|
|
40
40
|
trigger?: string;
|
|
41
41
|
allowEmpty?: boolean;
|
|
42
42
|
notify?: boolean;
|
|
43
|
+
allowPoolContention?: boolean;
|
|
43
44
|
}
|
|
44
45
|
export declare function register(program: Command): void;
|
|
45
46
|
export declare function runCommand(flags: RunFlags, cmd: Command): Promise<number>;
|
package/dist/commands/run.js
CHANGED
|
@@ -3,13 +3,14 @@ import { isAbsolute, join, relative, resolve as resolvePath } from 'node:path';
|
|
|
3
3
|
import { execa } from 'execa';
|
|
4
4
|
import pc from 'picocolors';
|
|
5
5
|
import { RecordingModeSchema, newRunId, runFiles, } from '@sdods/contracts';
|
|
6
|
-
import { SdodsError, CLI_OVERRIDES_ENV, VERSION, formatFindings, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
|
|
6
|
+
import { SdodsError, CLI_OVERRIDES_ENV, VERSION, checkPoolCapacity, collectGateEvidence, effectiveWorkers, evaluateGates, formatFindings, hasGates, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, poolAccountsFor, poolDemandByRole, poolTooSmallError, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
|
|
7
7
|
import { analyzeChangeImpact } from '@sdods/mcp';
|
|
8
8
|
import { createContext } from '../context.js';
|
|
9
9
|
import { browserStatuses } from './browsers.js';
|
|
10
|
+
import { gateFailedError, printGates } from '../gates.js';
|
|
10
11
|
import { maybeNotify, notifyRun } from '../notify.js';
|
|
11
12
|
import { installedPlaywrightVersion, stepResultsWarning } from '../runner-compat.js';
|
|
12
|
-
import { collect, json, out, parseIntFlag, table, warn } from '../ui.js';
|
|
13
|
+
import { collect, json, out, parseIntFlag, renderError, table, warn } from '../ui.js';
|
|
13
14
|
const RECORDING_MODES = RecordingModeSchema.options.join(' | ');
|
|
14
15
|
/** `--trace`/`--video` take Playwright's modes; refuse anything else before specs are generated. */
|
|
15
16
|
function recordingMode(flag, value) {
|
|
@@ -37,6 +38,7 @@ function addRunOptions(cmd) {
|
|
|
37
38
|
.option('--device <name>', 'device name for mobile emulation, e.g. "iPhone 15"')
|
|
38
39
|
.option('--headed', 'run headed')
|
|
39
40
|
.option('-w, --workers <n>', 'parallel workers', parseIntFlag('workers'))
|
|
41
|
+
.option('--allow-pool-contention', 'run even when a @user: role has fewer pool accounts than the workers that need it')
|
|
40
42
|
.option('--shard <i/n>', 'shard, e.g. 1/3')
|
|
41
43
|
.option('--retries <n>', 'retries per test', parseIntFlag('retries'))
|
|
42
44
|
.option('--trace <mode>', `Playwright trace: ${RECORDING_MODES} (default: evidence.trace)`)
|
|
@@ -54,7 +56,7 @@ function addRunOptions(cmd) {
|
|
|
54
56
|
.option('--har-update', 'record/refresh HAR files for @har scenarios')
|
|
55
57
|
.option('--har-replay', 'replay HAR files for @har scenarios')
|
|
56
58
|
.option('--strict', 'with --har-replay: abort on any request not in the HAR (offline)')
|
|
57
|
-
.option('--update-snapshots', '
|
|
59
|
+
.option('--update-snapshots', 'overwrite visual baselines without review (prefer `sdods baselines diff` / `accept`)')
|
|
58
60
|
.option('--ui', 'interactive UI mode')
|
|
59
61
|
.option('--debug', 'step debugger')
|
|
60
62
|
.option('--list', 'list the run targets and tests that would run')
|
|
@@ -150,6 +152,9 @@ export async function runCommand(flags, cmd) {
|
|
|
150
152
|
if (stepResults)
|
|
151
153
|
warn(stepResults);
|
|
152
154
|
const runId = flags.runId ?? newRunId();
|
|
155
|
+
if (flags.updateSnapshots)
|
|
156
|
+
warn('--update-snapshots overwrites every compared baseline without showing the diff. ' +
|
|
157
|
+
`Prefer a normal run, then \`sdods baselines diff --run ${runId}\` and \`sdods baselines accept <name> --run ${runId}\`.`);
|
|
153
158
|
const cli = {
|
|
154
159
|
env: envName,
|
|
155
160
|
headed: flags.headed,
|
|
@@ -206,6 +211,59 @@ export async function runCommand(flags, cmd) {
|
|
|
206
211
|
if (result.warnings.length && ctx.opts.verbose)
|
|
207
212
|
process.stderr.write(formatFindings(result) + '\n');
|
|
208
213
|
}
|
|
214
|
+
// --since selects features by git range; resolved here so the pool check counts the same selection.
|
|
215
|
+
const impact = flags.since
|
|
216
|
+
? analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since)
|
|
217
|
+
: undefined;
|
|
218
|
+
// #153: refuse, before generating anything, a run whose user pool cannot cover its workers.
|
|
219
|
+
// Listing runs nothing, and --ui / --debug run one scenario at a time by hand.
|
|
220
|
+
if (!flags.list && !flags.ui && !flags.debug) {
|
|
221
|
+
const poolCfg = cfg.project.data.userPool;
|
|
222
|
+
const accounts = poolCfg && poolCfg.mode !== 'shared' && poolCfg.leaseScope !== 'scenario'
|
|
223
|
+
? poolAccountsFor(cfg)
|
|
224
|
+
: undefined;
|
|
225
|
+
if (accounts) {
|
|
226
|
+
const moduleRels = modules.map((m) => relative(cfg.project.root, moduleDir(cfg.project.root, moduleByName(projectCfg, m)))
|
|
227
|
+
.replace(/\\/g, '/')
|
|
228
|
+
.concat('/'));
|
|
229
|
+
const impactedRels = impact ? new Set(impact.impacted.map((r) => r.feature)) : undefined;
|
|
230
|
+
const grep = flags.grep ? safeRegExp(flags.grep) : undefined;
|
|
231
|
+
const features = cfg.env.vars?.features;
|
|
232
|
+
const demand = poolDemandByRole(cfg.project, cfg.env.name, {
|
|
233
|
+
layers: layers.length ? layers : projectCfg.layers,
|
|
234
|
+
browsers: browsers.length ? browsers : projectCfg.browsers,
|
|
235
|
+
tags,
|
|
236
|
+
setupTags: setup?.tags,
|
|
237
|
+
flags: typeof features === 'string'
|
|
238
|
+
? features
|
|
239
|
+
.split(',')
|
|
240
|
+
.map((f) => f.trim())
|
|
241
|
+
.filter(Boolean)
|
|
242
|
+
: undefined,
|
|
243
|
+
quarantine: process.env.SDODS_QUARANTINE === 'run' ? 'run' : 'skip',
|
|
244
|
+
fullyParallel: proc?.fullyParallel ?? projectCfg.fullyParallel,
|
|
245
|
+
includeFeature: (rel) => (!featureRel || rel === featureRel) &&
|
|
246
|
+
(!moduleRels.length || moduleRels.some((m) => rel.startsWith(m))) &&
|
|
247
|
+
(!impactedRels || impactedRels.has(rel)),
|
|
248
|
+
includeScenario: (name) => (!flags.scenario || name.includes(flags.scenario)) &&
|
|
249
|
+
(!flags.grep || (grep?.test(name) ?? false)),
|
|
250
|
+
repeatEach: flags.repeatEach,
|
|
251
|
+
shardTotal: cfg.runtime.shard?.total,
|
|
252
|
+
});
|
|
253
|
+
const shortfalls = checkPoolCapacity({
|
|
254
|
+
pool: poolCfg,
|
|
255
|
+
accounts,
|
|
256
|
+
demand,
|
|
257
|
+
workers: effectiveWorkers(cfg.runtime.workers),
|
|
258
|
+
});
|
|
259
|
+
if (shortfalls.length) {
|
|
260
|
+
const error = poolTooSmallError(cfg, shortfalls, cfg.runtime.workers);
|
|
261
|
+
if (!flags.allowPoolContention)
|
|
262
|
+
throw error;
|
|
263
|
+
warn(`${error.message} Running anyway (--allow-pool-contention).`);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
209
267
|
const selection = {
|
|
210
268
|
project: entry.slug,
|
|
211
269
|
env: cfg.env.name,
|
|
@@ -318,8 +376,7 @@ export async function runCommand(flags, cmd) {
|
|
|
318
376
|
// `analyzeChangeImpact` has existed for a while and was reachable only
|
|
319
377
|
// through MCP, so it could advise a human and could not select a run. This is
|
|
320
378
|
// the seam that was missing; the mapping itself is unchanged.
|
|
321
|
-
if (flags.since) {
|
|
322
|
-
const impact = analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since);
|
|
379
|
+
if (flags.since && impact) {
|
|
323
380
|
if (impact.impacted.length === 0) {
|
|
324
381
|
// Deliberately NOT "run everything" and deliberately not a silent empty
|
|
325
382
|
// run. A run that registered zero scenarios exits 0 and looks identical
|
|
@@ -389,6 +446,30 @@ export async function runCommand(flags, cmd) {
|
|
|
389
446
|
warn(`trace redaction failed for ${relative(ctx.rootDir, e.file)}: ${e.error}`);
|
|
390
447
|
}
|
|
391
448
|
const summary = readSummary(runDir, runId, exitCode, Date.now() - started);
|
|
449
|
+
// Process gates are judged here, from the run that just finished, before anything publishes it.
|
|
450
|
+
// One shard of several holds only part of the run, so its pass rate and its a11y/perf evidence
|
|
451
|
+
// are not the run's; a sharded process is gated where the shards are merged
|
|
452
|
+
// (`sdods report merge --process`), not per shard.
|
|
453
|
+
const shardTotal = cfg.runtime.shard?.total ?? 1;
|
|
454
|
+
let gates;
|
|
455
|
+
if (proc && hasGates(proc.gates) && exitCode !== 130) {
|
|
456
|
+
if (shardTotal > 1) {
|
|
457
|
+
warn(`process "${proc.name}" gates are not evaluated on shard ${cfg.runtime.shard?.current}/${shardTotal}: one shard is not the whole run. Judge them on the merged shards with \`sdods report merge --process ${proc.name}\`.`);
|
|
458
|
+
}
|
|
459
|
+
else {
|
|
460
|
+
gates = evaluateGates({
|
|
461
|
+
process: proc.name,
|
|
462
|
+
gates: proc.gates,
|
|
463
|
+
totals: summary?.totals,
|
|
464
|
+
evidence: collectGateEvidence(runDir),
|
|
465
|
+
});
|
|
466
|
+
writeFileSync(join(runDir, runFiles.gates), JSON.stringify(gates, null, 2));
|
|
467
|
+
if (summary)
|
|
468
|
+
summary.gates = gates;
|
|
469
|
+
manifest.gatesPassed = gates.passed;
|
|
470
|
+
writeManifest();
|
|
471
|
+
}
|
|
472
|
+
}
|
|
392
473
|
if (summary)
|
|
393
474
|
writeFileSync(join(runDir, runFiles.summary), JSON.stringify(summary, null, 2));
|
|
394
475
|
const dbConfigured = process.env.DB_DRIVER === 'postgres'
|
|
@@ -428,6 +509,13 @@ export async function runCommand(flags, cmd) {
|
|
|
428
509
|
writeManifest();
|
|
429
510
|
}
|
|
430
511
|
}
|
|
512
|
+
// A breached gate fails a run whose scenarios passed. Exit 1, as the exit-code reference says:
|
|
513
|
+
// a gate not met is a failure, not a configuration error.
|
|
514
|
+
if (gates && !gates.passed && finalExit === 0) {
|
|
515
|
+
finalExit = 1;
|
|
516
|
+
manifest.exitCode = finalExit;
|
|
517
|
+
writeManifest();
|
|
518
|
+
}
|
|
431
519
|
// Enabled integrations (check runs, PR comment, issues) act on the run here, the same way
|
|
432
520
|
// `sdods integrations notify --run-id` does. Failures are reported and never change the exit code.
|
|
433
521
|
const notify = await maybeNotify({
|
|
@@ -461,9 +549,11 @@ export async function runCommand(flags, cmd) {
|
|
|
461
549
|
out(pc.dim(`integrations: not notified (${notify.reason})`));
|
|
462
550
|
}
|
|
463
551
|
if (ctx.opts.json) {
|
|
464
|
-
json({ runId, runDir, exitCode: finalExit, summary, manifest, notify });
|
|
552
|
+
json({ runId, runDir, exitCode: finalExit, summary, manifest, notify, gates });
|
|
465
553
|
}
|
|
466
554
|
else {
|
|
555
|
+
if (gates)
|
|
556
|
+
printGates(gates);
|
|
467
557
|
const t = summary?.totals;
|
|
468
558
|
out('');
|
|
469
559
|
out(`${finalExit === 0 ? pc.green('✔ passed') : finalExit === 130 ? pc.yellow('■ cancelled') : finalExit === 2 ? pc.yellow('■ nothing ran') : pc.red('✖ failed')} ${t ? `${t.passed} passed, ${t.failed} failed, ${t.skipped} skipped, ${t.flaky} flaky` : ''} ${pc.dim(`(${Math.round((Date.now() - started) / 1000)}s)`)}`);
|
|
@@ -471,6 +561,8 @@ export async function runCommand(flags, cmd) {
|
|
|
471
561
|
out(pc.dim(`html report: ${join(runDir, runFiles.htmlReport, 'index.html')}`));
|
|
472
562
|
out(pc.dim(`dashboard: ${join(runDir, runFiles.dashboard, 'index.html')}`));
|
|
473
563
|
}
|
|
564
|
+
if (gates && !gates.passed)
|
|
565
|
+
renderError(gateFailedError(gates, join(runDir, runFiles.gates)), Boolean(ctx.opts.json));
|
|
474
566
|
return finalExit;
|
|
475
567
|
}
|
|
476
568
|
/**
|
|
@@ -496,6 +588,15 @@ function projectFeaturePath(projectRoot, repoRoot, input) {
|
|
|
496
588
|
});
|
|
497
589
|
return rel;
|
|
498
590
|
}
|
|
591
|
+
/** `--grep` as the pool check reads it; a pattern Playwright would reject matches no scenario. */
|
|
592
|
+
function safeRegExp(pattern) {
|
|
593
|
+
try {
|
|
594
|
+
return new RegExp(pattern);
|
|
595
|
+
}
|
|
596
|
+
catch {
|
|
597
|
+
return undefined;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
499
600
|
function escapeRe(s) {
|
|
500
601
|
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
501
602
|
}
|