@sdods/cli 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/commands/baselines.d.ts +3 -0
- package/dist/commands/baselines.js +111 -0
- package/dist/commands/doctor.js +44 -2
- package/dist/commands/integrations.js +47 -1
- package/dist/commands/load.d.ts +3 -0
- package/dist/commands/load.js +95 -0
- package/dist/commands/matrix.d.ts +9 -0
- package/dist/commands/matrix.js +90 -0
- package/dist/commands/report.js +79 -4
- package/dist/commands/run.d.ts +1 -0
- package/dist/commands/run.js +119 -6
- package/dist/program.js +6 -0
- package/package.json +8 -8
- package/skills/sdods/SKILL.md +2 -1
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--chromium/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--firefox/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/__screenshots__/demo-shop--ui--webkit/linux/inventory.png +0 -0
- package/templates/projects/demo-shop/features/auth/user-pool.feature +1 -1
- package/templates/projects/demo-shop/features/cart/performance.feature +14 -0
- package/templates/projects/demo-shop/features/inventory/accessibility.feature +12 -0
- package/templates/projects/demo-shop/features/inventory/visual.feature +2 -1
- package/templates/projects/demo-shop/load/posts.yaml +17 -0
- package/templates/projects/demo-shop/sdods.project.yaml +3 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { relative, resolve as resolvePath } from 'node:path';
|
|
2
|
+
import pc from 'picocolors';
|
|
3
|
+
import { DEFAULT_ARTIFACTS_DIR, SdodsError, acceptVisualFailures, listVisualFailures, selectVisualFailures, } from '@sdods/core';
|
|
4
|
+
import { createContext } from '../context.js';
|
|
5
|
+
import { heading, json, ok, out, table } from '../ui.js';
|
|
6
|
+
import { listRuns } from './trace.js';
|
|
7
|
+
function artifactsDirOf(ctx, flag) {
|
|
8
|
+
return resolvePath(ctx.rootDir, flag ?? process.env.SDODS_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
|
|
9
|
+
}
|
|
10
|
+
function findRun(ctx, opts) {
|
|
11
|
+
const artifactsDir = artifactsDirOf(ctx, opts.artifactsDir);
|
|
12
|
+
const runs = listRuns(artifactsDir);
|
|
13
|
+
const run = opts.run ? runs.find((r) => r.runId === opts.run) : runs[0];
|
|
14
|
+
if (!run)
|
|
15
|
+
throw new SdodsError('CONFIG_NOT_FOUND', opts.run
|
|
16
|
+
? `Run ${opts.run} not found under ${artifactsDir}.`
|
|
17
|
+
: `No runs under ${artifactsDir}.`, {
|
|
18
|
+
hint: 'Run `sdods run -t @visual …` first, or download the CI run into .sdods/runs/<id>.',
|
|
19
|
+
exitCode: 2,
|
|
20
|
+
});
|
|
21
|
+
return run;
|
|
22
|
+
}
|
|
23
|
+
const pct = (r) => (r === undefined ? '' : `${(r * 100).toFixed(2)}%`);
|
|
24
|
+
export function register(program) {
|
|
25
|
+
const baselines = program
|
|
26
|
+
.command('baselines')
|
|
27
|
+
.description('Review failed visual baseline checks of a run and accept the new screenshots');
|
|
28
|
+
baselines
|
|
29
|
+
.command('diff')
|
|
30
|
+
.description('List the visual checks that failed in a run: expected, actual and diff images')
|
|
31
|
+
.option('--run <id>', 'run id (default: the most recent run)')
|
|
32
|
+
.option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
|
|
33
|
+
.action(async (opts, cmd) => {
|
|
34
|
+
const ctx = createContext(cmd);
|
|
35
|
+
const run = findRun(ctx, opts);
|
|
36
|
+
const failures = listVisualFailures(run.dir);
|
|
37
|
+
const abs = (p) => (p ? resolvePath(run.dir, p) : undefined);
|
|
38
|
+
if (ctx.opts.json)
|
|
39
|
+
return json({
|
|
40
|
+
runId: run.runId,
|
|
41
|
+
failures: failures.map((f) => ({
|
|
42
|
+
...f,
|
|
43
|
+
actual: abs(f.actual),
|
|
44
|
+
expected: abs(f.expected),
|
|
45
|
+
diff: abs(f.diff),
|
|
46
|
+
})),
|
|
47
|
+
});
|
|
48
|
+
if (!failures.length) {
|
|
49
|
+
ok(`No failed visual checks in run ${run.runId}.`);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
heading(`${failures.length} failed visual check(s) in run ${run.runId}`);
|
|
53
|
+
const rel = (p) => (p ? relative(ctx.rootDir, p) : '—');
|
|
54
|
+
for (const f of failures) {
|
|
55
|
+
out('');
|
|
56
|
+
out(`${pc.bold(`${f.runnerProject}/${f.name}`)} ${pc.dim(f.platform)} ${reasonLabel(f)}`);
|
|
57
|
+
out(` scenario ${f.scenarioName} ${pc.dim(`(${f.featureUri})`)}`);
|
|
58
|
+
out(` baseline ${f.baseline}`);
|
|
59
|
+
table([
|
|
60
|
+
{ image: 'expected', path: rel(abs(f.expected)) },
|
|
61
|
+
{ image: 'actual', path: rel(abs(f.actual)) },
|
|
62
|
+
{ image: 'diff', path: rel(abs(f.diff)) },
|
|
63
|
+
]);
|
|
64
|
+
}
|
|
65
|
+
out('');
|
|
66
|
+
out(pc.dim(`Accept after reviewing the diff: sdods baselines accept <name…> --run ${run.runId} (or --all)`));
|
|
67
|
+
});
|
|
68
|
+
baselines
|
|
69
|
+
.command('accept [names...]')
|
|
70
|
+
.description('Copy the actual screenshot of failed visual checks over their baselines (writes to the project tree)')
|
|
71
|
+
.requiredOption('--run <id>', 'run id the screenshots come from (required)')
|
|
72
|
+
.option('--all', 'accept every failed visual check of the run')
|
|
73
|
+
.option('--artifacts-dir <dir>', `artifacts directory (default: ${DEFAULT_ARTIFACTS_DIR})`)
|
|
74
|
+
.action(async (names, opts, cmd) => {
|
|
75
|
+
const ctx = createContext(cmd);
|
|
76
|
+
if (opts.all && names.length)
|
|
77
|
+
throw new SdodsError('CONFIG_INVALID', 'Pass baseline names or --all, not both.', {
|
|
78
|
+
exitCode: 2,
|
|
79
|
+
});
|
|
80
|
+
const run = findRun(ctx, opts);
|
|
81
|
+
const failures = listVisualFailures(run.dir);
|
|
82
|
+
if (!failures.length)
|
|
83
|
+
throw new SdodsError('CONFIG_NOT_FOUND', `No failed visual checks in run ${run.runId}.`, {
|
|
84
|
+
hint: 'Nothing to accept. `sdods baselines diff --run <id>` lists what failed.',
|
|
85
|
+
exitCode: 2,
|
|
86
|
+
});
|
|
87
|
+
const picked = selectVisualFailures(failures, { names, all: opts.all });
|
|
88
|
+
const accepted = acceptVisualFailures({
|
|
89
|
+
runDir: run.dir,
|
|
90
|
+
failures: picked,
|
|
91
|
+
projectRoot: (slug) => ctx.registry.rootOf(slug),
|
|
92
|
+
});
|
|
93
|
+
if (ctx.opts.json)
|
|
94
|
+
return json({ runId: run.runId, accepted });
|
|
95
|
+
for (const a of accepted) {
|
|
96
|
+
ok(`${a.created ? 'created' : 'updated'} ${relative(ctx.rootDir, a.to)} ${pc.dim(`${a.runnerProject}/${a.name} · ${a.platform} · ${reasonLabel(a)}`)}`);
|
|
97
|
+
out(pc.dim(` from ${relative(ctx.rootDir, a.from)}`));
|
|
98
|
+
if (a.diff)
|
|
99
|
+
out(pc.dim(` diff ${relative(ctx.rootDir, a.diff)}`));
|
|
100
|
+
}
|
|
101
|
+
out(pc.dim('Review the PNGs with `git diff --stat` and commit them with the change that moved the pixels.'));
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function reasonLabel(f) {
|
|
105
|
+
if (f.reason === 'missing')
|
|
106
|
+
return pc.cyan('new baseline');
|
|
107
|
+
if (f.reason === 'size')
|
|
108
|
+
return pc.yellow('size changed');
|
|
109
|
+
return pc.yellow(`changed ${pct(f.diffRatio)}`.trim());
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=baselines.js.map
|
package/dist/commands/doctor.js
CHANGED
|
@@ -2,17 +2,18 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { execa } from 'execa';
|
|
4
4
|
import pc from 'picocolors';
|
|
5
|
-
import { collectVarRefs, loadDotEnvLayer, loadEnvFile } from '@sdods/core';
|
|
5
|
+
import { collectVarRefs, describeWorkers, effectiveWorkers, loadDotEnvLayer, loadEnvFile, poolAccountsByEnv, } from '@sdods/core';
|
|
6
6
|
import { createContext } from '../context.js';
|
|
7
7
|
import { browserStatuses } from './browsers.js';
|
|
8
8
|
import { MIN_PLAYWRIGHT_FOR_STEP_RESULTS, installedPlaywrightVersion, stepResultsWarning, } from '../runner-compat.js';
|
|
9
|
-
import { json, out } from '../ui.js';
|
|
9
|
+
import { json, out, parseIntFlag } from '../ui.js';
|
|
10
10
|
export function registerDoctorCommand(program) {
|
|
11
11
|
program
|
|
12
12
|
.command('doctor')
|
|
13
13
|
.description('Check Node, Bun, browsers, projects, env vars and database reachability')
|
|
14
14
|
.option('-p, --project <slug>', 'limit env-var checks to one project')
|
|
15
15
|
.option('--fix', 'install missing browsers')
|
|
16
|
+
.option('-w, --workers <n>', 'worker count to check user pools against (default: SDODS_WORKERS, else Playwright default)', parseIntFlag('workers'))
|
|
16
17
|
.action(async (opts, cmd) => {
|
|
17
18
|
const ctx = createContext(cmd);
|
|
18
19
|
const checks = [];
|
|
@@ -45,6 +46,8 @@ export function registerDoctorCommand(program) {
|
|
|
45
46
|
cwd: ctx.rootDir,
|
|
46
47
|
}).catch(() => undefined);
|
|
47
48
|
}
|
|
49
|
+
const envWorkers = Number(process.env.SDODS_WORKERS);
|
|
50
|
+
const workers = opts.workers ?? (Number.isInteger(envWorkers) && envWorkers > 0 ? envWorkers : undefined);
|
|
48
51
|
const entries = opts.project
|
|
49
52
|
? [ctx.registry.entry(opts.project)]
|
|
50
53
|
: ctx.registry.entriesList();
|
|
@@ -80,6 +83,9 @@ export function registerDoctorCommand(program) {
|
|
|
80
83
|
? `Add ${missing.join(', ')} to ${join(e.root, `.env.${envName}`)}`
|
|
81
84
|
: undefined,
|
|
82
85
|
});
|
|
86
|
+
const pool = poolCheck(e.slug, envName, { ...e.config, root: e.root }, envCfg.users.poolSize, workers);
|
|
87
|
+
if (pool)
|
|
88
|
+
checks.push(pool);
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
91
|
const driver = process.env.DB_DRIVER ?? 'sqlite';
|
|
@@ -121,6 +127,42 @@ export function registerDoctorCommand(program) {
|
|
|
121
127
|
process.exitCode = 1;
|
|
122
128
|
});
|
|
123
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* #153: accounts per role in an exclusive, worker-held pool against the worker count. Advisory —
|
|
132
|
+
* doctor does not know which roles a run will select; `sdods run` makes the real decision and
|
|
133
|
+
* fails with USER_POOL_TOO_SMALL. Undefined when there is nothing to compare.
|
|
134
|
+
*/
|
|
135
|
+
function poolCheck(slug, envName, project, poolSize, workers) {
|
|
136
|
+
const pool = project.data.userPool;
|
|
137
|
+
if (!pool)
|
|
138
|
+
return undefined;
|
|
139
|
+
const name = `${slug}/${envName} user pool`;
|
|
140
|
+
if (pool.mode === 'shared' || pool.leaseScope === 'scenario') {
|
|
141
|
+
return {
|
|
142
|
+
name,
|
|
143
|
+
ok: true,
|
|
144
|
+
detail: pool.mode === 'shared' ? 'mode: shared (no leases)' : 'leaseScope: scenario (queues)',
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const accounts = poolAccountsByEnv(project, {
|
|
148
|
+
envs: [envName],
|
|
149
|
+
poolSize: () => poolSize,
|
|
150
|
+
})?.get(envName);
|
|
151
|
+
if (!accounts)
|
|
152
|
+
return undefined; // not file-backed
|
|
153
|
+
const n = effectiveWorkers(workers);
|
|
154
|
+
const short = [...accounts].filter(([, count]) => count < n);
|
|
155
|
+
return {
|
|
156
|
+
name,
|
|
157
|
+
ok: short.length === 0,
|
|
158
|
+
optional: true,
|
|
159
|
+
detail: `${[...accounts].map(([role, count]) => `${role} ${count}`).join(', ') || 'no accounts'} ` +
|
|
160
|
+
`vs ${describeWorkers(workers)}`,
|
|
161
|
+
fix: short.length
|
|
162
|
+
? `fewer accounts than workers for ${short.map(([r]) => r).join(', ')}: sdods run stops with USER_POOL_TOO_SMALL when more of their @user: scenarios could run at once than there are accounts. Set data.userPool.leaseScope: scenario, add accounts, or lower --workers.`
|
|
163
|
+
: undefined,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
124
166
|
/**
|
|
125
167
|
* The token matrix. "one-of" rows form a group: agents need ANY one of them.
|
|
126
168
|
* Platform features never need an external token.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import pc from 'picocolors';
|
|
4
|
-
import { DEFAULT_ARTIFACTS_DIR } from '@sdods/core';
|
|
4
|
+
import { DEFAULT_ARTIFACTS_DIR, SdodsError } from '@sdods/core';
|
|
5
5
|
import { createContext } from '../context.js';
|
|
6
6
|
import { LINKS_FILE, notifyRun } from '../notify.js';
|
|
7
7
|
import { json, ok, out, table, warn } from '../ui.js';
|
|
@@ -54,6 +54,7 @@ export function register(program) {
|
|
|
54
54
|
detail: res.detail,
|
|
55
55
|
secrets: fmtSecrets(p.secrets),
|
|
56
56
|
...(ctx.opts.json && res.labels ? { labels: res.labels } : {}),
|
|
57
|
+
...(ctx.opts.json && res.evidence ? { evidence: res.evidence } : {}),
|
|
57
58
|
});
|
|
58
59
|
}
|
|
59
60
|
if (ctx.opts.json)
|
|
@@ -190,6 +191,51 @@ export function register(program) {
|
|
|
190
191
|
return ok('Nothing to sync (no enabled providers or no links).');
|
|
191
192
|
table(rows);
|
|
192
193
|
});
|
|
194
|
+
const evidence = cmd
|
|
195
|
+
.command('evidence')
|
|
196
|
+
.description('The GitHub evidence branch (integrations.github.evidence.host: branch)');
|
|
197
|
+
evidence
|
|
198
|
+
.command('prune')
|
|
199
|
+
.description('Rewrite the evidence branch without runs uploaded before the cutoff (a new orphan commit, force-pushed)')
|
|
200
|
+
.requiredOption('-p, --project <slug>', 'project slug')
|
|
201
|
+
.option('--older-than <age>', '90min, 36h, 14d or 2w; a bare number is days (default: evidence.retainDays)')
|
|
202
|
+
.option('--dry-run', 'list the runs that would be removed without rewriting the branch')
|
|
203
|
+
.action(async (opts, c) => {
|
|
204
|
+
const ctx = createContext(c);
|
|
205
|
+
const entry = ctx.registry.entry(opts.project);
|
|
206
|
+
const { getProviders, GitHubProvider, parseAge } = await import('@sdods/integrations');
|
|
207
|
+
if (opts.olderThan) {
|
|
208
|
+
try {
|
|
209
|
+
parseAge(opts.olderThan);
|
|
210
|
+
}
|
|
211
|
+
catch (e) {
|
|
212
|
+
throw new SdodsError('CONFIG_INVALID', e.message);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const [gh] = await getProviders(entry.config, { only: ['github'], projectRoot: entry.root });
|
|
216
|
+
const docsPath = '/docs/guides/github-and-jira#evidence-that-renders-in-private-repositories';
|
|
217
|
+
if (!gh?.enabled || !(gh.provider instanceof GitHubProvider))
|
|
218
|
+
throw new SdodsError('CONFIG_INVALID', `integrations.github is not enabled for project ${entry.slug}.`, { docsPath });
|
|
219
|
+
if (gh.initError)
|
|
220
|
+
throw new SdodsError('CONFIG_INVALID', `github: ${gh.initError}`, { docsPath });
|
|
221
|
+
if (entry.config.integrations.github?.evidence?.host !== 'branch')
|
|
222
|
+
throw new SdodsError('CONFIG_INVALID', `integrations.github.evidence.host is not "branch" for project ${entry.slug}.`, { docsPath });
|
|
223
|
+
const res = await gh.provider.pruneEvidence({
|
|
224
|
+
olderThan: opts.olderThan,
|
|
225
|
+
dryRun: Boolean(opts.dryRun),
|
|
226
|
+
});
|
|
227
|
+
if (ctx.opts.json)
|
|
228
|
+
return json(res);
|
|
229
|
+
const verb = res.dryRun ? 'would remove' : 'removed';
|
|
230
|
+
if (!res.removed.length)
|
|
231
|
+
ok(`${res.branch}: nothing to prune (${res.kept.length} run(s) kept)`);
|
|
232
|
+
else {
|
|
233
|
+
table(res.removed.map((r) => ({ run: r.runId, uploaded: r.uploadedAt })));
|
|
234
|
+
ok(`${res.branch}: ${verb} ${res.removed.length} run(s), kept ${res.kept.length}${res.commitSha ? ` (new root ${res.commitSha.slice(0, 10)})` : ''}`);
|
|
235
|
+
}
|
|
236
|
+
if (res.unknown.length)
|
|
237
|
+
warn(`kept ${res.unknown.length} run(s) missing from runs/index.json (age unknown): ${res.unknown.join(', ')}`);
|
|
238
|
+
});
|
|
193
239
|
cmd
|
|
194
240
|
.command('links')
|
|
195
241
|
.description('List known issue links for a project')
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { relative } from 'node:path';
|
|
2
|
+
import pc from 'picocolors';
|
|
3
|
+
import { SdodsError } from '@sdods/core';
|
|
4
|
+
import { createContext } from '../context.js';
|
|
5
|
+
import { heading, json, ok, out, table, warn } from '../ui.js';
|
|
6
|
+
export function register(program) {
|
|
7
|
+
program
|
|
8
|
+
.command('load')
|
|
9
|
+
.description('Load test an API with k6: generate a script from load/<profile>.yaml and run it (opt-in per environment)')
|
|
10
|
+
.argument('[profile]', 'profile name: projects/<slug>/load/<profile>.yaml')
|
|
11
|
+
.option('-p, --project <slug>', 'project slug')
|
|
12
|
+
.option('-e, --env <name>', 'environment (must set load.allowed: true)')
|
|
13
|
+
.option('--dry-run', 'write the k6 script and stop; k6 is not needed')
|
|
14
|
+
.option('--out <dir>', 'output directory (default .sdods/runs/<id>/load/<profile>)')
|
|
15
|
+
.option('--runner <runner>', 'auto | k6 | docker (auto uses k6 on PATH)', 'auto')
|
|
16
|
+
.option('--image <image>', 'k6 image for --runner docker', 'grafana/k6:latest')
|
|
17
|
+
.option('--list', 'list the load profiles of the project')
|
|
18
|
+
.action(async (profile, opts, cmd) => {
|
|
19
|
+
const ctx = createContext(cmd);
|
|
20
|
+
const entry = ctx.registry.pick(opts.project);
|
|
21
|
+
const load = await import('@sdods/core/load');
|
|
22
|
+
if (opts.list) {
|
|
23
|
+
const rows = load.listLoadProfiles(entry.root).map((name) => ({ profile: name }));
|
|
24
|
+
if (ctx.opts.json)
|
|
25
|
+
return json(rows);
|
|
26
|
+
if (!rows.length)
|
|
27
|
+
return out(pc.dim(`no profiles in ${relative(ctx.rootDir, entry.root)}/load`));
|
|
28
|
+
return table(rows, ['profile']);
|
|
29
|
+
}
|
|
30
|
+
if (!profile) {
|
|
31
|
+
throw new SdodsError('CONFIG_INVALID', 'Name a load profile, or pass --list.', {
|
|
32
|
+
hint: `sdods load -p ${entry.slug} -e <env> <profile> --dry-run`,
|
|
33
|
+
exitCode: 2,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
if (!['auto', 'k6', 'docker'].includes(opts.runner)) {
|
|
37
|
+
throw new SdodsError('CONFIG_INVALID', `--runner must be auto, k6 or docker.`, {
|
|
38
|
+
exitCode: 2,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
const quiet = Boolean(ctx.opts.json || ctx.opts.quiet);
|
|
42
|
+
if (!quiet)
|
|
43
|
+
heading(`load · ${entry.slug} · ${profile}${opts.dryRun ? ' · dry run' : ''}`);
|
|
44
|
+
const res = await load.runLoad({
|
|
45
|
+
rootDir: ctx.rootDir,
|
|
46
|
+
projectRoot: entry.root,
|
|
47
|
+
profile,
|
|
48
|
+
env: opts.env,
|
|
49
|
+
outDir: opts.out,
|
|
50
|
+
dryRun: Boolean(opts.dryRun),
|
|
51
|
+
runner: opts.runner,
|
|
52
|
+
image: opts.image,
|
|
53
|
+
// --json keeps stdout for the result; k6's own output goes to stderr.
|
|
54
|
+
stdio: ctx.opts.json ? 'stderr' : 'inherit',
|
|
55
|
+
log: (line) => {
|
|
56
|
+
if (quiet)
|
|
57
|
+
return;
|
|
58
|
+
if (line.startsWith('warning: '))
|
|
59
|
+
warn(line.slice('warning: '.length));
|
|
60
|
+
else
|
|
61
|
+
out(line);
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
process.exitCode = res.exitCode;
|
|
65
|
+
if (ctx.opts.json)
|
|
66
|
+
return json(res);
|
|
67
|
+
const script = relative(ctx.rootDir, res.scriptFile);
|
|
68
|
+
if (res.dryRun) {
|
|
69
|
+
ok(`script written: ${script}`);
|
|
70
|
+
if (res.requiredEnv.length)
|
|
71
|
+
out(pc.dim(`reads from the environment: ${res.requiredEnv.join(', ')}`));
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const s = res.summary;
|
|
75
|
+
if (s) {
|
|
76
|
+
table([
|
|
77
|
+
{
|
|
78
|
+
requests: s.requests ?? '',
|
|
79
|
+
'p95 ms': s.p95Ms === undefined ? '' : Math.round(s.p95Ms),
|
|
80
|
+
failed: s.failedRate === undefined ? '' : `${(s.failedRate * 100).toFixed(2)}%`,
|
|
81
|
+
checks: s.checksRate === undefined ? '' : `${(s.checksRate * 100).toFixed(2)}%`,
|
|
82
|
+
},
|
|
83
|
+
], ['requests', 'p95 ms', 'failed', 'checks']);
|
|
84
|
+
}
|
|
85
|
+
if (res.summaryFile)
|
|
86
|
+
out(pc.dim(`summary: ${relative(ctx.rootDir, res.summaryFile)}`));
|
|
87
|
+
if (res.exitCode === 0)
|
|
88
|
+
ok('thresholds passed');
|
|
89
|
+
else if (res.thresholdsFailed)
|
|
90
|
+
warn('thresholds failed (k6 exit 99)');
|
|
91
|
+
else
|
|
92
|
+
warn(`k6 exited with ${res.k6ExitCode}`);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=load.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Command } from 'commander';
|
|
2
|
+
/**
|
|
3
|
+
* `sdods matrix expand` — write the `Examples:` blocks of every `@matrix:<name>` Scenario Outline
|
|
4
|
+
* from `roles.matrix.yaml` (#118). `--check` writes nothing and fails when a feature is out of date,
|
|
5
|
+
* for CI; `--dry-run` writes nothing and, with `--json`, returns the expanded text (the MCP
|
|
6
|
+
* `matrix_expand` tool turns that into a proposal instead of touching the working tree).
|
|
7
|
+
*/
|
|
8
|
+
export declare function register(program: Command): void;
|
|
9
|
+
//# sourceMappingURL=matrix.d.ts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { writeFileSync } from 'node:fs';
|
|
2
|
+
import { resolve as resolvePath, sep } from 'node:path';
|
|
3
|
+
import pc from 'picocolors';
|
|
4
|
+
import { ROLES_MATRIX_FILE, listFeatureFiles, planMatrixExpansion } from '@sdods/core';
|
|
5
|
+
import { createContext } from '../context.js';
|
|
6
|
+
import { collect, json, ok, out } from '../ui.js';
|
|
7
|
+
/**
|
|
8
|
+
* `sdods matrix expand` — write the `Examples:` blocks of every `@matrix:<name>` Scenario Outline
|
|
9
|
+
* from `roles.matrix.yaml` (#118). `--check` writes nothing and fails when a feature is out of date,
|
|
10
|
+
* for CI; `--dry-run` writes nothing and, with `--json`, returns the expanded text (the MCP
|
|
11
|
+
* `matrix_expand` tool turns that into a proposal instead of touching the working tree).
|
|
12
|
+
*/
|
|
13
|
+
export function register(program) {
|
|
14
|
+
const matrix = program
|
|
15
|
+
.command('matrix')
|
|
16
|
+
.description('Role × surface matrices from roles.matrix.yaml');
|
|
17
|
+
matrix
|
|
18
|
+
.command('expand')
|
|
19
|
+
.description('Write the generated Examples of every @matrix:<name> outline')
|
|
20
|
+
.option('-p, --project <slug>', 'project slug (default: all projects)')
|
|
21
|
+
.option('--check', 'write nothing; exit 3 if any feature is out of date or invalid')
|
|
22
|
+
.option('--dry-run', 'write nothing; report what would change (with --json, the new text)')
|
|
23
|
+
.option('--file <path>', 'only this feature file (repeatable)', collect, [])
|
|
24
|
+
.action((opts, cmd) => {
|
|
25
|
+
const ctx = createContext(cmd);
|
|
26
|
+
const entries = opts.project
|
|
27
|
+
? [ctx.registry.entry(opts.project)]
|
|
28
|
+
: ctx.registry.entriesList();
|
|
29
|
+
const only = opts.file.map((f) => resolvePath(f));
|
|
30
|
+
const write = !opts.check && !opts.dryRun;
|
|
31
|
+
const files = [];
|
|
32
|
+
const problems = [];
|
|
33
|
+
let configInvalid = false;
|
|
34
|
+
for (const e of entries) {
|
|
35
|
+
// The matrix needs the project file, not an environment: no env is resolved (or required).
|
|
36
|
+
const project = { ...e.config, root: e.root };
|
|
37
|
+
const featureFiles = only.length
|
|
38
|
+
? only.filter((f) => f.startsWith(resolvePath(project.root) + sep))
|
|
39
|
+
: listFeatureFiles(project.root);
|
|
40
|
+
const plan = planMatrixExpansion(project, featureFiles);
|
|
41
|
+
if (plan.loaded?.problems.length)
|
|
42
|
+
configInvalid = true;
|
|
43
|
+
for (const p of [...plan.problems, ...plan.files.flatMap((f) => f.problems)])
|
|
44
|
+
problems.push({ project: e.slug, ...p });
|
|
45
|
+
for (const f of plan.files) {
|
|
46
|
+
// A template with errors is reported, not rewritten: fix the outline, then expand.
|
|
47
|
+
const blocked = f.problems.some((p) => p.severity === 'error');
|
|
48
|
+
if (write && f.changed && !blocked)
|
|
49
|
+
writeFileSync(f.file, f.after);
|
|
50
|
+
files.push({
|
|
51
|
+
project: e.slug,
|
|
52
|
+
path: f.path,
|
|
53
|
+
status: !f.changed ? 'unchanged' : write && !blocked ? 'updated' : 'stale',
|
|
54
|
+
examples: f.examples,
|
|
55
|
+
...(opts.dryRun && ctx.opts.json ? { content: f.after } : {}),
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const errors = problems.filter((p) => p.severity === 'error');
|
|
60
|
+
const stale = files.filter((f) => f.status === 'stale');
|
|
61
|
+
const failed = opts.check ? stale.length > 0 || errors.length > 0 : errors.length > 0;
|
|
62
|
+
if (ctx.opts.json)
|
|
63
|
+
json({ ok: !failed, files, problems });
|
|
64
|
+
else {
|
|
65
|
+
for (const p of problems)
|
|
66
|
+
out(`${p.severity === 'error' ? pc.red('✖') : pc.yellow('⚠')} ${p.project}/${p.file}${p.line ? `:${p.line}` : ''} ${p.rule} ${p.message}`);
|
|
67
|
+
for (const f of files) {
|
|
68
|
+
const label = f.status === 'updated'
|
|
69
|
+
? pc.green('updated')
|
|
70
|
+
: f.status === 'stale'
|
|
71
|
+
? pc.yellow('out of date')
|
|
72
|
+
: pc.dim('up to date');
|
|
73
|
+
out(`${label} ${f.project}/${f.path} (${f.examples} generated example(s))`);
|
|
74
|
+
}
|
|
75
|
+
if (!files.length)
|
|
76
|
+
out(pc.dim(`No @matrix:<name> outlines found (${ROLES_MATRIX_FILE}).`));
|
|
77
|
+
else if (opts.check && !stale.length && !errors.length)
|
|
78
|
+
ok('matrix features are up to date');
|
|
79
|
+
if (opts.check && stale.length)
|
|
80
|
+
out(`Run \`sdods matrix expand${opts.project ? ` -p ${opts.project}` : ''}\` and commit the result.`);
|
|
81
|
+
}
|
|
82
|
+
if (opts.dryRun && !opts.check)
|
|
83
|
+
return;
|
|
84
|
+
if (configInvalid && !opts.check)
|
|
85
|
+
process.exitCode = 2;
|
|
86
|
+
else if (failed)
|
|
87
|
+
process.exitCode = 3;
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=matrix.js.map
|
package/dist/commands/report.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
-
import { join, resolve } from 'node:path';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, extname, join, resolve } from 'node:path';
|
|
3
3
|
import { execa } from 'execa';
|
|
4
4
|
import pc from 'picocolors';
|
|
5
5
|
import { runFiles } from '@sdods/contracts';
|
|
6
6
|
import { SdodsError } from '@sdods/core';
|
|
7
7
|
import { createContext } from '../context.js';
|
|
8
|
-
import { json, ok, out, table } from '../ui.js';
|
|
8
|
+
import { json, ok, out, table, warn } from '../ui.js';
|
|
9
9
|
function artifactsRoot(rootDir) {
|
|
10
10
|
return resolve(rootDir, process.env.SDODS_ARTIFACTS_DIR ?? '.sdods/runs');
|
|
11
11
|
}
|
|
@@ -195,7 +195,9 @@ export function register(program) {
|
|
|
195
195
|
.action(async (dirs, opts, cmd) => {
|
|
196
196
|
const ctx = createContext(cmd);
|
|
197
197
|
const root = artifactsRoot(ctx.rootDir);
|
|
198
|
-
|
|
198
|
+
// `report` defines --run too, and Commander hands it to the parent wherever it appears.
|
|
199
|
+
const parent = (cmd.parent?.opts() ?? {});
|
|
200
|
+
const runId = opts.run ?? parent.run ?? latestRunId(root);
|
|
199
201
|
if (!runId) {
|
|
200
202
|
throw new SdodsError('CONFIG_NOT_FOUND', 'No run to merge into.', {
|
|
201
203
|
hint: 'Pass --run <id>, or run a suite first.',
|
|
@@ -233,6 +235,79 @@ export function register(program) {
|
|
|
233
235
|
return json({ runId, merged: dirs.length, htmlReport: join(runDir, runFiles.htmlReport) });
|
|
234
236
|
ok(`Merged ${dirs.length} shard report dir(s) into ${join(runDir, runFiles.htmlReport)}`);
|
|
235
237
|
});
|
|
238
|
+
report
|
|
239
|
+
.command('traceability')
|
|
240
|
+
.description('Export requirement → scenario → result traceability for a run (@req:<id> tags), with an empty sign-off block')
|
|
241
|
+
.requiredOption('-p, --project <slug>', 'project slug')
|
|
242
|
+
.option('-e, --env <name>', 'only consider runs against this environment')
|
|
243
|
+
.option('--run <id>', 'run id (default: the latest run of the project)')
|
|
244
|
+
.option('--last', 'use the latest run of the project; fail if there is none')
|
|
245
|
+
.option('--format <fmt>', 'json|csv|md|html (default: from the -o extension, else json with --json, else md)')
|
|
246
|
+
.option('-o, --output <file>', 'write the export to a file instead of stdout')
|
|
247
|
+
.action(async (opts, cmd) => {
|
|
248
|
+
const ctx = createContext(cmd);
|
|
249
|
+
// `report` defines --run/--last itself, and Commander hands options it knows to the parent
|
|
250
|
+
// wherever they appear, so read them from both.
|
|
251
|
+
const parent = (cmd.parent?.opts() ?? {});
|
|
252
|
+
const runOpt = opts.run ?? parent.run;
|
|
253
|
+
const last = Boolean(opts.last ?? parent.last);
|
|
254
|
+
const { TRACEABILITY_FORMATS, buildTraceabilityReport, latestRunFor, readRunManifest, renderTraceability, } = await import('@sdods/core/analyze');
|
|
255
|
+
const output = opts.output
|
|
256
|
+
? resolve(process.cwd(), opts.output)
|
|
257
|
+
: undefined;
|
|
258
|
+
const fromExt = output
|
|
259
|
+
? {
|
|
260
|
+
'.json': 'json',
|
|
261
|
+
'.csv': 'csv',
|
|
262
|
+
'.md': 'md',
|
|
263
|
+
'.html': 'html',
|
|
264
|
+
'.htm': 'html',
|
|
265
|
+
}[extname(output).toLowerCase()]
|
|
266
|
+
: undefined;
|
|
267
|
+
// -o's extension decides the file format; --json then only shapes the confirmation line.
|
|
268
|
+
const format = (opts.format ?? fromExt ?? (ctx.opts.json ? 'json' : 'md'));
|
|
269
|
+
if (!TRACEABILITY_FORMATS.includes(format))
|
|
270
|
+
throw new SdodsError('CONFIG_INVALID', `Unknown format "${format}".`, {
|
|
271
|
+
hint: `Use one of: ${TRACEABILITY_FORMATS.join(', ')}.`,
|
|
272
|
+
exitCode: 2,
|
|
273
|
+
});
|
|
274
|
+
const slug = opts.project;
|
|
275
|
+
const project = { ...ctx.registry.get(slug), root: ctx.registry.rootOf(slug) };
|
|
276
|
+
const root = artifactsRoot(ctx.rootDir);
|
|
277
|
+
let run;
|
|
278
|
+
if (runOpt) {
|
|
279
|
+
const dir = join(root, runOpt);
|
|
280
|
+
if (!existsSync(dir))
|
|
281
|
+
throw new SdodsError('RUN_FAILED', `No run "${runOpt}" under ${root}.`, { exitCode: 2 });
|
|
282
|
+
const m = readRunManifest(dir);
|
|
283
|
+
if (m && m.projectSlug !== slug)
|
|
284
|
+
throw new SdodsError('CONFIG_INVALID', `Run ${runOpt} belongs to project "${m.projectSlug}", not "${slug}".`, { exitCode: 2 });
|
|
285
|
+
if (m && opts.env && m.env !== opts.env)
|
|
286
|
+
throw new SdodsError('CONFIG_INVALID', `Run ${runOpt} ran against "${m.env}", not "${opts.env}".`, { exitCode: 2 });
|
|
287
|
+
run = { runId: runOpt, dir };
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
run = latestRunFor(root, slug, opts.env) ?? undefined;
|
|
291
|
+
if (!run && last)
|
|
292
|
+
throw new SdodsError('RUN_FAILED', `No runs of ${slug}${opts.env ? ` on ${opts.env}` : ''} under ${root}.`, {
|
|
293
|
+
hint: `Run \`sdods run -p ${slug}\` first, or omit --last for a static matrix.`,
|
|
294
|
+
exitCode: 2,
|
|
295
|
+
});
|
|
296
|
+
if (!run)
|
|
297
|
+
warn(`No runs of ${slug}${opts.env ? ` on ${opts.env}` : ''} found: every scenario is reported as not run.`);
|
|
298
|
+
}
|
|
299
|
+
const traceReport = await buildTraceabilityReport({ project, run });
|
|
300
|
+
const text = renderTraceability(traceReport, format);
|
|
301
|
+
if (!output)
|
|
302
|
+
return out(text);
|
|
303
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
304
|
+
writeFileSync(output, text);
|
|
305
|
+
const s = traceReport.summary;
|
|
306
|
+
if (ctx.opts.json)
|
|
307
|
+
return json({ output, format, run: traceReport.run?.id ?? null, summary: s });
|
|
308
|
+
ok(`Wrote ${format} traceability export to ${output}`);
|
|
309
|
+
out(`${s.requirements} requirement(s): ${pc.green(`${s.passed} passed`)} ${pc.red(`${s.failed} failed`)} ${pc.yellow(`${s.notRun} not run`)} ${s.notCovered == null ? pc.dim('not covered: unknown') : pc.magenta(`${s.notCovered} not covered`)}`);
|
|
310
|
+
});
|
|
236
311
|
program
|
|
237
312
|
.command('show-report')
|
|
238
313
|
.description('Open the HTML report of a run')
|
package/dist/commands/run.d.ts
CHANGED
|
@@ -40,6 +40,7 @@ export interface RunFlags {
|
|
|
40
40
|
trigger?: string;
|
|
41
41
|
allowEmpty?: boolean;
|
|
42
42
|
notify?: boolean;
|
|
43
|
+
allowPoolContention?: boolean;
|
|
43
44
|
}
|
|
44
45
|
export declare function register(program: Command): void;
|
|
45
46
|
export declare function runCommand(flags: RunFlags, cmd: Command): Promise<number>;
|