@metasession.co/devaudit-cli 0.1.39 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/sdlc/files/_common/skills/e2e-test-engineer/SKILL.md +38 -1
- package/sdlc/files/_common/skills/e2e-test-engineer/references/evidence-shot-core.ts +31 -0
- package/sdlc/files/_common/skills/e2e-test-engineer/references/evidence.ts +15 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@metasession.co/devaudit-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.40",
|
|
4
4
|
"description": "DevAudit CLI — installs, syncs, and operates the Metasession SDLC across consumer projects.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@clack/prompts": "^0.8.2",
|
|
36
|
-
"@metasession.co/devaudit-plugin-sdk": "^0.1.
|
|
36
|
+
"@metasession.co/devaudit-plugin-sdk": "^0.1.40",
|
|
37
37
|
"commander": "^12.1.0",
|
|
38
38
|
"consola": "^3.2.3",
|
|
39
39
|
"env-paths": "^3.0.0",
|
|
@@ -327,7 +327,7 @@ test('AC1: edit dialog opens with fields pre-filled', async ({ page }) => {
|
|
|
327
327
|
- Call `evidenceShot` **immediately after** the AC-proving assertion, before navigating, closing dialogs, or any further interaction.
|
|
328
328
|
- AC number is a separate argument (`ac: number`) — the helper composes the filename `REQ-XXX-AC<n>-<slug>.png`. The slug describes what the screenshot proves (`edit-dialog-prefilled`), NOT the AC number.
|
|
329
329
|
- Slug is kebab-case lowercase (`[a-z0-9-]+`). Capitalised slugs, underscores, or spaces throw.
|
|
330
|
-
- One screenshot per AC
|
|
330
|
+
- One **canonical** screenshot per AC; additional stage screenshots are tier-gated — see *Screenshot density per spec role* below.
|
|
331
331
|
- Failure forensics stays untouched (`screenshot: 'only-on-failure'` + `trace: 'on-first-retry'`).
|
|
332
332
|
|
|
333
333
|
The helper is shipped automatically into `e2e/helpers/evidence.ts` by the SDLC sync (node-stack consumers). Output lands at `compliance/evidence/<REQ-ID>/screenshots/REQ-XXX-AC<n>-<slug>.png` — commit these PNGs as part of the evidence pack so reviewers can corroborate the test-plan AC mapping.
|
|
@@ -336,6 +336,43 @@ The helper also writes a sidecar `<filename>.meta.json` containing the AC mappin
|
|
|
336
336
|
|
|
337
337
|
The canonical helper source lives at `references/evidence.ts` in this skill.
|
|
338
338
|
|
|
339
|
+
### Screenshot density per spec role
|
|
340
|
+
|
|
341
|
+
The number of `evidenceShot` calls per spec should scale to the spec's role:
|
|
342
|
+
|
|
343
|
+
- **While the spec is a feature artefact** (newly authored on the branch, before merge to develop): capture multiple stages — every meaningful transition or state the AC documents. The dense evidence is what reviewers use to verify the AC was met end-to-end during the feature cycle.
|
|
344
|
+
- **Once the spec joins the regression pack** (post-merge, `git diff --diff-filter=A` no longer matches it): capture only the canonical "this still works" anchor per AC. Re-running the dense journey on every regression cycle is noise and inflates CI artefact storage with little signal.
|
|
345
|
+
|
|
346
|
+
The `EvidenceShotOrigin` signal (`'feature' | 'regression'`) auto-detects from `E2E_NEW_SPECS`. Mark stage screenshots with `{ tier: 'feature' }`; the helper auto-suppresses them on regression runs. The canonical anchor uses the default tier (`'always'`).
|
|
347
|
+
|
|
348
|
+
```ts
|
|
349
|
+
test('AC7: stock dial completes the transition', async ({ page }) => {
|
|
350
|
+
// Stage screenshots — fire while the spec is a feature artefact;
|
|
351
|
+
// auto-suppress once it graduates into the regression pack.
|
|
352
|
+
await openStockDial(page, item.id);
|
|
353
|
+
await evidenceShot(page, 'REQ-066', 7, 'dial-open', { tier: 'feature' });
|
|
354
|
+
await advanceDial(page);
|
|
355
|
+
await evidenceShot(page, 'REQ-066', 7, 'in-progress', { tier: 'feature' });
|
|
356
|
+
|
|
357
|
+
// Canonical anchor — always fires (default tier: 'always').
|
|
358
|
+
// This is the artefact every future regression run re-captures as
|
|
359
|
+
// proof the AC still holds.
|
|
360
|
+
await expect(dial.getByRole('status')).toHaveText('Completed');
|
|
361
|
+
await evidenceShot(page, 'REQ-066', 7, 'completed');
|
|
362
|
+
});
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
A reasonable default per AC:
|
|
366
|
+
|
|
367
|
+
- 1× canonical "completed / final state" shot (tier `'always'`).
|
|
368
|
+
- 1–3× stage shots covering the meaningful intermediate transitions (tier `'feature'`).
|
|
369
|
+
|
|
370
|
+
When to deviate:
|
|
371
|
+
|
|
372
|
+
- **Single-shot ACs** (one assertion that's its own proof — e.g. *"the form submits and returns to the list"*) need only the canonical anchor. Don't manufacture stages just to hit the 1–3 band.
|
|
373
|
+
- **Long flows** (>3 meaningful transitions) keep all stages tier `'feature'`. The post-merge regression run still has the canonical anchor to corroborate the AC; the dense journey is on the feature PR for reviewers and in the audit-pack download for that release forever.
|
|
374
|
+
- **Reviewer pushback that evidence feels thin** (single-shot per AC across a HIGH-risk REQ) almost always means tier `'feature'` stages are missing — add them on the feature branch where they actually fire, not after.
|
|
375
|
+
|
|
339
376
|
---
|
|
340
377
|
|
|
341
378
|
## Principles
|
|
@@ -7,6 +7,23 @@
|
|
|
7
7
|
|
|
8
8
|
export type EvidenceShotOrigin = 'feature' | 'regression';
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Capture density tier. Lets spec authors mark intermediate-state
|
|
12
|
+
* screenshots that only matter while the spec is being authored on a
|
|
13
|
+
* feature branch — once the spec joins the regression pack, those
|
|
14
|
+
* become noise and inflate CI artefact storage.
|
|
15
|
+
*
|
|
16
|
+
* - `'always'` (default) — capture on every run. Use for the canonical
|
|
17
|
+
* "this still works" anchor per AC; the artefact reviewers rely on
|
|
18
|
+
* to corroborate the test-plan mapping across every release.
|
|
19
|
+
* - `'feature'` — capture only when the spec's origin is `feature`
|
|
20
|
+
* (i.e. the spec was added on the current branch per
|
|
21
|
+
* `E2E_NEW_SPECS`). Auto-suppressed once the spec graduates into
|
|
22
|
+
* the regression pack. Use for stage screenshots covering meaningful
|
|
23
|
+
* intermediate transitions reviewers want during the feature cycle.
|
|
24
|
+
*/
|
|
25
|
+
export type EvidenceShotTier = 'always' | 'feature';
|
|
26
|
+
|
|
10
27
|
export interface EvidenceShotSidecar {
|
|
11
28
|
readonly origin: EvidenceShotOrigin;
|
|
12
29
|
readonly reqId: string;
|
|
@@ -16,6 +33,20 @@ export interface EvidenceShotSidecar {
|
|
|
16
33
|
readonly capturedAt: string;
|
|
17
34
|
}
|
|
18
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Pure decision: should the capture be suppressed?
|
|
38
|
+
*
|
|
39
|
+
* The only suppression case is `tier='feature'` × `origin='regression'`
|
|
40
|
+
* — a stage screenshot whose spec has graduated into the regression
|
|
41
|
+
* pack. Every other (tier, origin) combination captures.
|
|
42
|
+
*/
|
|
43
|
+
export function shouldSuppressEvidenceShot(
|
|
44
|
+
tier: EvidenceShotTier,
|
|
45
|
+
origin: EvidenceShotOrigin,
|
|
46
|
+
): boolean {
|
|
47
|
+
return tier === 'feature' && origin === 'regression';
|
|
48
|
+
}
|
|
49
|
+
|
|
19
50
|
const REQ_ID_RE = /^REQ-[A-Z0-9-]+$/;
|
|
20
51
|
const SLUG_RE = /^[a-z0-9-]+$/;
|
|
21
52
|
|
|
@@ -4,12 +4,14 @@ import { test, type Page } from '@playwright/test';
|
|
|
4
4
|
import {
|
|
5
5
|
autoDetectEvidenceShotOrigin,
|
|
6
6
|
composeScreenshotFilename,
|
|
7
|
+
shouldSuppressEvidenceShot,
|
|
7
8
|
validateEvidenceShotInputs,
|
|
8
9
|
type EvidenceShotOrigin,
|
|
9
10
|
type EvidenceShotSidecar,
|
|
11
|
+
type EvidenceShotTier,
|
|
10
12
|
} from './evidence-shot-core';
|
|
11
13
|
|
|
12
|
-
export type { EvidenceShotOrigin };
|
|
14
|
+
export type { EvidenceShotOrigin, EvidenceShotTier };
|
|
13
15
|
|
|
14
16
|
export interface EvidenceShotOptions {
|
|
15
17
|
/** Capture the full page rather than the viewport. Default: true. */
|
|
@@ -21,6 +23,14 @@ export interface EvidenceShotOptions {
|
|
|
21
23
|
* the calling spec's file appears in that list, else `regression`.
|
|
22
24
|
*/
|
|
23
25
|
readonly origin?: EvidenceShotOrigin;
|
|
26
|
+
/**
|
|
27
|
+
* Capture density tier. Default: `'always'`. Set to `'feature'` for
|
|
28
|
+
* intermediate-state screenshots that should only fire while the
|
|
29
|
+
* spec is on a feature branch — they auto-suppress once the spec
|
|
30
|
+
* graduates into the regression pack. See the SKILL.md "Screenshot
|
|
31
|
+
* density per spec role" section for the density policy.
|
|
32
|
+
*/
|
|
33
|
+
readonly tier?: EvidenceShotTier;
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
/**
|
|
@@ -59,15 +69,15 @@ export async function evidenceShot(
|
|
|
59
69
|
opts: EvidenceShotOptions = {},
|
|
60
70
|
): Promise<void> {
|
|
61
71
|
validateEvidenceShotInputs(reqId, ac, slug);
|
|
72
|
+
const tier: EvidenceShotTier = opts.tier ?? 'always';
|
|
73
|
+
const specFile = resolveSpecFile();
|
|
74
|
+
const origin = opts.origin ?? autoDetectEvidenceShotOrigin(specFile, process.env.E2E_NEW_SPECS);
|
|
75
|
+
if (shouldSuppressEvidenceShot(tier, origin)) return;
|
|
62
76
|
const fileName = composeScreenshotFilename(reqId, ac, slug);
|
|
63
77
|
const dir = path.join(process.cwd(), 'compliance/evidence', reqId, 'screenshots');
|
|
64
78
|
const pngPath = path.join(dir, fileName);
|
|
65
79
|
const sidecarPath = `${pngPath}.meta.json`;
|
|
66
|
-
|
|
67
80
|
await page.screenshot({ path: pngPath, fullPage: opts.fullPage ?? true });
|
|
68
|
-
|
|
69
|
-
const specFile = resolveSpecFile();
|
|
70
|
-
const origin = opts.origin ?? autoDetectEvidenceShotOrigin(specFile, process.env.E2E_NEW_SPECS);
|
|
71
81
|
const sidecar: EvidenceShotSidecar = {
|
|
72
82
|
origin,
|
|
73
83
|
reqId,
|