@metasession.co/devaudit-cli 0.1.64 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/sdlc/files/_common/scripts/validate-compliance-artifacts.sh +15 -3
- package/sdlc/files/_common/scripts/validate-compliance-artifacts.test.sh +30 -0
- package/sdlc/files/_common/skills/e2e-test-engineer/SKILL.md +30 -2
- package/sdlc/files/_common/skills/e2e-test-engineer/references/common-patterns.md +27 -0
- package/sdlc/files/_common/skills/e2e-test-engineer/references/test-tags.ts +55 -0
- package/sdlc/files/_common/skills/sdlc-implementer/SKILL.md +17 -0
- package/sdlc/files/ci/ci.yml.template +13 -1
- package/sdlc/files/ci/compliance-evidence.yml.template +15 -1
package/package.json
CHANGED
|
@@ -169,9 +169,21 @@ for REQ in $REQUIREMENTS; do
|
|
|
169
169
|
TICKET_PATTERN="compliance/pending-releases/RELEASE-TICKET-${REQ}*"
|
|
170
170
|
APPROVED_PATTERN="compliance/approved-releases/RELEASE-TICKET-${REQ}*"
|
|
171
171
|
SUPERSEDED_PATTERN="compliance/superseded-releases/RELEASE-TICKET-${REQ}*"
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
172
|
+
|
|
173
|
+
# devaudit-installer#193 — duplicate-ticket guard. A ticket must live
|
|
174
|
+
# in exactly ONE release directory. A pending copy left behind after
|
|
175
|
+
# close-out (e.g. carried back by a stale-branch merge) poisons the
|
|
176
|
+
# evidence-completeness gate for unrelated REQs (#192). Fail fast with
|
|
177
|
+
# an actionable message instead of letting the duplicate surface as an
|
|
178
|
+
# opaque gate failure downstream.
|
|
179
|
+
LOCATIONS=0
|
|
180
|
+
compgen -G "$TICKET_PATTERN" > /dev/null 2>&1 && LOCATIONS=$((LOCATIONS+1))
|
|
181
|
+
compgen -G "$APPROVED_PATTERN" > /dev/null 2>&1 && LOCATIONS=$((LOCATIONS+1))
|
|
182
|
+
compgen -G "$SUPERSEDED_PATTERN" > /dev/null 2>&1 && LOCATIONS=$((LOCATIONS+1))
|
|
183
|
+
if [ "$LOCATIONS" -gt 1 ]; then
|
|
184
|
+
echo " ERROR: RELEASE-TICKET-${REQ} exists in more than one release directory (pending/approved/superseded). Remove the stale pending copy — it will break the evidence-completeness gate (#192)."
|
|
185
|
+
EXIT_CODE=1
|
|
186
|
+
elif [ "$LOCATIONS" -eq 1 ]; then
|
|
175
187
|
echo " OK: Release ticket exists"
|
|
176
188
|
else
|
|
177
189
|
echo " ERROR: Release ticket missing: compliance/pending-releases/RELEASE-TICKET-${REQ}.md"
|
|
@@ -156,6 +156,36 @@ assert_grep "no missing-ticket ERROR" 'ERROR: Release ticket missing' 0
|
|
|
156
156
|
assert_exit "validator exits 0 with SUPERSEDED ticket location" 0
|
|
157
157
|
cd "$WORKDIR"
|
|
158
158
|
|
|
159
|
+
# --- case 7: duplicate ticket in pending + approved → ERROR + exit 1 ---
|
|
160
|
+
#
|
|
161
|
+
# Regression for devaudit-installer#193: a stale pending copy left behind
|
|
162
|
+
# after close-out (carried back by a stale-branch merge) must be caught
|
|
163
|
+
# here with an actionable message, not silently pass and poison the
|
|
164
|
+
# evidence-completeness gate downstream (#192).
|
|
165
|
+
|
|
166
|
+
echo "Case 7: duplicate ticket in pending + approved directories fails"
|
|
167
|
+
make_fixture "$WORKDIR/case7" "Ref: REQ-077"
|
|
168
|
+
{
|
|
169
|
+
echo '# RTM'
|
|
170
|
+
echo
|
|
171
|
+
echo '| ID | Description | Status |'
|
|
172
|
+
echo '| --- | --- | --- |'
|
|
173
|
+
echo '| REQ-077 | Duplicate ticket test | TESTED - PENDING SIGN-OFF |'
|
|
174
|
+
} > compliance/RTM.md
|
|
175
|
+
mkdir -p compliance/evidence/REQ-077
|
|
176
|
+
echo "scope" > compliance/evidence/REQ-077/test-scope.md
|
|
177
|
+
echo "plan" > compliance/evidence/REQ-077/test-plan.md
|
|
178
|
+
echo "summary" > compliance/evidence/REQ-077/test-execution-summary.md
|
|
179
|
+
# Ticket in BOTH pending and approved — the duplicate
|
|
180
|
+
touch compliance/pending-releases/RELEASE-TICKET-REQ-077.md
|
|
181
|
+
touch compliance/approved-releases/RELEASE-TICKET-REQ-077.md
|
|
182
|
+
git add . && git commit -q --amend --no-edit
|
|
183
|
+
run_validator
|
|
184
|
+
assert_grep "duplicate-ticket ERROR emitted" 'ERROR: RELEASE-TICKET-REQ-077 exists in more than one release directory' 1
|
|
185
|
+
assert_grep "no OK for duplicate ticket" 'OK: Release ticket exists' 0
|
|
186
|
+
assert_exit "validator exits 1 on duplicate ticket" 1
|
|
187
|
+
cd "$WORKDIR"
|
|
188
|
+
|
|
159
189
|
# --- case 5: bare-filename reference resolves to file at depth ≥2 ---
|
|
160
190
|
#
|
|
161
191
|
# Regression for the broken `compgen -G "**/$TF"` search: bash globstar
|
|
@@ -183,6 +183,27 @@ Write the tests in the project's existing style.
|
|
|
183
183
|
- **Read 2–3 nearby tests before writing.** Fastest way to absorb conventions you wouldn't have noticed otherwise.
|
|
184
184
|
- **Check `references/common-patterns.md` before writing role-based locators** for component-library UI (shadcn/ui, Radix, MUI, etc.). A short appendix of known framework × library gotchas — `CardTitle` is a `<div>` not a heading; Radix `<Select>` renders two `role="combobox"` nodes; Next.js `<Link>` clicks don't fire network requests — saves a round-trip through a failing selector each time.
|
|
185
185
|
|
|
186
|
+
#### Tag every test with its REQ and AC (devaudit-installer#196)
|
|
187
|
+
|
|
188
|
+
Every test spec covering an in-scope REQ **must** call `tagTest()` at the top of the test body, before any assertions or interactions:
|
|
189
|
+
|
|
190
|
+
```ts
|
|
191
|
+
import { tagTest } from './helpers/test-tags';
|
|
192
|
+
|
|
193
|
+
test('verification code submit', async ({ page }) => {
|
|
194
|
+
tagTest('REQ-083', 1); // REQ-083, acceptance criterion 1
|
|
195
|
+
// ... test body
|
|
196
|
+
});
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
For tests covering multiple ACs, pass an array: `tagTest('REQ-083', [1, 2])`.
|
|
200
|
+
|
|
201
|
+
For transport-layer specs (no `page` object), `tagTest` works identically — it only touches `test.info()`, no Playwright `page` required.
|
|
202
|
+
|
|
203
|
+
**Why this is mandatory:** The DevAudit portal's per-REQ approval card (`ReqApprovalCard`) joins test results with screenshots by acceptance criteria using `test.info().annotations`. Without `tagTest()`, every REQ shows "no tests in report tagged with this REQ" even though the test ran and passed — defeating the AC-by-AC breakdown that reviewers rely on (#196).
|
|
204
|
+
|
|
205
|
+
The helper is synced to `e2e/helpers/test-tags.ts` by `devaudit update` alongside `evidence.ts`. Do not inline annotation logic — use the helper so the annotation format stays consistent with what the portal parser expects.
|
|
206
|
+
|
|
186
207
|
For **visual regression** specifically:
|
|
187
208
|
- New tests need baseline images. Generate them, but **do not auto-approve** — surface them for the user to verify before they're committed.
|
|
188
209
|
- Use the project's existing breakpoints, viewports, and element-masking conventions.
|
|
@@ -199,7 +220,14 @@ Before running the suite (Phase 6), verify that the evidence traceability wiring
|
|
|
199
220
|
|
|
200
221
|
2. **Check `evidenceShot()` calls.** For each UI spec covering an in-scope REQ, grep for at least one `evidenceShot(page, 'REQ-XXX', <ac>, ...)` call per AC. The call must be placed **at the assertion that proves the AC**, before any further interaction or navigation. API-only specs that don't have a visual surface are exempt — note the exemption in the test-execution-summary.
|
|
201
222
|
|
|
202
|
-
3. **Check
|
|
223
|
+
3. **Check `tagTest()` calls (devaudit-installer#196).** Grep the authored/modified spec files for `tagTest('REQ-XXX', …)` imports and calls. Every test covering an in-scope REQ must call `tagTest()` at the top of the test body so the Playwright JSON reporter emits the REQ/AC association in `test.info().annotations`. The portal's `ReqApprovalCard` uses this to map test results to requirements — without it, the per-REQ approval card shows "no tests in report tagged with this REQ" even though the test ran and passed.
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
# Quick check — should find at least one tagTest call per in-scope REQ
|
|
227
|
+
grep -rn "tagTest('REQ-" e2e/ --include='*.spec.ts'
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Also verify the import is present: `import { tagTest } from './helpers/test-tags'` (or the correct relative path from the spec's location to `e2e/helpers/test-tags.ts`).
|
|
203
231
|
|
|
204
232
|
**If any check fails:**
|
|
205
233
|
|
|
@@ -207,7 +235,7 @@ Halt and report the gap to the user:
|
|
|
207
235
|
> Evidence wiring incomplete for REQ-XXX:
|
|
208
236
|
> - Missing `@requirement REQ-XXX` annotation in `e2e/<area>/foo.spec.ts`
|
|
209
237
|
> - Missing `evidenceShot()` call for AC2 in `e2e/<area>/bar.spec.ts`
|
|
210
|
-
> - Missing `
|
|
238
|
+
> - Missing `tagTest('REQ-XXX', …)` call in `e2e/<area>/baz.spec.ts`
|
|
211
239
|
>
|
|
212
240
|
> These must be fixed before running the suite — without them the portal will show zero screenshots and zero tagged tests for this REQ, and the CI gate (#169) will block the release.
|
|
213
241
|
|
|
@@ -96,3 +96,30 @@ await page.locator('[data-slot="card-title"]').filter({ hasText: 'Filter Tabs' }
|
|
|
96
96
|
The same pattern bites any title slot that mixes a text label with a
|
|
97
97
|
count-badge, status pill, or icon-with-tooltip sibling. Default to `exact: false`
|
|
98
98
|
for component-library titles, and use `getByTestId` when ambiguity is real.
|
|
99
|
+
|
|
100
|
+
## Missing `tagTest()` — portal shows "no tests tagged with this REQ"
|
|
101
|
+
|
|
102
|
+
**Symptom.** The DevAudit portal's per-REQ approval card shows "no tests
|
|
103
|
+
in report tagged with this REQ" even though the e2e test ran and passed
|
|
104
|
+
and `e2e-results.json` was uploaded successfully.
|
|
105
|
+
|
|
106
|
+
**Cause.** The test spec never called `tagTest()` to push REQ/AC annotations
|
|
107
|
+
onto `test.info().annotations`. The portal's `extractReqAndAc()` parser
|
|
108
|
+
checks three sources (annotations → bracket tags in title → bare regex)
|
|
109
|
+
and all three miss, so `reqId` is null for every test and
|
|
110
|
+
`composeReqApproval` filters them all out.
|
|
111
|
+
|
|
112
|
+
**Fix.** Import and call `tagTest()` at the top of every test body:
|
|
113
|
+
|
|
114
|
+
```ts
|
|
115
|
+
import { tagTest } from './helpers/test-tags';
|
|
116
|
+
|
|
117
|
+
test('verification code submit', async ({ page }) => {
|
|
118
|
+
tagTest('REQ-083', 1);
|
|
119
|
+
// ... test body
|
|
120
|
+
});
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
The helper is synced to `e2e/helpers/test-tags.ts` by `devaudit update`.
|
|
124
|
+
Do not inline the annotation logic — the helper ensures the format
|
|
125
|
+
matches what the portal parser expects (`{ type: 'req', description: 'REQ-XXX AC<n>' }`).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-REQ / per-AC test annotation helper (devaudit-installer#196).
|
|
3
|
+
*
|
|
4
|
+
* The DevAudit portal's Playwright parser (`playwright-parser.ts`) looks
|
|
5
|
+
* for REQ-XXX / AC<n> tokens in three places, in priority order:
|
|
6
|
+
*
|
|
7
|
+
* 1. `test.info().annotations` — `{ type: 'req', description: 'REQ-083 AC1' }`
|
|
8
|
+
* 2. Bracket tags in test title / suite path — `[REQ-083][AC2] …`
|
|
9
|
+
* 3. Bare regex match — `REQ-XXX` anywhere in title + suite
|
|
10
|
+
*
|
|
11
|
+
* Without annotations the per-REQ approval card shows "no tests in report
|
|
12
|
+
* tagged with this REQ" even though the test ran and passed. This helper
|
|
13
|
+
* makes source 1 trivial — one call at the top of the test body.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* import { tagTest } from './helpers/test-tags';
|
|
17
|
+
*
|
|
18
|
+
* test('verification code submit', async ({ page }) => {
|
|
19
|
+
* tagTest('REQ-083', 1);
|
|
20
|
+
* // ... test body
|
|
21
|
+
* });
|
|
22
|
+
*
|
|
23
|
+
* For tests covering multiple ACs, pass an array:
|
|
24
|
+
*
|
|
25
|
+
* tagTest('REQ-083', [1, 2]);
|
|
26
|
+
*
|
|
27
|
+
* For transport-layer specs (no page object) the helper works identically
|
|
28
|
+
* — it only touches `test.info()`, no Playwright `page` required.
|
|
29
|
+
*/
|
|
30
|
+
import { test } from '@playwright/test';
|
|
31
|
+
|
|
32
|
+
const REQ_ID_RE = /^REQ-[A-Z0-9-]+$/;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Tag the current test with its REQ ID and AC number(s) so the portal
|
|
36
|
+
* can join test results with screenshots by acceptance criteria.
|
|
37
|
+
*
|
|
38
|
+
* Must be called inside a `test(...)` body (uses `test.info()`).
|
|
39
|
+
*
|
|
40
|
+
* @param reqId `REQ-` prefixed requirement id (e.g. `REQ-083`)
|
|
41
|
+
* @param ac AC number (positive integer) or array of AC numbers
|
|
42
|
+
*/
|
|
43
|
+
export function tagTest(reqId: string, ac: number | readonly number[]): void {
|
|
44
|
+
if (!REQ_ID_RE.test(reqId)) {
|
|
45
|
+
throw new Error(`tagTest: invalid reqId "${reqId}" (must match ${REQ_ID_RE})`);
|
|
46
|
+
}
|
|
47
|
+
const acs = Array.isArray(ac) ? ac : [ac];
|
|
48
|
+
const info = test.info();
|
|
49
|
+
for (const n of acs) {
|
|
50
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
51
|
+
throw new Error(`tagTest: invalid ac "${n}" (must be a positive integer)`);
|
|
52
|
+
}
|
|
53
|
+
info.annotations.push({ type: 'req', description: `${reqId} AC${n}` });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -59,6 +59,17 @@ The only pauses in the whole workflow are the explicitly-named checkpoints:
|
|
|
59
59
|
|
|
60
60
|
Everything else is silent continuation. The rule is **opt-in-to-pause, not opt-out-of-pause**. If you find yourself stopping after a sub-skill's "Return to the running `sdlc-implementer` context" line and waiting for the operator to ask _"is anything happening?"_ — that is the bug this section exists to prevent. Keep going.
|
|
61
61
|
|
|
62
|
+
### False stopping points (devaudit-installer#194)
|
|
63
|
+
|
|
64
|
+
The following milestones **feel** like completion boundaries but are mid-flow — do NOT halt at any of them:
|
|
65
|
+
|
|
66
|
+
- **Sub-skill return** — covered above (#144).
|
|
67
|
+
- **Phase 2 integration PR merged to `develop`** — the merge is the trigger for CI to upload binary gate evidence, not a handoff to the operator. Once CI is green on `develop`, immediately begin Phase 3 (compile evidence). The next required operator handoff is Phase 4 step 5 (release PR opened, awaiting portal UAT approval), not the develop merge.
|
|
68
|
+
- **CI going green on `develop`** — this is the signal to proceed, not to stop and report.
|
|
69
|
+
- **Evidence upload completing** — the `devaudit push` calls in Phase 3 are intermediate steps; proceed to the next artefact or the next phase.
|
|
70
|
+
|
|
71
|
+
If you find yourself halting at any of these, state the literal line `Continuing — this is not a checkpoint.` and proceed to the next step in the same turn. The only thing that may interrupt the auto-continue is the scope-expansion halt gate (#171) firing on a new user request.
|
|
72
|
+
|
|
62
73
|
## Scope-expansion halt gate (devaudit-installer#171)
|
|
63
74
|
|
|
64
75
|
The change-request loop's scope-expansion halt rule (see [`references/change-request-loop.md`](./references/change-request-loop.md) §"If the change-request is fundamentally a different REQ") only fires during Phase 5 — after a UAT reviewer clicks "Request Changes" and the portal state is `uat_changes_requested`. In practice, scope-expanding requests also arrive from the user at any point after Phase 1:
|
|
@@ -342,6 +353,12 @@ Reached only on the **tracked** route from Phase 0 (the issue is already fetched
|
|
|
342
353
|
|
|
343
354
|
10. **Update SDLC status sticky** before exiting Phase 2: `bash scripts/update-sdlc-status.sh "$ISSUE_NUM" "Phase 2 complete — feat branch landed on $INTEGRATION_BRANCH; all gates green" "Phase 3 — sdlc-implementer auto-continuing (evidence compile)"`.
|
|
344
355
|
|
|
356
|
+
11. **Auto-continue to Phase 3 — mandatory, NOT a checkpoint (devaudit-installer#194).** After the integration PR merges and `develop` CI is green, do NOT return control to the operator. State the literal line:
|
|
357
|
+
|
|
358
|
+
> `Phase 2 complete — auto-continuing to Phase 3 (compile evidence).`
|
|
359
|
+
|
|
360
|
+
Then proceed to Phase 3 step 1 in the same turn. The integration-PR merge *feels* terminal — that is the inertia trap. The next required operator handoff is Phase 4 step 5 (release PR opened, awaiting portal UAT approval), not the develop merge. The only thing that interrupts this auto-continue is the scope-expansion halt gate (#171) firing on a new user request.
|
|
361
|
+
|
|
345
362
|
### Phase 3 — Compile evidence (SDLC stage 3)
|
|
346
363
|
|
|
347
364
|
1. **Invoke `requirements-aligner` to drop the per-REQ SRS-alignment artefact.** The skill's Phase 2 produces `compliance/evidence/REQ-XXX/srs-alignment.md` — the per-REQ trace from each AC to its SRS item, with an operator sign-off block. The artefact uploads with `evidence_type=srs_alignment` (visible in Documents tab + audit-pack export; v1 orphan-by-design per META-COMPLY framework-registry-auditor). Call via the standard Skill mechanism; don't inline the alignment logic.
|
|
@@ -554,7 +554,19 @@ jobs:
|
|
|
554
554
|
if [ -d compliance/pending-releases ]; then
|
|
555
555
|
for TICKET in compliance/pending-releases/RELEASE-TICKET-REQ-*.md; do
|
|
556
556
|
[ -f "$TICKET" ] || continue
|
|
557
|
-
|
|
557
|
+
REQ_ID=$(basename "$TICKET" .md | sed 's/^RELEASE-TICKET-//')
|
|
558
|
+
# devaudit-installer#192 — skip REQs already in a terminal
|
|
559
|
+
# release directory (approved/superseded). A stale pending
|
|
560
|
+
# duplicate (e.g. carried back by a stale-branch merge)
|
|
561
|
+
# retains its pre-close-out Status, so a Status-field check
|
|
562
|
+
# is insufficient — the approved/superseded ticket is the
|
|
563
|
+
# authoritative "this REQ is done" signal.
|
|
564
|
+
if compgen -G "compliance/approved-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1 \
|
|
565
|
+
|| compgen -G "compliance/superseded-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1; then
|
|
566
|
+
echo "Skipping ${REQ_ID}: already in a terminal release directory (stale pending ticket)"
|
|
567
|
+
continue
|
|
568
|
+
fi
|
|
569
|
+
SHOT_REQS+=("$REQ_ID")
|
|
558
570
|
done
|
|
559
571
|
fi
|
|
560
572
|
shopt -s nullglob
|
|
@@ -410,6 +410,13 @@ jobs:
|
|
|
410
410
|
for TICKET in compliance/pending-releases/RELEASE-TICKET-REQ-*.md; do
|
|
411
411
|
[ -f "$TICKET" ] || continue
|
|
412
412
|
REQ_ID=$(basename "$TICKET" .md | sed 's/^RELEASE-TICKET-//')
|
|
413
|
+
# devaudit-installer#192 — skip REQs already in a terminal
|
|
414
|
+
# release directory. See ci.yml.template for full rationale.
|
|
415
|
+
if compgen -G "compliance/approved-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1 \
|
|
416
|
+
|| compgen -G "compliance/superseded-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1; then
|
|
417
|
+
echo "Skipping ${REQ_ID}: already in a terminal release directory (stale pending ticket)"
|
|
418
|
+
continue
|
|
419
|
+
fi
|
|
413
420
|
IN_SCOPE_REQS+=("$REQ_ID")
|
|
414
421
|
done
|
|
415
422
|
fi
|
|
@@ -606,7 +613,14 @@ jobs:
|
|
|
606
613
|
if [ -d compliance/pending-releases ]; then
|
|
607
614
|
for TICKET in compliance/pending-releases/RELEASE-TICKET-REQ-*.md; do
|
|
608
615
|
[ -f "$TICKET" ] || continue
|
|
609
|
-
|
|
616
|
+
REQ_ID=$(basename "$TICKET" .md | sed 's/^RELEASE-TICKET-//')
|
|
617
|
+
# devaudit-installer#192 — skip REQs already in a terminal
|
|
618
|
+
# release directory. See ci.yml.template for full rationale.
|
|
619
|
+
if compgen -G "compliance/approved-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1 \
|
|
620
|
+
|| compgen -G "compliance/superseded-releases/RELEASE-TICKET-${REQ_ID}.md" > /dev/null 2>&1; then
|
|
621
|
+
continue
|
|
622
|
+
fi
|
|
623
|
+
REQS+=("$REQ_ID")
|
|
610
624
|
done
|
|
611
625
|
fi
|
|
612
626
|
if [ "${#REQS[@]}" -eq 0 ]; then
|