@metasession.co/devaudit-cli 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +43 -4
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/sdlc/ai-rules/SDLC_RULES.md +27 -7
- package/sdlc/files/_common/0-project-setup.md +4 -4
- package/sdlc/files/_common/scripts/extract-release-metadata.sh +38 -0
- package/sdlc/files/_common/scripts/extract-release-metadata.test.sh +26 -0
- package/sdlc/files/_common/scripts/generate-bundled-changes.sh +11 -0
- package/sdlc/files/_common/scripts/generate-bundled-changes.test.sh +4 -0
- package/sdlc/files/_common/scripts/upload-evidence.sh +31 -1
- package/sdlc/files/_common/scripts/upload-evidence.test.sh +119 -0
- package/sdlc/files/_common/scripts/validate-compliance-artifacts.sh +153 -2
- package/sdlc/files/_common/scripts/validate-compliance-artifacts.test.sh +155 -0
- package/sdlc/files/_common/skills/sdlc-implementer/SKILL.md +36 -1
- package/sdlc/files/ci/check-release-approval.yml.template +18 -28
- package/sdlc/files/ci/ci.yml.template +13 -7
- package/sdlc/files/ci/compliance-evidence.yml.template +30 -29
- package/sdlc/package.json +1 -1
- package/sdlc/src/blueprints/3-compile-evidence.raw.md +68 -0
- package/sdlc/src/blueprints/4-submit-for-review.raw.md +1 -1
|
@@ -126,6 +126,7 @@ make_fixture "$WORKDIR/case3" "Ref: REQ-030"
|
|
|
126
126
|
mkdir -p compliance/evidence/REQ-030
|
|
127
127
|
echo "scope" > compliance/evidence/REQ-030/test-scope.md
|
|
128
128
|
echo "plan" > compliance/evidence/REQ-030/test-plan.md
|
|
129
|
+
echo "summary" > compliance/evidence/REQ-030/test-execution-summary.md
|
|
129
130
|
touch compliance/pending-releases/RELEASE-TICKET-REQ-030.md
|
|
130
131
|
git add . && git commit -q --amend --no-edit
|
|
131
132
|
run_validator
|
|
@@ -147,6 +148,7 @@ make_fixture "$WORKDIR/case4" "Ref: REQ-031"
|
|
|
147
148
|
mkdir -p compliance/evidence/REQ-031
|
|
148
149
|
echo "scope" > compliance/evidence/REQ-031/test-scope.md
|
|
149
150
|
echo "plan" > compliance/evidence/REQ-031/test-plan.md
|
|
151
|
+
echo "summary" > compliance/evidence/REQ-031/test-execution-summary.md
|
|
150
152
|
# Only the superseded location — neither pending- nor approved-releases.
|
|
151
153
|
touch compliance/superseded-releases/RELEASE-TICKET-REQ-031.md
|
|
152
154
|
git add . && git commit -q --amend --no-edit
|
|
@@ -252,8 +254,161 @@ assert_grep "no evidence-dir ERROR for prose-only REQ-002" 'ERROR: Evidence dire
|
|
|
252
254
|
assert_exit "validator exits 0 when future REQ is only prose-mentioned" 0
|
|
253
255
|
cd "$WORKDIR"
|
|
254
256
|
|
|
257
|
+
# --- case 8: tracked REQ without test-execution-summary fails ---
|
|
258
|
+
#
|
|
259
|
+
# Regression for devaudit-installer#341: tracked releases must not pass
|
|
260
|
+
# validation without the per-release test report that satisfies the
|
|
261
|
+
# portal's Test Reports gate.
|
|
262
|
+
|
|
263
|
+
echo "Case 8: tracked REQ missing test-execution-summary fails"
|
|
264
|
+
make_fixture "$WORKDIR/case8" "Ref: REQ-341"
|
|
265
|
+
{
|
|
266
|
+
echo '# RTM'
|
|
267
|
+
echo
|
|
268
|
+
echo '| ID | Description | Status |'
|
|
269
|
+
echo '| --- | --- | --- |'
|
|
270
|
+
echo '| REQ-341 | Missing test summary | TESTED - PENDING SIGN-OFF |'
|
|
271
|
+
} > compliance/RTM.md
|
|
272
|
+
mkdir -p compliance/evidence/REQ-341
|
|
273
|
+
echo "scope" > compliance/evidence/REQ-341/test-scope.md
|
|
274
|
+
echo "plan" > compliance/evidence/REQ-341/test-plan.md
|
|
275
|
+
touch compliance/pending-releases/RELEASE-TICKET-REQ-341.md
|
|
276
|
+
git add . && git commit -q --amend --no-edit
|
|
277
|
+
run_validator
|
|
278
|
+
assert_grep "missing-summary ERROR emitted" 'ERROR: Test execution summary missing: compliance/evidence/REQ-341/test-execution-summary.md' 1
|
|
279
|
+
assert_exit "validator exits 1 when tracked REQ lacks test-execution-summary" 1
|
|
280
|
+
cd "$WORKDIR"
|
|
281
|
+
|
|
255
282
|
# --- summary ---
|
|
256
283
|
|
|
284
|
+
# --- case 9: bundled release context missing from canonical artefacts fails ---
|
|
285
|
+
echo "Case 9: bundled release artefacts without context headings fail"
|
|
286
|
+
make_fixture "$WORKDIR/case9" "Ref: REQ-344"
|
|
287
|
+
{
|
|
288
|
+
echo '# RTM'
|
|
289
|
+
echo
|
|
290
|
+
echo '| ID | Description | Status |'
|
|
291
|
+
echo '| --- | --- | --- |'
|
|
292
|
+
echo '| REQ-344 | Bundled release context missing | TESTED - PENDING SIGN-OFF |'
|
|
293
|
+
} > compliance/RTM.md
|
|
294
|
+
mkdir -p compliance/evidence/REQ-344
|
|
295
|
+
echo "scope" > compliance/evidence/REQ-344/test-scope.md
|
|
296
|
+
echo "plan" > compliance/evidence/REQ-344/test-plan.md
|
|
297
|
+
echo "summary" > compliance/evidence/REQ-344/test-execution-summary.md
|
|
298
|
+
echo "security" > compliance/evidence/REQ-344/security-summary.md
|
|
299
|
+
echo "ai note" > compliance/evidence/REQ-344/ai-use-note.md
|
|
300
|
+
cat > compliance/pending-releases/RELEASE-TICKET-REQ-344.md <<'EOF'
|
|
301
|
+
# Release Ticket — REQ-344
|
|
302
|
+
|
|
303
|
+
## Summary
|
|
304
|
+
Tracked release without bundle section.
|
|
305
|
+
EOF
|
|
306
|
+
echo "## Bundled Changes" > compliance/pending-releases/BUNDLED-CHANGES-REQ-344.md
|
|
307
|
+
git add . && git commit -q --amend --no-edit
|
|
308
|
+
run_validator
|
|
309
|
+
assert_grep "missing structured fields on bundled evidence are reported" "ERROR: Bundled release evidence is missing bundled field '\\*\\*Core tracked release:\\*\\*'" 1
|
|
310
|
+
assert_grep "missing structured fields on ticket are reported" "ERROR: Release ticket is missing bundled field '\\*\\*Core tracked release:\\*\\*'" 1
|
|
311
|
+
assert_grep "missing bundled section on ticket is reported" "ERROR: Bundled release evidence exists but the release ticket is missing" 1
|
|
312
|
+
assert_grep "missing bundled section on summary is reported" "ERROR: Bundled release evidence exists but test-execution-summary.md is missing" 1
|
|
313
|
+
assert_grep "missing bundled section on security summary is reported" "ERROR: Bundled release evidence exists but security-summary.md is missing" 1
|
|
314
|
+
assert_grep "missing bundled section on ai note is reported" "ERROR: Bundled release evidence exists but ai-use-note.md is missing" 1
|
|
315
|
+
assert_exit "validator exits 1 when bundled artefacts lack required headings" 1
|
|
316
|
+
cd "$WORKDIR"
|
|
317
|
+
|
|
318
|
+
# --- case 10: bundled release context present in canonical artefacts passes ---
|
|
319
|
+
echo "Case 10: bundled release artefacts with context headings pass"
|
|
320
|
+
make_fixture "$WORKDIR/case10" "Ref: REQ-345"
|
|
321
|
+
{
|
|
322
|
+
echo '# RTM'
|
|
323
|
+
echo
|
|
324
|
+
echo '| ID | Description | Status |'
|
|
325
|
+
echo '| --- | --- | --- |'
|
|
326
|
+
echo '| REQ-345 | Bundled release context present | TESTED - PENDING SIGN-OFF |'
|
|
327
|
+
} > compliance/RTM.md
|
|
328
|
+
mkdir -p compliance/evidence/REQ-345
|
|
329
|
+
echo "scope" > compliance/evidence/REQ-345/test-scope.md
|
|
330
|
+
echo "plan" > compliance/evidence/REQ-345/test-plan.md
|
|
331
|
+
cat > compliance/evidence/REQ-345/test-execution-summary.md <<'EOF'
|
|
332
|
+
# Test Execution Summary — REQ-345
|
|
333
|
+
|
|
334
|
+
## Bundled Release Context
|
|
335
|
+
- **Core tracked release:** REQ-345
|
|
336
|
+
- **Absorbed predecessor releases:** None
|
|
337
|
+
- **Absorbed non-release work:** housekeeping syncs only
|
|
338
|
+
- **Why bundled here:** housekeeping consolidation
|
|
339
|
+
- **Evidence impact:** core REQ proof unchanged; gate evidence covers full develop state
|
|
340
|
+
- **Reviewer impact:** reviewer is approving the tracked REQ plus absorbed housekeeping context
|
|
341
|
+
- **Security / risk impact:** None beyond core REQ
|
|
342
|
+
- **Reference:** compliance/pending-releases/BUNDLED-CHANGES-REQ-345.md
|
|
343
|
+
EOF
|
|
344
|
+
cat > compliance/evidence/REQ-345/security-summary.md <<'EOF'
|
|
345
|
+
# Security Summary — REQ-345
|
|
346
|
+
|
|
347
|
+
## Bundled Release Context
|
|
348
|
+
- **Core tracked release:** REQ-345
|
|
349
|
+
- **Absorbed predecessor releases:** None
|
|
350
|
+
- **Absorbed non-release work:** housekeeping syncs only
|
|
351
|
+
- **Why bundled here:** housekeeping consolidation
|
|
352
|
+
- **Evidence impact:** no extra security evidence beyond the shared gate outputs
|
|
353
|
+
- **Reviewer impact:** reviewer should treat the shared gate results as covering the absorbed housekeeping work too
|
|
354
|
+
- **Security / risk impact:** None beyond core REQ
|
|
355
|
+
- **Reference:** compliance/pending-releases/BUNDLED-CHANGES-REQ-345.md
|
|
356
|
+
EOF
|
|
357
|
+
cat > compliance/evidence/REQ-345/ai-use-note.md <<'EOF'
|
|
358
|
+
# AI Use Record — REQ-345
|
|
359
|
+
|
|
360
|
+
## Bundled Release Context
|
|
361
|
+
- **Core tracked release:** REQ-345
|
|
362
|
+
- **Absorbed predecessor releases:** None
|
|
363
|
+
- **Absorbed non-release work:** housekeeping syncs only
|
|
364
|
+
- **Why bundled here:** housekeeping consolidation
|
|
365
|
+
- **Evidence impact:** no AI-generated evidence split beyond the shared bundle context
|
|
366
|
+
- **Reviewer impact:** reviewer should read the bundle note as part of the approval scope
|
|
367
|
+
- **Security / risk impact:** None beyond core REQ
|
|
368
|
+
- **Reference:** compliance/pending-releases/BUNDLED-CHANGES-REQ-345.md
|
|
369
|
+
EOF
|
|
370
|
+
cat > compliance/pending-releases/RELEASE-TICKET-REQ-345.md <<'EOF'
|
|
371
|
+
# Release Ticket — REQ-345
|
|
372
|
+
|
|
373
|
+
## Summary
|
|
374
|
+
Tracked release with bundle section.
|
|
375
|
+
|
|
376
|
+
## Bundled Changes
|
|
377
|
+
- **Core tracked release:** REQ-345
|
|
378
|
+
- **Absorbed predecessor releases:** None
|
|
379
|
+
- **Absorbed non-release work:** housekeeping syncs only
|
|
380
|
+
- **Why bundled here:** housekeeping consolidation
|
|
381
|
+
- **Evidence impact:** gate evidence and release reports include the absorbed housekeeping changes
|
|
382
|
+
- **Reviewer impact:** approval scope is the tracked REQ plus the absorbed housekeeping context
|
|
383
|
+
- **Security / risk impact:** None beyond core REQ
|
|
384
|
+
- **Reference:** compliance/pending-releases/BUNDLED-CHANGES-REQ-345.md
|
|
385
|
+
EOF
|
|
386
|
+
cat > compliance/pending-releases/BUNDLED-CHANGES-REQ-345.md <<'EOF'
|
|
387
|
+
## Bundled Changes
|
|
388
|
+
|
|
389
|
+
- **Core tracked release:** `REQ-345`
|
|
390
|
+
- **Absorbed predecessor releases:** None
|
|
391
|
+
- **Absorbed non-release work:** housekeeping syncs only
|
|
392
|
+
- **Why bundled here:** housekeeping consolidation
|
|
393
|
+
- **Evidence impact:** gate evidence and release reports include the absorbed housekeeping changes
|
|
394
|
+
- **Reviewer impact:** approval scope is the tracked REQ plus the absorbed housekeeping context
|
|
395
|
+
- **Security / risk impact:** None beyond core REQ
|
|
396
|
+
- **Reference:** commit range `abc123..HEAD`
|
|
397
|
+
|
|
398
|
+
### Absorbed Non-Release Work
|
|
399
|
+
|
|
400
|
+
- `abc123` chore: sync templates
|
|
401
|
+
EOF
|
|
402
|
+
git add . && git commit -q --amend --no-edit
|
|
403
|
+
run_validator
|
|
404
|
+
assert_grep "bundled ticket section accepted" "OK: Release ticket documents bundled release context" 1
|
|
405
|
+
assert_grep "bundled test summary section accepted" "OK: test-execution-summary.md documents bundled release context" 1
|
|
406
|
+
assert_grep "bundled security summary section accepted" "OK: security-summary.md documents bundled release context" 1
|
|
407
|
+
assert_grep "bundled ai note section accepted" "OK: ai-use-note.md documents bundled release context" 1
|
|
408
|
+
assert_grep "structured bundled evidence accepted" "OK: Bundled release evidence carries the required structured fields" 1
|
|
409
|
+
assert_exit "validator exits 0 when bundled artefacts carry required headings" 0
|
|
410
|
+
cd "$WORKDIR"
|
|
411
|
+
|
|
257
412
|
echo
|
|
258
413
|
echo "=== validate-compliance-artifacts.test.sh: $PASS passed, $FAIL failed ==="
|
|
259
414
|
[ "$FAIL" -eq 0 ]
|
|
@@ -511,6 +511,41 @@ Reached only on the **tracked** route from Phase 0 (the issue is already fetched
|
|
|
511
511
|
|
|
512
512
|
If the validator fails, fix the summary before proceeding. E2E gate results must be one of: `PASS`, `FAIL`, `NOT_NEEDED` (with reason), or `SKIPPED` (with operator-approved rationale). The word "deferred" must never appear in `test-execution-summary.md` — not as a gate state, not in prose, not in final assessment. "Deferred to CI" and "Playwright browsers not installed locally" are environment issues, not gate states. The CI validator (`validate-test-summary.sh`) will reject any summary containing "deferred" or "browsers not installed" on PRs to main.
|
|
513
513
|
|
|
514
|
+
5c. **Run the Phase 3 completion guard before any PR/release-review step (devaudit-installer#341).** The tracked-release evidence pack is incomplete until the required Git artefacts exist. Before proceeding past Phase 3, verify:
|
|
515
|
+
|
|
516
|
+
- `compliance/evidence/REQ-XXX/test-scope.md`
|
|
517
|
+
- `compliance/evidence/REQ-XXX/test-plan.md`
|
|
518
|
+
- `compliance/evidence/REQ-XXX/test-execution-summary.md`
|
|
519
|
+
- `compliance/evidence/REQ-XXX/security-summary.md`
|
|
520
|
+
- `compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md`
|
|
521
|
+
- `compliance/evidence/REQ-XXX/implementation-plan.md` when the REQ risk class requires it
|
|
522
|
+
- any already-mandatory AI artefacts required by the risk/usage rules
|
|
523
|
+
- if `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md` exists, the canonical artefacts must carry the bundle narrative too:
|
|
524
|
+
- release ticket includes `## Bundled Changes` or `## Absorbed Predecessor Releases`
|
|
525
|
+
- `test-execution-summary.md` includes `## Bundled Release Context`
|
|
526
|
+
- `security-summary.md` includes `## Bundled Release Context`
|
|
527
|
+
- `ai-use-note.md` includes `## Bundled Release Context` when AI use is documented
|
|
528
|
+
|
|
529
|
+
Run:
|
|
530
|
+
|
|
531
|
+
```bash
|
|
532
|
+
MISSING=0
|
|
533
|
+
for f in \
|
|
534
|
+
compliance/evidence/REQ-XXX/test-scope.md \
|
|
535
|
+
compliance/evidence/REQ-XXX/test-plan.md \
|
|
536
|
+
compliance/evidence/REQ-XXX/test-execution-summary.md \
|
|
537
|
+
compliance/evidence/REQ-XXX/security-summary.md \
|
|
538
|
+
compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md
|
|
539
|
+
do
|
|
540
|
+
[ -f "$f" ] || { echo "MISSING: $f"; MISSING=1; }
|
|
541
|
+
done
|
|
542
|
+
[ "$MISSING" -eq 0 ] || exit 1
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
If any required artefact is missing, halt with an explicit error and do **not** continue to Phase 4:
|
|
546
|
+
|
|
547
|
+
> "Phase 3 incomplete — `test-execution-summary.md` is missing. This file satisfies the portal's Test Reports gate (`evidence_type=test_report`). Create it before opening the release PR."
|
|
548
|
+
|
|
514
549
|
6. **Organise artefacts** under `compliance/evidence/REQ-XXX/` with date-prefixed naming:
|
|
515
550
|
|
|
516
551
|
```
|
|
@@ -620,7 +655,7 @@ Reached only on the **tracked** route from Phase 0 (the issue is already fetched
|
|
|
620
655
|
|
|
621
656
|
**Release Approval Gate retry (devaudit-installer#211 Gap 17).** If the Release Approval Gate check fails on the PR and the portal approval was already given (API sync delay, stale cache), retry logic: re-run the Release Approval Gate workflow up to 3 times with 30-second intervals (`gh workflow run check-release-approval.yml` or trigger via `workflow_dispatch`). If it still fails after 3 retries: halt — "Release Approval Gate still failing after 3 retries. Portal may show approval but the gate can't verify it. Operator action — check the portal release status manually, verify the API key is valid, and re-run the workflow from GitHub Actions if needed."
|
|
622
657
|
|
|
623
|
-
**Auto-refresh on UAT approval (devaudit#562, devaudit-installer#283).** When the portal's UAT approval is granted, the portal automatically sends a `repository_dispatch('release-approved')` event to the consuming project's repo. This triggers `check-release-approval.yml` to re-run without manual intervention. The retry logic above is still needed for edge cases (API sync delays, network issues), but the common path is now fully automated. If the auto-refresh doesn't fire (e.g. portal GitHub trigger misconfigured), the manual retry sequence above remains the fallback.
|
|
658
|
+
**Auto-refresh on UAT approval (devaudit#562, devaudit-installer#283, #351).** When the portal's UAT approval is granted, the portal automatically sends a `repository_dispatch('release-approved')` event to the consuming project's repo. This triggers `check-release-approval.yml` to re-run without manual intervention and create a fresh passing `DevAudit Release Approval` check run on the approved PR head SHA. The retry logic above is still needed for edge cases (API sync delays, network issues), but the common path is now fully automated. If the auto-refresh doesn't fire (e.g. portal GitHub trigger misconfigured), the manual retry sequence above remains the fallback.
|
|
624
659
|
|
|
625
660
|
### Phase 5 — Finalise or change-request loop (SDLC stage 5)
|
|
626
661
|
|
|
@@ -255,7 +255,7 @@ jobs:
|
|
|
255
255
|
echo "::warning::Approved SHA (${APPROVED_SHA:0:8}) differs from PR HEAD (${PR_HEAD_SHA:0:8}). Code may have changed after approval."
|
|
256
256
|
fi
|
|
257
257
|
|
|
258
|
-
- name:
|
|
258
|
+
- name: Create refreshed PR check run (workflow_dispatch / repository_dispatch re-trigger)
|
|
259
259
|
if: env.BOOTSTRAP_MODE != 'true' && (github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch')
|
|
260
260
|
env:
|
|
261
261
|
GH_TOKEN: ${{ github.token }}
|
|
@@ -283,31 +283,21 @@ jobs:
|
|
|
283
283
|
CHECK_NAME="DevAudit Release Approval"
|
|
284
284
|
DETAILS_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
285
285
|
COMPLETED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
286
|
-
EXISTING_ID=$(gh api \
|
|
287
|
-
"/repos/${{ github.repository }}/commits/${TARGET_SHA}/check-runs?name=DevAudit%20Release%20Approval" \
|
|
288
|
-
--jq '.check_runs[] | select(.name == "DevAudit Release Approval") | .id' \
|
|
289
|
-
| head -1 || true)
|
|
290
286
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
-f completed_at="$COMPLETED_AT" \
|
|
309
|
-
-f details_url="$DETAILS_URL" \
|
|
310
|
-
-f "output[title]=DevAudit release approval confirmed" \
|
|
311
|
-
-f "output[summary]=Release ${{ steps.release.outputs.version }} is approved in DevAudit."
|
|
312
|
-
echo "Created ${CHECK_NAME} check run on ${TARGET_SHA}."
|
|
313
|
-
fi
|
|
287
|
+
# DevAudit-Installer#351 — GitHub's Feb 2025 check-run changes
|
|
288
|
+
# forbid externally mutating an existing workflow-owned check run
|
|
289
|
+
# from a workflow_dispatch / repository_dispatch flow. The refresh
|
|
290
|
+
# path therefore creates a fresh success check run on the approved
|
|
291
|
+
# SHA instead of PATCHing the original failed PR check.
|
|
292
|
+
gh api "/repos/${{ github.repository }}/check-runs" \
|
|
293
|
+
-X POST \
|
|
294
|
+
-f name="$CHECK_NAME" \
|
|
295
|
+
-f head_sha="$TARGET_SHA" \
|
|
296
|
+
-f status=completed \
|
|
297
|
+
-f conclusion=success \
|
|
298
|
+
-f completed_at="$COMPLETED_AT" \
|
|
299
|
+
-f details_url="$DETAILS_URL" \
|
|
300
|
+
-f external_id="${{ github.run_id }}" \
|
|
301
|
+
-f "output[title]=DevAudit release approval confirmed" \
|
|
302
|
+
-f "output[summary]=Release ${{ steps.release.outputs.version }} is approved in DevAudit."
|
|
303
|
+
echo "Created refreshed ${CHECK_NAME} check run on ${TARGET_SHA}."
|
|
@@ -793,9 +793,9 @@ jobs:
|
|
|
793
793
|
|
|
794
794
|
# devaudit-installer#237 — check for test-execution-summary.md
|
|
795
795
|
# as a secondary signal. This is a human-authored document and
|
|
796
|
-
# weaker than machine-generated evidence,
|
|
797
|
-
#
|
|
798
|
-
#
|
|
796
|
+
# weaker than machine-generated evidence, but it is still the
|
|
797
|
+
# designated evidence for test-maintenance REQs that update an
|
|
798
|
+
# existing tagged spec for a different requirement.
|
|
799
799
|
HAS_SUMMARY=false
|
|
800
800
|
if [ -f "compliance/evidence/${REQ}/test-execution-summary.md" ]; then
|
|
801
801
|
HAS_SUMMARY=true
|
|
@@ -804,10 +804,16 @@ jobs:
|
|
|
804
804
|
if [ "$SPECS_ON_DISK" -gt 0 ]; then
|
|
805
805
|
# Scenario B: E2E specs exist on disk but outside the smoke tier
|
|
806
806
|
echo "::warning::Evidence completeness gate: ${REQ} has zero tagged tests in this smoke run and zero per-AC screenshots, but ${SPECS_ON_DISK} spec file(s) on disk reference it (outside the smoke project). Evidence exists but was not executed in this CI tier. Consider running --project=critical or --grep \"${REQ}\" to capture evidence."
|
|
807
|
-
elif [ "$
|
|
808
|
-
# Scenario C:
|
|
809
|
-
#
|
|
810
|
-
|
|
807
|
+
elif [ "$HAS_SUMMARY" = "true" ]; then
|
|
808
|
+
# Scenario C: a test-execution-summary.md documents coverage.
|
|
809
|
+
# This covers both API/backend-only REQs with unit tests on disk
|
|
810
|
+
# and test-maintenance REQs whose evidence is an existing spec
|
|
811
|
+
# tagged to an earlier feature requirement.
|
|
812
|
+
if [ "$UNIT_TESTS_ON_DISK" -gt 0 ]; then
|
|
813
|
+
echo "::warning::Evidence completeness gate: ${REQ} has zero E2E tagged tests and zero screenshots, but ${UNIT_TESTS_ON_DISK} unit test file(s) on disk reference it and a test-execution-summary.md exists. Unit test evidence found but execution was not verified in this run. If this is an API-only or backend-only change, this is expected."
|
|
814
|
+
else
|
|
815
|
+
echo "::warning::Evidence completeness gate: ${REQ} has zero tagged tests and zero screenshots, but a test-execution-summary.md exists documenting coverage. Accepted as evidence for test-maintenance or execution-only REQs."
|
|
816
|
+
fi
|
|
811
817
|
else
|
|
812
818
|
# Scenario A: genuinely no evidence — no E2E specs, no unit tests,
|
|
813
819
|
# no test-execution-summary. Hard error preserved (#237 Constraint 3).
|
|
@@ -746,38 +746,39 @@ jobs:
|
|
|
746
746
|
fi
|
|
747
747
|
|
|
748
748
|
if [ "${#REQS[@]}" -eq 0 ] && [ -f e2e-artifacts/e2e-regression-results.json ]; then
|
|
749
|
+
{
|
|
750
|
+
printf '%s\n' 'import json'
|
|
751
|
+
printf '%s\n' 'import re'
|
|
752
|
+
printf '%s\n' 'import sys'
|
|
753
|
+
printf '%s\n' ''
|
|
754
|
+
printf '%s\n' 'try:'
|
|
755
|
+
printf '%s\n' " with open('e2e-artifacts/e2e-regression-results.json', 'r', encoding='utf-8') as handle:"
|
|
756
|
+
printf '%s\n' ' data = json.load(handle)'
|
|
757
|
+
printf '%s\n' 'except Exception:'
|
|
758
|
+
printf '%s\n' ' sys.exit(0)'
|
|
759
|
+
printf '%s\n' ''
|
|
760
|
+
printf '%s\n' 'seen = set()'
|
|
761
|
+
printf '%s\n' ''
|
|
762
|
+
printf '%s\n' 'def walk(value):'
|
|
763
|
+
printf '%s\n' ' if isinstance(value, str):'
|
|
764
|
+
printf '%s\n' " for req in re.findall(r'REQ-[0-9]+', value):"
|
|
765
|
+
printf '%s\n' ' seen.add(req)'
|
|
766
|
+
printf '%s\n' ' elif isinstance(value, dict):'
|
|
767
|
+
printf '%s\n' ' for item in value.values():'
|
|
768
|
+
printf '%s\n' ' walk(item)'
|
|
769
|
+
printf '%s\n' ' elif isinstance(value, list):'
|
|
770
|
+
printf '%s\n' ' for item in value:'
|
|
771
|
+
printf '%s\n' ' walk(item)'
|
|
772
|
+
printf '%s\n' ''
|
|
773
|
+
printf '%s\n' 'walk(data)'
|
|
774
|
+
printf '%s\n' 'for req in sorted(seen):'
|
|
775
|
+
printf '%s\n' ' print(req)'
|
|
776
|
+
} > /tmp/devaudit-extract-e2e-reqs.py
|
|
749
777
|
while IFS= read -r REQ_ID; do
|
|
750
778
|
[ -n "$REQ_ID" ] || continue
|
|
751
779
|
REQS+=("$REQ_ID")
|
|
752
|
-
done < <(python3 -
|
|
753
|
-
|
|
754
|
-
import re
|
|
755
|
-
import sys
|
|
756
|
-
|
|
757
|
-
try:
|
|
758
|
-
with open('e2e-artifacts/e2e-regression-results.json', 'r', encoding='utf-8') as handle:
|
|
759
|
-
data = json.load(handle)
|
|
760
|
-
except Exception:
|
|
761
|
-
sys.exit(0)
|
|
762
|
-
|
|
763
|
-
seen = set()
|
|
764
|
-
|
|
765
|
-
def walk(value):
|
|
766
|
-
if isinstance(value, str):
|
|
767
|
-
for req in re.findall(r'REQ-[0-9]+', value):
|
|
768
|
-
seen.add(req)
|
|
769
|
-
elif isinstance(value, dict):
|
|
770
|
-
for item in value.values():
|
|
771
|
-
walk(item)
|
|
772
|
-
elif isinstance(value, list):
|
|
773
|
-
for item in value:
|
|
774
|
-
walk(item)
|
|
775
|
-
|
|
776
|
-
walk(data)
|
|
777
|
-
for req in sorted(seen):
|
|
778
|
-
print(req)
|
|
779
|
-
PY
|
|
780
|
-
)
|
|
780
|
+
done < <(python3 /tmp/devaudit-extract-e2e-reqs.py)
|
|
781
|
+
rm -f /tmp/devaudit-extract-e2e-reqs.py
|
|
781
782
|
fi
|
|
782
783
|
|
|
783
784
|
if [ "${#REQS[@]}" -eq 1 ] && echo "$DERIVED_RELEASE" | grep -qE '^v[0-9]{4}\.[0-9]{2}\.[0-9]{2}([.][0-9]+)?$'; then
|
package/sdlc/package.json
CHANGED
|
@@ -197,6 +197,19 @@ Rules:
|
|
|
197
197
|
|
|
198
198
|
**Final assessment:** [All cycles passed / N cycles failed — see incidents]
|
|
199
199
|
|
|
200
|
+
## Bundled Release Context
|
|
201
|
+
|
|
202
|
+
Required when `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md` exists for this REQ.
|
|
203
|
+
|
|
204
|
+
- **Core tracked release:** [REQ-XXX + what the tracked change itself delivers]
|
|
205
|
+
- **Absorbed predecessor releases:** [versions or `None`]
|
|
206
|
+
- **Absorbed non-release work:** [follow-up PRs, housekeeping, CI fixes, process fixes, docs-only release-path work — include issue/PR/commit references]
|
|
207
|
+
- **Why bundled here:** [release re-anchoring / CI fix / framework sync / housekeeping consolidation / review-window consolidation]
|
|
208
|
+
- **Evidence impact:** [which test/security results prove the core REQ vs which ones simply keep the broader release train green]
|
|
209
|
+
- **Reviewer impact:** [how the approval scope is broader than the core REQ alone]
|
|
210
|
+
- **Security / risk impact:** [`None beyond core REQ` or explicit impact]
|
|
211
|
+
- **Reference:** `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md`
|
|
212
|
+
|
|
200
213
|
## Test Changes in This Release
|
|
201
214
|
|
|
202
215
|
**Added:**
|
|
@@ -287,6 +300,17 @@ cat > compliance/evidence/REQ-XXX/security-summary.md << EOF
|
|
|
287
300
|
**SAST High/Critical Findings:** 0
|
|
288
301
|
**Dependency Audit High/Critical:** 0
|
|
289
302
|
Evidence uploaded to DevAudit project: [PROJECT_SLUG]
|
|
303
|
+
|
|
304
|
+
## Bundled Release Context
|
|
305
|
+
[Only include this section when `BUNDLED-CHANGES-REQ-XXX.md` exists.]
|
|
306
|
+
- **Core tracked release:** [REQ-XXX]
|
|
307
|
+
- **Absorbed predecessor releases:** [versions or `None`]
|
|
308
|
+
- **Absorbed non-release work:** [template sync / CI fix / housekeeping / predecessor release absorbed]
|
|
309
|
+
- **Why bundled here:** [why the non-core work ships in this cut]
|
|
310
|
+
- **Evidence impact:** [what shared gate/security evidence now covers]
|
|
311
|
+
- **Reviewer impact:** [how the review scope broadens beyond the core REQ]
|
|
312
|
+
- **Security / risk impact:** [core REQ impact + whether any absorbed work changes the security posture]
|
|
313
|
+
- **Reference:** `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md`
|
|
290
314
|
EOF
|
|
291
315
|
```
|
|
292
316
|
|
|
@@ -365,6 +389,17 @@ ai_contributors:
|
|
|
365
389
|
## Summary
|
|
366
390
|
[Brief description of what the AI tool was used for]
|
|
367
391
|
|
|
392
|
+
## Bundled Release Context
|
|
393
|
+
[Only include this section when `BUNDLED-CHANGES-REQ-XXX.md` exists.]
|
|
394
|
+
- **Core tracked release:** [REQ-XXX]
|
|
395
|
+
- **Absorbed predecessor releases:** [versions or `None`]
|
|
396
|
+
- **Absorbed non-release work:** [follow-up PRs / housekeeping / CI fixes / process fixes]
|
|
397
|
+
- **Why bundled here:** [release re-anchoring / CI fix / framework sync / housekeeping consolidation]
|
|
398
|
+
- **Evidence impact:** [which AI-authored artefacts changed because of the bundle]
|
|
399
|
+
- **Reviewer impact:** [how the approval scope broadens beyond the core REQ]
|
|
400
|
+
- **Security / risk impact:** [`None beyond core REQ` or explicit impact]
|
|
401
|
+
- **Reference:** `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md`
|
|
402
|
+
|
|
368
403
|
## Code Generated By AI
|
|
369
404
|
- [list files]
|
|
370
405
|
|
|
@@ -487,6 +522,19 @@ Create `compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md`:
|
|
|
487
522
|
## Summary
|
|
488
523
|
[1-3 sentences]
|
|
489
524
|
|
|
525
|
+
## Bundled Changes
|
|
526
|
+
|
|
527
|
+
Required when `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md` exists or when predecessor bare-date releases are being superseded into this REQ.
|
|
528
|
+
|
|
529
|
+
- **Core tracked release:** [REQ-XXX]
|
|
530
|
+
- **Absorbed predecessor releases:** [versions or `None`]
|
|
531
|
+
- **Absorbed non-release work:** [PR numbers / short descriptions / commit ranges]
|
|
532
|
+
- **Why bundled here:** [release re-anchoring / CI fix / framework sync / housekeeping consolidation]
|
|
533
|
+
- **Evidence impact:** [which release reports/evidence are broader because of the bundle]
|
|
534
|
+
- **Reviewer impact:** [how the approval scope is broader than the core REQ alone]
|
|
535
|
+
- **Security / risk impact:** [`None beyond core REQ` or explicit impact]
|
|
536
|
+
- **Reference:** `compliance/pending-releases/BUNDLED-CHANGES-REQ-XXX.md`
|
|
537
|
+
|
|
490
538
|
## AI Contributors
|
|
491
539
|
|
|
492
540
|
| Tool | Version | Session | Commits | Date Range |
|
|
@@ -591,6 +639,26 @@ If using DevAudit, commit only compliance documents (RTM, release ticket, test s
|
|
|
591
639
|
- [ ] `compliance/evidence/REQ-XXX/security-summary.md`
|
|
592
640
|
- [ ] `compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md`
|
|
593
641
|
|
|
642
|
+
```bash
|
|
643
|
+
# Phase 3 completion guard — do not proceed to Phase 4 / release PR
|
|
644
|
+
# creation while any required tracked-release artefact is missing.
|
|
645
|
+
MISSING=0
|
|
646
|
+
for f in \
|
|
647
|
+
compliance/evidence/REQ-XXX/test-scope.md \
|
|
648
|
+
compliance/evidence/REQ-XXX/test-plan.md \
|
|
649
|
+
compliance/evidence/REQ-XXX/test-execution-summary.md \
|
|
650
|
+
compliance/evidence/REQ-XXX/security-summary.md \
|
|
651
|
+
compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md
|
|
652
|
+
do
|
|
653
|
+
[ -f "$f" ] || { echo "MISSING: $f"; MISSING=1; }
|
|
654
|
+
done
|
|
655
|
+
[ "$MISSING" -eq 0 ] || {
|
|
656
|
+
echo "Fix missing files before proceeding to Phase 4."
|
|
657
|
+
echo "test-execution-summary.md satisfies the portal's Test Reports gate."
|
|
658
|
+
exit 1
|
|
659
|
+
}
|
|
660
|
+
```
|
|
661
|
+
|
|
594
662
|
```bash
|
|
595
663
|
# DevAudit projects — commit + push compliance docs
|
|
596
664
|
git add compliance/RTM.md compliance/pending-releases/RELEASE-TICKET-REQ-XXX.md \
|
|
@@ -38,7 +38,7 @@ The PR is the **merge request**, not the development workspace. Develop on `deve
|
|
|
38
38
|
|
|
39
39
|
When you create the PR, CI runs automatically — GitHub Actions re-executes the verification gates (TypeScript, SAST, dependency audit, E2E) independently. The `Release Approval Gate` workflow (renamed from `UAT Approval Gate` in sdlc-v1.22.0) also runs, verifying the release has been approved in DevAudit. This produces tamper-resistant evidence verified by GitHub's infrastructure.
|
|
40
40
|
|
|
41
|
-
> **Note:** The Release Approval Gate check will fail initially if the release hasn't been approved in DevAudit yet. This is expected. After approving the release in DevAudit (Stage 3 Step 11), re-run the `Release Approval Gate` workflow from GitHub Actions (or use `workflow_dispatch`) to
|
|
41
|
+
> **Note:** The Release Approval Gate check will fail initially if the release hasn't been approved in DevAudit yet. This is expected. After approving the release in DevAudit (Stage 3 Step 11), re-run the `Release Approval Gate` workflow from GitHub Actions (or use `workflow_dispatch`) to create a fresh passing `DevAudit Release Approval` check on the approved PR head SHA. The PR cannot be merged until this check passes — it is a required status check.
|
|
42
42
|
|
|
43
43
|
What happens next depends on the risk level of the requirements in the PR:
|
|
44
44
|
|