@metasession.co/devaudit-cli 0.3.17 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +36 -34
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/scripts/upload-evidence.sh +23 -19
- package/sdlc/files/_common/governance/incident-report.md.template +6 -1
- package/sdlc/files/_common/governance/nil-incident-report.md.template +5 -2
- package/sdlc/files/_common/scripts/check-host-deployment.sh +58 -8
- package/sdlc/files/_common/scripts/check-host-deployment.test.sh +35 -3
- package/sdlc/files/_common/scripts/check-release-pr-scope.sh +8 -1
- package/sdlc/files/_common/scripts/check-release-pr-scope.test.sh +61 -12
- package/sdlc/files/_common/scripts/check-self-hosted-runner.sh +103 -0
- package/sdlc/files/_common/scripts/check-self-hosted-runner.test.sh +74 -0
- package/sdlc/files/_common/scripts/derive-release-version.sh +7 -9
- package/sdlc/files/_common/scripts/derive-release-version.test.sh +18 -18
- package/sdlc/files/_common/scripts/generate-bundled-changes.sh +24 -3
- package/sdlc/files/_common/scripts/generate-bundled-changes.test.sh +9 -5
- package/sdlc/files/_common/scripts/reconcile-railway-deployment.sh +37 -0
- package/sdlc/files/_common/scripts/reconcile-railway-deployment.test.sh +30 -0
- package/sdlc/files/_common/scripts/record-uat-execution.sh +195 -0
- package/sdlc/files/_common/scripts/record-uat-execution.test.sh +144 -0
- package/sdlc/files/_common/scripts/render-test-executions.sh +146 -0
- package/sdlc/files/_common/scripts/{render-test-cycles.test.sh → render-test-executions.test.sh} +12 -18
- package/sdlc/files/_common/scripts/{report-test-cycle.sh → report-test-execution.sh} +66 -74
- package/sdlc/files/_common/scripts/report-test-execution.test.sh +177 -0
- package/sdlc/files/_common/scripts/submit-for-uat-review.sh +21 -0
- package/sdlc/files/_common/scripts/upload-evidence.sh +23 -19
- package/sdlc/files/_common/scripts/validate-commits.sh +6 -4
- package/sdlc/files/_common/scripts/validate-commits.test.sh +15 -0
- package/sdlc/files/_common/skills/e2e-test-engineer/SKILL.md +9 -7
- package/sdlc/files/_common/skills/e2e-test-engineer/references/e2e-regression-3-tier.yml +62 -10
- package/sdlc/files/_common/skills/sdlc-implementer/SKILL.md +19 -16
- package/sdlc/files/ci/ci.yml.template +86 -46
- package/sdlc/files/ci/compliance-evidence.yml.template +173 -49
- package/sdlc/files/ci/feature-e2e.yml.template +17 -17
- package/sdlc/files/ci/incident-export.yml.template +18 -4
- package/sdlc/files/ci/post-deploy-prod.yml.template +92 -48
- package/sdlc/files/ci/python/ci.yml.template +14 -14
- package/sdlc/files/ci/quality-gates-provenance.yml.template +3 -0
- package/sdlc/files/ci/reconcile-deployment.yml.template +59 -0
- package/sdlc/package.json +1 -1
- package/sdlc/src/blueprints/3-compile-evidence.raw.md +10 -10
- package/sdlc/src/blueprints/4-submit-for-review.raw.md +1 -1
- package/sdlc/src/blueprints/5-deploy-main.raw.md +1 -1
- package/sdlc/src/blueprints/implementing-an-sdlc-issue.raw.md +3 -3
- package/sdlc/files/_common/scripts/render-test-cycles.sh +0 -227
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# report-test-execution.test.sh - Focused regression tests for the test execution lifecycle helper.
|
|
3
|
+
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
|
|
6
|
+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
7
|
+
HELPER="$SCRIPT_DIR/report-test-execution.sh"
|
|
8
|
+
[ -x "$HELPER" ] || chmod +x "$HELPER"
|
|
9
|
+
|
|
10
|
+
PASS=0
|
|
11
|
+
FAIL=0
|
|
12
|
+
|
|
13
|
+
ok() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
|
14
|
+
no() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
15
|
+
|
|
16
|
+
make_fixture() {
|
|
17
|
+
local dir="$1"
|
|
18
|
+
rm -rf "$dir"
|
|
19
|
+
mkdir -p "$dir/bin"
|
|
20
|
+
cat > "$dir/bin/curl" <<'EOF'
|
|
21
|
+
#!/usr/bin/env bash
|
|
22
|
+
set -euo pipefail
|
|
23
|
+
LOG_FILE="${EXECUTION_TEST_LOG:?}"
|
|
24
|
+
URL=""
|
|
25
|
+
OUT_FILE=""
|
|
26
|
+
WRITE_OUT=""
|
|
27
|
+
BODY=""
|
|
28
|
+
ARGS=("$@")
|
|
29
|
+
for ((i=0; i<${#ARGS[@]}; i++)); do
|
|
30
|
+
arg="${ARGS[$i]}"
|
|
31
|
+
case "$arg" in
|
|
32
|
+
-o) OUT_FILE="${ARGS[$((i+1))]}" ;;
|
|
33
|
+
-w) WRITE_OUT="${ARGS[$((i+1))]}" ;;
|
|
34
|
+
-d) BODY="${ARGS[$((i+1))]}" ;;
|
|
35
|
+
http://*|https://*) URL="$arg" ;;
|
|
36
|
+
esac
|
|
37
|
+
done
|
|
38
|
+
printf 'URL:%s\n' "$URL" >> "$LOG_FILE"
|
|
39
|
+
if [ -n "$BODY" ]; then
|
|
40
|
+
printf 'BODY:%s\n' "$BODY" >> "$LOG_FILE"
|
|
41
|
+
fi
|
|
42
|
+
|
|
43
|
+
status=500
|
|
44
|
+
payload='{"error":"unhandled"}'
|
|
45
|
+
case "$URL" in
|
|
46
|
+
*"/api/ci/releases/resolve"*)
|
|
47
|
+
status="${RESOLVE_STATUS:-200}"
|
|
48
|
+
payload="${RESOLVE_BODY:-{}}"
|
|
49
|
+
;;
|
|
50
|
+
*"/cycles/start")
|
|
51
|
+
status="${START_STATUS:-201}"
|
|
52
|
+
payload="${START_BODY:-{}}"
|
|
53
|
+
;;
|
|
54
|
+
*"/cycles/complete")
|
|
55
|
+
status="${COMPLETE_STATUS:-200}"
|
|
56
|
+
payload="${COMPLETE_BODY:-{}}"
|
|
57
|
+
;;
|
|
58
|
+
*"/cycles/reconcile")
|
|
59
|
+
status="${RECONCILE_STATUS:-200}"
|
|
60
|
+
payload="${RECONCILE_BODY:-{}}"
|
|
61
|
+
;;
|
|
62
|
+
esac
|
|
63
|
+
|
|
64
|
+
if [ -n "$OUT_FILE" ]; then
|
|
65
|
+
printf '%s' "$payload" > "$OUT_FILE"
|
|
66
|
+
fi
|
|
67
|
+
if [ -n "$WRITE_OUT" ]; then
|
|
68
|
+
printf '%s' "$status"
|
|
69
|
+
fi
|
|
70
|
+
EOF
|
|
71
|
+
chmod +x "$dir/bin/curl"
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
run_helper() {
|
|
75
|
+
PATH="$1/bin:$PATH" \
|
|
76
|
+
EXECUTION_TEST_LOG="$1/curl.log" \
|
|
77
|
+
DEVAUDIT_BASE_URL="https://devaudit.example.test" \
|
|
78
|
+
DEVAUDIT_API_KEY="mc_test_dummy" \
|
|
79
|
+
"$HELPER" "${@:2}"
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
assert_contains() {
|
|
83
|
+
local label="$1" needle="$2" file="$3"
|
|
84
|
+
if grep -Fq -- "$needle" "$file"; then ok "$label"; else no "$label"; fi
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
echo "=== report-test-execution.sh tests ==="
|
|
88
|
+
|
|
89
|
+
case_start_success() {
|
|
90
|
+
echo "case: start resolves exact release and records execution"
|
|
91
|
+
local dir out exit_code
|
|
92
|
+
dir=$(mktemp -d)
|
|
93
|
+
make_fixture "$dir"
|
|
94
|
+
out="$dir/out.env"
|
|
95
|
+
RESOLVE_STATUS=200 \
|
|
96
|
+
RESOLVE_BODY='{"latest":{"id":"release-1","version":"REQ-100"}}' \
|
|
97
|
+
START_STATUS=201 \
|
|
98
|
+
START_BODY='{"id":"execution-1"}' \
|
|
99
|
+
run_helper "$dir" start \
|
|
100
|
+
--project-slug fixture-app \
|
|
101
|
+
--release REQ-100 \
|
|
102
|
+
--sdlc-stage 2 \
|
|
103
|
+
--environment uat \
|
|
104
|
+
--suite-kind e2e \
|
|
105
|
+
--iteration-key rework-1 \
|
|
106
|
+
--iteration-ordinal 1 \
|
|
107
|
+
--idempotency-key github:fixture/repo:feature-e2e.e2e:12345:1:2:REQ-100 \
|
|
108
|
+
--output-file "$out" >"$dir/stdout.log" 2>"$dir/stderr.log" && exit_code=0 || exit_code=$?
|
|
109
|
+
if [ "$exit_code" -eq 0 ]; then ok "exit code 0"; else no "expected exit 0"; fi
|
|
110
|
+
assert_contains "outputs supported=true" "execution_supported=true" "$out"
|
|
111
|
+
assert_contains "outputs execution_record_id" "execution_record_id=execution-1" "$out"
|
|
112
|
+
assert_contains "called start endpoint" "/cycles/start" "$dir/curl.log"
|
|
113
|
+
assert_contains "payload has suite kind" '"suiteKind": "e2e"' "$dir/curl.log"
|
|
114
|
+
assert_contains "payload carries portal transport kind" '"cycleKind": "e2e"' "$dir/curl.log"
|
|
115
|
+
assert_contains "payload has iteration key" '"iterationKey": "rework-1"' "$dir/curl.log"
|
|
116
|
+
rm -rf "$dir"
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
case_complete_reconciles_terminal_conflict() {
|
|
120
|
+
echo "case: complete falls back to reconcile on terminal-outcome conflict"
|
|
121
|
+
local dir out exit_code
|
|
122
|
+
dir=$(mktemp -d)
|
|
123
|
+
make_fixture "$dir"
|
|
124
|
+
out="$dir/out.env"
|
|
125
|
+
RESOLVE_STATUS=200 \
|
|
126
|
+
RESOLVE_BODY='{"latest":{"id":"release-1","version":"REQ-100"}}' \
|
|
127
|
+
COMPLETE_STATUS=400 \
|
|
128
|
+
COMPLETE_BODY='{"error":"Test execution already has a different terminal outcome; use reconcile"}' \
|
|
129
|
+
RECONCILE_STATUS=200 \
|
|
130
|
+
RECONCILE_BODY='{"id":"execution-1"}' \
|
|
131
|
+
run_helper "$dir" complete \
|
|
132
|
+
--project-slug fixture-app \
|
|
133
|
+
--release REQ-100 \
|
|
134
|
+
--sdlc-stage 2 \
|
|
135
|
+
--environment uat \
|
|
136
|
+
--suite-kind e2e \
|
|
137
|
+
--idempotency-key github:fixture/repo:feature-e2e.e2e:12345:1:2:REQ-100 \
|
|
138
|
+
--started-at 2026-07-16T10:00:00Z \
|
|
139
|
+
--outcome failed \
|
|
140
|
+
--output-file "$out" >"$dir/stdout.log" 2>"$dir/stderr.log" && exit_code=0 || exit_code=$?
|
|
141
|
+
if [ "$exit_code" -eq 0 ]; then ok "exit code 0"; else no "expected exit 0"; fi
|
|
142
|
+
assert_contains "called complete endpoint" "/cycles/complete" "$dir/curl.log"
|
|
143
|
+
assert_contains "called reconcile endpoint" "/cycles/reconcile" "$dir/curl.log"
|
|
144
|
+
assert_contains "outputs reconcile endpoint" "execution_endpoint=reconcile" "$out"
|
|
145
|
+
rm -rf "$dir"
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
case_missing_release_fails() {
|
|
149
|
+
echo "case: release resolve failure is not treated as successful fallback"
|
|
150
|
+
local dir out exit_code
|
|
151
|
+
dir=$(mktemp -d)
|
|
152
|
+
make_fixture "$dir"
|
|
153
|
+
out="$dir/out.env"
|
|
154
|
+
RESOLVE_STATUS=404 \
|
|
155
|
+
RESOLVE_BODY='{"error":"missing"}' \
|
|
156
|
+
run_helper "$dir" start \
|
|
157
|
+
--project-slug fixture-app \
|
|
158
|
+
--release REQ-100 \
|
|
159
|
+
--sdlc-stage 2 \
|
|
160
|
+
--environment uat \
|
|
161
|
+
--suite-kind e2e \
|
|
162
|
+
--idempotency-key key \
|
|
163
|
+
--output-file "$out" >"$dir/stdout.log" 2>"$dir/stderr.log" && exit_code=0 || exit_code=$?
|
|
164
|
+
if [ "$exit_code" -ne 0 ]; then ok "exit code non-zero"; else no "expected non-zero"; fi
|
|
165
|
+
if grep -q '/cycles/start' "$dir/curl.log"; then no "unexpected start endpoint call"; else ok "no start endpoint call"; fi
|
|
166
|
+
rm -rf "$dir"
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
case_start_success
|
|
170
|
+
case_complete_reconciles_terminal_conflict
|
|
171
|
+
case_missing_release_fails
|
|
172
|
+
|
|
173
|
+
echo ""
|
|
174
|
+
echo "=== report-test-execution.test.sh: $PASS passed, $FAIL failed ==="
|
|
175
|
+
if [ "$FAIL" -gt 0 ]; then
|
|
176
|
+
exit 1
|
|
177
|
+
fi
|
|
@@ -27,6 +27,8 @@
|
|
|
27
27
|
# If status is uat_review → idempotent no-op (exit 0, "already submitted").
|
|
28
28
|
# If status is uat_approved → idempotent no-op (exit 0, "already approved").
|
|
29
29
|
# Any other status → refuse with an explanatory message.
|
|
30
|
+
# 6. Print the explicit Stage 4 UAT execution command to run after the
|
|
31
|
+
# reviewer has completed UAT. Submission is not UAT execution.
|
|
30
32
|
|
|
31
33
|
set -euo pipefail
|
|
32
34
|
|
|
@@ -50,6 +52,23 @@ note() { printf ' - %s\n' "$*"; }
|
|
|
50
52
|
fail() { printf ' ✗ %s\n' "$*"; FAILED=$((FAILED + 1)); }
|
|
51
53
|
ok() { printf ' ✓ %s\n' "$*"; }
|
|
52
54
|
|
|
55
|
+
print_uat_execution_next_step() {
|
|
56
|
+
local tested_sha="${1:-}"
|
|
57
|
+
[ -n "$tested_sha" ] || tested_sha="$(git rev-parse HEAD 2>/dev/null || true)"
|
|
58
|
+
echo ""
|
|
59
|
+
echo "Next required control: after the reviewer performs UAT, record the Stage 4 execution explicitly."
|
|
60
|
+
echo "Submission only moves the release into review; it does not claim UAT passed."
|
|
61
|
+
echo ""
|
|
62
|
+
echo " ./scripts/record-uat-execution.sh \\"
|
|
63
|
+
echo " --project-slug ${PROJECT_SLUG} \\"
|
|
64
|
+
echo " --release ${RELEASE_VERSION:-$VERSION_PREFIX} \\"
|
|
65
|
+
echo " --outcome passed \\"
|
|
66
|
+
echo " --executor \"<reviewer identity>\" \\"
|
|
67
|
+
echo " --tested-sha ${tested_sha:-<tested sha>} \\"
|
|
68
|
+
echo " --checklist-ref \"<UAT checklist or portal note>\" \\"
|
|
69
|
+
echo " --evidence-ref \"<supporting evidence path or URL>\""
|
|
70
|
+
}
|
|
71
|
+
|
|
53
72
|
echo "Readiness checks for ${PROJECT_SLUG} ${VERSION_PREFIX}"
|
|
54
73
|
|
|
55
74
|
# 1. Working tree clean
|
|
@@ -134,6 +153,7 @@ case "$RELEASE_STATUS" in
|
|
|
134
153
|
NEW_STATUS=$(echo "$BODY" | jq -r '.status')
|
|
135
154
|
echo " Status: draft → ${NEW_STATUS}"
|
|
136
155
|
echo " Review page: ${BASE_URL}/projects/${PROJECT_SLUG}/releases/${RELEASE_ID}?env=uat"
|
|
156
|
+
print_uat_execution_next_step "$REMOTE_SHA"
|
|
137
157
|
else
|
|
138
158
|
echo " Submission failed (HTTP ${HTTP_CODE}): $(echo "$BODY" | jq -r '.error // .')" >&2
|
|
139
159
|
exit 1
|
|
@@ -143,6 +163,7 @@ case "$RELEASE_STATUS" in
|
|
|
143
163
|
echo ""
|
|
144
164
|
echo "Release already submitted for UAT review (status: uat_review). No action taken."
|
|
145
165
|
echo "Review page: ${BASE_URL}/projects/${PROJECT_SLUG}/releases/${RELEASE_ID}?env=uat"
|
|
166
|
+
print_uat_execution_next_step "$REMOTE_SHA"
|
|
146
167
|
;;
|
|
147
168
|
uat_approved | prod_review | prod_approved | released)
|
|
148
169
|
echo ""
|
|
@@ -42,19 +42,19 @@
|
|
|
42
42
|
# 4 submit-for-review, 5 deploy. Forwarded as
|
|
43
43
|
# `sdlcStage`; unknown to older portals (ignored
|
|
44
44
|
# server-side, no error).
|
|
45
|
-
# --test-
|
|
46
|
-
# ID). Forwarded
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
# portals ignore the field (no error).
|
|
45
|
+
# --test-execution <id> Test execution identifier (typically the CI run
|
|
46
|
+
# ID). Forwarded to the portal transport as
|
|
47
|
+
# `testCycleId` until the upload API field is
|
|
48
|
+
# renamed.
|
|
50
49
|
# --evidence-scope <scope> Evidence ownership scope: release | stage |
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
# --test-
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
50
|
+
# execution | approval. `execution` is mapped
|
|
51
|
+
# to the portal transport value `cycle` until
|
|
52
|
+
# the upload API field is renamed.
|
|
53
|
+
# --test-execution-record-id <id>
|
|
54
|
+
# First-class portal test execution UUID.
|
|
55
|
+
# Requires `--evidence-scope execution`.
|
|
56
|
+
# Forwarded to the portal transport as
|
|
57
|
+
# `testCycleRecordId`.
|
|
58
58
|
#
|
|
59
59
|
# Required environment variables:
|
|
60
60
|
# DEVAUDIT_BASE_URL e.g. https://meta-comply-production.up.railway.app
|
|
@@ -136,9 +136,9 @@ while [ "$#" -gt 0 ]; do
|
|
|
136
136
|
# DevAudit-Installer#96.
|
|
137
137
|
--gate-status) GATE_STATUS="$2"; shift 2 ;;
|
|
138
138
|
--sdlc-stage) SDLC_STAGE="$2"; shift 2 ;;
|
|
139
|
-
--test-
|
|
139
|
+
--test-execution) TEST_CYCLE="$2"; shift 2 ;;
|
|
140
140
|
--evidence-scope) EVIDENCE_SCOPE="$2"; shift 2 ;;
|
|
141
|
-
--test-
|
|
141
|
+
--test-execution-record-id) TEST_CYCLE_RECORD_ID="$2"; shift 2 ;;
|
|
142
142
|
# --meta-key key=value (repeatable). Merged into the metadata JSON
|
|
143
143
|
# before posting. Validates the `key=value` shape; rejects bare
|
|
144
144
|
# keys without `=`.
|
|
@@ -166,14 +166,18 @@ if [ -n "$SDLC_STAGE" ] && ! [[ "$SDLC_STAGE" =~ ^[1-5]$ ]]; then
|
|
|
166
166
|
echo "Error: --sdlc-stage must be an integer 1-5 (got: $SDLC_STAGE)"
|
|
167
167
|
exit 1
|
|
168
168
|
fi
|
|
169
|
-
if [ -n "$EVIDENCE_SCOPE" ] && ! [[ "$EVIDENCE_SCOPE" =~ ^(release|stage|
|
|
170
|
-
echo "Error: --evidence-scope must be one of: release, stage,
|
|
169
|
+
if [ -n "$EVIDENCE_SCOPE" ] && ! [[ "$EVIDENCE_SCOPE" =~ ^(release|stage|execution|approval)$ ]]; then
|
|
170
|
+
echo "Error: --evidence-scope must be one of: release, stage, execution, approval"
|
|
171
171
|
exit 1
|
|
172
172
|
fi
|
|
173
|
-
if [ -n "$TEST_CYCLE_RECORD_ID" ] && [ "$EVIDENCE_SCOPE" != "
|
|
174
|
-
echo "Error: --test-
|
|
173
|
+
if [ -n "$TEST_CYCLE_RECORD_ID" ] && [ "$EVIDENCE_SCOPE" != "execution" ]; then
|
|
174
|
+
echo "Error: --test-execution-record-id requires --evidence-scope execution"
|
|
175
175
|
exit 1
|
|
176
176
|
fi
|
|
177
|
+
TRANSPORT_EVIDENCE_SCOPE="$EVIDENCE_SCOPE"
|
|
178
|
+
if [ "$TRANSPORT_EVIDENCE_SCOPE" = "execution" ]; then
|
|
179
|
+
TRANSPORT_EVIDENCE_SCOPE="cycle"
|
|
180
|
+
fi
|
|
177
181
|
|
|
178
182
|
if [ -z "${DEVAUDIT_BASE_URL:-}" ]; then
|
|
179
183
|
echo "Error: DEVAUDIT_BASE_URL environment variable is required"
|
|
@@ -553,7 +557,7 @@ for FILE in "${FILES[@]}"; do
|
|
|
553
557
|
[ -n "$GATE_STATUS" ] && CURL_ARGS+=(-F "gateStatus=${GATE_STATUS}")
|
|
554
558
|
[ -n "$SDLC_STAGE" ] && CURL_ARGS+=(-F "sdlcStage=${SDLC_STAGE}")
|
|
555
559
|
[ -n "$TEST_CYCLE" ] && CURL_ARGS+=(-F "testCycleId=${TEST_CYCLE}")
|
|
556
|
-
[ -n "$
|
|
560
|
+
[ -n "$TRANSPORT_EVIDENCE_SCOPE" ] && CURL_ARGS+=(-F "evidenceScope=${TRANSPORT_EVIDENCE_SCOPE}")
|
|
557
561
|
[ -n "$TEST_CYCLE_RECORD_ID" ] && CURL_ARGS+=(-F "testCycleRecordId=${TEST_CYCLE_RECORD_ID}")
|
|
558
562
|
[ -n "$SENTINEL_CONTENT" ] && CURL_ARGS+=(-F "sentinelContent=${SENTINEL_CONTENT}")
|
|
559
563
|
[ -n "$COMMIT_TIMESTAMP" ] && CURL_ARGS+=(-F "commitTimestamp=${COMMIT_TIMESTAMP}")
|
|
@@ -21,11 +21,12 @@ echo "=== Commit Convention Validation ==="
|
|
|
21
21
|
echo "Comparing: $BASE_BRANCH...HEAD"
|
|
22
22
|
echo ""
|
|
23
23
|
|
|
24
|
-
# Conventional Commit regex:
|
|
24
|
+
# Conventional Commit regex: optional [REQ-XXX] prefix, then
|
|
25
|
+
# type(optional-scope): description.
|
|
25
26
|
# Scope accepts anything except `)` so multi-scope subjects like
|
|
26
27
|
# `feat(auth,profile):` and `fix(rewards/expiry):` validate. The closing-paren
|
|
27
|
-
# guard prevents pathological inputs. DevAudit-Installer#93.
|
|
28
|
-
CC_REGEX='^(feat|fix|docs|test|refactor|chore|compliance|security|perf|ci|build|revert)(\([^)]+\))?!?: .+'
|
|
28
|
+
# guard prevents pathological inputs. DevAudit-Installer#93/#440.
|
|
29
|
+
CC_REGEX='^(\[REQ-[0-9]{3,}\][[:space:]]+)?(feat|fix|docs|test|refactor|chore|compliance|security|perf|ci|build|revert)(\([^)]+\))?!?: .+'
|
|
29
30
|
|
|
30
31
|
COMMITS=$(git log "$BASE_BRANCH"..HEAD --format='%H' || true)
|
|
31
32
|
|
|
@@ -78,7 +79,8 @@ while IFS= read -r sha; do
|
|
|
78
79
|
# are exempt. Mirrors the commitlint rule; this is the PR-CI half that
|
|
79
80
|
# `--no-verify` can't skip. Work starts from a requirement (which starts
|
|
80
81
|
# from an issue) — use the sdlc-implementer skill to assign one.
|
|
81
|
-
|
|
82
|
+
SUBJECT_FOR_TYPE=$(echo "$SUBJECT" | sed -E 's/^\[REQ-[0-9]{3,}\][[:space:]]+//')
|
|
83
|
+
TYPE=$(echo "$SUBJECT_FOR_TYPE" | grep -oE '^[a-z]+' || true)
|
|
82
84
|
case "$TYPE" in
|
|
83
85
|
feat|fix|refactor|perf)
|
|
84
86
|
if ! echo "$SUBJECT" | grep -qP '\[REQ-\d{3,}\]' \
|
|
@@ -138,6 +138,21 @@ run_validator
|
|
|
138
138
|
assert_exit "no active context exits 1" 1
|
|
139
139
|
assert_grep "hard error emitted without any active release context" "ERROR .*implementation commit but cites no requirement" 1
|
|
140
140
|
|
|
141
|
+
# Case 5: leading REQ prefix is accepted before the Conventional Commit type.
|
|
142
|
+
echo "Case 5: leading REQ prefix before type is accepted"
|
|
143
|
+
make_fixture "$WORKDIR/case5" "[REQ-123] fix(reports): map dynamic category reports" "Co-Authored-By: Test <test@example.com>"
|
|
144
|
+
run_validator
|
|
145
|
+
assert_exit "leading REQ prefix exits 0" 0
|
|
146
|
+
assert_grep "no conventional-commit error for leading REQ prefix" "Not Conventional Commits format" 0
|
|
147
|
+
assert_grep "no missing-requirement error for leading REQ prefix" "implementation commit but cites no requirement" 0
|
|
148
|
+
|
|
149
|
+
# Case 6: malformed leading REQ prefixes remain invalid.
|
|
150
|
+
echo "Case 6: malformed leading REQ prefix is rejected"
|
|
151
|
+
make_fixture "$WORKDIR/case6" "[REQ-12] fix: invalid short req prefix" "Co-Authored-By: Test <test@example.com>"
|
|
152
|
+
run_validator
|
|
153
|
+
assert_exit "malformed leading REQ prefix exits 1" 1
|
|
154
|
+
assert_grep "malformed prefix is not treated as conventional" "Not Conventional Commits format" 1
|
|
155
|
+
|
|
141
156
|
echo
|
|
142
157
|
echo "Result: $PASS passed, $FAIL failed"
|
|
143
158
|
[ "$FAIL" = "0" ]
|
|
@@ -229,9 +229,9 @@ Before running the suite (Phase 6), verify that the evidence traceability wiring
|
|
|
229
229
|
|
|
230
230
|
Also verify the import is present: `import { tagTest } from './helpers/test-tags'` (or the correct relative path from the spec's location to `e2e/helpers/test-tags.ts`).
|
|
231
231
|
|
|
232
|
-
4. **Check release and
|
|
232
|
+
4. **Check release and execution provenance.** Before handing the work back to `sdlc-implementer`, identify the tracked `REQ-XXX`, the source branch SHA, and the expected SDLC stage/test execution for the run. The CI evidence upload must retain those values so the portal can show the result in the correct numbered execution for the release that produced it.
|
|
233
233
|
- A passing local run is not portal evidence by itself. Report it as local verification and let the configured CI workflow upload the resulting report and screenshots.
|
|
234
|
-
- For a bundled release, evidence remains attributed to its originating REQ and
|
|
234
|
+
- For a bundled release, evidence remains attributed to its originating REQ and execution. The absorbing release may reference that source release through bundle lineage, but must not relabel the source E2E run as newly produced by the absorbing REQ.
|
|
235
235
|
- Confirm the CI invocation has an unambiguous release context (`REQ-XXX` or the documented standalone-housekeeping declaration). A bare-date integration housekeeping run is historical CI only; it must not create an approval-ready E2E evidence set.
|
|
236
236
|
- If the project workflow cannot provide this context, halt and report a workflow-contract gap. Do not claim that screenshots or E2E results will appear correctly on the portal.
|
|
237
237
|
|
|
@@ -247,7 +247,7 @@ Halt and report the gap to the user:
|
|
|
247
247
|
|
|
248
248
|
Do **not** proceed to Phase 6 until all gaps are resolved. The user may choose to skip an AC (e.g. API-only, transport-only) — that's valid, but it must be an explicit decision recorded in the test-execution-summary, not an omission.
|
|
249
249
|
|
|
250
|
-
**Write the evidence-wiring sentinel (devaudit-installer#226).** After all Phase 5½ checks pass (all `@requirement`, `evidenceShot()`, `tagTest()`, and release/
|
|
250
|
+
**Write the evidence-wiring sentinel (devaudit-installer#226).** After all Phase 5½ checks pass (all `@requirement`, `evidenceShot()`, `tagTest()`, and release/execution provenance checks verified), write a `.e2e-evidence-wired` sentinel file in the repo root so the pre-push hook and `sdlc-implementer` Phase 2 step 5b can verify evidence wiring was validated:
|
|
251
251
|
|
|
252
252
|
```bash
|
|
253
253
|
echo "WIRED $(date -u +%Y-%m-%dT%H:%M:%SZ) REQ-XXX" > .e2e-evidence-wired
|
|
@@ -366,7 +366,7 @@ Each filed issue needs:
|
|
|
366
366
|
|
|
367
367
|
#### Framework classification + the `incident` label
|
|
368
368
|
|
|
369
|
-
Every defect filed from Phase 6 becomes `incident_report` evidence when (a) the issue is labelled `incident` and (b) the issue is closed. The flow: closed-with-label → `incident-export.yml` exports the body to `compliance/governance/incident-report-<N>.md` → `compliance-evidence.yml` uploads as `incident_report` evidence → portal flips the attributed clauses MISSING → COVERED.
|
|
369
|
+
Every defect filed from Phase 6 becomes `incident_report` evidence when (a) the issue is labelled `incident` and (b) the issue is closed. The flow: closed-with-label → `incident-export.yml` exports the body to `compliance/governance/incident-report-<N>.md` with `incident_kind`, `source_release`, `source_issue`, and `semantic_id` frontmatter → `compliance-evidence.yml` uploads it as source-owned `incident_report` evidence → portal flips the attributed clauses MISSING → COVERED for that source release.
|
|
370
370
|
|
|
371
371
|
Classify the defect against this table when filing — the canonical version lives at `governance-doc-author/references/incident-classification.md`, mirrored here for the e2e workflow:
|
|
372
372
|
|
|
@@ -411,6 +411,8 @@ Once closed, the `incident-export.yml` workflow exports this issue's body to `co
|
|
|
411
411
|
- **Path B (any of SOC2/GDPR/EUAIA ticked):** auto-files a PR with the GDPR triage + sign-off sections to fill in. Merge that PR → `compliance-evidence.yml` uploads as `incident_report`.
|
|
412
412
|
```
|
|
413
413
|
|
|
414
|
+
The issue body must identify the owning `REQ-XXX` or standalone housekeeping release so the exported report's `source_release` is correct. Do not let a later bundled release claim the incident as newly produced evidence; bundles inherit by explicit lineage.
|
|
415
|
+
|
|
414
416
|
Pre-tick boxes you're confident about. Leave the operator-judgement ones (GDPR triage, AI-failure classification) for the operator to confirm in the export PR.
|
|
415
417
|
|
|
416
418
|
#### Worked examples
|
|
@@ -448,7 +450,7 @@ Wrap up with a summary the user can drop into the PR or ticket:
|
|
|
448
450
|
- Defects filed — count, with links.
|
|
449
451
|
- Requirements gap reports — count, with details (devaudit-installer#212 Gap 4). These are ACs classified as "impossible to test" or "missing AC" — returned to `sdlc-implementer` for the requirements gap flow, not filed as defects.
|
|
450
452
|
- Missed requirements — count, with links.
|
|
451
|
-
- **Release provenance** — originating `REQ-XXX`, local run result, expected CI workflow/run, and expected portal stage/
|
|
453
|
+
- **Release provenance** — originating `REQ-XXX`, local run result, expected CI workflow/run, and expected portal stage/execution. For bundled work, state the source REQ explicitly and that the absorbing release will reference it through lineage rather than overwrite its attribution.
|
|
452
454
|
|
|
453
455
|
**Then feed the test-design record (devaudit#50).** The Stage 3 `test-execution-summary.md` (generated per `3-compile-evidence.md` Step 1a) carries a `## Test design` section at the top. Before Stage 3 finalises the file, populate that section with the design-time decisions this skill made, so the SDLC has a recorded trace that scope was *decided*, not implicit:
|
|
454
456
|
|
|
@@ -496,8 +498,8 @@ The canonical helper source lives at `references/evidence.ts` in this skill.
|
|
|
496
498
|
|
|
497
499
|
The number of `evidenceShot` calls per spec should scale to the spec's role:
|
|
498
500
|
|
|
499
|
-
- **While the spec is a feature artefact** (newly authored on the branch, before merge to develop): capture multiple stages
|
|
500
|
-
- **Once the spec joins the regression pack** (post-merge, `git diff --diff-filter=A` no longer matches it): capture only the canonical "this still works" anchor per AC. Re-running the dense journey on every regression
|
|
501
|
+
- **While the spec is a feature artefact** (newly authored on the branch, before merge to develop): capture multiple stages - every meaningful transition or state the AC documents. The dense evidence is what reviewers use to verify the AC was met end-to-end during the feature iteration.
|
|
502
|
+
- **Once the spec joins the regression pack** (post-merge, `git diff --diff-filter=A` no longer matches it): capture only the canonical "this still works" anchor per AC. Re-running the dense journey on every regression execution is noise and inflates CI artefact storage with little signal.
|
|
501
503
|
|
|
502
504
|
The `EvidenceShotOrigin` signal (`'feature' | 'regression'`) auto-detects from `E2E_NEW_SPECS`. Mark stage screenshots with `{ tier: 'feature' }`; the helper auto-suppresses them on regression runs. The canonical anchor uses the default tier (`'always'`).
|
|
503
505
|
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
#
|
|
3
3
|
# Copy this into your consumer-owned .github/workflows/e2e-regression.yml
|
|
4
4
|
# to adopt the 3-tier model: smoke (every develop push, fast) / critical
|
|
5
|
-
# (consumer-enabled release PR, ~10-15 min target) / regression
|
|
6
|
-
#
|
|
5
|
+
# (consumer-enabled release PR, ~10-15 min target) / regression after a
|
|
6
|
+
# successful production deployment status (or manual dispatch).
|
|
7
7
|
#
|
|
8
8
|
# The framework does NOT sync this file automatically — your consumer
|
|
9
9
|
# owns its e2e-regression.yml. Apply the patterns below to your own
|
|
@@ -17,14 +17,18 @@
|
|
|
17
17
|
# playwright.config.ts must define the `critical` project for this to
|
|
18
18
|
# fire; if it doesn't, the gate falls back to the existing `smoke`
|
|
19
19
|
# project so PR-to-main stays green during migration.
|
|
20
|
+
#
|
|
21
|
+
# A post-merge regression is release evidence, not a best-effort background
|
|
22
|
+
# task. Keep its timeout below the job timeout so the workflow can retain
|
|
23
|
+
# partial evidence and report a terminal timeout outcome to DevAudit.
|
|
20
24
|
|
|
21
25
|
name: E2E Regression
|
|
22
26
|
|
|
23
27
|
on:
|
|
24
28
|
pull_request:
|
|
25
29
|
branches: [main] # critical-tier gate before merge
|
|
26
|
-
|
|
27
|
-
|
|
30
|
+
deployment_status:
|
|
31
|
+
types: [created] # full regression only after successful production deploy
|
|
28
32
|
workflow_dispatch:
|
|
29
33
|
inputs:
|
|
30
34
|
specs:
|
|
@@ -42,11 +46,22 @@ concurrency:
|
|
|
42
46
|
jobs:
|
|
43
47
|
e2e:
|
|
44
48
|
name: E2E Regression Tests
|
|
49
|
+
if: >-
|
|
50
|
+
github.event_name != 'deployment_status' ||
|
|
51
|
+
(github.event.deployment_status.state == 'success' &&
|
|
52
|
+
(github.event.deployment.environment == 'production' ||
|
|
53
|
+
github.event.deployment.environment == 'prod' ||
|
|
54
|
+
endsWith(github.event.deployment.environment, '/ production') ||
|
|
55
|
+
endsWith(github.event.deployment.environment, '/production')))
|
|
45
56
|
runs-on: ubuntu-latest # adapt to your runner; e.g. self-hosted, ubuntu-24.04
|
|
57
|
+
# The full regression target is about 35 minutes. Leave time to archive
|
|
58
|
+
# partial output before GitHub terminates the job.
|
|
59
|
+
timeout-minutes: 55
|
|
46
60
|
steps:
|
|
47
61
|
- uses: actions/checkout@v6
|
|
48
62
|
with:
|
|
49
63
|
fetch-depth: 0 # for E2E_NEW_SPECS computation
|
|
64
|
+
ref: ${{ github.event.deployment.sha || github.sha }}
|
|
50
65
|
|
|
51
66
|
- uses: actions/setup-node@v6
|
|
52
67
|
with:
|
|
@@ -79,7 +94,7 @@ jobs:
|
|
|
79
94
|
fi
|
|
80
95
|
echo "specs=" >> "$GITHUB_OUTPUT"
|
|
81
96
|
;;
|
|
82
|
-
|
|
97
|
+
deployment_status|schedule)
|
|
83
98
|
echo "project=regression" >> "$GITHUB_OUTPUT"
|
|
84
99
|
echo "specs=" >> "$GITHUB_OUTPUT"
|
|
85
100
|
echo "Running full regression project"
|
|
@@ -93,6 +108,25 @@ jobs:
|
|
|
93
108
|
;;
|
|
94
109
|
esac
|
|
95
110
|
|
|
111
|
+
- name: Record E2E execution context
|
|
112
|
+
id: context
|
|
113
|
+
env:
|
|
114
|
+
E2E_TARGET_URL: ${{ vars.E2E_TARGET_URL }}
|
|
115
|
+
run: |
|
|
116
|
+
set -euo pipefail
|
|
117
|
+
TARGET_URL="${E2E_TARGET_URL:-${PLAYWRIGHT_BASE_URL:-${BASE_URL:-http://localhost:3000}}}"
|
|
118
|
+
jq -n \
|
|
119
|
+
--arg targetUrl "$TARGET_URL" \
|
|
120
|
+
--arg project "${{ steps.select.outputs.project }}" \
|
|
121
|
+
--arg specs "${{ steps.select.outputs.specs }}" \
|
|
122
|
+
--arg serverStart "consumer-managed-or-external" \
|
|
123
|
+
--arg serverStop "consumer-managed-or-external" \
|
|
124
|
+
--arg startedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
125
|
+
--argjson timeoutMinutes 40 \
|
|
126
|
+
'{target_url: $targetUrl, project: $project, selected_specs: $specs, test_server_start: $serverStart, test_server_stop: $serverStop, timeout_minutes: $timeoutMinutes, started_at: $startedAt, outcome: "running"}' \
|
|
127
|
+
> e2e-regression-metadata.json
|
|
128
|
+
echo "target_url=$TARGET_URL" >> "$GITHUB_OUTPUT"
|
|
129
|
+
|
|
96
130
|
- name: Run E2E suite
|
|
97
131
|
id: run
|
|
98
132
|
env:
|
|
@@ -100,14 +134,28 @@ jobs:
|
|
|
100
134
|
PLAYWRIGHT_JSON_OUTPUT_NAME: e2e-regression-results.json
|
|
101
135
|
# Add your e2e_env values here as needed (DEVAUDIT_BASE_URL etc.)
|
|
102
136
|
run: |
|
|
103
|
-
set -
|
|
137
|
+
set -uo pipefail
|
|
104
138
|
PROJECT="${{ steps.select.outputs.project }}"
|
|
105
139
|
SPECS="${{ steps.select.outputs.specs }}"
|
|
106
140
|
if [ -n "$SPECS" ]; then
|
|
107
|
-
npx playwright test --project="$PROJECT" --reporter=json,html $SPECS
|
|
141
|
+
timeout --signal=TERM --kill-after=60s 40m npx playwright test --project="$PROJECT" --reporter=json,html $SPECS
|
|
142
|
+
else
|
|
143
|
+
timeout --signal=TERM --kill-after=60s 40m npx playwright test --project="$PROJECT" --reporter=json,html
|
|
144
|
+
fi
|
|
145
|
+
STATUS=$?
|
|
146
|
+
if [ "$STATUS" -eq 124 ]; then
|
|
147
|
+
OUTCOME="timed_out"
|
|
148
|
+
echo "::error::E2E regression exceeded its 40-minute execution budget. Partial evidence will be uploaded."
|
|
149
|
+
elif [ "$STATUS" -eq 0 ]; then
|
|
150
|
+
OUTCOME="passed"
|
|
108
151
|
else
|
|
109
|
-
|
|
152
|
+
OUTCOME="failed"
|
|
110
153
|
fi
|
|
154
|
+
jq --arg outcome "$OUTCOME" --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson exitCode "$STATUS" \
|
|
155
|
+
'. + {outcome: $outcome, completed_at: $completedAt, exit_code: $exitCode}' \
|
|
156
|
+
e2e-regression-metadata.json > e2e-regression-metadata.tmp
|
|
157
|
+
mv e2e-regression-metadata.tmp e2e-regression-metadata.json
|
|
158
|
+
exit "$STATUS"
|
|
111
159
|
|
|
112
160
|
- uses: actions/upload-artifact@v7
|
|
113
161
|
if: always()
|
|
@@ -115,11 +163,15 @@ jobs:
|
|
|
115
163
|
name: e2e-regression-report
|
|
116
164
|
path: |
|
|
117
165
|
e2e-regression-results.json
|
|
166
|
+
e2e-regression-metadata.json
|
|
118
167
|
playwright-report/
|
|
119
168
|
test-results/
|
|
169
|
+
e2e-server.log
|
|
170
|
+
server.log
|
|
171
|
+
if-no-files-found: warn
|
|
120
172
|
|
|
121
173
|
# ─────────────────────────────────────────────────────────────
|
|
122
|
-
# Post-
|
|
174
|
+
# Post-deployment auto-issue on regression failure.
|
|
123
175
|
#
|
|
124
176
|
# Catches regressions that slipped past the critical-tier PR gate.
|
|
125
177
|
# Opens a high-priority issue tagging the merge commit + the
|
|
@@ -127,7 +179,7 @@ jobs:
|
|
|
127
179
|
# No auto-revert — that's intentionally an operator decision.
|
|
128
180
|
# ─────────────────────────────────────────────────────────────
|
|
129
181
|
- name: Open hotfix issue on post-merge regression
|
|
130
|
-
if: failure() && github.event_name == '
|
|
182
|
+
if: failure() && github.event_name == 'deployment_status' && github.event.deployment_status.state == 'success'
|
|
131
183
|
env:
|
|
132
184
|
GH_TOKEN: ${{ github.token }}
|
|
133
185
|
run: |
|