mandrel 2.40.0 → 2.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/README.md +2 -2
- package/.agents/agents/story-worker.md +24 -14
- package/.agents/docs/agentrc-reference.json +7 -2
- package/.agents/docs/configuration.md +5 -2
- package/.agents/instructions.md +5 -6
- package/.agents/rules/api-conventions.md +43 -7
- package/.agents/rules/ci-remediation.md +3 -14
- package/.agents/rules/gherkin-standards.md +21 -6
- package/.agents/rules/git-conventions.md +6 -5
- package/.agents/rules/security-baseline.md +6 -7
- package/.agents/rules/testing-standards.md +75 -198
- package/.agents/schemas/agentrc.schema.json +17 -2
- package/.agents/schemas/validation-evidence.schema.json +3 -1
- package/.agents/scripts/acceptance-eval.js +68 -3
- package/.agents/scripts/coverage-capture.js +25 -8
- package/.agents/scripts/install-matrix-assert.js +2 -2
- package/.agents/scripts/lib/audit-to-stories/epic-grouping-directive.js +6 -0
- package/.agents/scripts/lib/baselines/crap-preview-incremental.js +7 -2
- package/.agents/scripts/lib/baselines/git-base.js +74 -38
- package/.agents/scripts/lib/close-validation/gates.js +153 -25
- package/.agents/scripts/lib/close-validation/process.js +30 -1
- package/.agents/scripts/lib/close-validation/runner.js +5 -0
- package/.agents/scripts/lib/config/gates/crap-incremental-coverage.schema.js +33 -12
- package/.agents/scripts/lib/config/quality.js +36 -21
- package/.agents/scripts/lib/config-settings-schema-delivery.js +6 -0
- package/.agents/scripts/lib/coverage-capture-incremental.js +12 -6
- package/.agents/scripts/lib/crap-baseline-join.js +11 -7
- package/.agents/scripts/lib/full-suite-lock.js +311 -0
- package/.agents/scripts/lib/generated/agentrc-validator.js +1 -1
- package/.agents/scripts/lib/orchestration/check-baselines/phases/evaluate.js +11 -104
- package/.agents/scripts/lib/orchestration/check-baselines/phases/refresh-ack.js +320 -0
- package/.agents/scripts/lib/orchestration/check-baselines/phases/report.js +8 -1
- package/.agents/scripts/lib/orchestration/dependency-candidates.js +139 -0
- package/.agents/scripts/lib/orchestration/epic-candidates.js +159 -0
- package/.agents/scripts/lib/orchestration/epic-checklist.js +103 -0
- package/.agents/scripts/lib/orchestration/epic-container.js +18 -2
- package/.agents/scripts/lib/orchestration/plan-context.js +97 -36
- package/.agents/scripts/lib/orchestration/plan-persist/cross-plan-links.js +80 -0
- package/.agents/scripts/lib/orchestration/plan-persist/epic-adoption.js +192 -0
- package/.agents/scripts/lib/orchestration/plan-persist/epic-ops.js +5 -1
- package/.agents/scripts/lib/orchestration/plan-persist/external-deps.js +164 -0
- package/.agents/scripts/lib/orchestration/plan-persist/run-plan-persist.js +14 -2
- package/.agents/scripts/lib/orchestration/plan-persist/story-ops.js +26 -5
- package/.agents/scripts/lib/orchestration/single-story-close/failed-terminal.js +83 -4
- package/.agents/scripts/lib/orchestration/single-story-close/phases/close-validation.js +39 -7
- package/.agents/scripts/lib/orchestration/single-story-close/runner.js +70 -18
- package/.agents/scripts/lib/orchestration/ticket-validator.js +11 -1
- package/.agents/scripts/lib/orchestration/verify-credit.js +207 -0
- package/.agents/scripts/lib/single-story-sweep/sweep-lock.js +24 -0
- package/.agents/scripts/plan-persist.js +60 -1
- package/.agents/skills/core/browser-testing-with-devtools/SKILL.md +5 -10
- package/.agents/skills/core/browser-testing-with-devtools/reference.md +7 -46
- package/.agents/skills/core/code-review-and-quality/SKILL.md +0 -5
- package/.agents/skills/core/documentation-and-adrs/SKILL.md +0 -3
- package/.agents/skills/core/gates-and-baselines/SKILL.md +10 -137
- package/.agents/skills/core/gates-and-baselines/reference.md +103 -0
- package/.agents/skills/core/idea-refinement/SKILL.md +2 -18
- package/.agents/skills/core/security-and-hardening/SKILL.md +2 -4
- package/.agents/skills/core/security-and-hardening/reference.md +0 -70
- package/.agents/skills/skills.index.json +10 -70
- package/.agents/skills/stack/qa/gherkin-authoring/SKILL.md +4 -10
- package/.agents/skills/stack/qa/gherkin-authoring/reference.md +9 -113
- package/.agents/skills/stack/qa/playwright-bdd/SKILL.md +29 -154
- package/.agents/skills/stack/qa/qa-harness/SKILL.md +157 -98
- package/.agents/workflows/git-cleanup.md +3 -2
- package/.agents/workflows/git-deliver.md +3 -2
- package/.agents/workflows/helpers/acceptance-self-eval.md +12 -0
- package/.agents/workflows/helpers/deliver-digest.md +31 -10
- package/.agents/workflows/helpers/deliver-story-reference.md +50 -30
- package/.agents/workflows/helpers/deliver-story.md +23 -21
- package/.agents/workflows/helpers/plan-reference.md +82 -2
- package/.agents/workflows/mandrel-plan.md +45 -45
- package/.agents/workflows/qa-assist.md +20 -17
- package/.agents/workflows/qa-explore.md +30 -29
- package/.agents/workflows/qa-run.md +2 -1
- package/docs/CHANGELOG.md +32 -0
- package/package.json +1 -1
- package/.agents/rules/changelog-style.md +0 -180
- package/.agents/rules/shell-conventions.md +0 -61
- package/.agents/scripts/lib/qa/coverage-verdict.js +0 -214
- package/.agents/skills/core/api-and-interface-design/SKILL.md +0 -55
- package/.agents/skills/core/api-and-interface-design/reference.md +0 -76
- package/.agents/skills/core/debugging-and-error-recovery/SKILL.md +0 -45
- package/.agents/skills/core/debugging-and-error-recovery/reference.md +0 -56
- package/.agents/skills/core/git-workflow-and-versioning/SKILL.md +0 -54
- package/.agents/skills/core/idea-refinement/refinement-criteria.md +0 -155
- package/.agents/skills/core/idea-refinement/scripts/idea-refine.sh +0 -15
- package/.agents/skills/core/qa-coverage-mapping/SKILL.md +0 -105
- package/.agents/skills/stack/qa/qa-explore-driving/SKILL.md +0 -152
- package/.agents/skills/stack/qa/vitest/SKILL.md +0 -22
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: qa-harness
|
|
3
3
|
description:
|
|
4
|
-
Conventions for
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
4
|
+
Conventions for agent-driven QA driving — the one driving-conventions
|
|
5
|
+
reference behind both `/qa-run` (the known-scenario sweep through a real
|
|
6
|
+
browser) and `/qa-explore` (agent-led exploratory driving). Use when
|
|
7
|
+
instrumenting a live surface — covers navigation-first execution,
|
|
8
|
+
per-surface console and network capture, design-token visual checks, the
|
|
9
|
+
`F#` finding shape, per-environment resolution, and static driving as the
|
|
10
|
+
documented interim. The run procedures live in `.agents/workflows/qa-run.md`
|
|
11
|
+
and `.agents/workflows/qa-explore.md`; this skill is the conventions
|
|
12
|
+
reference they lean on.
|
|
11
13
|
---
|
|
12
14
|
|
|
13
15
|
# Skill: qa-harness
|
|
@@ -15,47 +17,110 @@ description:
|
|
|
15
17
|
## Policy Capsule
|
|
16
18
|
|
|
17
19
|
- Driving rules (navigation-first, semantic `Then`, mandatory redaction, sequential-only) live in one prose home — [`qa-run-scenario.md`](../../../../workflows/helpers/qa-run-scenario.md); apply them, do not restate them.
|
|
20
|
+
- Resolve the `qa` contract first and fail loudly when it is absent or malformed; there is no auto-detection fallback and no headless degrade.
|
|
21
|
+
- Resolve the target **environment** before driving (via `resolveQaEnvironment`); each environment keys its own `baseUrl`, `signInSeam`, and `allowWrites`, and its name is recorded on the ledger.
|
|
22
|
+
- Sign in only through the resolved `signInSeam` (`kind: 'url'` dev impersonation or `kind: 'skill'` with a stored `credentialRef`). **Never** type real credentials inline and never fabricate a session.
|
|
23
|
+
- Hold the read-only capture invariant absolutely: no source edits, no product mutations — the only write is appending to the `temp/qa/<sessionId>` ledger.
|
|
24
|
+
- Scrub captured console, network, and headers of tokens, session cookies, and PII before any finding reaches disk or GitHub; findings are posted to GitHub at approval time.
|
|
18
25
|
- Capture console and network per surface; turn each non-allowlisted console error and each failed/error-status request into one structured `F#` finding.
|
|
19
26
|
- Filter console through `qa.consoleAllowlist` via `filterConsoleMessages`; treat the allowlist as a benign-noise filter, never as a security control to silence genuine errors.
|
|
20
27
|
- Spot-check surfaces against `qa.designTokens` when set; flag gross token violations (off-palette colors, off-scale spacing/typography) as findings.
|
|
21
|
-
-
|
|
22
|
-
- Record findings as `QaLedgerItem`s and route them through the shared classify/route/promote core ([`qa-core.md`](../../../../workflows/helpers/qa-core.md)); the harness never files tickets autonomously.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
[
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
- Choose **static driving** only at Plan time and only where no seam resolves — the documented interim, never a silent fallback — and record the partial coverage.
|
|
29
|
+
- Record findings as `QaLedgerItem`s and route them through the shared classify/route/promote core ([`qa-core.md`](../../../../workflows/helpers/qa-core.md)); the harness never files tickets autonomously, and every phase transition is HITL-gated.
|
|
30
|
+
|
|
31
|
+
Guidance for driving a live surface through a real browser (the chrome-devtools
|
|
32
|
+
MCP surface). Two workflows lean on this one skill: the **known-scenario
|
|
33
|
+
sweep** ([`qa-run.md`](../../../../workflows/qa-run.md)) walks a resolved
|
|
34
|
+
Gherkin scenario set, and **exploratory driving**
|
|
35
|
+
([`qa-explore.md`](../../../../workflows/qa-explore.md)) walks a named surface
|
|
36
|
+
the agent has no script for. Each **procedure** — argument parsing, phase
|
|
37
|
+
gates, contract resolution sequence — is the SSOT in its own workflow; this
|
|
38
|
+
skill shows **how** to apply the driving and instrumentation conventions both
|
|
39
|
+
depend on. Assertion-tier rules live in
|
|
40
|
+
[`testing-standards.md`](../../../../rules/testing-standards.md)
|
|
32
41
|
(§ Assertion Placement); scenario prose conventions live in
|
|
33
42
|
[`gherkin-authoring`](../gherkin-authoring/SKILL.md); browser-locator
|
|
34
|
-
discipline is shared with [`playwright`](../playwright/SKILL.md)
|
|
35
|
-
|
|
43
|
+
discipline is shared with [`playwright`](../playwright/SKILL.md); browser
|
|
44
|
+
instrumentation lives in
|
|
45
|
+
[`browser-testing-with-devtools`](../../../core/browser-testing-with-devtools/SKILL.md).
|
|
46
|
+
Read this skill before instrumenting a live surface; read the workflow for the
|
|
47
|
+
run order.
|
|
36
48
|
|
|
37
49
|
## 1. Driving Rules Live in One Prose Home
|
|
38
50
|
|
|
39
|
-
The driving rules
|
|
40
|
-
**semantic `Then
|
|
41
|
-
browser-action mapping, the per-`When` write guard, mandatory
|
|
42
|
-
redaction, and the **sequential-only** browser rule — are stated once
|
|
43
|
-
[`qa-run-scenario.md`](../../../../workflows/helpers/qa-run-scenario.md)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
+
The driving rules both modes depend on — **navigation-first / never
|
|
52
|
+
URL-jump**, **semantic `Then`** assertion against the accessibility snapshot,
|
|
53
|
+
the Gherkin → browser-action mapping, the per-`When` write guard, mandatory
|
|
54
|
+
evidence redaction, and the **sequential-only** browser rule — are stated once
|
|
55
|
+
in [`qa-run-scenario.md`](../../../../workflows/helpers/qa-run-scenario.md).
|
|
56
|
+
Apply them from there; this skill does not restate them. In short: reach every
|
|
57
|
+
surface the way a real user would (start at a root, click affordances, never
|
|
58
|
+
deep-link a starting state), assert semantically (roles, accessible names,
|
|
59
|
+
visible text — never DOM/CSS/XPath selectors, HTTP status, response bodies, or
|
|
60
|
+
DB rows), and treat broken navigation — a missing affordance, a nav 404, a
|
|
61
|
+
guard redirect loop — as a **finding, not a workaround**. Assertion-tier rules
|
|
62
|
+
are in
|
|
51
63
|
[`testing-standards.md` § Assertion Placement](../../../../rules/testing-standards.md#assertion-placement).
|
|
52
64
|
|
|
53
|
-
## 2.
|
|
65
|
+
## 2. The Read-Only Capture Invariant
|
|
66
|
+
|
|
67
|
+
Capture is **strictly read-only** in both modes. This invariant is inviolable
|
|
68
|
+
per [`security-baseline.md`](../../../../rules/security-baseline.md) — it is
|
|
69
|
+
not a soft preference.
|
|
70
|
+
|
|
71
|
+
- **No source edits.** The agent does not modify application code, config, or
|
|
72
|
+
tests while driving. Driving observes; it never repairs.
|
|
73
|
+
- **No product mutations.** No creating, updating, or deleting product data, no
|
|
74
|
+
destructive form submissions, no irreversible actions "to see what happens".
|
|
75
|
+
When a surface's only path forward is a mutating action, record the boundary
|
|
76
|
+
as the finding and stop — do not cross it.
|
|
77
|
+
- **The only write is the ledger.** The single permitted side effect is
|
|
78
|
+
appending finding lines under `temp/qa/<sessionId>`.
|
|
79
|
+
- **Scrub before persisting.** Strip tokens, session cookies, `Authorization`
|
|
80
|
+
headers, and PII from captured console and network evidence via the shared
|
|
81
|
+
redaction path **before** any finding reaches disk or GitHub. Captured
|
|
82
|
+
evidence is untrusted until scrubbed.
|
|
83
|
+
- **HITL gates every write outward.** Phase transitions and GitHub writes
|
|
84
|
+
(ticket creation, promotion) happen only behind an operator confirmation
|
|
85
|
+
gate; the agent never files or promotes findings autonomously.
|
|
86
|
+
|
|
87
|
+
## 3. Contract, Environment & Sign-In
|
|
54
88
|
|
|
55
|
-
|
|
56
|
-
|
|
89
|
+
- **Resolve the `qa` contract first.** Before any browser work, resolve the
|
|
90
|
+
contract via `resolveQaContract(config)`. When the block is absent,
|
|
91
|
+
malformed, or missing a required field, the resolver **throws** — relay its
|
|
92
|
+
verbatim message and STOP. There is no auto-detection fallback.
|
|
93
|
+
- **Resolve the environment.** `resolveQaEnvironment` keys each deployment
|
|
94
|
+
target to `{ name, baseUrl, signInSeam, allowWrites }`. Where the operator's
|
|
95
|
+
input does not pin an unambiguous target and the contract declares more than
|
|
96
|
+
one environment, prompt (or accept `defaultEnvironment`) — never silently
|
|
97
|
+
pick one. Record the resolved name on the ledger.
|
|
98
|
+
- **Sign in through the seam, never inline.** The two seam kinds:
|
|
99
|
+
- **`kind: 'url'` (dev impersonation).** Substitute the persona **name** into
|
|
100
|
+
the seam's URL template and navigate there. The persona name is the sole
|
|
101
|
+
input; no per-persona auth material is read.
|
|
102
|
+
- **`kind: 'skill'` (procedural / credential).** Invoke the named consumer
|
|
103
|
+
sign-in skill, which reads a per-persona **`credentialRef`** — an indirect
|
|
104
|
+
handle to a stored credential, never an inline secret. Read that skill's
|
|
105
|
+
`SKILL.md` and follow it.
|
|
106
|
+
|
|
107
|
+
The agent MUST NOT type real usernames, passwords, or tokens, and MUST NOT
|
|
108
|
+
fabricate or forge a session. This is a hard security boundary, not a
|
|
109
|
+
convenience to work around. Confirm authenticated state with a
|
|
110
|
+
`take_snapshot` before driving.
|
|
111
|
+
- **No headless fallback.** The chrome-devtools MCP surface is a host-provided
|
|
112
|
+
runtime dependency. If it is unavailable, degrade with a clear error and stop
|
|
113
|
+
— never fall back to the retired headless BDD runner.
|
|
57
114
|
|
|
58
|
-
|
|
115
|
+
## 4. Mode — Known-Scenario Sweep (`/qa-run`)
|
|
116
|
+
|
|
117
|
+
The sweep drives a resolved scenario set and instruments each surface the
|
|
118
|
+
moment it lands on it, before moving on. Capture is **per surface** so evidence
|
|
119
|
+
is attributable to a concrete user-reachable state. Record each scenario's
|
|
120
|
+
result (pass / fail / blocked), the surface it ended on, and a one-line symptom
|
|
121
|
+
for any failure.
|
|
122
|
+
|
|
123
|
+
### 4.1 Console
|
|
59
124
|
|
|
60
125
|
1. Capture with `list_console_messages` on the current surface.
|
|
61
126
|
2. Filter through the contract's `consoleAllowlist` using
|
|
@@ -75,13 +140,13 @@ Allowlist matching is case-sensitive substring matching, so patterns stay
|
|
|
75
140
|
readable in `.agentrc.json` without regex escaping; a blank pattern is ignored
|
|
76
141
|
rather than matching everything.
|
|
77
142
|
|
|
78
|
-
###
|
|
143
|
+
### 4.2 Network
|
|
79
144
|
|
|
80
145
|
Capture with `list_network_requests` on the surface. Failed requests and
|
|
81
146
|
error-status responses (4xx / 5xx) become findings alongside the
|
|
82
147
|
console-derived set, sharing the same `F#` numbering across the surface.
|
|
83
148
|
|
|
84
|
-
###
|
|
149
|
+
### 4.3 Design-token visual check
|
|
85
150
|
|
|
86
151
|
When the contract's `designTokens` pointer is set (it defaults to `null`),
|
|
87
152
|
spot-check the rendered surface against the token source. Flag **gross** token
|
|
@@ -98,10 +163,10 @@ A gross violation is one a designer would call a regression on sight; subtle
|
|
|
98
163
|
sub-pixel differences are not harness findings. When `designTokens` is `null`,
|
|
99
164
|
skip this check entirely — do not invent a token source.
|
|
100
165
|
|
|
101
|
-
|
|
166
|
+
### 4.4 The `F#` finding shape
|
|
102
167
|
|
|
103
|
-
Every captured problem is normalized into the structured `F#`
|
|
104
|
-
|
|
168
|
+
Every captured problem is normalized into the structured `F#` shape so the
|
|
169
|
+
sweep can record it onto the shared ledger (each `F#` finding becomes one
|
|
105
170
|
`QaLedgerItem` — see [`qa-core.md`](../../../../workflows/helpers/qa-core.md))
|
|
106
171
|
and the schema validates:
|
|
107
172
|
|
|
@@ -111,7 +176,7 @@ and the schema validates:
|
|
|
111
176
|
"classification": "console-error", // console-error | network-error | visual-token | ...
|
|
112
177
|
"surface": "/invoices", // the user-reachable surface, not a deep link
|
|
113
178
|
"symptom": "...", // one-line user-visible / captured symptom
|
|
114
|
-
"likelyRootCause": null, //
|
|
179
|
+
"likelyRootCause": null, // null unless the cause is evidenced, not guessed
|
|
115
180
|
"disposition": "follow-up", // blocker | follow-up
|
|
116
181
|
"acceptance": null, // AC this folds into, when known
|
|
117
182
|
"evidence": {
|
|
@@ -124,77 +189,71 @@ and the schema validates:
|
|
|
124
189
|
- **Determinism is load-bearing.** Re-running the same selector over the same
|
|
125
190
|
captured console with the same allowlist yields the same findings in the same
|
|
126
191
|
order. Do not reorder or renumber findings between sweeps.
|
|
127
|
-
- **Scrub before rendering.** Before any finding's `evidence` is rendered or
|
|
128
|
-
drafted, strip tokens, session cookies, Authorization headers, and PII from
|
|
129
|
-
the captured console and network per
|
|
130
|
-
[`security-baseline.md`](../../../../rules/security-baseline.md). Findings are
|
|
131
|
-
posted to GitHub at approval time — captured evidence is untrusted until
|
|
132
|
-
scrubbed.
|
|
133
|
-
|
|
134
|
-
## 4. Framework-Generic Heuristic Cards
|
|
135
|
-
|
|
136
|
-
The harness ships **framework-generic** root-cause heuristics — they reason
|
|
137
|
-
about symptoms, not about any one frontend framework. Use a card to populate
|
|
138
|
-
`likelyRootCause` and to set `disposition`. The cards are guidance, not a
|
|
139
|
-
classifier: when a symptom matches none cleanly, leave `likelyRootCause: null`
|
|
140
|
-
and let the operator triage from the symptom.
|
|
141
|
-
|
|
142
|
-
| Symptom pattern | Likely root cause | Default disposition |
|
|
143
|
-
| --- | --- | --- |
|
|
144
|
-
| `404` / `Not Found` on a navigation or asset request | Dead route, broken link, or missing build artifact | follow-up (blocker if it breaks the scenario path) |
|
|
145
|
-
| `401` / `403` reaching a surface the persona should see | Missing or over-tight authorization check; guard misconfig | blocker |
|
|
146
|
-
| `500` / `502` / `503` on a user action | Server-side fault behind the action | blocker |
|
|
147
|
-
| Uncaught `TypeError` / `ReferenceError` in console | Null/undefined dereference or missing binding in client code | blocker when it breaks the surface, else follow-up |
|
|
148
|
-
| `Failed to fetch` / `NetworkError` / CORS-rejected request | Misconfigured CORS allowlist, wrong origin, or a downed dependency | follow-up |
|
|
149
|
-
| Hydration / mismatch warning escalated to error | Server/client render divergence | follow-up |
|
|
150
|
-
| Off-palette color, off-scale spacing/typography | Design-token drift — hard-coded value bypassing the token | follow-up |
|
|
151
|
-
| Repeated identical console error across many surfaces | A shared component or global bootstrap fault | record once; the shared route/dedup core collapses duplicates at triage |
|
|
152
|
-
|
|
153
|
-
Heuristics for working the cards:
|
|
154
|
-
|
|
155
|
-
- **Record once, let dedup collapse.** When the same error fires on many
|
|
156
|
-
surfaces, record it once rather than filing N copies; the shared
|
|
157
|
-
classify/route/dedup core ([`qa-core.md`](../../../../workflows/helpers/qa-core.md))
|
|
158
|
-
collapses duplicates at triage against the fingerprint footer.
|
|
159
192
|
- **Blocker vs. follow-up.** A finding is a **blocker** when it breaks the
|
|
160
193
|
scenario's user-visible outcome or exposes an authorization gap. Everything
|
|
161
|
-
else
|
|
194
|
+
else — noise that does not break the journey, cosmetic token drift — is a
|
|
162
195
|
**follow-up**.
|
|
163
196
|
- **Symptom over diagnosis.** When unsure of the root cause, record the precise
|
|
164
197
|
symptom and leave `likelyRootCause: null`. A wrong guess is worse than an
|
|
165
198
|
honest "unknown" the operator can triage.
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
199
|
+
- **Record once, let dedup collapse.** When the same error fires on many
|
|
200
|
+
surfaces, record it once rather than filing N copies; the shared
|
|
201
|
+
classify/route/dedup core collapses duplicates at triage against the
|
|
202
|
+
fingerprint footer.
|
|
203
|
+
|
|
204
|
+
## 5. Mode — Exploratory Driving (`/qa-explore`)
|
|
205
|
+
|
|
206
|
+
Exploratory driving walks a named surface with no scenario script, capturing
|
|
207
|
+
what it observes. It is the agent-led half of exploratory QA; its human-led
|
|
208
|
+
sibling is `/qa-assist` (the human drives, the agent scribes). Everything in
|
|
209
|
+
§§ 1–3 applies unchanged; the deltas are the method choice and the per-surface
|
|
210
|
+
boundary.
|
|
211
|
+
|
|
212
|
+
- **Pick the driving method explicitly at Plan time** — drive (default) vs.
|
|
213
|
+
static — and record it in the ledger. Do not switch methods mid-surface
|
|
214
|
+
without a new Plan note.
|
|
215
|
+
- **Drive is the default.** When the resolved environment carries a
|
|
216
|
+
`signInSeam`, authenticated surfaces — including deployed hosts — are
|
|
217
|
+
**driven** through that seam, not statically deferred.
|
|
218
|
+
- **Static driving is the documented interim**, chosen **only where no seam
|
|
219
|
+
resolves** for the target environment. It walks the surface from source,
|
|
220
|
+
route definitions, and rendered markup rather than a running browser:
|
|
221
|
+
- **Never a silent fallback.** Static is a deliberate Plan-phase decision
|
|
222
|
+
recorded with its reason ("environment: preview, method: static, reason: no
|
|
223
|
+
seam resolves"), not something the agent slips into when the browser MCP
|
|
224
|
+
hiccups.
|
|
225
|
+
- **Interim, not equivalent.** Static driving cannot exercise real
|
|
226
|
+
authorization, routing guards, or runtime console/network signal. Treat its
|
|
227
|
+
coverage as partial and say so in the ledger; a static pass does not close
|
|
228
|
+
the same coverage a driven pass would.
|
|
229
|
+
- **Same read-only invariant** (§ 2) applies identically.
|
|
230
|
+
- **Promote to driving when a seam lands.** Re-run the surface driven rather
|
|
231
|
+
than leaving it permanently static.
|
|
232
|
+
- **A surface that could not be driven is itself a signal.** Where the
|
|
233
|
+
environment resolves no seam, drive the unauthenticated surface or fall back
|
|
234
|
+
to static and record the gap — never a silent skip.
|
|
235
|
+
- **Observe, do not fabricate.** Never script the runtime to manufacture an
|
|
236
|
+
outcome the exploration is meant to discover.
|
|
237
|
+
|
|
238
|
+
## 6. Record onto the Ledger & Triage (Never File Autonomously)
|
|
239
|
+
|
|
240
|
+
Record each finding as a `QaLedgerItem` on the shared session ledger under
|
|
170
241
|
`temp/qa/`, then route the ledger through the shared classify → route →
|
|
171
242
|
disposition → promote core — both stated once in
|
|
172
243
|
[`qa-core.md`](../../../../workflows/helpers/qa-core.md). The harness **MUST
|
|
173
|
-
NOT** create tickets autonomously: findings are promoted through
|
|
174
|
-
after the operator confirms each disposition at the HITL
|
|
175
|
-
is the safety boundary against spurious filing.
|
|
176
|
-
|
|
177
|
-
## 6. Sign-In & Contract Discipline
|
|
178
|
-
|
|
179
|
-
- **Resolve the `qa` contract first.** Before any browser work, resolve the
|
|
180
|
-
contract via `resolveQaContract(config)`. When the block is absent,
|
|
181
|
-
malformed, or missing a required field, the resolver **throws** — relay its
|
|
182
|
-
verbatim message and STOP. There is no auto-detection fallback.
|
|
183
|
-
- **Dev seam only.** Sign in once per persona via the contract's `signInSeam`
|
|
184
|
-
(`kind: 'url'` dev seam or `kind: 'skill'`). **Never** enter real
|
|
185
|
-
credentials. Confirm authenticated state with a `take_snapshot` before
|
|
186
|
-
driving any scenario.
|
|
187
|
-
- **No headless fallback.** The chrome-devtools MCP surface is a host-provided
|
|
188
|
-
runtime dependency. If it is unavailable, degrade with a clear error and stop
|
|
189
|
-
— never fall back to the retired headless BDD runner.
|
|
244
|
+
NOT** create tickets autonomously: findings are promoted through
|
|
245
|
+
`/mandrel-plan` only after the operator confirms each disposition at the HITL
|
|
246
|
+
write gate. That gate is the safety boundary against spurious filing.
|
|
190
247
|
|
|
191
248
|
## 7. Cross-References
|
|
192
249
|
|
|
193
|
-
-
|
|
250
|
+
- Sweep procedure (SSOT): [`qa-run.md`](../../../../workflows/qa-run.md).
|
|
251
|
+
- Exploration procedure (SSOT): [`qa-explore.md`](../../../../workflows/qa-explore.md).
|
|
194
252
|
- Driving rules (one prose home): [`qa-run-scenario.md`](../../../../workflows/helpers/qa-run-scenario.md).
|
|
195
253
|
- Shared QA core (contract/session/redaction/ledger/triage/HITL): [`qa-core.md`](../../../../workflows/helpers/qa-core.md).
|
|
196
254
|
- Console filter module: [`console-allowlist.js`](../../../../scripts/lib/qa/console-allowlist.js).
|
|
197
255
|
- Assertion-tier rules: [`testing-standards.md`](../../../../rules/testing-standards.md).
|
|
198
256
|
- Scenario prose: [`gherkin-authoring`](../gherkin-authoring/SKILL.md).
|
|
199
257
|
- Browser-locator discipline: [`playwright`](../playwright/SKILL.md).
|
|
200
|
-
-
|
|
258
|
+
- Browser instrumentation: [`browser-testing-with-devtools`](../../../core/browser-testing-with-devtools/SKILL.md).
|
|
259
|
+
- Evidence scrubbing / read-only boundary: [`security-baseline.md`](../../../../rules/security-baseline.md).
|
|
@@ -17,8 +17,9 @@ Reach for it when the automated hygiene left an unusual state behind.
|
|
|
17
17
|
> **When to run**: after a session that landed several PRs, or before starting a
|
|
18
18
|
> new Story, to put the local checkout into a known tidy state.
|
|
19
19
|
>
|
|
20
|
-
> **Persona**: `devops-engineer`
|
|
21
|
-
> `
|
|
20
|
+
> **Persona**: `devops-engineer` — git guidance comes from the always-on
|
|
21
|
+
> [`rules/git-conventions.md`](../rules/git-conventions.md), which needs no
|
|
22
|
+
> skill citation.
|
|
22
23
|
|
|
23
24
|
The enumeration + reap logic lives in
|
|
24
25
|
[`git-cleanup.js`](../scripts/git-cleanup.js) — it computes the candidate list,
|
|
@@ -19,8 +19,9 @@ explicitly; the interactive choice prompt fires **only** when the detected
|
|
|
19
19
|
state is genuinely ambiguous, so the common path stays non-interactive and
|
|
20
20
|
scriptable.
|
|
21
21
|
|
|
22
|
-
> **Persona**: `devops-engineer`
|
|
23
|
-
> `
|
|
22
|
+
> **Persona**: `devops-engineer` — git guidance comes from the always-on
|
|
23
|
+
> [`rules/git-conventions.md`](../rules/git-conventions.md), which needs no
|
|
24
|
+
> skill citation.
|
|
24
25
|
|
|
25
26
|
---
|
|
26
27
|
|
|
@@ -113,6 +113,18 @@ mid-delivery, and evaluates the actual work product.
|
|
|
113
113
|
optional advisory pre-flight — a criterion cannot be scored `met` without
|
|
114
114
|
the supporting `verify[]` evidence where a `verify[]` command is relevant
|
|
115
115
|
to it.
|
|
116
|
+
- **Reuses the credited full-suite run instead of re-paying for it.**
|
|
117
|
+
Before spawning a `verify[]` entry, classify it with `resolveVerifyCredit`
|
|
118
|
+
from
|
|
119
|
+
[`verify-credit.js`](../../scripts/lib/orchestration/verify-credit.js): an
|
|
120
|
+
entry that is itself a full-suite command (`npm test`, `pnpm run test`,
|
|
121
|
+
a bare `node --test`) is consulted against the **same stamp close reads**
|
|
122
|
+
and, when that stamp is fresh, recorded as `pass` with a `detail` naming
|
|
123
|
+
the credit — **never respawned**. A stale or absent stamp reports
|
|
124
|
+
`spawn: true` and the command runs for real, so the credit can never
|
|
125
|
+
manufacture a pass. The gate warns on any such entry: the intended shape
|
|
126
|
+
is scoped `verify[]` entries **plus** the one credited run
|
|
127
|
+
([`deliver-digest.md`](deliver-digest.md) § 5).
|
|
116
128
|
- **Shares `lint` / `typecheck` evidence with close.** When a
|
|
117
129
|
`verify[]` command is **byte-identical** to a close-validation gate — in
|
|
118
130
|
practice only the cheap, command-identical `lint` and `typecheck` gates
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
description: >-
|
|
3
3
|
The deliver path's one bundled framework read. Carries what
|
|
4
4
|
every Story delivery always needs — dispatch decision, engine invariants,
|
|
5
|
-
the change-set/ceremony incantation, the acceptance-eval gate,
|
|
6
|
-
terminal envelope contract — so the engine reads one
|
|
7
|
-
re-reading the helper/schema set each session.
|
|
5
|
+
the change-set/ceremony incantation, the acceptance-eval gate, the credited
|
|
6
|
+
full-suite run, and the terminal envelope contract — so the engine reads one
|
|
7
|
+
file instead of re-reading the helper/schema set each session.
|
|
8
8
|
---
|
|
9
9
|
|
|
10
10
|
# Deliver digest (read once per session)
|
|
@@ -27,10 +27,8 @@ rule produces it:
|
|
|
27
27
|
whatever its shape — sub-agent isolation is load-bearing only against a
|
|
28
28
|
*concurrent* sibling racing the same checkout, and a one-Story run has none.
|
|
29
29
|
2. **Every other run is `subagent`.** A multi-Story run dispatches every Story
|
|
30
|
-
as a sub-agent however trivial its shape
|
|
31
|
-
|
|
32
|
-
on one beat. Shape still sets ceremony; the `route::lite` label is a
|
|
33
|
-
human-visible hint, never the control signal.
|
|
30
|
+
as a sub-agent however trivial its shape. Shape still sets ceremony; the
|
|
31
|
+
`route::lite` label is a human-visible hint, never the control signal.
|
|
34
32
|
|
|
35
33
|
`inline` removes model-side fan-out only — no `story-worker` boot, no fresh
|
|
36
34
|
acceptance-critic spawn. **`subagent` and `inline` run the same engine**: same
|
|
@@ -112,7 +110,30 @@ an unmerged cluster verdict scores a fraction of the criteria and still reports
|
|
|
112
110
|
not close**: post a `friction` comment and flip `agent::blocked`.
|
|
113
111
|
Per-round mechanics: [`acceptance-self-eval.md`](acceptance-self-eval.md).
|
|
114
112
|
|
|
115
|
-
## 5.
|
|
113
|
+
## 5. The one creditable full-suite run
|
|
114
|
+
|
|
115
|
+
**After the self-eval loop's last fix commit, immediately before the push** —
|
|
116
|
+
the credit is keyed on the tree, so any later commit invalidates it. Redraft
|
|
117
|
+
rounds run scoped tests; only this final run needs credit, and a bare
|
|
118
|
+
`npm test` / `pnpm run test` deposits **none**, so close re-runs the identical
|
|
119
|
+
suite. Shape it by the predicate `close-validation/gates.js` uses for its test
|
|
120
|
+
gate:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
# CRAP gate on (default) + a `test:coverage` script — writes the stamp the
|
|
124
|
+
# close `coverage-capture` gate reads:
|
|
125
|
+
node <main-repo>/.agents/scripts/coverage-capture.js --cwd <workCwd>
|
|
126
|
+
# otherwise — the record the close `test` gate reads. <workCwd> ABSOLUTE,
|
|
127
|
+
# runner exactly `npm test`: both sides hash {cmd, args, cwd}.
|
|
128
|
+
node <main-repo>/.agents/scripts/evidence-gate.js --standalone \
|
|
129
|
+
--scope-id <storyId> --gate test --worktree <workCwd> -- npm test
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`verify[]` is scoped entries **plus** this one run: an entry that is itself a
|
|
133
|
+
full-suite command is reported credited against the same stamp, never
|
|
134
|
+
respawned.
|
|
135
|
+
|
|
136
|
+
## 6. Terminal envelope — the return contract
|
|
116
137
|
|
|
117
138
|
`single-story-close.js` emits exactly one envelope on stdout between
|
|
118
139
|
`--- STORY DELIVER TERMINAL ---` markers, schema-validated against
|
|
@@ -129,7 +150,7 @@ Relay it verbatim; never hand-compose one, never substitute prose.
|
|
|
129
150
|
|
|
130
151
|
Required fields: `kind` (`story-deliver-terminal`), `storyId`, `status`,
|
|
131
152
|
`phase`, `elapsedSeconds`, `nextCommand`. `phase` is one of `init`,
|
|
132
|
-
`wrong-tree-guard`, `
|
|
153
|
+
`wrong-tree-guard`, `base-sync`, `close-validation`, `push`, `pull-request`,
|
|
133
154
|
`code-review`, `auto-merge`, `confirm-merge`, `post-land`, `done`. `gates`
|
|
134
155
|
reports every gate as `passed` / `failed` / `skipped` — a skipped gate is
|
|
135
156
|
reported, never omitted, so a missing gate is never read as a passing one.
|
|
@@ -139,7 +160,7 @@ reported, never omitted, so a missing gate is never read as a passing one.
|
|
|
139
160
|
success; a failed gate replays its tail inline. `AGENT_LOG_LEVEL=verbose`
|
|
140
161
|
restores live streaming.
|
|
141
162
|
|
|
142
|
-
##
|
|
163
|
+
## 7. When to leave this file
|
|
143
164
|
|
|
144
165
|
- Unclear state / a re-run refusal → `deliver-recover.js --story <id>` (read-only).
|
|
145
166
|
- Lease, sweep, worktree-scope detail → [`deliver-story-reference.md`](deliver-story-reference.md).
|
|
@@ -233,17 +233,19 @@ runs maker-blind at Story-scope review inside the close subprocess. The
|
|
|
233
233
|
dispatch step produces `checklistPath` from the Story's predicted footprint
|
|
234
234
|
before it spawns the worker — see [`/mandrel-deliver`](../mandrel-deliver.md).
|
|
235
235
|
|
|
236
|
-
**
|
|
236
|
+
**Full-suite discipline (spine Step 2.5).** Repo-invariant guards —
|
|
237
237
|
drift-guard and schema tests living outside the Story's scoped greps — are
|
|
238
238
|
the failure class that actually bounces deliveries: close-validation
|
|
239
239
|
discovers them only after the whole close pipeline has run, at several times
|
|
240
|
-
the cost of one
|
|
240
|
+
the cost of one full-suite run in the worktree.
|
|
241
241
|
|
|
242
|
-
**Run it so close can credit it.**
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
242
|
+
**Run it once, last, so close can credit it.** The run belongs **after** the
|
|
243
|
+
self-eval loop's last fix commit and immediately **before** the hand-off push,
|
|
244
|
+
so its stamp describes the tree that is pushed; redraft rounds run scoped
|
|
245
|
+
tests. Close skips a gate that already passed at the current HEAD, but a bare
|
|
246
|
+
`npm test` deposits no such record — the suite then runs twice per delivery,
|
|
247
|
+
once here and once in the close gate chain. Pick the invocation by the same
|
|
248
|
+
predicate `close-validation/gates.js` uses to choose its test gate:
|
|
247
249
|
|
|
248
250
|
```bash
|
|
249
251
|
# CRAP gate enabled (default) + a `test:coverage` script — writes the stamp
|
|
@@ -259,7 +261,15 @@ node <main-repo>/.agents/scripts/evidence-gate.js --standalone \
|
|
|
259
261
|
The credit expires the moment it stops describing the tree: evidence is keyed
|
|
260
262
|
on HEAD, the capture stamp on a content digest of `crap.targetDirs`. A
|
|
261
263
|
self-eval fix — or any commit — invalidates it and close re-runs the suite for
|
|
262
|
-
real, so this never trades away the gate.
|
|
264
|
+
real, so this never trades away the gate. That keying is exactly why the run
|
|
265
|
+
comes last.
|
|
266
|
+
|
|
267
|
+
**`verify[]` reuses the same stamp.** A `verify[]` entry that is itself a
|
|
268
|
+
full-suite command is reported **credited** against that stamp rather than
|
|
269
|
+
respawned (`resolveVerifyCredit` in
|
|
270
|
+
[`verify-credit.js`](../../scripts/lib/orchestration/verify-credit.js)), and the
|
|
271
|
+
self-eval gate warns when it sees one: the intended shape is scoped `verify[]`
|
|
272
|
+
entries **plus** the single credited run.
|
|
263
273
|
|
|
264
274
|
**Conflict with `main` mid-implementation** → resolve as you would any branch
|
|
265
275
|
rebase. There is no `epic/<id>` intermediate, so the rebase base is `main`
|
|
@@ -500,22 +510,16 @@ judgment that help text cannot carry.
|
|
|
500
510
|
|
|
501
511
|
The `single-story-close.js` script, in order:
|
|
502
512
|
|
|
503
|
-
1.
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
goes to `temp/orchestration/close-gates-<storyId>.log`; a clean run reports
|
|
507
|
-
one digest line naming that artifact, and a **failed** gate replays its
|
|
508
|
-
captured tail inline so the evidence is in front of you without opening a
|
|
509
|
-
file. Read the artifact when you need the full text — or re-run under
|
|
510
|
-
`AGENT_LOG_LEVEL=verbose` for live streaming.
|
|
511
|
-
1a. **Syncs the Story branch from `origin/<baseBranch>`** before push.
|
|
512
|
-
Runs `git fetch origin <baseBranch>` followed by
|
|
513
|
-
`git merge --no-edit origin/<baseBranch>` inside the worktree. This
|
|
513
|
+
1. **Syncs the Story branch from `origin/<baseBranch>`** — before the gates,
|
|
514
|
+
not after them. Runs `git fetch origin <baseBranch>` followed
|
|
515
|
+
by `git merge --no-edit origin/<baseBranch>` inside the worktree. This
|
|
514
516
|
defends against the parallel-`/deliver-story` race: when
|
|
515
517
|
multiple sessions run in parallel, the Story that auto-merges first
|
|
516
518
|
bumps `baseBranch`, and without this sync the lagging Stories open
|
|
517
519
|
PRs that are "behind base" and stall against branch-protection's
|
|
518
|
-
`up-to-date branch` rule.
|
|
520
|
+
`up-to-date branch` rule. Running it first also means a conflict costs
|
|
521
|
+
no gate run at all, and — the load-bearing half — the tree the gates
|
|
522
|
+
validate is the tree the push sends. Outcomes:
|
|
519
523
|
- **No-op / fast-forward / clean merge-commit** → close proceeds to
|
|
520
524
|
push.
|
|
521
525
|
- **Merge conflict** → the merge is aborted, a `friction` structured
|
|
@@ -533,12 +537,26 @@ The `single-story-close.js` script, in order:
|
|
|
533
537
|
closes the PR-open-time race but a residual race remains between PR
|
|
534
538
|
open and auto-merge fire.
|
|
535
539
|
|
|
536
|
-
2.
|
|
537
|
-
|
|
540
|
+
2. Runs the close-validation gates against `baseBranch` as the baseline.
|
|
541
|
+
On any gate failure it throws — the operator fixes and re-runs close.
|
|
542
|
+
The chain fails cheapest-first: `typecheck`, `lint`, `format` and the
|
|
543
|
+
coverage-independent half of the baselines gate
|
|
544
|
+
(`check-baselines-independent`) run in parallel, and only once they are
|
|
545
|
+
green does the serial walk pay for `coverage-capture` and the
|
|
546
|
+
coverage-consuming half (`check-baselines-coverage`).
|
|
547
|
+
**Gate output is captured, not streamed.** Every gate line
|
|
548
|
+
goes to `temp/orchestration/close-gates-<storyId>.log`; a clean run reports
|
|
549
|
+
one digest line naming that artifact, and a **failed** gate replays its
|
|
550
|
+
captured tail inline so the evidence is in front of you without opening a
|
|
551
|
+
file. Read the artifact when you need the full text — or re-run under
|
|
552
|
+
`AGENT_LOG_LEVEL=verbose` for live streaming.
|
|
553
|
+
|
|
554
|
+
3. Pushes `story-<id>` to `origin`.
|
|
555
|
+
4. Probes for an existing open PR with `head = story-<id>`. If none
|
|
538
556
|
exists, opens one via `gh pr create --base <baseBranch>`. The PR
|
|
539
557
|
body carries `Closes #<storyId>` so the GitHub merge auto-closes the
|
|
540
558
|
issue.
|
|
541
|
-
|
|
559
|
+
4a. **Enables GitHub native auto-merge by default** via
|
|
542
560
|
`gh pr merge <prNumber> --auto --squash --delete-branch`. Once CI's
|
|
543
561
|
required checks turn green, GitHub squash-merges the PR and deletes
|
|
544
562
|
the source branch — the operator does not need to babysit the merge
|
|
@@ -546,7 +564,7 @@ The `single-story-close.js` script, in order:
|
|
|
546
564
|
non-fatal: the operator retains the manual merge surface in the
|
|
547
565
|
GitHub UI. Pass `--no-auto-merge` to opt out when the PR needs a
|
|
548
566
|
pre-merge eyeball.
|
|
549
|
-
|
|
567
|
+
5. Flips the Story to **`agent::closing`** (NOT `agent::done`) and leaves
|
|
550
568
|
the GitHub issue **OPEN**. Auto-merge completes
|
|
551
569
|
asynchronously _after_ this script exits, so closing the issue here
|
|
552
570
|
would strand a CLOSED issue with no merged work if the PR later failed
|
|
@@ -557,9 +575,9 @@ The `single-story-close.js` script, in order:
|
|
|
557
575
|
`--no-wait-merge` run, or the in-close confirm phase on the
|
|
558
576
|
close-and-land default. (Step 5.5 is the Status-column resync.) A Story
|
|
559
577
|
only reaches `agent::done` once its PR to `main` is confirmed merged.
|
|
560
|
-
|
|
578
|
+
6. Reaps the worktree when `delivery.worktreeIsolation.reapOnSuccess`
|
|
561
579
|
is enabled.
|
|
562
|
-
|
|
580
|
+
7. **Releases the Story lease.** Clears the Story assignment
|
|
563
581
|
that init claimed so the next `/deliver-story` run sees an
|
|
564
582
|
unclaimed ticket. The release is a no-op when the operator no longer
|
|
565
583
|
holds the claim (a later run took over via reclaim/steal), so a late
|
|
@@ -572,13 +590,15 @@ The `single-story-close.js` script, in order:
|
|
|
572
590
|
de-assigning the ticket. The close result carries
|
|
573
591
|
`leaseReleased: <boolean>`.
|
|
574
592
|
|
|
575
|
-
`--skip-validation` bypasses
|
|
593
|
+
`--skip-validation` bypasses the gate step. Use only when re-running
|
|
576
594
|
close after a fixed gate failure that's already known to pass.
|
|
577
595
|
|
|
578
|
-
`--skip-sync` bypasses
|
|
579
|
-
close after a hand-resolved sync, or in tests.
|
|
596
|
+
`--skip-sync` bypasses the base-sync step. Use only when re-running
|
|
597
|
+
close after a hand-resolved sync, or in tests. The two flags are
|
|
598
|
+
independent: either, both or neither may be set, and each elides exactly
|
|
599
|
+
its own phase.
|
|
580
600
|
|
|
581
|
-
`--no-auto-merge` disables
|
|
601
|
+
`--no-auto-merge` disables the auto-merge arm (step 4a). Use when the PR materially changes
|
|
582
602
|
behaviour and warrants pre-merge review.
|
|
583
603
|
|
|
584
604
|
---
|