@mmerterden/multi-agent-pipeline 20.27.2 → 20.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +6 -5
- package/README.tr.md +6 -5
- package/docs/adr/0008-installer-modularization-and-secret-leak-defense.md +0 -1
- package/docs/architecture.md +4 -4
- package/docs/ecosystem.md +5 -5
- package/docs/facts.json +14 -12
- package/docs/features.md +2 -0
- package/install/_dev-only-files.mjs +1 -0
- package/install/catalog-history.json +1 -1
- package/install/index.mjs +0 -2
- package/manifest.json +67 -44
- package/package.json +4 -1
- package/pipeline/agents/code-reviewer.md +1 -0
- package/pipeline/agents/test-gen-analyst.md +98 -0
- package/pipeline/commands/multi-agent/sync/SKILL.md +3 -3
- package/pipeline/commands/multi-agent/test-gen/SKILL.md +124 -0
- package/pipeline/multi-agent-refs/cross-cli-contract.md +11 -5
- package/pipeline/multi-agent-refs/features/doctor.md +13 -0
- package/pipeline/multi-agent-refs/features/test-classification.md +76 -0
- package/pipeline/multi-agent-refs/features/usage-reporting.md +17 -0
- package/pipeline/multi-agent-refs/help/en.md +1 -0
- package/pipeline/multi-agent-refs/help/tr.md +1 -0
- package/pipeline/multi-agent-refs/phases/operations.md +2 -2
- package/pipeline/multi-agent-refs/phases/phase-2-dev.md +6 -7
- package/pipeline/multi-agent-refs/phases/phase-3-review.md +3 -3
- package/pipeline/multi-agent-refs/phases/phase-5-report.md +6 -5
- package/pipeline/multi-agent-refs/test-gen/stages.md +227 -0
- package/pipeline/preferences-template.json +3 -0
- package/pipeline/schemas/prefs.schema.json +14 -0
- package/pipeline/schemas/test-gen-discovery.schema.json +73 -0
- package/pipeline/schemas/test-gen-gaps.schema.json +122 -0
- package/pipeline/schemas/test-gen-inventory.schema.json +59 -0
- package/pipeline/schemas/test-gen-results.schema.json +44 -0
- package/pipeline/scripts/_handle-patterns.mjs +166 -0
- package/pipeline/scripts/_http-post.mjs +141 -0
- package/pipeline/scripts/_stack-routing.mjs +4 -0
- package/pipeline/scripts/_test-gaming.mjs +227 -0
- package/pipeline/scripts/_unified-diff.mjs +104 -0
- package/pipeline/scripts/_worktree-snapshot.mjs +113 -0
- package/pipeline/scripts/doctor.mjs +51 -0
- package/pipeline/scripts/evidence-gate.mjs +74 -3
- package/pipeline/scripts/handle-diff-audit.mjs +188 -0
- package/pipeline/scripts/routine-registry.mjs +5 -1
- package/pipeline/scripts/run-ui-tests.sh +27 -3
- package/pipeline/scripts/test-classify.mjs +296 -0
- package/pipeline/scripts/test-gen-gate.mjs +409 -0
- package/pipeline/scripts/test-gen-state.mjs +313 -0
- package/pipeline/scripts/test-integrity-gate.mjs +76 -11
- package/pipeline/scripts/uninstall.mjs +1 -0
- package/pipeline/scripts/usage-register-beacon.mjs +239 -0
- package/pipeline/scripts/usage-register.mjs +128 -33
- package/pipeline/scripts/usage-report.mjs +43 -33
- package/pipeline/skills/.skill-manifest.json +23 -3
- package/pipeline/skills/.skills-index.json +57 -2
- package/pipeline/skills/shared/README.md +10 -5
- package/pipeline/skills/shared/core/multi-agent-sync/SKILL.md +3 -3
- package/pipeline/skills/shared/core/multi-agent-test-gen/SKILL.md +120 -0
- package/pipeline/skills/shared/external/fix-review-verdicts/SKILL.md +133 -0
- package/pipeline/skills/shared/external/handle-request/SKILL.md +97 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/SKILL.md +129 -0
- package/pipeline/skills/shared/external/scenario-gap-analysis/references/scenario-schema.md +60 -0
- package/pipeline/skills/shared/external/test-result-triage/SKILL.md +160 -0
- package/pipeline/skills/skills-index.md +7 -2
- package/install/_telemetry.mjs +0 -61
|
@@ -33,7 +33,7 @@ Run all steps automatically:
|
|
|
33
33
|
```
|
|
34
34
|
Step 0: DOCTOR node $HOME/.claude/scripts/doctor.mjs - exit 2 or 4 STOPS the sync
|
|
35
35
|
Step 1: DETECT Compare timestamps, find stale targets
|
|
36
|
-
Step 2: COPILOT Claude Code -> Copilot CLI (instructions +
|
|
36
|
+
Step 2: COPILOT Claude Code -> Copilot CLI (instructions + 66 sub-command skills)
|
|
37
37
|
Step 2b: CODEX Claude Code -> Codex CLI (1 router skill + sub-command specs as refs + agent TOML)
|
|
38
38
|
Step 3: REPO Claude Code -> pipeline repo (genericized, personal data scrub)
|
|
39
39
|
Step 3d: DEV-TOOLKIT Companion MCP server -> detect movement, ship gates, commit + publish
|
|
@@ -229,7 +229,7 @@ When invoked with the `release` argument:
|
|
|
229
229
|
|-------------|-------------|
|
|
230
230
|
| `~/.claude/commands/multi-agent/{cmd}/SKILL.md` | `~/.copilot/skills/multi-agent-{cmd}/SKILL.md` |
|
|
231
231
|
|
|
232
|
-
**
|
|
232
|
+
**66 commands are synced** (canonical inventory - must match `cross-cli-contract.md` section 1; drift = contract violation):
|
|
233
233
|
|
|
234
234
|
```
|
|
235
235
|
analysis, analysis-jira, analysis-resolve, autopilot, autopilot-off,
|
|
@@ -241,7 +241,7 @@ prune-prompts, purge, refactor, research, resume, review,
|
|
|
241
241
|
review-analysis, review-issue, review-jira, route-off, route-on,
|
|
242
242
|
route-status, routines, save, scaffold, scan, scenario-audit, search,
|
|
243
243
|
security-review, serve, setup, stack, status, steer, store-ready, sync, test, test-accessibility,
|
|
244
|
-
test-dark-mode, test-dynamic-type, test-screenshots, testflight-validation,
|
|
244
|
+
test-dark-mode, test-dynamic-type, test-gen, test-screenshots, testflight-validation,
|
|
245
245
|
uninstall, update
|
|
246
246
|
```
|
|
247
247
|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: multi-agent-test-gen
|
|
3
|
+
description: "Write the tests an app lacks, stage by stage, each gated on its artefact: inventory, read-only gap analysis, handles, tests, runs, per-test classification. Use when screens, flows or a change need tests."
|
|
4
|
+
user-invocable: true
|
|
5
|
+
argument-hint: "[scope] [--platform ios|android|web] [--repo <path>] - scope: a screen, route or flow, or empty for what changed since the last test-gen run"
|
|
6
|
+
metadata:
|
|
7
|
+
language: en
|
|
8
|
+
not-for: test, bug-bash, scenario-audit, manual-test
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
# multi-agent-test-gen
|
|
12
|
+
|
|
13
|
+
**Input**: $ARGUMENTS
|
|
14
|
+
|
|
15
|
+
Writes the tests an app is missing, in the repo's own frameworks, and reports
|
|
16
|
+
what they found. Seven stages run as subagents, each leaving one artefact that a
|
|
17
|
+
script gates before the next stage starts: inventory, discovery, gaps, handles,
|
|
18
|
+
write, run, classify. The two analysis stages run as a read-only agent type, so
|
|
19
|
+
nothing they do can change the repository.
|
|
20
|
+
|
|
21
|
+
Stage contract, read before Step 1:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
$HOME/.claude/multi-agent-refs/test-gen/stages.md
|
|
25
|
+
$HOME/.claude/multi-agent-refs/features/test-classification.md
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Scripts: `test-gen-gate.mjs` (artefact, handle and write gates),
|
|
29
|
+
`handle-diff-audit.mjs` (handles only), `test-classify.mjs`,
|
|
30
|
+
`test-gen-state.mjs` (change base, revision cap), `evidence-gate.mjs`, all under
|
|
31
|
+
`$HOME/.claude/scripts/`. Schemas: `test-gen-inventory`, `test-gen-gaps`,
|
|
32
|
+
`test-gen-results` under `$HOME/.claude/schemas/`.
|
|
33
|
+
|
|
34
|
+
## Gotchas
|
|
35
|
+
|
|
36
|
+
- An analysis stage that wants to "just fix" something is out of its lane. Discovery and gaps go to `test-gen-analyst`, whose grant is an allowlist of read-only tools.
|
|
37
|
+
- The handle stage adds handles and nothing else. A wrapped view, a renamed element or a changed condition fails `handle-diff-audit.mjs`; revert the hunk, do not argue with the audit.
|
|
38
|
+
- `HANDLES: none-needed` is a claim that the handles exist. It needs the `file:line` of each.
|
|
39
|
+
- A test that passed once is not green. Unstable is a fail, and a red test with no base to compare is `not_measured`, not a regression.
|
|
40
|
+
- Test data is named in `dataNeeds[]`, never filled in by the run. A credential is a `secretRef`.
|
|
41
|
+
- A product bug is reported, never fixed here. In autopilot it parks the run.
|
|
42
|
+
|
|
43
|
+
## Step 1 - Prepare
|
|
44
|
+
|
|
45
|
+
1. Repo: `--repo`, else the current directory; confirm it in one line.
|
|
46
|
+
Platform: `--platform`, else detect as `multi-agent-test` does. Run id
|
|
47
|
+
`tg-<timestamp>`; `RUN="$HOME/.claude/logs/test-gen/<repo>/<timestamp>"`,
|
|
48
|
+
created with `mkdir -p "$RUN"`.
|
|
49
|
+
2. Toolkit: `test_results_classify`, `coverage_gap` and `ui_locator_codegen`
|
|
50
|
+
need `@mmerterden/multi-agent-toolkit-mcp` >= v3.25.0. Check the tool list
|
|
51
|
+
once. Each missing tool has a fallback in `stages.md` (classify from the
|
|
52
|
+
repeats, the analyst matches coverage itself with every match `inferred`,
|
|
53
|
+
locators written by hand from the handles); say which fallbacks are in use,
|
|
54
|
+
once.
|
|
55
|
+
3. Change base. `TG_STATE="$HOME/.claude/knowledge/<project>/test-gen-state.json"`:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
node "$HOME/.claude/scripts/test-gen-state.mjs" base-scan --state "$TG_STATE" --repo "$REPO" \
|
|
59
|
+
--graph "$HOME/.claude/knowledge/<project>/code-graph.json" > "$RUN/base-scan.json"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Exit 0: the changed files and their dependents narrow an empty scope. Exit 1
|
|
63
|
+
(`measured: false`, first run or base gone): nothing is narrowed, and the
|
|
64
|
+
reason goes in the report.
|
|
65
|
+
|
|
66
|
+
## Step 2 - Inventory and priority
|
|
67
|
+
|
|
68
|
+
Stage 1 of `stages.md`, then the priority pick: ask which items to cover
|
|
69
|
+
(one question to the user; autopilot: the top three, recorded as `autopilot-top3`). Gate:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
node "$HOME/.claude/scripts/test-gen-gate.mjs" check --stage inventory --file "$RUN/inventory.json"
|
|
73
|
+
node "$HOME/.claude/scripts/evidence-gate.mjs" --claim artefact --status passed \
|
|
74
|
+
--evidence "$RUN/inventory.json" --stage-start "$STAGE_START"
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Step 3 - Discovery and gaps (read-only)
|
|
78
|
+
|
|
79
|
+
Stages 2 and 3, both with `subagent_type: "test-gen-analyst"`. Write each answer
|
|
80
|
+
with the Write tool, then gate each file with `evidence-gate.mjs --claim
|
|
81
|
+
artefact` against its own stage start, and each with `test-gen-gate.mjs check`
|
|
82
|
+
(`--stage discovery`, `--stage gaps`). Show the gaps (bucket, priority,
|
|
83
|
+
expectation) and the `dataNeeds[]`; ask for the data or the credential names
|
|
84
|
+
(autopilot: write only the scenarios whose needs are met).
|
|
85
|
+
|
|
86
|
+
## Step 4 - Handles
|
|
87
|
+
|
|
88
|
+
Stage 4. Snapshot first (`test-gen-gate.mjs baseline`), then the subagent, then
|
|
89
|
+
both gates against that snapshot: `test-gen-gate.mjs handles` and
|
|
90
|
+
`handle-diff-audit.mjs --baseline` exit 0. The gates read only the stage's own
|
|
91
|
+
change; the person's uncommitted work is never the stage's to revert. Skip the
|
|
92
|
+
stage when no gap has `handlesNeeded`, and say so.
|
|
93
|
+
|
|
94
|
+
## Step 5 - Write, run, classify
|
|
95
|
+
|
|
96
|
+
Stages 5 to 7: a new snapshot, the tests and `written.json`, the
|
|
97
|
+
`test-gen-gate.mjs written --baseline` gate (test files only, inside the repo
|
|
98
|
+
and changed by the stage, no test gaming, a discovered runner, more than zero
|
|
99
|
+
tests run), the repeated runs, then
|
|
100
|
+
`results.json` through `test-classify.mjs` and `test-gen-gate.mjs results`.
|
|
101
|
+
|
|
102
|
+
## Step 6 - Triage
|
|
103
|
+
|
|
104
|
+
Stage 8, one verdict per non-pass result, each through `test-gen-state.mjs
|
|
105
|
+
revise`. Exit 3 parks the run: stop, keep every artefact, report why, and record
|
|
106
|
+
no change base, because the run did not finish.
|
|
107
|
+
|
|
108
|
+
## Step 7 - Report and base
|
|
109
|
+
|
|
110
|
+
Print, in this order: the tests written (file, scenario, expectation), the
|
|
111
|
+
results table from `results.json` with every `fail` and `not_measured` reason,
|
|
112
|
+
the triage verdicts, the handles added (`handles.md`), the scenarios blocked by
|
|
113
|
+
data, and `gaps.json`'s `unmeasured[]`. State which parts are mechanical (the
|
|
114
|
+
gates, the classification, the handle audit) and which are judgement (the
|
|
115
|
+
inventory, the gaps, the triage). Then record the change base
|
|
116
|
+
(`test-gen-state.mjs base-record`).
|
|
117
|
+
|
|
118
|
+
Offer, one question: open the product bugs through `multi-agent` with each
|
|
119
|
+
failing test as the acceptance check, or stop. **No commit, no push, no issue
|
|
120
|
+
from this command.**
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: fix-review-verdicts
|
|
3
|
+
description: "Independent fix review with three verdicts, test-gaming, scope and root-cause, each backed by file:line findings; the result is computed from findings, never from a claim. Use when reviewing a fix made to turn tests green. Not a full review checklist."
|
|
4
|
+
metadata:
|
|
5
|
+
source: multi-agent-pipeline
|
|
6
|
+
version: 1.0.0
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Fix review verdicts
|
|
10
|
+
|
|
11
|
+
Tests can be made green without fixing anything: hide the element only under
|
|
12
|
+
test, branch on an automation flag, loosen a check, swallow the error. The
|
|
13
|
+
tests pass and report nothing. Only someone reading the diff catches it, and
|
|
14
|
+
that reader should not be the one who wrote the fix.
|
|
15
|
+
|
|
16
|
+
## 1. Inputs
|
|
17
|
+
|
|
18
|
+
The reviewer does not see the author's conversation. It reads:
|
|
19
|
+
|
|
20
|
+
- the approved root-cause statement: what broke, where, and the agreed fix;
|
|
21
|
+
- the diff;
|
|
22
|
+
- the evidence runs: did the expected tests turn green, did the guard tests
|
|
23
|
+
stay green;
|
|
24
|
+
- machine check results (lint, types, unit tests) from the project's own
|
|
25
|
+
approved commands;
|
|
26
|
+
- the author's report, as a claim to verify, not as evidence.
|
|
27
|
+
|
|
28
|
+
The reviewer changes no code. Its only output is the review.
|
|
29
|
+
|
|
30
|
+
## 2. The three required verdicts
|
|
31
|
+
|
|
32
|
+
Each one gets an explicit verdict and a reason.
|
|
33
|
+
|
|
34
|
+
**test-gaming: `clean` | `violation`.** A violation is any of:
|
|
35
|
+
|
|
36
|
+
- an environment branch in product code: `navigator.webdriver`,
|
|
37
|
+
`process.env.CI`, `window.Cypress`, a user-agent or URL-parameter check,
|
|
38
|
+
`ProcessInfo` arguments or environment, an `isRunningTests` flag, a
|
|
39
|
+
test-only build flag;
|
|
40
|
+
- detecting automation, or showing or hiding an element only under test;
|
|
41
|
+
- hard-coding the text or value the test looks for;
|
|
42
|
+
- a swallowed error: an empty `catch`, a discarded `try?`, an ignored
|
|
43
|
+
`runCatching`;
|
|
44
|
+
- a loosened product validation, an inflated timeout, a skipped or disabled
|
|
45
|
+
test;
|
|
46
|
+
- an existing assertion, fixture or snapshot changed so that it accepts what
|
|
47
|
+
it used to reject (a weaker matcher, a widened tolerance, a re-recorded
|
|
48
|
+
snapshot, a fixture edited to dodge the bug) in a fix whose cause is in
|
|
49
|
+
product code.
|
|
50
|
+
|
|
51
|
+
Not a violation: a new regression test that encodes the bug, fails on the
|
|
52
|
+
unfixed code and passes on the fix. That is the bug-fix protocol (a failing
|
|
53
|
+
test first), and it is expected. Check it actually fails without the fix; a
|
|
54
|
+
new test that passes on the unfixed base proves nothing and is a suggestion
|
|
55
|
+
finding.
|
|
56
|
+
|
|
57
|
+
**scope: `clean` | `violation`.** The diff touches only the approved cause.
|
|
58
|
+
Unrelated files, reformatting, opportunistic refactors, lockfile or dependency
|
|
59
|
+
changes and configuration changes are violations.
|
|
60
|
+
|
|
61
|
+
**root-cause: `root` | `symptom`.** Does the fix remove the cause or suppress
|
|
62
|
+
the result? Fixing contrast by hiding the element, overflow by clipping it, an
|
|
63
|
+
error by deleting its message, a race by retrying until it passes, or bad
|
|
64
|
+
input by clamping it are symptoms.
|
|
65
|
+
|
|
66
|
+
## 3. Findings
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{ "severity": "blocking", "area": "test-gaming", "file": "src/cart/Total.tsx", "line": 42, "note": "returns the expected total when navigator.webdriver is set" }
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
- `severity`: `blocking` or `suggestion`.
|
|
73
|
+
- `area`: `test-gaming`, `scope`, `symptom`, `correctness`, `security`,
|
|
74
|
+
`performance`, `maintainability`, `accessibility`.
|
|
75
|
+
- Each verdict maps to one finding area: test-gaming `violation` to area
|
|
76
|
+
`test-gaming`, scope `violation` to area `scope`, root-cause `symptom` to
|
|
77
|
+
area `symptom`. Every such verdict comes with at least one blocking finding
|
|
78
|
+
in its area. A correctness or security defect is blocking.
|
|
79
|
+
- A suspicion without a line to show is a suggestion with its reason. A line
|
|
80
|
+
that shows test-gaming, scope creep or a symptom fix is blocking, without
|
|
81
|
+
hedging.
|
|
82
|
+
|
|
83
|
+
## 4. Computing the result
|
|
84
|
+
|
|
85
|
+
The result is derived, never declared. An author's "all green", a reviewer's
|
|
86
|
+
"looks good" and a summary field saying passed are ignored. It has three
|
|
87
|
+
states: `passed`, `blocked`, `not_measured`.
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
VERDICT_AREA = {test-gaming: violation -> test-gaming,
|
|
91
|
+
scope: violation -> scope,
|
|
92
|
+
root-cause: symptom -> symptom}
|
|
93
|
+
|
|
94
|
+
if the review was never submitted:
|
|
95
|
+
result = not_measured
|
|
96
|
+
elif any of the three verdicts is missing or outside its allowed values:
|
|
97
|
+
result = not_measured # an invalid review measures nothing
|
|
98
|
+
else:
|
|
99
|
+
for each (verdict, value) -> area in VERDICT_AREA that matches:
|
|
100
|
+
if no blocking finding exists in that area:
|
|
101
|
+
add a blocking finding in that area, built from the verdict's reason
|
|
102
|
+
for each machine check:
|
|
103
|
+
failed on the fix and not on the base -> blocking finding (correctness)
|
|
104
|
+
changed the working tree -> blocking finding (checks read, never write)
|
|
105
|
+
failed on the base as well -> preexisting: reported, not blocking
|
|
106
|
+
had no approved command (undefined) -> unmeasured check
|
|
107
|
+
could not run (tool missing, timeout) -> unmeasured check
|
|
108
|
+
if any finding is blocking:
|
|
109
|
+
result = blocked
|
|
110
|
+
elif any unmeasured check:
|
|
111
|
+
result = not_measured
|
|
112
|
+
else:
|
|
113
|
+
result = passed
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`blocked` wins over `not_measured`: a known defect is reported even when part
|
|
117
|
+
of the review could not run. `not_measured` never counts as `passed`; the fix
|
|
118
|
+
does not ship on it, and the report names what was not measured and why.
|
|
119
|
+
|
|
120
|
+
## 5. Output
|
|
121
|
+
|
|
122
|
+
The structured review (three verdicts with reasons, findings, a two to four
|
|
123
|
+
sentence summary) and a report: each verdict with its reason, the findings
|
|
124
|
+
with `file:line`, and a "could not measure" section.
|
|
125
|
+
|
|
126
|
+
## Related
|
|
127
|
+
|
|
128
|
+
- The stack's review rubric for the general pass: `ai-android-toolkit`
|
|
129
|
+
`reference/review-checklist`, `ai-ios-toolkit:ios-coding-standard`, and the
|
|
130
|
+
built-in `/code-review`.
|
|
131
|
+
- `security-review` when the diff touches auth, input handling or secrets.
|
|
132
|
+
- `test-result-triage`, which decides whether a red test was a product bug
|
|
133
|
+
before any fix is written.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: handle-request
|
|
3
|
+
description: "Add the test handles a scenario needs and answer each request with ADDED, ALREADY or COULD NOT ADD plus file:line or reason. Identity values only, attribute-only diffs; iOS accessibilityIdentifier, Compose testTag, web data-testid. Use when a test needs an element handle the UI lacks."
|
|
4
|
+
metadata:
|
|
5
|
+
source: multi-agent-pipeline
|
|
6
|
+
version: 1.0.0
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Handle request
|
|
10
|
+
|
|
11
|
+
A handle is the identifier a test finds an element by: it does not change with
|
|
12
|
+
copy, locale or layout. This step adds the handles a scenario list asks for,
|
|
13
|
+
and nothing else.
|
|
14
|
+
|
|
15
|
+
## 1. Input
|
|
16
|
+
|
|
17
|
+
A list of requests, usually the "handles needed" section of
|
|
18
|
+
`scenario-gap-analysis`. Each request names the element, the screen, a
|
|
19
|
+
proposed identifier and the scenario it unblocks.
|
|
20
|
+
|
|
21
|
+
## 2. One answer per request
|
|
22
|
+
|
|
23
|
+
| Answer | When |
|
|
24
|
+
|---|---|
|
|
25
|
+
| `ADDED file:line` | The handle was added at that line. |
|
|
26
|
+
| `ALREADY file:line` | A usable handle already exists there; give its value. |
|
|
27
|
+
| `COULD NOT ADD reason` | It was not added. The reason is specific: element rendered by a third-party view, identifier must come from a shared schema that lacks the key, repository is read-only, the element is not exposed to the accessibility tree and exposing it would change what assistive technology hears. |
|
|
28
|
+
|
|
29
|
+
No other answer exists. A request that is half done is split into the part
|
|
30
|
+
added and the part that could not be.
|
|
31
|
+
|
|
32
|
+
## 3. Rules
|
|
33
|
+
|
|
34
|
+
- **Identity, never position.** A row carries the item's id
|
|
35
|
+
(`order_row_{orderId}`), never the loop index, which shifts when the list is
|
|
36
|
+
reordered or filtered. Do not rewrite a list-rendering expression to obtain
|
|
37
|
+
an index or an id; if the id is not in scope, answer COULD NOT ADD.
|
|
38
|
+
- **Attribute-only diffs.** Add identifier attributes or modifiers. No logic,
|
|
39
|
+
state, flow, data, styling, renames or deletions. Anything a line loses must
|
|
40
|
+
reappear in the added line.
|
|
41
|
+
- **Stable names.** Screen-scoped, descriptive, independent of visible text
|
|
42
|
+
and locale (`cart_checkout_button`). Follow the project's existing naming
|
|
43
|
+
scheme and, where it generates identifiers from a shared schema, use the
|
|
44
|
+
generated constant; a key missing from the schema is added upstream, so the
|
|
45
|
+
answer is COULD NOT ADD with that reason.
|
|
46
|
+
- **One pass.** Write every requested handle before building. On mobile each
|
|
47
|
+
build costs minutes; one handle per build turns a step into hours.
|
|
48
|
+
- **Out of bounds.** Build files, manifests and permissions, signing,
|
|
49
|
+
dependencies. No commit: leave the diff for a human.
|
|
50
|
+
|
|
51
|
+
## 4. The handle has to be visible where the test runs
|
|
52
|
+
|
|
53
|
+
A handle in source that the test driver cannot see is worse than none: it
|
|
54
|
+
looks done. For each handle, confirm it reaches the tree the test reads:
|
|
55
|
+
|
|
56
|
+
| Platform | Handle | Seen by the driver only when |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| iOS (SwiftUI, UIKit) | `accessibilityIdentifier` | the element is in the accessibility tree: not hidden, not merged into a parent element |
|
|
59
|
+
| Android (Compose) | `Modifier.testTag` | inside the app's Compose tests always; for UI Automator based drivers only under `testTagsAsResourceId` on an ancestor |
|
|
60
|
+
| Android (Views) | `android:id` | the view is laid out and not excluded from accessibility |
|
|
61
|
+
| Web | `data-testid` (or the project's configured test id attribute) | the component forwards it to the DOM and the build under test does not strip it |
|
|
62
|
+
| Flutter | `Semantics(identifier: ...)` for external drivers; `Key` for in-process tests | a `Key` works with `find.byKey` inside `flutter test` and integration tests, but never reaches the accessibility tree, so an external driver (XCUITest, UI Automator, Maestro, Appium) needs the `Semantics` identifier |
|
|
63
|
+
|
|
64
|
+
Verify against the running app when a build loop exists: a UI tree dump
|
|
65
|
+
(`ios_get_ui_tree`, `android_get_ui_tree`), a DOM query or `web_find`. When no
|
|
66
|
+
build loop is available in this step, the report says "not verified on device"
|
|
67
|
+
and gives the build and install commands for a human. Never claim a check that
|
|
68
|
+
did not run.
|
|
69
|
+
|
|
70
|
+
The stack plugins carry the platform detail: `ai-ios-toolkit:handle-request`,
|
|
71
|
+
`ai-android-toolkit:handle-request`, `ai-frontend-toolkit:handle-request`.
|
|
72
|
+
|
|
73
|
+
## 5. Check the diff before finishing
|
|
74
|
+
|
|
75
|
+
- `git diff -U0` shows added handle lines; their count is greater than zero
|
|
76
|
+
unless every answer is ALREADY or COULD NOT ADD.
|
|
77
|
+
- Every removed line comes back in an added line with only the identifier
|
|
78
|
+
added.
|
|
79
|
+
- No file outside the UI sources the requests point to.
|
|
80
|
+
|
|
81
|
+
## 6. Report
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
cart_checkout_button ADDED src/cart/CartFooter.tsx:31
|
|
85
|
+
order_row_{orderId} ALREADY src/orders/OrderRow.tsx:12 (data-testid="order-row-{id}")
|
|
86
|
+
map_pin COULD NOT ADD rendered by a third-party map view with no identifier API
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Then: a diff summary (files, lines changed), the scenarios these handles
|
|
90
|
+
unblock, and what was not verified on device, with the reason.
|
|
91
|
+
|
|
92
|
+
## Related
|
|
93
|
+
|
|
94
|
+
- `scenario-gap-analysis` produces the requests.
|
|
95
|
+
- `test-result-triage` sends a locator-not-found result here before calling a
|
|
96
|
+
test stale.
|
|
97
|
+
- `e2e-testing` for the locator order a test uses once the handle exists.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: scenario-gap-analysis
|
|
3
|
+
description: "Set an inventory against existing tests, sort each behaviour into COVERED, WRONGLY COVERED or MISSING, and write gaps as measurable, prioritised scenarios with cross-screen flows. Use when planning tests or auditing coverage. Not for writing tests or judging a run."
|
|
4
|
+
metadata:
|
|
5
|
+
source: multi-agent-pipeline
|
|
6
|
+
version: 1.0.0
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Scenario gap analysis
|
|
10
|
+
|
|
11
|
+
Decide **what** has to be tested, and why. How to test it belongs to the
|
|
12
|
+
writing step. The output is a ranked list of measurable scenarios that a test
|
|
13
|
+
writer can turn into code without asking what was meant.
|
|
14
|
+
|
|
15
|
+
## 1. Read before you call anything missing
|
|
16
|
+
|
|
17
|
+
Gather the inputs first, and read all of them:
|
|
18
|
+
|
|
19
|
+
- **Inventory**: screens, modules, flows or endpoints in scope, from a design
|
|
20
|
+
scenario inventory, a site map, a UI tree, or exploration reports.
|
|
21
|
+
- **Existing tests**: the test tree and any test catalogue. Search it for every
|
|
22
|
+
behaviour before listing that behaviour as a gap.
|
|
23
|
+
- **Regression history** where the project keeps one: past bugs, regression
|
|
24
|
+
maps, incident notes. A case that broke before is the first candidate.
|
|
25
|
+
- **Exploration notes**: what someone saw the product actually do.
|
|
26
|
+
|
|
27
|
+
Match tests to inventory items by **identifier** (accessibility identifier,
|
|
28
|
+
test id, route, endpoint path), never by a name that merely sounds similar.
|
|
29
|
+
When a match is inferred rather than read, say so on that line.
|
|
30
|
+
|
|
31
|
+
## 2. Three buckets
|
|
32
|
+
|
|
33
|
+
| Bucket | Meaning |
|
|
34
|
+
|---|---|
|
|
35
|
+
| **COVERED** | A test exercises the behaviour and its assertion would fail if the behaviour broke. |
|
|
36
|
+
| **WRONGLY COVERED** | A test exists and stays green while the defect is present. |
|
|
37
|
+
| **MISSING** | No test exercises the behaviour. |
|
|
38
|
+
|
|
39
|
+
WRONGLY COVERED is the most valuable bucket, because the suite reports safety
|
|
40
|
+
it does not provide. Typical shapes:
|
|
41
|
+
|
|
42
|
+
- A text scan over rendered text that never sees content injected as markup.
|
|
43
|
+
- A visibility filter that passes a broken image because its width is zero.
|
|
44
|
+
- An assertion on the HTTP status that passes while the body carries an error.
|
|
45
|
+
- A test that seeds the store or mocks the handoff it claims to verify.
|
|
46
|
+
- An existence check where the value is what matters (a total, a date, a count).
|
|
47
|
+
|
|
48
|
+
For every WRONGLY COVERED line, name the test (`file:line`) and the defect it
|
|
49
|
+
cannot see.
|
|
50
|
+
|
|
51
|
+
## 3. Scenario schema
|
|
52
|
+
|
|
53
|
+
Each MISSING or WRONGLY COVERED behaviour becomes one scenario:
|
|
54
|
+
|
|
55
|
+
| Field | Content |
|
|
56
|
+
|---|---|
|
|
57
|
+
| Title | Usable as the test name. |
|
|
58
|
+
| Priority | `bug-catching` (broke before or likely to, hurts the user directly: data, money, dates, a dead end in the funnel, leaked copy keys) / `edge` (empty, error, boundary, slow network, back navigation, session expiry, flag combinations) / `baseline` (happy path, element present). |
|
|
59
|
+
| Business reason | Who loses what, or which revenue or compliance risk it carries. "Nice to have" is not a reason. |
|
|
60
|
+
| Preconditions | Account, data and state the scenario needs, and where each comes from. |
|
|
61
|
+
| Steps | What the user does. |
|
|
62
|
+
| Measurable expectation | A claim a test can check exactly: a value, a label, a route, an element present or absent. A scenario with no measurable expectation is not written. |
|
|
63
|
+
| Automation | `yes` / `no` (stays a manual check, still listed) / `needs-code` (name the missing handle or hook). |
|
|
64
|
+
| Handles needed | Identity-carrying handles the test will locate by, for the handle step. |
|
|
65
|
+
| Data needs | What test data the scenario requires. The agent states the need; a human supplies the value. Never invent a credential. |
|
|
66
|
+
| Flakiness risk | Third-party scripts, live content, timing. Say it up front. |
|
|
67
|
+
|
|
68
|
+
The machine-readable shape and a worked example are in
|
|
69
|
+
[references/scenario-schema.md](references/scenario-schema.md).
|
|
70
|
+
|
|
71
|
+
## 4. Cross-screen flows (at least two)
|
|
72
|
+
|
|
73
|
+
Write at least two scenarios that walk a user path across screens. Only an
|
|
74
|
+
analysis that has seen the whole module can write them, and they catch the
|
|
75
|
+
class of defect screen tests cannot: screen A does not write the state screen
|
|
76
|
+
B reads. Both screens are green on their own and the flow is broken.
|
|
77
|
+
|
|
78
|
+
Rules for a flow scenario:
|
|
79
|
+
|
|
80
|
+
- Every step uses the state the previous step left. No seeding a store, no
|
|
81
|
+
injecting state, no deep link or direct route to an intermediate screen.
|
|
82
|
+
- Enter from the first screen and move forward the way a user does.
|
|
83
|
+
- Mock only noise outside the flow (a cookie banner, an analytics beacon),
|
|
84
|
+
never the data the flow carries: whether it is carried is the measurement.
|
|
85
|
+
- A long flow may be split into segments, each starting where the previous one
|
|
86
|
+
ended, with no seeding inside a segment.
|
|
87
|
+
|
|
88
|
+
Screen scenarios may seed state and deep link; they measure the screen's own
|
|
89
|
+
resilience. Flow scenarios may not.
|
|
90
|
+
|
|
91
|
+
When the scope cannot hold two flows (a single screen, a module with one
|
|
92
|
+
entry and no onward path), write the ones it holds and record the shortfall
|
|
93
|
+
under could-not-measure with the reason. Never pad the count with a flow that
|
|
94
|
+
seeds its own state.
|
|
95
|
+
|
|
96
|
+
## 5. Judgement rules
|
|
97
|
+
|
|
98
|
+
- Do not propose what is already covered. A covered case listed as missing is
|
|
99
|
+
noise, not analysis.
|
|
100
|
+
- Platforms differ. Device permissions and backgrounding have no web
|
|
101
|
+
equivalent; browser back, reload, deep links, horizontal overflow and
|
|
102
|
+
keyboard navigation are web-specific.
|
|
103
|
+
- Ask for handles that carry identity (`data-order-id`, a row keyed by the
|
|
104
|
+
item id), never a list index, which moves when the order changes.
|
|
105
|
+
- Open with "do these three first". An unranked list of twenty is not usable.
|
|
106
|
+
|
|
107
|
+
## 6. Could not measure
|
|
108
|
+
|
|
109
|
+
A separate section, always present. Each line names what could not be
|
|
110
|
+
measured, why (no account, environment down, screen unreachable, data
|
|
111
|
+
missing), and what would make it measurable. The absence of a measurement is
|
|
112
|
+
not a positive result and never folds into COVERED.
|
|
113
|
+
|
|
114
|
+
## 7. Output order
|
|
115
|
+
|
|
116
|
+
1. The top three, with one line each on why.
|
|
117
|
+
2. Bucket table: behaviour, bucket, test `file:line` where one exists.
|
|
118
|
+
3. Scenarios in priority order, flows included and marked as flows.
|
|
119
|
+
4. Handles needed, grouped by screen.
|
|
120
|
+
5. Data needs.
|
|
121
|
+
6. Could not measure.
|
|
122
|
+
|
|
123
|
+
## Related
|
|
124
|
+
|
|
125
|
+
- API and service gaps: `ai-backend-toolkit:scenario-gap-analysis` adds the
|
|
126
|
+
envelope, contract and service-chain rules.
|
|
127
|
+
- Adding the handles the scenarios ask for: `handle-request`.
|
|
128
|
+
- Judging a run of the written tests: `test-result-triage`.
|
|
129
|
+
- Driving a running app to observe behaviour first: `e2e-testing`.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Scenario schema
|
|
2
|
+
|
|
3
|
+
One JSON object per scenario. Field names are the contract a test writer or a
|
|
4
|
+
pipeline step reads; values follow the rules in `SKILL.md`.
|
|
5
|
+
|
|
6
|
+
```json
|
|
7
|
+
{
|
|
8
|
+
"id": "cart-total-after-coupon",
|
|
9
|
+
"title": "Cart total reflects a percentage coupon after quantity change",
|
|
10
|
+
"bucket": "WRONGLY_COVERED",
|
|
11
|
+
"coveredBy": "tests/cart/coupon.spec.ts:42",
|
|
12
|
+
"defectNotSeen": "asserts the coupon badge exists, never the total",
|
|
13
|
+
"priority": "bug-catching",
|
|
14
|
+
"businessReason": "A wrong total charges the customer a different amount than shown",
|
|
15
|
+
"kind": "screen",
|
|
16
|
+
"preconditions": ["signed-in account with an empty cart", "coupon code valid for 10 percent"],
|
|
17
|
+
"steps": ["add item A", "apply coupon", "change quantity to 3"],
|
|
18
|
+
"expectation": "total label equals 3 x unit price x 0.9, rounded to 2 decimals",
|
|
19
|
+
"automation": "yes",
|
|
20
|
+
"handles": ["cart_total_label", "cart_item_quantity_{itemId}"],
|
|
21
|
+
"dataNeeds": [{ "name": "coupon-10-percent", "supplied": "human" }],
|
|
22
|
+
"flakinessRisk": "none known"
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
| Field | Values |
|
|
27
|
+
|---|---|
|
|
28
|
+
| `bucket` | `COVERED`, `WRONGLY_COVERED`, `MISSING` |
|
|
29
|
+
| `coveredBy` | `file:line` of the existing test, or null for MISSING |
|
|
30
|
+
| `priority` | `bug-catching`, `edge`, `baseline` |
|
|
31
|
+
| `kind` | `screen` or `flow` |
|
|
32
|
+
| `automation` | `yes`, `no`, `needs-code` (with the reason in `automationNote`) |
|
|
33
|
+
| `dataNeeds[].supplied` | always `human` for credentials and personal data |
|
|
34
|
+
|
|
35
|
+
## A flow scenario
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{
|
|
39
|
+
"id": "flow-browse-to-confirmation",
|
|
40
|
+
"title": "Item chosen on the list arrives unchanged on the confirmation screen",
|
|
41
|
+
"bucket": "MISSING",
|
|
42
|
+
"priority": "bug-catching",
|
|
43
|
+
"kind": "flow",
|
|
44
|
+
"steps": [
|
|
45
|
+
"open the catalogue from the home screen",
|
|
46
|
+
"open item A and pick size M",
|
|
47
|
+
"add to cart and go to checkout",
|
|
48
|
+
"confirm"
|
|
49
|
+
],
|
|
50
|
+
"expectation": "confirmation shows item A, size M, the price shown on the item screen",
|
|
51
|
+
"rules": "no store seeding, no direct route to checkout, no mock of the cart request",
|
|
52
|
+
"automation": "yes"
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Could-not-measure entry
|
|
57
|
+
|
|
58
|
+
```json
|
|
59
|
+
{ "what": "refund screen", "why": "no account with a completed order", "needs": "a seeded order supplied by a human" }
|
|
60
|
+
```
|