@ivorycanvas/qamap 0.3.5 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +19 -14
  3. package/dist/behavior-intent.d.ts +6 -0
  4. package/dist/behavior-intent.js +172 -0
  5. package/dist/behavior-intent.js.map +1 -0
  6. package/dist/behavior-manifest.d.ts +6 -0
  7. package/dist/behavior-manifest.js +221 -0
  8. package/dist/behavior-manifest.js.map +1 -0
  9. package/dist/behavior.d.ts +143 -0
  10. package/dist/behavior.js +458 -0
  11. package/dist/behavior.js.map +1 -0
  12. package/dist/change-intent.d.ts +71 -0
  13. package/dist/change-intent.js +744 -0
  14. package/dist/change-intent.js.map +1 -0
  15. package/dist/cli.js +6 -5
  16. package/dist/cli.js.map +1 -1
  17. package/dist/e2e.d.ts +16 -1
  18. package/dist/e2e.js +286 -23
  19. package/dist/e2e.js.map +1 -1
  20. package/dist/index.d.ts +9 -1
  21. package/dist/index.js +4 -0
  22. package/dist/index.js.map +1 -1
  23. package/dist/qa.d.ts +1 -0
  24. package/dist/qa.js +73 -1
  25. package/dist/qa.js.map +1 -1
  26. package/dist/terminal.js +1 -1
  27. package/dist/terminal.js.map +1 -1
  28. package/dist/version.d.ts +1 -1
  29. package/dist/version.js +1 -1
  30. package/docs/adoption.md +11 -10
  31. package/docs/agent-format.md +9 -7
  32. package/docs/agent-skill.md +5 -2
  33. package/docs/architecture.md +131 -0
  34. package/docs/benchmarking.md +20 -3
  35. package/docs/commands.md +13 -7
  36. package/docs/e2e-output-examples.md +22 -8
  37. package/docs/quickstart-demo.md +9 -2
  38. package/docs/release-validation.md +32 -6
  39. package/docs/releasing.md +13 -0
  40. package/docs/roadmap.md +20 -10
  41. package/package.json +4 -2
  42. package/schema/qamap-agent.schema.json +42 -1
  43. package/schema/qamap-behavior.schema.json +307 -0
  44. package/skills/qamap-pr-qa/SKILL.md +16 -11
@@ -2,7 +2,7 @@
2
2
 
3
3
  QAMap's unit tests prove that the implementation behaves as coded. The benchmark contract checks a different question: does a representative PR receive a useful QA answer?
4
4
 
5
- `bench.config.json` is committed and runs in CI. Each target under `test/benchmarks/` contains a `base/` repository snapshot and a `head/` overlay. The runner materializes them as a temporary Git repository with a `main` baseline and one feature commit. QAMap reads those repositories but never installs dependencies or executes their code.
5
+ `bench.config.json` is committed and runs in CI. Each target under `test/benchmarks/` contains a `base/` repository snapshot and a `head/` overlay. The runner materializes them as a temporary Git repository with a `main` baseline and one feature commit. Intent fixtures set a synthetic `commitMessage` so commit-to-lifecycle behavior is part of the contract. QAMap reads those repositories but never installs dependencies or executes their code.
6
6
 
7
7
  ## Run the public contract
8
8
 
@@ -14,6 +14,9 @@ The command fails when any target violates its declared expectations. The initia
14
14
 
15
15
  - a web app with no tests;
16
16
  - a web app with Playwright and an existing mock handler;
17
+ - Vue and SvelteKit web changes with framework-native route files;
18
+ - a web preferences change that must become submit, persistence, request-failure, and re-entry QA instead of a generic journey;
19
+ - a mobile reminder change that must become scheduling, calendar, duplicate, resynchronization, and entry-routing QA;
17
20
  - an Expo app with Maestro;
18
21
  - an API service that should produce a contract checklist instead of a browser journey;
19
22
  - a design-token repository that should stay on artifact verification;
@@ -27,13 +30,24 @@ Each target can declare:
27
30
 
28
31
  | Field | Meaning |
29
32
  | --- | --- |
30
- | `runner` | Expected `playwright`, `maestro`, or `manual` recommendation. |
33
+ | `commitMessage` | Synthetic feature-commit message used when materializing a fixture. |
34
+ | `runner` | Expected `playwright`, `maestro`, or `manual` output adapter. Runner correctness alone is not a useful intent benchmark. |
31
35
  | `minFlows` | Minimum number of affected flows. |
36
+ | `minChangeIntents` | Minimum evidence-backed Change Intents. |
37
+ | `minHighConfidenceIntents` | Minimum intents supported strongly enough by commit and diff evidence to avoid mandatory review. |
38
+ | `minCommitBehaviorNodes` | Minimum Behavior Graph nodes carrying commit provenance. |
32
39
  | `minImportPropagatedFlows` | Minimum flows discovered through reverse imports. |
33
40
  | `minDiffAnchoredFlows` | Minimum flows using selector evidence introduced by the diff. |
34
41
  | `minManifestMatches` | Minimum domain, flow, and check matches from an external base manifest. |
35
42
  | `minManifestFlowMatches` | Minimum flow-level matches from the external base manifest. |
36
43
  | `minManifestBackedFlows` | Minimum QA flows that preserve manifest provenance. |
44
+ | `minManifestBehaviorNodes` | Minimum Behavior Graph nodes carrying verification-manifest evidence. |
45
+ | `mustHaveBehaviorKinds` | Behavior Graph node kinds that must be present, such as `flow`, `surface`, `source`, `assertion`, or `locator`. |
46
+ | `mustNameIntents` | Concrete terms that must appear in the inferred intent title. |
47
+ | `mustNotNameIntents` | Misleading terms that must not appear in inferred intent titles. |
48
+ | `mustIncludeLifecycle` | Trigger, condition, action, state, effect, or outcome terms that must survive in the ordered lifecycle. |
49
+ | `mustIncludeQaScenarios` | Failure, boundary, state-transition, or primary QA terms that must be proposed before runner compilation. |
50
+ | `mustFindIntentEvidence` | Commit or diff terms that must remain attached to intent provenance. |
37
51
  | `mustReachFiles` | Files that the selected flows must reach. |
38
52
  | `mustNameFlows` | Product terms that must appear in a user-facing flow title. |
39
53
  | `mustNotNameFlows` | Misleading flow-title terms that must not be emitted. |
@@ -41,6 +55,7 @@ Each target can declare:
41
55
  | `mustIncludeSteps` | Behavior terms that must appear in draft steps. |
42
56
  | `mustFindSelectors` | Stable selector evidence that must be recovered from the repository. |
43
57
  | `mustFindSuccessSignals` | Observable outcome text that must appear in the flow's success criteria. |
58
+ | `mustFindEntrypoints` | Route, screen, or command entrypoints that affected flows must recover. |
44
59
  | `mustFindEvidence` | Required evidence or fixture terms that must be reported. |
45
60
  | `mustNotFindEvidence` | Evidence terms that would be false positives for this change. |
46
61
  | `mustRecommendCommands` | Commands the setup or validation path must expose. |
@@ -62,7 +77,9 @@ node scripts/bench.mjs --baseline bench-results/<file>.json
62
77
 
63
78
  When both files exist, `pnpm bench` prefers the gitignored local config. CI always passes `--config bench.config.json --assert`, so private paths cannot affect the public quality gate.
64
79
 
65
- Saved results include flow titles, draft paths, recall gaps, readiness, agent payload size, and timing. Use a saved baseline to see heuristic movement, but treat the committed expectation contract as the merge gate.
80
+ Saved results include intent titles, lifecycle and scenario terms, flow titles, draft paths, recall gaps, readiness, agent payload size, and timing. Use a saved baseline to see heuristic movement, but treat the committed expectation contract as the merge gate.
81
+
82
+ Every benchmark target also enforces the Behavior Graph base contract: graph schema version 1, at least one graph flow for every planned flow, at least one impacted node for a non-empty diff, and no edge whose endpoint is missing. The table reports `graph n/i` as total nodes versus impacted nodes. These checks keep the graph connected to real PR analysis while framework-specific adapters are introduced incrementally.
66
83
 
67
84
  ## Adding a regression
68
85
 
package/docs/commands.md CHANGED
@@ -21,7 +21,7 @@ Use `pnpm dlx @ivorycanvas/qamap ...` for one-off runs without installing QAMap
21
21
 
22
22
  ## Reading The Output
23
23
 
24
- Human-facing reports (`text` and `markdown` formats) open with an **At a Glance** section: affected behavior, the reviewer question to answer before merge, concrete repository evidence, the proposed draft path, the next command, and the one or two missing trust requirements. When printed to an interactive terminal the report is colorized (headings, statuses, priority tags, inline commands); files written with `--output`, pipes, CI logs, and the machine formats (`json`, `agent`, `sarif`) are always plain. The standard `NO_COLOR` and `FORCE_COLOR` environment variables are honored.
24
+ Human-facing reports (`text` and `markdown` formats) open with an **At a Glance** section: evidence-backed change intent, behavior lifecycle, affected behavior, the reviewer question to answer before merge, concrete repository evidence, the proposed draft path, and missing trust requirements. Runner information appears later as an automation adapter. When printed to an interactive terminal the report is colorized (headings, statuses, priority tags, inline commands); files written with `--output`, pipes, CI logs, and the machine formats (`json`, `agent`, `sarif`) are always plain. The standard `NO_COLOR` and `FORCE_COLOR` environment variables are honored.
25
25
 
26
26
  Draft readiness is reported as a **stage on a fixed four-step journey**, for example `Stage: setup needed (1 of 4) — readiness 0/100`. A fresh repository usually starts at stage 1 — that is the expected starting point, not a failure. Each stage maps to a stable `readiness.level` value in the `json` and `agent` formats, which keeps machine output unchanged:
27
27
 
@@ -37,7 +37,9 @@ Draft readiness is reported as a **stage on a fixed four-step journey**, for exa
37
37
 
38
38
  On a changed branch, QAMap tries to produce reviewable verification artifacts instead of only saying "write more tests":
39
39
 
40
- - a branch-aware verification plan that names the changed domain, actor, trigger, goal, success signal, and edge cases
40
+ - a commit-and-diff-backed change intent with confidence, review requirements, and source evidence
41
+ - an ordered trigger, condition, action, state-change, side-effect, and observable-outcome lifecycle
42
+ - runner-independent primary, failure, boundary, and state-transition QA scenarios
41
43
  - draft Playwright, Maestro, CLI command, or manual checklist files when the repository shape supports them
42
44
  - a repo-level verification manifest loop where humans correct durable flows once and later PRs get sharper route/check/test draft suggestions
43
45
  - a runner setup proposal that explains why Playwright or Maestro fits the changed surface and which files/commands would be created if the team accepts it
@@ -61,9 +63,9 @@ That means QAMap is most valuable when it becomes the team's verification base:
61
63
  | `qamap eval . --base origin/main --head HEAD --pr-body-file pr-body.md` | Score change readiness across intent, risk, tests, and review size. |
62
64
  | `qamap github-action . --mode review --base origin/main --head HEAD` | Generate GitHub Action annotations, step summary, and PR comment body. |
63
65
  | `qamap test-plan . --base origin/main --head HEAD --include-working-tree` | Suggest domain test scenarios for changed files. |
64
- | `qamap qa . --base origin/main --head HEAD` | One-command PR QA: affected flows, missing QA evidence, a PR checklist, and E2E starter drafts, opening with At a Glance. |
65
- | `qamap qa . --base origin/main --head HEAD --format agent` | The same decision content as one ~2-4KB JSON line for coding agents — a versioned contract documented in [docs/agent-format.md](agent-format.md). |
66
- | `qamap e2e plan . --base origin/main --head HEAD` | Suggest E2E runner, bootstrap steps, user flows, coverage targets, existing test evidence, and missing testability hooks for changed files. |
66
+ | `qamap qa . --base origin/main --head HEAD` | One-command PR QA: change intent, behavior lifecycle, QA scenarios, affected flows, missing evidence, and optional automation drafts. |
67
+ | `qamap qa . --base origin/main --head HEAD --format agent` | The same decision content as one compact JSON line for coding agents — a versioned contract documented in [docs/agent-format.md](agent-format.md). |
68
+ | `qamap e2e plan . --base origin/main --head HEAD` | Derive change intent and QA scenarios, then map them to coverage, test evidence, testability gaps, and an automation adapter. |
67
69
  | `qamap e2e plan . --base origin/main --head HEAD --record-history` | Save a compact local run snapshot under `.qamap/runs/` while keeping JSON/Markdown output usable. |
68
70
  | `qamap e2e setup . --runner playwright` | Explicitly apply the accepted runner setup and create the first changed-flow E2E draft without overwriting existing files. |
69
71
  | `qamap e2e draft . --base origin/main --head HEAD --dry-run` | Preview generated Maestro, Playwright, or manual E2E drafts without writing files. |
@@ -90,7 +92,9 @@ For monorepos, pass `--workspace-root` when scanning a package. Package-local ch
90
92
 
91
93
  `qamap test-plan` turns changed file paths into a review-ready domain test checklist. It also discovers common validation commands from `package.json`, `pyproject.toml`, `go.mod`, `Cargo.toml`, Gradle files, and Maven `pom.xml`. Add `--include-working-tree` for local, uncommitted changes while iterating.
92
94
 
93
- `qamap e2e plan` turns changed file paths into a first-pass E2E testing plan. It detects whether a project looks like Expo/React Native, web, API/service, or CLI package, recommends a runner such as Maestro or Playwright, starts backend services with an API contract checklist, starts executable packages with a CLI command verification checklist, suggests bootstrap steps for repos with little or no test history, suggests domain language for the changed behavior, suggests candidate user flows, adds coverage targets, compares those targets with existing test-suite evidence when tests are present, flags API-dependent flows that need mock or fixture responses, and points out missing stable selectors such as `testID` or `data-testid` before anyone starts writing tests from a blank file.
95
+ `qamap e2e plan` first reads behavior-bearing commits in the selected base/head range. Related `feat`, `fix`, `hotfix`, `perf`, and supporting `refactor` commits are grouped into change intents, then connected to added diff symbols. The result is an evidence-backed lifecycle and a set of runner-independent primary, failure, boundary, and state-transition scenarios. Low-confidence or working-tree-only intent remains review-required instead of becoming trusted regression policy.
96
+
97
+ After that judgment, QAMap detects whether the target looks like Expo/React Native, web, API/service, CLI, or another repository shape and selects a Maestro, Playwright, or manual output adapter. The plan compares scenario targets with existing test evidence, identifies required mocks and fixtures, and reports missing stable selectors such as `testID` or `data-testid`. Runner detection is an implementation detail, not the first value shown to the user.
94
98
 
95
99
  The plan also includes an execution profile: detected start command, test command, Playwright `baseURL`, mobile app id, runner config files, env fixture files, confidence, and blockers. This keeps generated E2E drafts honest about whether they are runnable candidates or still review-only scaffolds.
96
100
 
@@ -100,7 +104,7 @@ When a repository does not already have the selected E2E runner, the plan includ
100
104
 
101
105
  When run at a monorepo root, the E2E plan also reports changed app/package targets. This helps a maintainer move from a broad workspace diff to scoped commands such as `qamap e2e plan services/listing --workspace-root . --base origin/main --head HEAD`, where package-specific runner detection and flow naming are usually sharper.
102
106
 
103
- Each candidate flow also includes a flow language brief: actor, trigger, goal, success signal, reviewer question, and edge cases. The brief keeps generated tests tied to product behavior rather than only changed file names.
107
+ Each candidate flow also includes a flow language brief: actor, trigger, goal, success signal, reviewer question, and edge cases. Intent-backed flows additionally retain the original commits, ordered lifecycle, QA scenarios, confidence, and review requirement so generated tests can be traced to evidence rather than only changed file names.
104
108
 
105
109
  The bootstrap section answers what must happen before generated drafts can be treated as real regression coverage. For example, a testless web project can get required steps for Playwright setup, first draft generation, stable selector work, fixture/mock data, and missing validation evidence, plus recommended steps for `.qamap/manifest.yaml`, `.qamap/domains.yml`, `.qamap/flows.yml`, and local history recording.
106
110
 
@@ -207,6 +211,8 @@ Pass `--record-history` when you want QAMap to keep a compact local snapshot of
207
211
 
208
212
  The draft result is meant to be useful as a PR artifact, not only as generated files. Markdown and JSON output include:
209
213
 
214
+ - `changeAnalysis`: commit-backed intents, confidence, review requirements, source evidence, lifecycle stages, and runner-independent QA scenarios
215
+ - intent-backed draft files retain `intentId`, `intentConfidence`, `lifecycle`, and `qaScenarios`; generated files include the same evidence as comments
210
216
  - `languageBrief`: actor, trigger, goal, success signal, reviewer question, and edge cases for each draft file
211
217
  - `promotionStatus`: whether the draft is a `commit-candidate`, `needs-review`, or `low-signal`
212
218
  - `runnableStatus`: whether the draft is a `runnable-candidate`, `near-runnable`, or `review-only`
@@ -15,20 +15,31 @@ The output should be specific enough to paste into a PR comment:
15
15
  ```txt
16
16
  # QAMap QA Draft
17
17
 
18
+ At a Glance
19
+ - Change intent: Submit checkout and persist the confirmed order [high]
20
+ - Behavior lifecycle: trigger: Submit checkout -> side-effect: Create order -> observable-outcome: Show order confirmation
21
+ - Affected behavior: Submit checkout and persist the confirmed order
22
+
23
+ Change Intent Evidence
24
+ - Commit: feat: submit checkout and persist the confirmed order
25
+ - Critical scenario: Submit checkout and persist the confirmed order
26
+ - Assert: order confirmation is visible and persisted
27
+ - Recommended scenario: Failure, timeout, and retry handling
28
+
18
29
  Summary
19
30
  - Project: Web
20
- - Recommended runner: Playwright
31
+ - Automation adapter: Playwright
21
32
  - Manifest: not found; using repo signals and PR diff only
22
33
  - Stage: almost runnable (3 of 4)
23
34
 
24
35
  PR Comment Draft
25
- - Affected flow: Checkout UI smoke flow
36
+ - Affected flow: Submit checkout and persist the confirmed order
26
37
  - User journey: Customer -> Open route /checkout -> Complete checkout with realistic form data
27
38
  - Success signal: confirmation state is visible after submit
28
39
  - Changed files: src/pages/checkout/index.tsx
29
40
 
30
41
  Suggested E2E / QA Draft
31
- - tests/e2e/checkout-ui-smoke-flow.spec.ts: near runnable
42
+ - tests/e2e/submit-checkout-and-persist-the-confirmed-order.spec.ts: near runnable
32
43
  - Open route /checkout.
33
44
  - Fill checkout email.
34
45
  - Submit checkout.
@@ -139,7 +150,7 @@ flows:
139
150
 
140
151
  ```txt
141
152
  Project: Web
142
- Recommended runner: Playwright
153
+ Automation adapter: Playwright
143
154
  Execution profile: high
144
155
  Start command: pnpm run dev
145
156
  Test command: pnpm run test:e2e
@@ -224,7 +235,7 @@ For an Expo or React Native change, QAMap should recommend Maestro and carry mob
224
235
 
225
236
  ```txt
226
237
  Project: Expo / React Native
227
- Recommended runner: Maestro
238
+ Automation adapter: Maestro
228
239
 
229
240
  Flow: Ink Drawing UI smoke flow
230
241
  Actor: User
@@ -269,7 +280,7 @@ For backend changes such as `src/v1/listing/utils.ts`, QAMap should not invent a
269
280
 
270
281
  ```txt
271
282
  Project: API / service
272
- Recommended runner: Manual
283
+ Automation adapter: Manual
273
284
 
274
285
  Flow: Listing API contract smoke checklist
275
286
  Actor: API consumer or upstream service
@@ -296,7 +307,7 @@ For an npm package that exposes `package.json` bin entries, QAMap should not inv
296
307
 
297
308
  ```txt
298
309
  Project: CLI
299
- Recommended runner: Manual
310
+ Automation adapter: Manual
300
311
 
301
312
  Flow: CLI command verification checklist
302
313
  Actor: CLI user or maintainer
@@ -460,9 +471,12 @@ Related Changed Files
460
471
  Good QAMap output should answer these questions quickly:
461
472
 
462
473
  - What behavior did this branch probably change?
474
+ - Which commits and diff symbols support that intent, and how confident is the inference?
475
+ - What trigger, condition, state change, side effect, and observable outcome form its lifecycle?
476
+ - Which primary, failure, boundary, and state-transition checks follow from that lifecycle?
463
477
  - What does the team call that behavior?
464
478
  - Who or what exercises it?
465
- - Which runner or checklist is the right first shape?
479
+ - Which existing automation adapter can compile the selected QA scenario?
466
480
  - Which setup, selector, fixture, or validation gap blocks this from becoming real regression coverage?
467
481
 
468
482
  If the output only says "make an E2E test" without these answers, it is not ready for the current release bar.
@@ -145,10 +145,13 @@ Input
145
145
  ```txt
146
146
  Output
147
147
  - PR comment/checklist draft
148
+ - commit-backed change intent and confidence
149
+ - ordered behavior lifecycle
150
+ - primary, failure, boundary, and state-transition QA scenarios
148
151
  - changed domain language
149
152
  - candidate user flow
150
153
  - manifest evidence when a repo-local flow/check matches
151
- - recommended runner
154
+ - automation adapter selected after QA design
152
155
  - draft file path
153
156
  - flow language brief
154
157
  - runnable status
@@ -164,9 +167,13 @@ The result should not stop at "selector needed" or "fixture needed." A useful QA
164
167
  ```txt
165
168
  # QAMap QA Draft
166
169
 
170
+ At a Glance
171
+ - Change intent: Submit checkout and persist the confirmed order [high]
172
+ - Behavior lifecycle: trigger: submit checkout -> side-effect: create order -> observable-outcome: show confirmation
173
+
167
174
  Summary
168
175
  - Project: Web
169
- - Recommended runner: Playwright
176
+ - Automation adapter: Playwright
170
177
  - Manifest: .qamap/manifest.yaml
171
178
  - Stage: almost runnable (3 of 4)
172
179
 
@@ -1,5 +1,19 @@
1
1
  # Release Validation
2
2
 
3
+ ## 0.4.0 - 2026-07-12
4
+
5
+ Validated as the first intent-first QA design release. The release contract starts with behavior-bearing commits and diff evidence, reconstructs an ordered behavior lifecycle, proposes runner-independent QA scenarios, and only then compiles an optional automation draft:
6
+
7
+ | Gate | Current result |
8
+ | --- | --- |
9
+ | `pnpm test` | 139/139 passing |
10
+ | `pnpm bench:ci` | 12/12 synthetic PR targets pass; dedicated web and mobile lifecycle targets require commit intent, ordered lifecycle phases, failure/boundary/state QA, observable success text, and commit-backed Behavior Graph evidence |
11
+ | Coverage | Lines 86.80%, branches 83.37%, functions 94.86% |
12
+ | Change Intent coverage | Lines 94.76%, branches 88.57%, functions 98.04% |
13
+ | `npm publish --dry-run --access public` | Passed for `@ivorycanvas/qamap@0.4.0`; 129 files, 813.8 kB packed |
14
+ | Intent-first output | Markdown, JSON, agent output, Behavior Graph, and generated drafts preserve confidence, commit evidence, lifecycle, and QA scenarios before automation-adapter guidance |
15
+ | Read-only safety | Public regression coverage uses synthetic repositories; optional local smoke validation runs from temporary copies and does not modify target repositories |
16
+
3
17
  ## 0.3.5 - 2026-07-11
4
18
 
5
19
  Validated as a manifest-feedback reliability patch:
@@ -83,7 +97,9 @@ For each target, record:
83
97
 
84
98
  - command used
85
99
  - base/head refs or working-tree mode
86
- - recommended runner
100
+ - inferred change intent, confidence, commit evidence, and review requirement
101
+ - lifecycle and runner-independent QA scenario quality
102
+ - selected automation adapter
87
103
  - generated flow language brief quality
88
104
  - draft readiness summary
89
105
  - draft self-check status and blockers
@@ -140,7 +156,9 @@ node dist/cli.js e2e draft <package> --workspace-root <repo-root> --base <base>
140
156
 
141
157
  The E2E plan should show:
142
158
 
143
- - runner recommendation with clear evidence
159
+ - change intent with commit and diff evidence, confidence, and review requirement
160
+ - ordered behavior lifecycle and concrete primary, failure, boundary, and state-transition QA scenarios
161
+ - automation adapter selection after the runner-independent QA sections
144
162
  - execution profile with start command, test command, base URL or app id when discoverable, confidence, and blockers
145
163
  - runner setup proposal with install commands, explicit `qamap e2e setup` acceptance command, files to create/update, and next commands when the repo lacks an E2E runner
146
164
  - setup output that reports the first generated changed-flow draft file after the accepted runner setup is applied
@@ -151,6 +169,7 @@ The E2E plan should show:
151
169
 
152
170
  The E2E draft should show:
153
171
 
172
+ - intent evidence, lifecycle, and QA scenario comments inside generated artifacts
154
173
  - `verification-manifest` as the draft source when a matched manifest flow is strong enough
155
174
  - manifest evidence comments inside generated drafts
156
175
  - external `--manifest <file>` previews that let teams test generated manifest quality without writing `.qamap/manifest.yaml` into target repos
@@ -170,9 +189,11 @@ The QA draft should show:
170
189
 
171
190
  - no-write PR comment/checklist output from `qamap qa`
172
191
  - no cloud and no LLM token positioning in the output
192
+ - change intent and lifecycle before automation setup
193
+ - primary, failure, boundary, and state-transition QA scenarios when evidence supports them
173
194
  - affected flow language with changed files and reviewer question
174
195
  - suggested E2E or manual checklist path
175
- - runner recommendation inherited from the E2E planner
196
+ - automation adapter inherited from the E2E planner
176
197
  - missing fixture, selector, assertion, runner, validation, or manifest evidence
177
198
  - PR checklist items that can be pasted into a pull request
178
199
  - optional manifest repair path when a manifest-backed recommendation is wrong
@@ -198,6 +219,9 @@ The manifest commands should show:
198
219
  The public demo must prove the product shape, not only command execution. Before promoting a release, run or update a small demo where a realistic PR diff produces a concrete E2E starting point:
199
220
 
200
221
  - The output names the affected product feature and user flow in domain language.
222
+ - The output shows the behavior-bearing commit evidence, confidence, and review requirement behind the inferred intent.
223
+ - The output orders trigger, condition, action, state change, side effect, and observable outcome before selecting a runner.
224
+ - The output proposes concrete primary, failure, boundary, and state-transition QA where evidence supports them.
201
225
  - The output names the draft file that would be created or previewed.
202
226
  - The draft includes route or screen entry, realistic actions, and at least one meaningful assertion.
203
227
  - The report explains why the test was recommended from changed files and manifest evidence.
@@ -213,11 +237,12 @@ The matrix below is public, fixture-backed evidence from the repository test sui
213
237
 
214
238
  | Target | Fixture-backed coverage | Expected output |
215
239
  | --- | --- | --- |
240
+ | Commit intent and lifecycle | `change intent clusters related commits into one evidence-backed lifecycle`; `change intent keeps unrelated feature commits separate`; `change intent ignores release-only commit metadata`; `E2E planning promotes commit intent before runner-specific draft generation` | Related feature commits become one evidence-backed intent; unrelated features remain separate; release-only changes do not become product intent; lifecycle and QA scenarios appear before Playwright or Maestro compilation; generic `primary journey` and `smoke flow` titles are suppressed when evidence supports a concrete name. |
216
241
  | Manifest-free QA skill entrypoint | `qa command emits a PR comment draft without requiring a manifest` | `qamap qa` works without `.qamap/manifest.yaml`, emits a local-first PR QA draft, names affected flow, changed files, suggested E2E/checklist path, missing evidence, PR checklist, agent handoff, and says manifest is optional upgrade rather than a first-use gate. |
217
242
  | Packaged PR QA skill template | `package metadata includes the portable PR QA skill template` | npm package metadata includes `skills`, and `skills/qamap-pr-qa/SKILL.md` contains the local PR QA workflow, `qamap qa` command, and manifest repair guidance. |
218
243
  | Verification manifest loop | `manifest init creates a baseline verification manifest`; `manifest init keeps Expo app file domains specific`; `manifest init captures advisory instruction context`; `manifest bootstrap produces concrete PR E2E draft from repo QA memory`; `e2e draft can use an external verification manifest for read-only adoption preview`; `manifest matches explain e2e and verify recommendations`; `manifest validate reports missing and stale manifest policy` | Generated `.qamap/manifest.yaml` includes `$schema`, domains, flows, anchors, checks, runner, source, and confidence; context preview reports repo-local instruction sources, role summaries, validation commands, safety rules, and diagnostics; validator reports missing/stale/duplicate policy; explain output maps branch changes to manifest domains/flows/checks; E2E drafts prefer `verification-manifest` sources with manifest evidence, route entry, detected input/action selectors, required checks, and manifest repair paths; external manifests can be passed with `--manifest` for read-only adoption smoke tests. |
219
- | Web app with Playwright routes | `generateE2ePlan matches committed core flow definitions`; `generateE2eDraft uses web selectors in Playwright specs`; `generateE2eDraft dry run previews files without writing drafts`; `generateE2eDraft asserts changed HTML success copy in Playwright specs`; `generateE2ePlan captures Playwright execution profile and self-check blockers`; `generateE2ePlan infers Playwright base URLs from dev scripts`; `generateE2eDraft supports Next app router route groups and concrete route hints`; `generateE2ePlan reads React Router object route paths`; `generateE2eDraft fills dynamic route params from concrete route hints`; `generateE2eDraft emits runnable Playwright role and input actions` | `Web` project profile, `playwright` runner, core-flow names such as `Checkout purchase`, route-aware Playwright drafts, dry-run preview status without filesystem writes, stable selector hints, changed HTML success copy assertions, execution profile, dev-script base URL hints, opt-in Playwright setup proposal, Next App Router route groups, React Router object paths, dynamic route params, draft self-check status, action items, and validation gaps. |
220
- | Expo / React Native mobile app | `generateE2ePlan recommends mobile flows for Expo changes`; `generateE2ePlan detects Maestro app ids from app config files`; `generateE2eDraft scopes entrypoint hints to each domain scenario`; `generateE2eDraft names changed component actions before generic primary journeys` | `Expo / React Native` project profile, `maestro` runner, app id and launch command hints from `app.json` or `app.config.*`, Maestro YAML drafts, `testID`/`accessibilityLabel` selector hints, action-specific scenario names such as `Listing Media Link Submit`, and mobile setup actions. |
244
+ | Web app with Playwright routes | `generateE2ePlan matches committed core flow definitions`; `generateE2eDraft uses web selectors in Playwright specs`; `generateE2eDraft dry run previews files without writing drafts`; `generateE2eDraft asserts changed HTML success copy in Playwright specs`; `generateE2ePlan captures Playwright execution profile and self-check blockers`; `generateE2ePlan infers Playwright base URLs from dev scripts`; `generateE2eDraft supports Next app router route groups and concrete route hints`; `generateE2ePlan reads React Router object route paths`; `generateE2eDraft fills dynamic route params from concrete route hints`; `generateE2eDraft emits runnable Playwright role and input actions` | `Web` project profile, Playwright output adapter, intent-backed or core-flow names, route-aware drafts, dry-run preview status without filesystem writes, stable selector hints, changed HTML success copy assertions, execution profile, dev-script base URL hints, opt-in setup proposal, route groups, object paths, dynamic params, draft self-check status, action items, and validation gaps. |
245
+ | Expo / React Native mobile app | `generateE2ePlan recommends mobile flows for Expo changes`; `generateE2ePlan detects Maestro app ids from app config files`; `generateE2eDraft scopes entrypoint hints to each domain scenario`; `generateE2eDraft names changed component actions before generic primary journeys` | `Expo / React Native` project profile, Maestro output adapter, app id and launch command hints from `app.json` or `app.config.*`, YAML drafts, `testID`/`accessibilityLabel` selector hints, and mobile setup actions after runner-independent QA intent. |
221
246
  | API or backend service | `generateE2ePlan detects API service projects and suggests contract checklists`; `generateE2ePlan detects Django service apps from a workspace root`; `generateE2ePlan names versioned API service paths with domain language`; `generateE2ePlan uses matched core flow names for API service contracts` | `API / service` project profile, manual contract checklist, Django/FastAPI-style service signals when present, domain-aware titles such as `Listing API contract`, API consumer actor, endpoint/handler/service-path trigger, service start/test command hints, and contract failure coverage. |
222
247
  | CLI package | `generateE2ePlan detects CLI packages and suggests command verification checklists` | `CLI` project profile from `package.json` bin entries, manual command verification checklist, CLI user or maintainer actor language, command invocation trigger, stdout/stderr/generated-file/exit-code success signal, valid and invalid argument coverage, and no required API fixture action unless the changed command path explicitly exposes network or fixture evidence. |
223
248
  | Design tokens and data catalogs | `generateE2ePlan detects design token packages and suggests artifact validation`; `generateE2ePlan detects data catalog repositories and suggests catalog verification` | `Design tokens` and `Data catalog` project profiles, manual artifact/catalog checklist, token or catalog actor language, schema/generated output/consumer fixture coverage, fixture readiness marked not needed for API mocks, and validation matrix rows that do not require browser/device selectors. |
@@ -239,7 +264,8 @@ Private repositories may be used as local, read-only diagnostics, but their raw
239
264
  The release candidate must pass the fixture-backed suite, package dry-run, and read-only smoke protocol. Record only public synthetic evidence in this document or in release notes:
240
265
 
241
266
  - whether the generated flow names match the team's domain language
242
- - whether the recommended runner is plausible
267
+ - whether commit evidence supports the inferred intent and lifecycle
268
+ - whether the automation adapter is plausible without being mistaken for the product value
243
269
  - whether generated drafts identify the right actor, trigger, success signal, and edge cases
244
270
  - whether action items are concrete enough for a developer to convert into runnable tests
245
271
  - whether false positives are caused by missing manifests, weak selectors, or unsupported project structure
package/docs/releasing.md CHANGED
@@ -2,6 +2,19 @@
2
2
 
3
3
  This runbook defines the release process for `qamap`. It is intentionally conservative: the package should be published only when the local release gate, documentation, and representative repository validation all agree.
4
4
 
5
+ ## Pre-1.0 Version Policy
6
+
7
+ QAMap keeps major and minor changes deliberately rare during `0.x` development.
8
+
9
+ - Patch is the default for bug fixes, inference quality, performance, internal architecture, benchmarks, documentation, and additional adapters that preserve the existing CLI, schema, and safety contracts.
10
+ - Minor is reserved for a new product-level capability, an incompatible CLI or manifest contract, or a meaningful change to default execution and safety behavior.
11
+ - Risky minor work should ship through `alpha`, `beta`, and `rc` prereleases before the final minor.
12
+ - Do not pre-allocate a minor version to every roadmap phase. Define the next minor release bar and continue compatible work as patches until that bar is met.
13
+
14
+ Version `0.4.0` is earned by the first commit-to-intent-to-scenario vertical slice: behavior-bearing commits and diff symbols become an evidence-backed lifecycle and concrete runner-independent QA, then existing Playwright, Maestro, or manual adapters compile the result. The next minor is reserved for explicit temporary execution and normalized evidence without modifying the target repository.
15
+
16
+ Version `1.0.0` requires a stable public contract and external adoption, not implementation volume alone. CLI commands, exit codes, machine output, manifest migration, adapter compatibility, no-LLM/no-upload guarantees, and release operations must be dependable. Repository stars are useful social proof, but repeated use in unrelated repositories and reported QA value are stronger release evidence.
17
+
5
18
  ## Release Owner Checklist
6
19
 
7
20
  Before publishing, confirm:
package/docs/roadmap.md CHANGED
@@ -1,47 +1,56 @@
1
1
  # Roadmap
2
2
 
3
- QAMap starts as a local CLI for repo-level AI agent readiness. The project can grow in layers without becoming heavy.
3
+ QAMap is a local CLI for evidence-backed PR QA design. The project can grow in layers without becoming a hosted platform or token-dependent agent.
4
4
 
5
5
  ## North Star
6
6
 
7
- QAMap should become a local-first QA skill plus optional QA memory layer: it maps PR changes to affected flows, drafts the E2E/checklist work needed before merge, and improves over time through a repository-level verification manifest. The goal is not to replace reviewers or QA. The goal is to remove the repeated blank-page work that makes developers skip good verification.
7
+ QAMap should become a local, zero-LLM, change-aware QA engine. It should map any PR change to affected behavior, select the smallest relevant QA set, and eventually execute that QA with local evidence. A repository-level verification manifest supplies reviewed product intent that static code evidence cannot prove. The goal is not to replace reviewers or QA. The goal is to remove the repeated blank-page and manual verification work that makes developers skip good QA.
8
8
 
9
9
  The sharp product position is:
10
10
 
11
11
  ```txt
12
- Do not re-prompt AI for the same QA context.
13
- Run a local PR QA draft first, then capture team QA memory in the repo when repeated corrections should improve future recommendations.
12
+ Read the change, find the affected behavior, and produce local QA evidence.
13
+ No source upload. No LLM token. Let reviewed repository memory improve every later PR.
14
14
  ```
15
15
 
16
- This means QAMap should be judged less like a generic test recorder and more like a manifest-backed PR verification assistant. A recorder captures what a person just clicked. QAMap should explain what the PR changed, which durable flow/check knowledge applies, and what test artifact should now exist.
16
+ This means QAMap should be judged by whether it identifies the right behavior and catches a seeded regression, not only whether it writes plausible test code. Playwright, Maestro, and other runners are implementation details behind that product contract.
17
17
 
18
18
  ## Release Bar
19
19
 
20
20
  Before treating the next public release as ready, the golden demo must satisfy these conditions:
21
21
 
22
22
  - First-run output is concrete, not broad: it names the affected feature, flow, draft file, and checks.
23
+ - Commit evidence becomes an explicit change intent and ordered behavior lifecycle before any runner is recommended.
24
+ - Generic `primary journey` and `smoke flow` names fail the benchmark when commit and diff evidence can support a concrete lifecycle.
23
25
  - Manifest authoring burden stays low: `manifest context` and `manifest init` provide a useful baseline before a human edits YAML.
24
26
  - Generated E2E draft is a usable starting point: it has route/screen entry, meaningful actions, assertions, and clear TODOs only where repo data is missing.
25
27
  - Recommendation evidence is explainable: output shows the changed file, manifest flow/check, and manifest path to repair when wrong.
26
28
  - README demo shows the full loop: manifest-free PR QA draft, optional repo context baseline, PR mapping, E2E draft, and remaining validation gaps.
27
29
  - One manifest correction should improve future PR recommendations without another LLM prompt.
28
30
 
29
- ## Now
31
+ ## Now: 0.4.x
30
32
 
33
+ - Stabilize commit-range Change Intent analysis: related behavior commits, diff symbols, confidence, review requirements, lifecycle stages, and runner-independent QA scenarios.
34
+ - Keep `qamap.change-intent` as a direct Behavior Graph adapter while moving more source observations out of the compatibility adapter.
35
+ - Preserve `qa` as the static, read-only product surface while designing explicit execution behind `verify`; never turn a scan into implicit project-code execution.
31
36
  - Treat the committed [benchmark contract](benchmarking.md) as the quality gate for recommendations, not only implementation correctness. Reduce real failures into public fixtures and require `pnpm bench:ci` on every PR.
32
- - Make `qa` the primary product surface. Its first screen and `--format agent` payload should agree on affected behavior, reviewer question, repository evidence, draft path, and missing trust requirements.
37
+ - Make `qa` the primary product surface. Its first screen and `--format agent` payload must agree on change intent, lifecycle, QA scenarios, affected behavior, repository evidence, draft path, and missing trust requirements.
33
38
  - Improve changed-file impact mapping from shared symbols and components to consuming routes, screens, API contracts, and manifest flows.
34
39
  - Keep the [release validation checklist](release-validation.md), [manifest guide](manifest.md), public [E2E output examples](e2e-output-examples.md), and README examples aligned with captured output from the public fixtures.
35
40
  - Stabilize the manifest feedback loop with `.qamap/manifest.yaml`, `manifest init`, `manifest validate`, `manifest explain`, JSON Schema, and manifest-driven E2E draft shaping.
36
41
  - Keep `manifest context` useful as a pre-init sanity check for repo-local QA memory, harness docs, agent instructions, and runbooks.
37
- - Improve generated drafts until the golden demo feels like a real starting point, not a generic checklist.
42
+ - Improve Playwright and Maestro compilation from intent scenarios while keeping runner choice behind the runner-independent QA contract.
38
43
  - Keep `verify`, `e2e`, and `manifest` as deeper layers behind `qa`; freeze new scanner, doctor, eval, domains, flows, and history features until the core QA contract is consistently useful.
39
44
 
40
45
  ## Next
41
46
 
47
+ - Move route, screen, endpoint, selector, fixture, test, and contract discovery into analyzer adapters, starting with TypeScript web stacks and reusing one web behavior model across Next.js, React Router, Vue/Nuxt, and SvelteKit.
48
+ - Compare base and head Behavior Graphs, then select impacted graph paths before refining deterministic success, validation, failure, empty, loading, auth, and contract scenarios.
49
+ - Add policy-controlled temporary execution so selected scenarios can produce normalized pass, fail, blocked, and not-verifiable evidence without modifying the target repository.
50
+ - Add an execution benchmark that injects known regressions and requires QAMap to select and fail the relevant scenario without changing the fixture repository.
42
51
  - Add symbol-level anchors for exported components, hooks, API clients, handlers, schemas, and queries after the public import-impact fixture stays stable.
43
52
  - Add a manifest correction command that proposes the exact flow/anchor patch and applies it only after human approval, avoiding routine hand-edits to YAML.
44
- - Add stronger deterministic draft adapters for Playwright and Maestro while keeping `manual` output for API, CLI, token, and catalog repositories.
53
+ - Add stronger deterministic draft adapters for Playwright and Maestro while keeping `manual` output for API, CLI, token, and catalog repositories; runner detection itself is not a product success metric.
45
54
  - Expand the public benchmark corpus with package-scoped monorepos, auth/session changes, dynamic routes, API failure fixtures, and non-JavaScript services.
46
55
  - Keep the `--format agent` output a stable, versioned contract that skills and MCP wrappers can rely on.
47
56
  - Continue expanding agent surface detection across popular coding-agent tools without making the public workflow depend on a single vendor.
@@ -55,7 +64,8 @@ Before treating the next public release as ready, the golden demo must satisfy t
55
64
 
56
65
  ## Non-Goals
57
66
 
58
- - QAMap will not execute untrusted project code.
67
+ - Static commands such as `qa`, `scan`, and planning modes will not execute scanned project code. Future execution requires an explicit command and a visible, policy-controlled execution plan.
68
+ - QAMap will not implement its own browser or device automation engine; it will orchestrate proven local executors behind a framework-neutral QA contract.
59
69
  - QAMap will not replace tests, review, branch protection, threat modeling, or security review.
60
70
  - QAMap will not become a general-purpose code style linter.
61
71
  - QAMap will not become a deep MCP server analysis engine.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@ivorycanvas/qamap",
3
- "version": "0.3.5",
4
- "description": "Local-first QA skill that turns PR diffs into affected flows, missing evidence, and E2E drafts with no cloud or LLM token.",
3
+ "version": "0.4.0",
4
+ "description": "Local zero-LLM PR QA designer that turns commit intent and diffs into behavior lifecycles, QA scenarios, and deterministic automation drafts.",
5
5
  "type": "module",
6
6
  "packageManager": "pnpm@10.32.1",
7
7
  "bin": {
@@ -39,6 +39,8 @@
39
39
  "mcp",
40
40
  "preflight",
41
41
  "pull-request",
42
+ "change-intent",
43
+ "behavior-graph",
42
44
  "e2e",
43
45
  "e2e-testing",
44
46
  "qa",
@@ -45,7 +45,7 @@
45
45
  "runner": {
46
46
  "type": "string",
47
47
  "enum": ["maestro", "playwright", "manual"],
48
- "description": "Recommended E2E runner for this repository."
48
+ "description": "Automation output adapter selected after runner-independent QA intent analysis."
49
49
  },
50
50
  "manifest": {
51
51
  "type": ["string", "null"],
@@ -71,6 +71,47 @@
71
71
  "files": { "type": "number", "minimum": 0 }
72
72
  }
73
73
  },
74
+ "intents": {
75
+ "type": "array",
76
+ "description": "Evidence-backed change intents derived before runner selection (capped).",
77
+ "items": {
78
+ "type": "object",
79
+ "required": ["title", "confidence", "reviewRequired", "evidence", "lifecycle", "scenarios"],
80
+ "properties": {
81
+ "title": { "type": "string" },
82
+ "confidence": { "type": "string", "enum": ["low", "medium", "high"] },
83
+ "reviewRequired": { "type": "boolean" },
84
+ "evidence": { "type": "array", "items": { "type": "string" } },
85
+ "lifecycle": {
86
+ "type": "array",
87
+ "items": {
88
+ "type": "object",
89
+ "required": ["phase", "label"],
90
+ "properties": {
91
+ "phase": {
92
+ "type": "string",
93
+ "enum": ["trigger", "condition", "action", "state-change", "side-effect", "observable-outcome"]
94
+ },
95
+ "label": { "type": "string" }
96
+ }
97
+ }
98
+ },
99
+ "scenarios": {
100
+ "type": "array",
101
+ "items": {
102
+ "type": "object",
103
+ "required": ["priority", "kind", "title", "assertions"],
104
+ "properties": {
105
+ "priority": { "type": "string", "enum": ["critical", "recommended"] },
106
+ "kind": { "type": "string", "enum": ["primary", "failure", "boundary", "state-transition"] },
107
+ "title": { "type": "string" },
108
+ "assertions": { "type": "array", "items": { "type": "string" } }
109
+ }
110
+ }
111
+ }
112
+ }
113
+ }
114
+ },
74
115
  "firstDraftCommand": {
75
116
  "type": "string",
76
117
  "description": "One command that creates the first E2E draft. Only present when the repository has no test suite and the diff reaches a product journey."