@ivorycanvas/qamap 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -1
- package/README.md +19 -14
- package/dist/behavior-intent.d.ts +6 -0
- package/dist/behavior-intent.js +172 -0
- package/dist/behavior-intent.js.map +1 -0
- package/dist/behavior-manifest.d.ts +6 -0
- package/dist/behavior-manifest.js +221 -0
- package/dist/behavior-manifest.js.map +1 -0
- package/dist/behavior.d.ts +143 -0
- package/dist/behavior.js +458 -0
- package/dist/behavior.js.map +1 -0
- package/dist/change-intent.d.ts +71 -0
- package/dist/change-intent.js +744 -0
- package/dist/change-intent.js.map +1 -0
- package/dist/cli.js +6 -5
- package/dist/cli.js.map +1 -1
- package/dist/e2e.d.ts +16 -1
- package/dist/e2e.js +329 -24
- package/dist/e2e.js.map +1 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/manifest.js +138 -4
- package/dist/manifest.js.map +1 -1
- package/dist/qa.d.ts +1 -0
- package/dist/qa.js +73 -1
- package/dist/qa.js.map +1 -1
- package/dist/terminal.js +1 -1
- package/dist/terminal.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/adoption.md +11 -10
- package/docs/agent-format.md +9 -7
- package/docs/agent-skill.md +5 -2
- package/docs/architecture.md +131 -0
- package/docs/benchmarking.md +26 -4
- package/docs/commands.md +13 -7
- package/docs/e2e-output-examples.md +23 -9
- package/docs/manifest.md +4 -0
- package/docs/quickstart-demo.md +9 -2
- package/docs/release-validation.md +50 -37
- package/docs/releasing.md +13 -0
- package/docs/roadmap.md +20 -10
- package/package.json +4 -2
- package/schema/qamap-agent.schema.json +42 -1
- package/schema/qamap-behavior.schema.json +307 -0
- package/skills/qamap-pr-qa/SKILL.md +16 -11
|
@@ -15,20 +15,31 @@ The output should be specific enough to paste into a PR comment:
|
|
|
15
15
|
```txt
|
|
16
16
|
# QAMap QA Draft
|
|
17
17
|
|
|
18
|
+
At a Glance
|
|
19
|
+
- Change intent: Submit checkout and persist the confirmed order [high]
|
|
20
|
+
- Behavior lifecycle: trigger: Submit checkout -> side-effect: Create order -> observable-outcome: Show order confirmation
|
|
21
|
+
- Affected behavior: Submit checkout and persist the confirmed order
|
|
22
|
+
|
|
23
|
+
Change Intent Evidence
|
|
24
|
+
- Commit: feat: submit checkout and persist the confirmed order
|
|
25
|
+
- Critical scenario: Submit checkout and persist the confirmed order
|
|
26
|
+
- Assert: order confirmation is visible and persisted
|
|
27
|
+
- Recommended scenario: Failure, timeout, and retry handling
|
|
28
|
+
|
|
18
29
|
Summary
|
|
19
30
|
- Project: Web
|
|
20
|
-
-
|
|
31
|
+
- Automation adapter: Playwright
|
|
21
32
|
- Manifest: not found; using repo signals and PR diff only
|
|
22
33
|
- Stage: almost runnable (3 of 4)
|
|
23
34
|
|
|
24
35
|
PR Comment Draft
|
|
25
|
-
- Affected flow:
|
|
36
|
+
- Affected flow: Submit checkout and persist the confirmed order
|
|
26
37
|
- User journey: Customer -> Open route /checkout -> Complete checkout with realistic form data
|
|
27
38
|
- Success signal: confirmation state is visible after submit
|
|
28
39
|
- Changed files: src/pages/checkout/index.tsx
|
|
29
40
|
|
|
30
41
|
Suggested E2E / QA Draft
|
|
31
|
-
- tests/e2e/checkout-
|
|
42
|
+
- tests/e2e/submit-checkout-and-persist-the-confirmed-order.spec.ts: near runnable
|
|
32
43
|
- Open route /checkout.
|
|
33
44
|
- Fill checkout email.
|
|
34
45
|
- Submit checkout.
|
|
@@ -139,7 +150,7 @@ flows:
|
|
|
139
150
|
|
|
140
151
|
```txt
|
|
141
152
|
Project: Web
|
|
142
|
-
|
|
153
|
+
Automation adapter: Playwright
|
|
143
154
|
Execution profile: high
|
|
144
155
|
Start command: pnpm run dev
|
|
145
156
|
Test command: pnpm run test:e2e
|
|
@@ -224,7 +235,7 @@ For an Expo or React Native change, QAMap should recommend Maestro and carry mob
|
|
|
224
235
|
|
|
225
236
|
```txt
|
|
226
237
|
Project: Expo / React Native
|
|
227
|
-
|
|
238
|
+
Automation adapter: Maestro
|
|
228
239
|
|
|
229
240
|
Flow: Ink Drawing UI smoke flow
|
|
230
241
|
Actor: User
|
|
@@ -269,7 +280,7 @@ For backend changes such as `src/v1/listing/utils.ts`, QAMap should not invent a
|
|
|
269
280
|
|
|
270
281
|
```txt
|
|
271
282
|
Project: API / service
|
|
272
|
-
|
|
283
|
+
Automation adapter: Manual
|
|
273
284
|
|
|
274
285
|
Flow: Listing API contract smoke checklist
|
|
275
286
|
Actor: API consumer or upstream service
|
|
@@ -296,7 +307,7 @@ For an npm package that exposes `package.json` bin entries, QAMap should not inv
|
|
|
296
307
|
|
|
297
308
|
```txt
|
|
298
309
|
Project: CLI
|
|
299
|
-
|
|
310
|
+
Automation adapter: Manual
|
|
300
311
|
|
|
301
312
|
Flow: CLI command verification checklist
|
|
302
313
|
Actor: CLI user or maintainer
|
|
@@ -367,7 +378,7 @@ If reusable repo-local evidence already exists, the PR QA output reads its conte
|
|
|
367
378
|
|
|
368
379
|
```txt
|
|
369
380
|
Missing evidence before trusting this PR
|
|
370
|
-
- [recommended] fixture: Confirm fixture coverage for /api/
|
|
381
|
+
- [recommended] fixture: Confirm fixture coverage for /api/sample/status - Reuse src/services/sampleSeed.ts (exports sampleSeed) to build a deterministic response for /api/sample/status.
|
|
371
382
|
```
|
|
372
383
|
|
|
373
384
|
When an existing handler file already covers part of the flow, the next action names the file and the still-uncovered endpoints, for example `Extend src/mocks/handlers.ts (already handles /api/invoices) to also cover /api/payments/summary`. The generated Playwright mock bodies then reuse the response keys observed in that file (`invoices: "qamap-invoices"`) instead of the generic placeholder below, with a comment noting the source file.
|
|
@@ -460,9 +471,12 @@ Related Changed Files
|
|
|
460
471
|
Good QAMap output should answer these questions quickly:
|
|
461
472
|
|
|
462
473
|
- What behavior did this branch probably change?
|
|
474
|
+
- Which commits and diff symbols support that intent, and how confident is the inference?
|
|
475
|
+
- What trigger, condition, state change, side effect, and observable outcome form its lifecycle?
|
|
476
|
+
- Which primary, failure, boundary, and state-transition checks follow from that lifecycle?
|
|
463
477
|
- What does the team call that behavior?
|
|
464
478
|
- Who or what exercises it?
|
|
465
|
-
- Which
|
|
479
|
+
- Which existing automation adapter can compile the selected QA scenario?
|
|
466
480
|
- Which setup, selector, fixture, or validation gap blocks this from becoming real regression coverage?
|
|
467
481
|
|
|
468
482
|
If the output only says "make an E2E test" without these answers, it is not ready for the current release bar.
|
package/docs/manifest.md
CHANGED
|
@@ -56,6 +56,10 @@ QAMap cannot know every team's product priorities from file paths alone. The man
|
|
|
56
56
|
|
|
57
57
|
When a recommendation is wrong, edit the manifest path shown in QAMap output. The next branch should get better recommendations without another explanation.
|
|
58
58
|
|
|
59
|
+
For server projects, the generated baseline can include manual API contract flows inferred from common route, controller, handler, and framework-backed service modules. These flows use file anchors and start with success-contract and invalid-request checks; maintainers should still review names, criticality, and project-specific failure behavior before committing them as team policy.
|
|
60
|
+
|
|
61
|
+
A domain match is useful even when no declared flow anchor matches. In that case QAMap keeps the existing code-derived flow, steps, selectors, and entrypoint, and adds the manifest domain as provenance. It does not fabricate flow-level checks. Add or correct a flow anchor when the team wants the manifest to supply durable checks and runner details on later PRs.
|
|
62
|
+
|
|
59
63
|
## Concrete Bootstrap Example
|
|
60
64
|
|
|
61
65
|
A first baseline can be useful before a human writes YAML by hand. Suppose the default branch has:
|
package/docs/quickstart-demo.md
CHANGED
|
@@ -145,10 +145,13 @@ Input
|
|
|
145
145
|
```txt
|
|
146
146
|
Output
|
|
147
147
|
- PR comment/checklist draft
|
|
148
|
+
- commit-backed change intent and confidence
|
|
149
|
+
- ordered behavior lifecycle
|
|
150
|
+
- primary, failure, boundary, and state-transition QA scenarios
|
|
148
151
|
- changed domain language
|
|
149
152
|
- candidate user flow
|
|
150
153
|
- manifest evidence when a repo-local flow/check matches
|
|
151
|
-
-
|
|
154
|
+
- automation adapter selected after QA design
|
|
152
155
|
- draft file path
|
|
153
156
|
- flow language brief
|
|
154
157
|
- runnable status
|
|
@@ -164,9 +167,13 @@ The result should not stop at "selector needed" or "fixture needed." A useful QA
|
|
|
164
167
|
```txt
|
|
165
168
|
# QAMap QA Draft
|
|
166
169
|
|
|
170
|
+
At a Glance
|
|
171
|
+
- Change intent: Submit checkout and persist the confirmed order [high]
|
|
172
|
+
- Behavior lifecycle: trigger: submit checkout -> side-effect: create order -> observable-outcome: show confirmation
|
|
173
|
+
|
|
167
174
|
Summary
|
|
168
175
|
- Project: Web
|
|
169
|
-
-
|
|
176
|
+
- Automation adapter: Playwright
|
|
170
177
|
- Manifest: .qamap/manifest.yaml
|
|
171
178
|
- Stage: almost runnable (3 of 4)
|
|
172
179
|
|
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# Release Validation
|
|
2
2
|
|
|
3
|
+
## 0.4.0 - 2026-07-12
|
|
4
|
+
|
|
5
|
+
Validated as the first intent-first QA design release. The release contract starts with behavior-bearing commits and diff evidence, reconstructs an ordered behavior lifecycle, proposes runner-independent QA scenarios, and only then compiles an optional automation draft:
|
|
6
|
+
|
|
7
|
+
| Gate | Current result |
|
|
8
|
+
| --- | --- |
|
|
9
|
+
| `pnpm test` | 139/139 passing |
|
|
10
|
+
| `pnpm bench:ci` | 12/12 synthetic PR targets pass; dedicated web and mobile lifecycle targets require commit intent, ordered lifecycle phases, failure/boundary/state QA, observable success text, and commit-backed Behavior Graph evidence |
|
|
11
|
+
| Coverage | Lines 86.80%, branches 83.37%, functions 94.86% |
|
|
12
|
+
| Change Intent coverage | Lines 94.76%, branches 88.57%, functions 98.04% |
|
|
13
|
+
| `npm publish --dry-run --access public` | Passed for `@ivorycanvas/qamap@0.4.0`; 129 files, 813.8 kB packed |
|
|
14
|
+
| Intent-first output | Markdown, JSON, agent output, Behavior Graph, and generated drafts preserve confidence, commit evidence, lifecycle, and QA scenarios before automation-adapter guidance |
|
|
15
|
+
| Read-only safety | Public regression coverage uses synthetic repositories; optional local smoke validation runs from temporary copies and does not modify target repositories |
|
|
16
|
+
|
|
17
|
+
## 0.3.5 - 2026-07-11
|
|
18
|
+
|
|
19
|
+
Validated as a manifest-feedback reliability patch:
|
|
20
|
+
|
|
21
|
+
| Gate | Current result |
|
|
22
|
+
| --- | --- |
|
|
23
|
+
| `pnpm test` | 127/127 passing |
|
|
24
|
+
| `pnpm bench:ci` | Eight public PR fixtures pass; API and reverse-import fixtures also generate an external base manifest and require a manifest-backed head flow |
|
|
25
|
+
| Coverage | Lines 86.09%, branches 83.18%, functions 94.66% |
|
|
26
|
+
| `npm publish --dry-run --access public` | Passed for `@ivorycanvas/qamap@0.3.5`; 115 files, 772.8 kB packed |
|
|
27
|
+
| API manifest baseline | Common server modules produce manual contract flows with API anchors and success/failure checks |
|
|
28
|
+
| Domain fallback | Domain-only matches preserve manifest provenance without claiming unrelated files or inventing manifest checks |
|
|
29
|
+
| Privacy | Public changes use synthetic fixtures; private smoke output remains outside the repository |
|
|
30
|
+
|
|
3
31
|
## 0.3.4 - 2026-07-10
|
|
4
32
|
|
|
5
33
|
Validated before publishing `0.3.4` with a committed, CI-enforced recommendation contract instead of relying only on private smoke repositories:
|
|
@@ -69,7 +97,9 @@ For each target, record:
|
|
|
69
97
|
|
|
70
98
|
- command used
|
|
71
99
|
- base/head refs or working-tree mode
|
|
72
|
-
-
|
|
100
|
+
- inferred change intent, confidence, commit evidence, and review requirement
|
|
101
|
+
- lifecycle and runner-independent QA scenario quality
|
|
102
|
+
- selected automation adapter
|
|
73
103
|
- generated flow language brief quality
|
|
74
104
|
- draft readiness summary
|
|
75
105
|
- draft self-check status and blockers
|
|
@@ -126,7 +156,9 @@ node dist/cli.js e2e draft <package> --workspace-root <repo-root> --base <base>
|
|
|
126
156
|
|
|
127
157
|
The E2E plan should show:
|
|
128
158
|
|
|
129
|
-
-
|
|
159
|
+
- change intent with commit and diff evidence, confidence, and review requirement
|
|
160
|
+
- ordered behavior lifecycle and concrete primary, failure, boundary, and state-transition QA scenarios
|
|
161
|
+
- automation adapter selection after the runner-independent QA sections
|
|
130
162
|
- execution profile with start command, test command, base URL or app id when discoverable, confidence, and blockers
|
|
131
163
|
- runner setup proposal with install commands, explicit `qamap e2e setup` acceptance command, files to create/update, and next commands when the repo lacks an E2E runner
|
|
132
164
|
- setup output that reports the first generated changed-flow draft file after the accepted runner setup is applied
|
|
@@ -137,6 +169,7 @@ The E2E plan should show:
|
|
|
137
169
|
|
|
138
170
|
The E2E draft should show:
|
|
139
171
|
|
|
172
|
+
- intent evidence, lifecycle, and QA scenario comments inside generated artifacts
|
|
140
173
|
- `verification-manifest` as the draft source when a matched manifest flow is strong enough
|
|
141
174
|
- manifest evidence comments inside generated drafts
|
|
142
175
|
- external `--manifest <file>` previews that let teams test generated manifest quality without writing `.qamap/manifest.yaml` into target repos
|
|
@@ -156,9 +189,11 @@ The QA draft should show:
|
|
|
156
189
|
|
|
157
190
|
- no-write PR comment/checklist output from `qamap qa`
|
|
158
191
|
- no cloud and no LLM token positioning in the output
|
|
192
|
+
- change intent and lifecycle before automation setup
|
|
193
|
+
- primary, failure, boundary, and state-transition QA scenarios when evidence supports them
|
|
159
194
|
- affected flow language with changed files and reviewer question
|
|
160
195
|
- suggested E2E or manual checklist path
|
|
161
|
-
-
|
|
196
|
+
- automation adapter inherited from the E2E planner
|
|
162
197
|
- missing fixture, selector, assertion, runner, validation, or manifest evidence
|
|
163
198
|
- PR checklist items that can be pasted into a pull request
|
|
164
199
|
- optional manifest repair path when a manifest-backed recommendation is wrong
|
|
@@ -184,6 +219,9 @@ The manifest commands should show:
|
|
|
184
219
|
The public demo must prove the product shape, not only command execution. Before promoting a release, run or update a small demo where a realistic PR diff produces a concrete E2E starting point:
|
|
185
220
|
|
|
186
221
|
- The output names the affected product feature and user flow in domain language.
|
|
222
|
+
- The output shows the behavior-bearing commit evidence, confidence, and review requirement behind the inferred intent.
|
|
223
|
+
- The output orders trigger, condition, action, state change, side effect, and observable outcome before selecting a runner.
|
|
224
|
+
- The output proposes concrete primary, failure, boundary, and state-transition QA where evidence supports them.
|
|
187
225
|
- The output names the draft file that would be created or previewed.
|
|
188
226
|
- The draft includes route or screen entry, realistic actions, and at least one meaningful assertion.
|
|
189
227
|
- The report explains why the test was recommended from changed files and manifest evidence.
|
|
@@ -199,11 +237,12 @@ The matrix below is public, fixture-backed evidence from the repository test sui
|
|
|
199
237
|
|
|
200
238
|
| Target | Fixture-backed coverage | Expected output |
|
|
201
239
|
| --- | --- | --- |
|
|
240
|
+
| Commit intent and lifecycle | `change intent clusters related commits into one evidence-backed lifecycle`; `change intent keeps unrelated feature commits separate`; `change intent ignores release-only commit metadata`; `E2E planning promotes commit intent before runner-specific draft generation` | Related feature commits become one evidence-backed intent; unrelated features remain separate; release-only changes do not become product intent; lifecycle and QA scenarios appear before Playwright or Maestro compilation; generic `primary journey` and `smoke flow` titles are suppressed when evidence supports a concrete name. |
|
|
202
241
|
| Manifest-free QA skill entrypoint | `qa command emits a PR comment draft without requiring a manifest` | `qamap qa` works without `.qamap/manifest.yaml`, emits a local-first PR QA draft, names affected flow, changed files, suggested E2E/checklist path, missing evidence, PR checklist, agent handoff, and says manifest is optional upgrade rather than a first-use gate. |
|
|
203
242
|
| Packaged PR QA skill template | `package metadata includes the portable PR QA skill template` | npm package metadata includes `skills`, and `skills/qamap-pr-qa/SKILL.md` contains the local PR QA workflow, `qamap qa` command, and manifest repair guidance. |
|
|
204
243
|
| Verification manifest loop | `manifest init creates a baseline verification manifest`; `manifest init keeps Expo app file domains specific`; `manifest init captures advisory instruction context`; `manifest bootstrap produces concrete PR E2E draft from repo QA memory`; `e2e draft can use an external verification manifest for read-only adoption preview`; `manifest matches explain e2e and verify recommendations`; `manifest validate reports missing and stale manifest policy` | Generated `.qamap/manifest.yaml` includes `$schema`, domains, flows, anchors, checks, runner, source, and confidence; context preview reports repo-local instruction sources, role summaries, validation commands, safety rules, and diagnostics; validator reports missing/stale/duplicate policy; explain output maps branch changes to manifest domains/flows/checks; E2E drafts prefer `verification-manifest` sources with manifest evidence, route entry, detected input/action selectors, required checks, and manifest repair paths; external manifests can be passed with `--manifest` for read-only adoption smoke tests. |
|
|
205
|
-
| Web app with Playwright routes | `generateE2ePlan matches committed core flow definitions`; `generateE2eDraft uses web selectors in Playwright specs`; `generateE2eDraft dry run previews files without writing drafts`; `generateE2eDraft asserts changed HTML success copy in Playwright specs`; `generateE2ePlan captures Playwright execution profile and self-check blockers`; `generateE2ePlan infers Playwright base URLs from dev scripts`; `generateE2eDraft supports Next app router route groups and concrete route hints`; `generateE2ePlan reads React Router object route paths`; `generateE2eDraft fills dynamic route params from concrete route hints`; `generateE2eDraft emits runnable Playwright role and input actions` | `Web` project profile,
|
|
206
|
-
| Expo / React Native mobile app | `generateE2ePlan recommends mobile flows for Expo changes`; `generateE2ePlan detects Maestro app ids from app config files`; `generateE2eDraft scopes entrypoint hints to each domain scenario`; `generateE2eDraft names changed component actions before generic primary journeys` | `Expo / React Native` project profile,
|
|
244
|
+
| Web app with Playwright routes | `generateE2ePlan matches committed core flow definitions`; `generateE2eDraft uses web selectors in Playwright specs`; `generateE2eDraft dry run previews files without writing drafts`; `generateE2eDraft asserts changed HTML success copy in Playwright specs`; `generateE2ePlan captures Playwright execution profile and self-check blockers`; `generateE2ePlan infers Playwright base URLs from dev scripts`; `generateE2eDraft supports Next app router route groups and concrete route hints`; `generateE2ePlan reads React Router object route paths`; `generateE2eDraft fills dynamic route params from concrete route hints`; `generateE2eDraft emits runnable Playwright role and input actions` | `Web` project profile, Playwright output adapter, intent-backed or core-flow names, route-aware drafts, dry-run preview status without filesystem writes, stable selector hints, changed HTML success copy assertions, execution profile, dev-script base URL hints, opt-in setup proposal, route groups, object paths, dynamic params, draft self-check status, action items, and validation gaps. |
|
|
245
|
+
| Expo / React Native mobile app | `generateE2ePlan recommends mobile flows for Expo changes`; `generateE2ePlan detects Maestro app ids from app config files`; `generateE2eDraft scopes entrypoint hints to each domain scenario`; `generateE2eDraft names changed component actions before generic primary journeys` | `Expo / React Native` project profile, Maestro output adapter, app id and launch command hints from `app.json` or `app.config.*`, YAML drafts, `testID`/`accessibilityLabel` selector hints, and mobile setup actions after runner-independent QA intent. |
|
|
207
246
|
| API or backend service | `generateE2ePlan detects API service projects and suggests contract checklists`; `generateE2ePlan detects Django service apps from a workspace root`; `generateE2ePlan names versioned API service paths with domain language`; `generateE2ePlan uses matched core flow names for API service contracts` | `API / service` project profile, manual contract checklist, Django/FastAPI-style service signals when present, domain-aware titles such as `Listing API contract`, API consumer actor, endpoint/handler/service-path trigger, service start/test command hints, and contract failure coverage. |
|
|
208
247
|
| CLI package | `generateE2ePlan detects CLI packages and suggests command verification checklists` | `CLI` project profile from `package.json` bin entries, manual command verification checklist, CLI user or maintainer actor language, command invocation trigger, stdout/stderr/generated-file/exit-code success signal, valid and invalid argument coverage, and no required API fixture action unless the changed command path explicitly exposes network or fixture evidence. |
|
|
209
248
|
| Design tokens and data catalogs | `generateE2ePlan detects design token packages and suggests artifact validation`; `generateE2ePlan detects data catalog repositories and suggests catalog verification` | `Design tokens` and `Data catalog` project profiles, manual artifact/catalog checklist, token or catalog actor language, schema/generated output/consumer fixture coverage, fixture readiness marked not needed for API mocks, and validation matrix rows that do not require browser/device selectors. |
|
|
@@ -216,43 +255,17 @@ The matrix below is public, fixture-backed evidence from the repository test sui
|
|
|
216
255
|
|
|
217
256
|
See [E2E output examples](e2e-output-examples.md) for the kind of plan and draft snippets users should see from the current release.
|
|
218
257
|
|
|
219
|
-
##
|
|
220
|
-
|
|
221
|
-
Last verified on 2026-07-03 for the 0.3.1 README demo patch after removing the stale animated GIF from the README and npm tarball:
|
|
222
|
-
|
|
223
|
-
| Check | Result |
|
|
224
|
-
| --- | --- |
|
|
225
|
-
| `pnpm test` | 95 tests passed. |
|
|
226
|
-
| `git diff --check` | Passed. |
|
|
227
|
-
| `pnpm pack --dry-run` | Passed; tarball includes 97 files and no committed demo GIF. |
|
|
228
|
-
| `npm publish --dry-run --access public` | Passed for `@ivorycanvas/qamap@0.3.1`; package size is 289.1 kB and includes the manifest schema, quick-start docs, release docs, and `skills/qamap-pr-qa/SKILL.md`. |
|
|
229
|
-
|
|
230
|
-
## Real Repository Smoke Snapshot
|
|
231
|
-
|
|
232
|
-
The latest smoke run used private representative repositories and wrote draft output only under `/tmp/qamap-preview-*`. The smoke commands did not run `e2e setup` or write generated files into the target repositories. The table records public-safe target shapes rather than private repository names.
|
|
233
|
-
|
|
234
|
-
2026-07-02 follow-up smoke on PR #71 ran `manifest context`, `manifest init --write /tmp/...`, `e2e plan`, and `e2e draft --dry-run` across 14 local Git repositories plus package-scoped scans for a large frontend monorepo. All commands completed without changing target repository status. The run exposed one important adoption need: users should be able to generate a manifest outside the target repo and pass it back into PR commands. PR #71 now supports that read-only preview through `--manifest <file>`.
|
|
235
|
-
|
|
236
|
-
The same follow-up also showed the next quality gap. External manifest previews work, but many real branch changes did not match route/page-only manifest anchors, so generated drafts often stayed `domain-language`, `review-only`, or `near-runnable` instead of `verification-manifest`. The next release bar should improve component, function, API client, query, schema, and test anchors so manifest matches cover more than route files.
|
|
237
|
-
|
|
238
|
-
| Target shape | Base/head mode | Result | Follow-up signal |
|
|
239
|
-
| --- | --- | --- | --- |
|
|
240
|
-
| Expo / React Native manifest baseline | `manifest init` wrote only to `/tmp/qamap-journal-note-manifest.yaml` | Generated 9 domains, 8 flows, 8 anchors, and 16 checks without changing the target repo. Direct `app/*.tsx` screens now produce specific paths such as `app/SentimentChatPage.tsx`; `+not-found.tsx` is not promoted as a product domain. | Good 0.2.x signal for baseline quality. Remaining work is richer screen/route semantics and selector-specific checks. |
|
|
241
|
-
| Web monorepo package | Package scan with `--workspace-root`, feature branch compared with `main`, working tree included | Detected `web`, recommended Playwright, inferred a concrete route, produced changed-flow specs, and correctly blocked promotion because Playwright config, deterministic fixture/mock data, selector evidence, and validation evidence were missing. | Flow naming improved, but docs/design-only files can still create low-signal drafts that should be filtered or demoted. |
|
|
242
|
-
| Expo / React Native app | Feature branch compared with `develop`, working tree included | Detected `expo-react-native`, recommended Maestro, found an existing Jest suite, produced multiple near-runnable YAML drafts, and gave useful blockers for missing Maestro directory plus missing stable mobile selectors. | Strongest current real-repo result; remaining gap is selector and app-specific setup quality. |
|
|
243
|
-
| Nuxt / Vue web app with existing Playwright tests | `origin/develop` to `HEAD`, working tree included | Detected `web`, recommended Playwright, recognized existing test evidence, and blocked draft promotion because no Playwright config or runnable route/screen entrypoint was inferred. | Test-only or generated-test changes can still produce generic smoke draft names; QAMap should better distinguish changed tests from changed product behavior. |
|
|
244
|
-
| Django-style API service | `origin/develop` to `HEAD`, working tree included | Detected `api-service`, selected manual output, found a large pytest suite, generated API contract and configuration checklists with zero TODOs, and avoided browser/device runner assumptions. | Good service classification; fixture readiness should become more endpoint-specific. |
|
|
245
|
-
| Design token repository | `main` to `HEAD`, working tree included | Detected `design-tokens`, selected manual output, avoided browser/device selector requirements, and produced review-only artifact validation output. | Docs-only or style-only changes may still surface content/theme wording instead of token artifact language. |
|
|
246
|
-
| Taxonomy / data catalog repository | `main` to `HEAD`, working tree included | Detected `data-catalog`, selected manual output, avoided API mock requirements, and produced review-only checklist output with no TODOs. | Catalog-specific changes are handled, but docs/config-only changes should be labeled more explicitly as low-signal. |
|
|
258
|
+
## Private Smoke Protocol
|
|
247
259
|
|
|
248
|
-
|
|
260
|
+
Private repositories may be used as local, read-only diagnostics, but their raw output is never release documentation or fixture input. Write manifests and drafts only to an operating-system temp directory, compare target `git status` before and after, and retain no repository names, paths, flow titles, source excerpts, or domain vocabulary in commits or pull requests. Reduce every confirmed defect to a minimal synthetic fixture before it enters the public suite.
|
|
249
261
|
|
|
250
262
|
## Ongoing Validation Notes
|
|
251
263
|
|
|
252
|
-
The release candidate
|
|
264
|
+
The release candidate must pass the fixture-backed suite, package dry-run, and read-only smoke protocol. Record only public synthetic evidence in this document or in release notes:
|
|
253
265
|
|
|
254
266
|
- whether the generated flow names match the team's domain language
|
|
255
|
-
- whether the
|
|
267
|
+
- whether commit evidence supports the inferred intent and lifecycle
|
|
268
|
+
- whether the automation adapter is plausible without being mistaken for the product value
|
|
256
269
|
- whether generated drafts identify the right actor, trigger, success signal, and edge cases
|
|
257
270
|
- whether action items are concrete enough for a developer to convert into runnable tests
|
|
258
271
|
- whether false positives are caused by missing manifests, weak selectors, or unsupported project structure
|
|
@@ -260,7 +273,7 @@ The release candidate has passed the fixture-backed suite, package dry-run, npm
|
|
|
260
273
|
|
|
261
274
|
## Stop Conditions
|
|
262
275
|
|
|
263
|
-
Do not publish
|
|
276
|
+
Do not publish the current candidate if any representative target shows one of these problems:
|
|
264
277
|
|
|
265
278
|
- generated flow names are dominated by generic folder names instead of product language
|
|
266
279
|
- test-only or docs-only changes are presented as confident product journeys without low-signal wording
|
package/docs/releasing.md
CHANGED
|
@@ -2,6 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
This runbook defines the release process for `qamap`. It is intentionally conservative: the package should be published only when the local release gate, documentation, and representative repository validation all agree.
|
|
4
4
|
|
|
5
|
+
## Pre-1.0 Version Policy
|
|
6
|
+
|
|
7
|
+
QAMap keeps major and minor changes deliberately rare during `0.x` development.
|
|
8
|
+
|
|
9
|
+
- Patch is the default for bug fixes, inference quality, performance, internal architecture, benchmarks, documentation, and additional adapters that preserve the existing CLI, schema, and safety contracts.
|
|
10
|
+
- Minor is reserved for a new product-level capability, an incompatible CLI or manifest contract, or a meaningful change to default execution and safety behavior.
|
|
11
|
+
- Risky minor work should ship through `alpha`, `beta`, and `rc` prereleases before the final minor.
|
|
12
|
+
- Do not pre-allocate a minor version to every roadmap phase. Define the next minor release bar and continue compatible work as patches until that bar is met.
|
|
13
|
+
|
|
14
|
+
Version `0.4.0` is earned by the first commit-to-intent-to-scenario vertical slice: behavior-bearing commits and diff symbols become an evidence-backed lifecycle and concrete runner-independent QA, then existing Playwright, Maestro, or manual adapters compile the result. The next minor is reserved for explicit temporary execution and normalized evidence without modifying the target repository.
|
|
15
|
+
|
|
16
|
+
Version `1.0.0` requires a stable public contract and external adoption, not implementation volume alone. CLI commands, exit codes, machine output, manifest migration, adapter compatibility, no-LLM/no-upload guarantees, and release operations must be dependable. Repository stars are useful social proof, but repeated use in unrelated repositories and reported QA value are stronger release evidence.
|
|
17
|
+
|
|
5
18
|
## Release Owner Checklist
|
|
6
19
|
|
|
7
20
|
Before publishing, confirm:
|
package/docs/roadmap.md
CHANGED
|
@@ -1,47 +1,56 @@
|
|
|
1
1
|
# Roadmap
|
|
2
2
|
|
|
3
|
-
QAMap
|
|
3
|
+
QAMap is a local CLI for evidence-backed PR QA design. The project can grow in layers without becoming a hosted platform or token-dependent agent.
|
|
4
4
|
|
|
5
5
|
## North Star
|
|
6
6
|
|
|
7
|
-
QAMap should become a local-
|
|
7
|
+
QAMap should become a local, zero-LLM, change-aware QA engine. It should map any PR change to affected behavior, select the smallest relevant QA set, and eventually execute that QA with local evidence. A repository-level verification manifest supplies reviewed product intent that static code evidence cannot prove. The goal is not to replace reviewers or QA. The goal is to remove the repeated blank-page and manual verification work that makes developers skip good QA.
|
|
8
8
|
|
|
9
9
|
The sharp product position is:
|
|
10
10
|
|
|
11
11
|
```txt
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
Read the change, find the affected behavior, and produce local QA evidence.
|
|
13
|
+
No source upload. No LLM token. Let reviewed repository memory improve every later PR.
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
This means QAMap should be judged
|
|
16
|
+
This means QAMap should be judged by whether it identifies the right behavior and catches a seeded regression, not only whether it writes plausible test code. Playwright, Maestro, and other runners are implementation details behind that product contract.
|
|
17
17
|
|
|
18
18
|
## Release Bar
|
|
19
19
|
|
|
20
20
|
Before treating the next public release as ready, the golden demo must satisfy these conditions:
|
|
21
21
|
|
|
22
22
|
- First-run output is concrete, not broad: it names the affected feature, flow, draft file, and checks.
|
|
23
|
+
- Commit evidence becomes an explicit change intent and ordered behavior lifecycle before any runner is recommended.
|
|
24
|
+
- Generic `primary journey` and `smoke flow` names fail the benchmark when commit and diff evidence can support a concrete lifecycle.
|
|
23
25
|
- Manifest authoring burden stays low: `manifest context` and `manifest init` provide a useful baseline before a human edits YAML.
|
|
24
26
|
- Generated E2E draft is a usable starting point: it has route/screen entry, meaningful actions, assertions, and clear TODOs only where repo data is missing.
|
|
25
27
|
- Recommendation evidence is explainable: output shows the changed file, manifest flow/check, and manifest path to repair when wrong.
|
|
26
28
|
- README demo shows the full loop: manifest-free PR QA draft, optional repo context baseline, PR mapping, E2E draft, and remaining validation gaps.
|
|
27
29
|
- One manifest correction should improve future PR recommendations without another LLM prompt.
|
|
28
30
|
|
|
29
|
-
## Now
|
|
31
|
+
## Now: 0.4.x
|
|
30
32
|
|
|
33
|
+
- Stabilize commit-range Change Intent analysis: related behavior commits, diff symbols, confidence, review requirements, lifecycle stages, and runner-independent QA scenarios.
|
|
34
|
+
- Keep `qamap.change-intent` as a direct Behavior Graph adapter while moving more source observations out of the compatibility adapter.
|
|
35
|
+
- Preserve `qa` as the static, read-only product surface while designing explicit execution behind `verify`; never turn a scan into implicit project-code execution.
|
|
31
36
|
- Treat the committed [benchmark contract](benchmarking.md) as the quality gate for recommendations, not only implementation correctness. Reduce real failures into public fixtures and require `pnpm bench:ci` on every PR.
|
|
32
|
-
- Make `qa` the primary product surface. Its first screen and `--format agent` payload
|
|
37
|
+
- Make `qa` the primary product surface. Its first screen and `--format agent` payload must agree on change intent, lifecycle, QA scenarios, affected behavior, repository evidence, draft path, and missing trust requirements.
|
|
33
38
|
- Improve changed-file impact mapping from shared symbols and components to consuming routes, screens, API contracts, and manifest flows.
|
|
34
39
|
- Keep the [release validation checklist](release-validation.md), [manifest guide](manifest.md), public [E2E output examples](e2e-output-examples.md), and README examples aligned with captured output from the public fixtures.
|
|
35
40
|
- Stabilize the manifest feedback loop with `.qamap/manifest.yaml`, `manifest init`, `manifest validate`, `manifest explain`, JSON Schema, and manifest-driven E2E draft shaping.
|
|
36
41
|
- Keep `manifest context` useful as a pre-init sanity check for repo-local QA memory, harness docs, agent instructions, and runbooks.
|
|
37
|
-
- Improve
|
|
42
|
+
- Improve Playwright and Maestro compilation from intent scenarios while keeping runner choice behind the runner-independent QA contract.
|
|
38
43
|
- Keep `verify`, `e2e`, and `manifest` as deeper layers behind `qa`; freeze new scanner, doctor, eval, domains, flows, and history features until the core QA contract is consistently useful.
|
|
39
44
|
|
|
40
45
|
## Next
|
|
41
46
|
|
|
47
|
+
- Move route, screen, endpoint, selector, fixture, test, and contract discovery into analyzer adapters, starting with TypeScript web stacks and reusing one web behavior model across Next.js, React Router, Vue/Nuxt, and SvelteKit.
|
|
48
|
+
- Compare base and head Behavior Graphs, then select impacted graph paths before refining deterministic success, validation, failure, empty, loading, auth, and contract scenarios.
|
|
49
|
+
- Add policy-controlled temporary execution so selected scenarios can produce normalized pass, fail, blocked, and not-verifiable evidence without modifying the target repository.
|
|
50
|
+
- Add an execution benchmark that injects known regressions and requires QAMap to select and fail the relevant scenario without changing the fixture repository.
|
|
42
51
|
- Add symbol-level anchors for exported components, hooks, API clients, handlers, schemas, and queries after the public import-impact fixture stays stable.
|
|
43
52
|
- Add a manifest correction command that proposes the exact flow/anchor patch and applies it only after human approval, avoiding routine hand-edits to YAML.
|
|
44
|
-
- Add stronger deterministic draft adapters for Playwright and Maestro while keeping `manual` output for API, CLI, token, and catalog repositories.
|
|
53
|
+
- Add stronger deterministic draft adapters for Playwright and Maestro while keeping `manual` output for API, CLI, token, and catalog repositories; runner detection itself is not a product success metric.
|
|
45
54
|
- Expand the public benchmark corpus with package-scoped monorepos, auth/session changes, dynamic routes, API failure fixtures, and non-JavaScript services.
|
|
46
55
|
- Keep the `--format agent` output a stable, versioned contract that skills and MCP wrappers can rely on.
|
|
47
56
|
- Continue expanding agent surface detection across popular coding-agent tools without making the public workflow depend on a single vendor.
|
|
@@ -55,7 +64,8 @@ Before treating the next public release as ready, the golden demo must satisfy t
|
|
|
55
64
|
|
|
56
65
|
## Non-Goals
|
|
57
66
|
|
|
58
|
-
-
|
|
67
|
+
- Static commands such as `qa`, `scan`, and planning modes will not execute scanned project code. Future execution requires an explicit command and a visible, policy-controlled execution plan.
|
|
68
|
+
- QAMap will not implement its own browser or device automation engine; it will orchestrate proven local executors behind a framework-neutral QA contract.
|
|
59
69
|
- QAMap will not replace tests, review, branch protection, threat modeling, or security review.
|
|
60
70
|
- QAMap will not become a general-purpose code style linter.
|
|
61
71
|
- QAMap will not become a deep MCP server analysis engine.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ivorycanvas/qamap",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Local-
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Local zero-LLM PR QA designer that turns commit intent and diffs into behavior lifecycles, QA scenarios, and deterministic automation drafts.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "pnpm@10.32.1",
|
|
7
7
|
"bin": {
|
|
@@ -39,6 +39,8 @@
|
|
|
39
39
|
"mcp",
|
|
40
40
|
"preflight",
|
|
41
41
|
"pull-request",
|
|
42
|
+
"change-intent",
|
|
43
|
+
"behavior-graph",
|
|
42
44
|
"e2e",
|
|
43
45
|
"e2e-testing",
|
|
44
46
|
"qa",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"runner": {
|
|
46
46
|
"type": "string",
|
|
47
47
|
"enum": ["maestro", "playwright", "manual"],
|
|
48
|
-
"description": "
|
|
48
|
+
"description": "Automation output adapter selected after runner-independent QA intent analysis."
|
|
49
49
|
},
|
|
50
50
|
"manifest": {
|
|
51
51
|
"type": ["string", "null"],
|
|
@@ -71,6 +71,47 @@
|
|
|
71
71
|
"files": { "type": "number", "minimum": 0 }
|
|
72
72
|
}
|
|
73
73
|
},
|
|
74
|
+
"intents": {
|
|
75
|
+
"type": "array",
|
|
76
|
+
"description": "Evidence-backed change intents derived before runner selection (capped).",
|
|
77
|
+
"items": {
|
|
78
|
+
"type": "object",
|
|
79
|
+
"required": ["title", "confidence", "reviewRequired", "evidence", "lifecycle", "scenarios"],
|
|
80
|
+
"properties": {
|
|
81
|
+
"title": { "type": "string" },
|
|
82
|
+
"confidence": { "type": "string", "enum": ["low", "medium", "high"] },
|
|
83
|
+
"reviewRequired": { "type": "boolean" },
|
|
84
|
+
"evidence": { "type": "array", "items": { "type": "string" } },
|
|
85
|
+
"lifecycle": {
|
|
86
|
+
"type": "array",
|
|
87
|
+
"items": {
|
|
88
|
+
"type": "object",
|
|
89
|
+
"required": ["phase", "label"],
|
|
90
|
+
"properties": {
|
|
91
|
+
"phase": {
|
|
92
|
+
"type": "string",
|
|
93
|
+
"enum": ["trigger", "condition", "action", "state-change", "side-effect", "observable-outcome"]
|
|
94
|
+
},
|
|
95
|
+
"label": { "type": "string" }
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
"scenarios": {
|
|
100
|
+
"type": "array",
|
|
101
|
+
"items": {
|
|
102
|
+
"type": "object",
|
|
103
|
+
"required": ["priority", "kind", "title", "assertions"],
|
|
104
|
+
"properties": {
|
|
105
|
+
"priority": { "type": "string", "enum": ["critical", "recommended"] },
|
|
106
|
+
"kind": { "type": "string", "enum": ["primary", "failure", "boundary", "state-transition"] },
|
|
107
|
+
"title": { "type": "string" },
|
|
108
|
+
"assertions": { "type": "array", "items": { "type": "string" } }
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
},
|
|
74
115
|
"firstDraftCommand": {
|
|
75
116
|
"type": "string",
|
|
76
117
|
"description": "One command that creates the first E2E draft. Only present when the repository has no test suite and the diff reaches a product journey."
|