@asrunjs/asrun 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/explore-case.md +2 -1
- package/.claude/commands/gen-spec.md +2 -1
- package/.claude/commands/heal-spec.md +2 -1
- package/.claude/commands/make-case.md +2 -1
- package/.claude/skills/exploring-cases/SKILL.md +59 -64
- package/.claude/skills/exploring-cases/references/case-rules.md +5 -0
- package/.claude/skills/exploring-cases/references/notes.md +87 -0
- package/.claude/skills/exploring-cases/references/open-questions.md +8 -0
- package/.claude/skills/exploring-cases/references/readiness.md +3 -2
- package/.claude/skills/exploring-cases/references/tools.md +34 -0
- package/.claude/skills/exploring-cases/references/user-handoff.md +13 -9
- package/.claude/skills/generating-specs/SKILL.md +61 -47
- package/.claude/skills/generating-specs/references/conventions.md +6 -2
- package/.claude/skills/making-cases/SKILL.md +12 -5
- package/.claude/skills/making-cases/references/writes-and-email.md +5 -2
- package/AGENTS.md +30 -14
- package/README.md +16 -11
- package/bin/asrun.mjs +1 -0
- package/dist/lib/case-doc.d.ts +1 -1
- package/dist/lib/config.d.ts +26 -0
- package/dist/lib/config.js +51 -0
- package/dist/lib/explore-notes.d.ts +74 -0
- package/dist/lib/explore-notes.js +242 -0
- package/dist/lib/fixtures.d.ts +1 -1
- package/dist/lib/fixtures.js +7 -4
- package/dist/lib/paths.d.ts +2 -0
- package/dist/lib/paths.js +2 -0
- package/dist/lib/records.d.ts +17 -2
- package/dist/lib/records.js +32 -2
- package/dist/lib/rules.d.ts +57 -0
- package/dist/lib/rules.js +140 -0
- package/dist/scripts/cleanup.js +10 -1
- package/dist/scripts/cli.js +17 -2
- package/dist/scripts/doctor.js +27 -48
- package/dist/scripts/explore-notes.d.ts +1 -0
- package/dist/scripts/explore-notes.js +131 -0
- package/dist/scripts/lint-specs.js +23 -1
- package/dist/scripts/rules.d.ts +1 -0
- package/dist/scripts/rules.js +148 -0
- package/docs/agents/case-format.md +9 -5
- package/docs/agents/framework-dev.md +4 -0
- package/docs/ai-workflow.md +30 -13
- package/docs/cases.md +14 -10
- package/docs/commands.md +33 -4
- package/docs/install.md +2 -1
- package/docs/theme-guide.html +79 -2
- package/docs/troubleshooting.md +11 -2
- package/docs/web-ui.md +40 -12
- package/package.json +3 -1
- package/templates/.claude/commands/explore-case.md +3 -1
- package/templates/.claude/commands/gen-spec.md +3 -1
- package/templates/.claude/commands/heal-spec.md +3 -1
- package/templates/.claude/commands/make-case.md +3 -1
- package/templates/AGENTS.md +17 -8
- package/templates/cases/_template.md +3 -3
- package/templates/package.json +5 -0
- package/ui/dist/assets/{dist-BmWaNjqO.js → dist-BX88bMYN.js} +1 -1
- package/ui/dist/assets/{dist-2msWI-5M.js → dist-Bfv586Or.js} +1 -1
- package/ui/dist/assets/{dist-e-KhTWHJ.js → dist-BjlbPf3t.js} +1 -1
- package/ui/dist/assets/{dist-BVvHVJ3N.js → dist-CJnqQMvX.js} +1 -1
- package/ui/dist/assets/{dist-BMfJatHo.js → dist-CaAcCNRS.js} +1 -1
- package/ui/dist/assets/{dist-sfZKBSMm.js → dist-Clj0wGFy.js} +1 -1
- package/ui/dist/assets/{dist-J36xayOv.js → dist-D8TUm3Ai.js} +1 -1
- package/ui/dist/assets/{dist-4U_gPHXf.js → dist-DUDqh_3e.js} +1 -1
- package/ui/dist/assets/{dist-BwNz_xW2.js → dist-DkAx5a-Y.js} +1 -1
- package/ui/dist/assets/{dist-wjB7g6OJ.js → dist-cuboTVh-.js} +1 -1
- package/ui/dist/assets/index-DaUQORCy.js +353 -0
- package/ui/dist/assets/index-j3W4BBCj.css +2 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/index-CGbRJT3b.js +0 -348
- package/ui/dist/assets/index-CI0i0LWc.css +0 -2
|
@@ -9,8 +9,10 @@ Read `references/conventions.md` first. It defines the spec shape, fixtures, ass
|
|
|
9
9
|
the read-only rules. Read the workspace `AGENTS.md` for the hard rules and the project rules. Read `../exploring-cases/references/open-questions.md`: it
|
|
10
10
|
defines how open questions are written into the case file, resolved and reported. Also read
|
|
11
11
|
`../exploring-cases/references/case-rules.md` (how a case's `## Rules` are read, applied and maintained),
|
|
12
|
-
`../exploring-cases/references/assertion-targets.md` (the exact thing each check looks at)
|
|
13
|
-
`../exploring-cases/references/readiness.md` (wait for the content, verify downloads in five parts)
|
|
12
|
+
`../exploring-cases/references/assertion-targets.md` (the exact thing each check looks at),
|
|
13
|
+
`../exploring-cases/references/readiness.md` (wait for the content, verify downloads in five parts),
|
|
14
|
+
`../exploring-cases/references/notes.md` (the local exploration note this run reads first) and
|
|
15
|
+
`../exploring-cases/references/tools.md` (the browser tools this MCP can run). All six are binding here.
|
|
14
16
|
|
|
15
17
|
This skill never stops to ask the user in the chat, because it may run without anybody present (a one-shot
|
|
16
18
|
`claude -p` / `opencode run`). This holds whatever mode or agent the run is in: plan mode changes what you may write,
|
|
@@ -35,10 +37,15 @@ spec afterwards (see the report). Healing (below) is the exception: it reads the
|
|
|
35
37
|
and the spec, and runs only the profiles that are logged in, and only when the report is not enough.
|
|
36
38
|
|
|
37
39
|
## Before you start
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
`AGENTS.md`
|
|
41
|
-
|
|
40
|
+
Run `pnpm asrun rules --json` (or `pnpm asrun rules` for people): it prints the framework baseline, the project's
|
|
41
|
+
`AGENTS.md` §2 rules and the project settings (the test-data prefix). Those are the effective rules; if the command is
|
|
42
|
+
unavailable, read the workspace `AGENTS.md` directly. Also read `AGENTS.md` §2, and `.claude/skills/generating-specs.md`
|
|
43
|
+
in the workspace, applying the override after this skill and §2 after both (the baseline, "Precedence"). F1–F6 and R2
|
|
44
|
+
always win. **Never turn a project rule into a question:** a difference between §2 and a framework default is not a
|
|
45
|
+
conflict; apply §2 and, if a row's id does not fit its text, apply the text and report the mismatch. This is not a stop
|
|
46
|
+
condition: continue with the procedure below. If `pnpm asrun rules` exits 1 (an invalid §2 row or a prefix that the
|
|
47
|
+
document and `package.json` disagree on), do not guess and do not turn it into a case question: report its errors and
|
|
48
|
+
end with `RESULT: needs-input`; a person fixes `AGENTS.md` or `package.json` first.
|
|
42
49
|
|
|
43
50
|
## Inputs
|
|
44
51
|
- The case id and the environment to explore in (`staging` or `production`). There is no default. If the
|
|
@@ -116,42 +123,42 @@ Decide this first, before exploring anything. The spec path is `tests/<id>.spec.
|
|
|
116
123
|
never for what the check asserts about it: that goes to `test.fixme` with the reason, or `needs-input`.
|
|
117
124
|
- Do not invent expected values. If the case's `## Values` and the spec disagree, the case wins: fix the spec and
|
|
118
125
|
mention the change in the report.
|
|
119
|
-
3. **Check sessions** if `login: required`:
|
|
120
|
-
|
|
121
|
-
information for is never dropped because it is not logged in: write its part from
|
|
122
|
-
`not verified on the page` in the report.
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
126
|
+
3. **Check sessions** if `login: required`: follow `../exploring-cases/references/tools.md`, "Starting a session". A
|
|
127
|
+
profile that needs login is named with `pnpm signin <profile-id>`, and the run continues with the profiles that are
|
|
128
|
+
ready. A profile the case holds information for is never dropped because it is not logged in: write its part from
|
|
129
|
+
the case and mark it `not verified on the page` in the report.
|
|
130
|
+
4. **Replay the case on the real page** with the `playwright-test` MCP, following
|
|
131
|
+
`../exploring-cases/references/tools.md` ("Starting a session", "Short calls") and `readiness.md` ("Exploring"). If
|
|
132
|
+
its tools are not registered in this session at all (no `browser_*` / `generator_*` tool) or it reports that it
|
|
133
|
+
failed to start, do not fall back to ad-hoc Playwright scripts: stop, tell the user which MCP server is missing and
|
|
134
|
+
to run `pnpm asrun doctor`, write no spec, and end with `RESULT: needs-input`.
|
|
135
|
+
**Read each profile's exploration note first** (`../exploring-cases/references/notes.md`): run
|
|
136
|
+
`pnpm asrun explore:notes <case-id> --json`. A fresh note (its `caseSha256` matches the case) supplies the locator,
|
|
137
|
+
the request pattern and the target's match count for a step; confirm each on the page instead of rediscovering it,
|
|
138
|
+
and skip what it does not hold. A missing or stale note means the plain replay below. After confirming an entry of
|
|
139
|
+
a fresh note, you may write the verified locator or request back into that file (`updatedAt` gets the time); never
|
|
140
|
+
add a step or a target, and never touch a stale note. Say in the report what came from a note and what was
|
|
141
|
+
refreshed. Perform the steps in the environment you were given (never ask for it; there is no default).
|
|
134
142
|
Do the read-only steps as written, and follow the `## Rules` that say how to prepare, wait or which control to
|
|
135
143
|
avoid. **Never perform a step that writes or needs a choice**: the agent does not write
|
|
136
144
|
data, in staging or in production, and it does not hand off to a person here or wait for one. A rule never
|
|
137
145
|
authorizes one of those steps. Write those steps
|
|
138
|
-
of the spec from the `- Recipe:` lines under the step (positions,
|
|
146
|
+
of the spec from the `- Recipe:` lines under the step (positions, created names, control labels and roles that
|
|
139
147
|
`/explore-case` recorded). Pages that only exist after such a step cannot be seen from here: write their locators
|
|
140
|
-
from
|
|
141
|
-
leave the first run of the spec (or `/heal-spec`) to confirm
|
|
142
|
-
per domain (example.com and exampleshop.com differ in markup). A profile the case holds
|
|
143
|
-
not logged in is skipped and named in the report; its part is written from the case and
|
|
144
|
-
the assertion table. Use this exploration to answer the open points of step 2, and read each
|
|
145
|
-
the case's `## Values`: open that profile's page only when a value or a behavior still has to be
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
question, never `.first()`.
|
|
148
|
+
from a fresh note first, then from the recipe, and mark them in the report as "observed by explore on <profile>" or
|
|
149
|
+
"not verified on the page"; leave the first run of the spec (or `/heal-spec`) to confirm the latter. Do this for at
|
|
150
|
+
least one logged-in profile per domain (example.com and exampleshop.com differ in markup). A profile the case holds
|
|
151
|
+
information for that is not logged in is skipped and named in the report; its part is written from the case and
|
|
152
|
+
marked `not verified` in the assertion table. Use this exploration to answer the open points of step 2, and read each
|
|
153
|
+
profile's value from the case's `## Values`: open that profile's page only when a value or a behavior still has to be
|
|
154
|
+
resolved for it.
|
|
155
|
+
**Ready before you read, here too**: follow `readiness.md`. The request URL patterns for `actAndWait` come from a
|
|
156
|
+
fresh note or from `browser_network_requests` during this replay, never from a guess; the `Ready when` signal comes
|
|
157
|
+
from the case.
|
|
158
|
+
**Check every target on the page:** follow `../exploring-cases/references/assertion-targets.md`, "What the spec does
|
|
159
|
+
with a card (generating)". Confirm that the card's `Identified by` matches exactly one element and that this element
|
|
160
|
+
is what `What` says (outline it with `locator.highlight()` and look at the screenshot, see `tools.md`). A locator that
|
|
161
|
+
matches several elements or the wrong one is fixed or becomes a question, never `.first()`.
|
|
155
162
|
Prefer `getByTestId` (the project uses `data-tn`), then role/label locators. No CSS chains or XPath.
|
|
156
163
|
Update the case file with what you learned: fold observations into the right section, marked `(seen on <profile>)`
|
|
157
164
|
or, for a literal the spec asserts, `(observed on <profile>, 🔍 to be confirmed: <what to decide>)`; remove the
|
|
@@ -179,7 +186,8 @@ Decide this first, before exploring anything. The spec path is `tests/<id>.spec.
|
|
|
179
186
|
never decided by one `isVisible()` look (its `timeout` option is ignored): use `clickOptionalUntilHidden`
|
|
180
187
|
(`@asrunjs/asrun/ready`) or a waiting `expect(...).toBeVisible({timeout})`; see `readiness.md`, "Optional confirmations".
|
|
181
188
|
7. **Check the files**: run `pnpm typecheck`, `pnpm spec:lint <id>` and `pnpm case:check` (or `node --import tsx <framework>/scripts/check-cases.ts` if
|
|
182
|
-
it fails with a sandbox error such as `listen EPERM`).
|
|
189
|
+
it fails with a sandbox error such as `listen EPERM`). When you wrote an exploration note back, also run
|
|
190
|
+
`pnpm asrun explore:notes <id> --check --profile <profile-id>` for each profile you refreshed. Fix every error, including every `FAIL` of the spec lint (a lint failure is never silenced: fix the spec or turn the check
|
|
183
191
|
into a question). Every lint `WARN` is handled too: fix it, or name it in the report with the reason it is acceptable.
|
|
184
192
|
Then do the **self-review**: for each assertion answer in writing "if the product were broken, would this fail?"
|
|
185
193
|
and "if the product is right, could this fail by timing?" (a check that cannot fail, or that races the page, is rewritten).
|
|
@@ -188,7 +196,8 @@ Decide this first, before exploring anything. The spec path is `tests/<id>.spec.
|
|
|
188
196
|
Do NOT run `pnpm asrun`, do not use
|
|
189
197
|
`--update-snapshots` and do not create baselines: the first run of the spec does that, and a person reviews them.
|
|
190
198
|
8. **Report** in the user's language: spec path, **an assertion table** (step, what is asserted, target card, ready signal, and
|
|
191
|
-
`observed on the page` or `not verified`; no row may be missing), profiles explored,
|
|
199
|
+
`observed on the page` or `not verified`; no row may be missing), profiles explored, which steps were written from a
|
|
200
|
+
fresh exploration note and which notes were refreshed (or that there were none), profiles skipped (and why),
|
|
192
201
|
the profiles of `profiles.yaml` the case holds no information for (they run on `default` alone), whether the case file was
|
|
193
202
|
translated, what was added to the case file as `(seen on <profile>)` and which lines still need a person
|
|
194
203
|
(`🔍 to be confirmed`, with the reason), checks generated from observations, visual checks that need a baseline on
|
|
@@ -203,11 +212,13 @@ Decide this first, before exploring anything. The spec path is `tests/<id>.spec.
|
|
|
203
212
|
|
|
204
213
|
## Cases that write data
|
|
205
214
|
Only when the case front matter lists environments under `writes` (for example `writes: [staging]`). Never add
|
|
206
|
-
`--allow-writes` or set `ASRUN_ALLOW_WRITES`. Use the `records` fixture, `configureCase
|
|
207
|
-
described in `references/conventions.md`.
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
215
|
+
`--allow-writes` or set `ASRUN_ALLOW_WRITES`. Use the `records` fixture, `configureCase`, `testDataName` and
|
|
216
|
+
`emailStepAllowed` as described in `references/conventions.md`. Every name the case creates is
|
|
217
|
+
`testDataName(expected.<key>)`, using the base literal from `## Values` and the project prefix from
|
|
218
|
+
`pnpm asrun rules`; use the composed value for the form, the locators and `records.track`. The agent does not perform
|
|
219
|
+
write steps while generating (see step 4); the spec does, when a person runs it in an environment that `writes` lists.
|
|
220
|
+
Elsewhere the runner skips the case with a notice. Never send email in production; in other environments only if
|
|
221
|
+
`profile.email` is true, otherwise skip that step and say so in the report.
|
|
211
222
|
|
|
212
223
|
## Healing and refining a spec
|
|
213
224
|
Inputs: the case id and the environment (both required, there is no default: without an environment stop and say so),
|
|
@@ -250,12 +261,15 @@ Heal from evidence, not from a habit of rerunning. A report that already exists
|
|
|
250
261
|
leaves out the profiles that are not logged in instead of failing them. A profile that is not logged in is never healed:
|
|
251
262
|
name it in the report with `pnpm signin <profile-id>`.
|
|
252
263
|
4. **Look at the pages when the change depends on them.** Reading a page is not a run and is always allowed, also for an
|
|
253
|
-
improvement request with no failure behind it. Use the `playwright-test` MCP exactly as `/gen-spec` step 4 describes
|
|
254
|
-
|
|
264
|
+
improvement request with no failure behind it. Use the `playwright-test` MCP exactly as `/gen-spec` step 4 describes,
|
|
265
|
+
following `../exploring-cases/references/tools.md` ("Starting a session", "Short calls"), `readiness.md` and the
|
|
266
|
+
exploration note (`../exploring-cases/references/notes.md`). Do the read-only
|
|
255
267
|
steps only: **never perform a step that writes or needs a choice**, in any environment. What you see goes into the
|
|
256
268
|
case by the gate in `open-questions.md` ("Which sentences carry a marker"): `(seen on <profile>)` for a locator, a
|
|
257
269
|
flow or a request, and `(observed on <profile>, 🔍 to be confirmed: <what a person must decide>)` only for a
|
|
258
|
-
literal the spec asserts, so the person settles that in the review gate below.
|
|
270
|
+
literal the spec asserts, so the person settles that in the review gate below. If the note was stale, edit the case
|
|
271
|
+
as usual; if it was fresh, a verified locator may be written back
|
|
272
|
+
(`../exploring-cases/references/notes.md`).
|
|
259
273
|
5. **Audit the assertion before you blame anyone.** Read the `assertion-targets.md` and `readiness.md` sections on healing.
|
|
260
274
|
- Is the locator still what the target card says (`What`, `Identified by`)? Open the failure screenshot and the
|
|
261
275
|
baseline (`pnpm baselines:review <case-id>`) and look at each: a baseline that shows something other than the card
|
|
@@ -47,8 +47,12 @@ and `@writes` for cases that write data (drives `--read-only`).
|
|
|
47
47
|
A case that writes only in some environments is skipped (not failed) in the others, with a notice.
|
|
48
48
|
|
|
49
49
|
## Cases that write data (`writes: [<env>, ...]`)
|
|
50
|
-
- Name everything the case creates `
|
|
51
|
-
|
|
50
|
+
- Name everything the case creates through `testDataName(base)` (`@asrunjs/asrun/records`), where `base` is the
|
|
51
|
+
literal from the case's `## Values`: it returns `<prefix> <base> (<UTC run stamp>)`, for example
|
|
52
|
+
`[asrun] Draft CMA (2026-11-09 12-34-56)`. The prefix is the project's `asrun.testDataPrefix` (default `[asrun]`,
|
|
53
|
+
printed by `pnpm asrun rules`). `records.track` refuses a name without the prefix, and the stamp keeps two runs
|
|
54
|
+
apart so a person can find and delete one run's data. Compose a name once and use that variable for the form, the
|
|
55
|
+
locators, the assertions and `records.track`; never pass a raw `expected.*` value as the created name.
|
|
52
56
|
- Register data with the `records` fixture right after creating it, together with the function that removes it:
|
|
53
57
|
`await records.track({kind: 'Document draft', name, where: '/app/documents'}, async () => {...delete it from the documents list...});`
|
|
54
58
|
Everything tracked is removed after the test, newest first, also when the test failed. The remove function opens
|
|
@@ -16,10 +16,15 @@ Read the workspace `AGENTS.md` (hard rules and project rules) and `cases/_templa
|
|
|
16
16
|
This skill writes `cases/*.md` only. It never writes specs, `profiles.yaml`, or anything under `tests/`.
|
|
17
17
|
|
|
18
18
|
## Before you start
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
Run `pnpm asrun rules --json` (or `pnpm asrun rules` for people): it prints the framework baseline, the project's
|
|
20
|
+
`AGENTS.md` §2 rules and the project settings (the test-data prefix). Those are the effective rules; if the command is
|
|
21
|
+
unavailable, read the workspace `AGENTS.md` directly. Also read `AGENTS.md` §2, and `.claude/skills/making-cases.md`
|
|
22
|
+
in the workspace, applying the override after this skill and §2 after both (the baseline, "Precedence"). F1–F6 and R2
|
|
23
|
+
always win. **Never turn a project rule into a question:** a difference between §2 and a framework default is not a
|
|
24
|
+
conflict; apply §2 and, if a row's id does not fit its text, apply the text and report the mismatch. This is not a stop
|
|
25
|
+
condition: continue with the procedure below. If `pnpm asrun rules` exits 1 (an invalid §2 row or a prefix that the
|
|
26
|
+
document and `package.json` disagree on), do not guess and do not turn it into a case question: report its errors and
|
|
27
|
+
stop before writing anything; a person fixes `AGENTS.md` or `package.json` first.
|
|
23
28
|
|
|
24
29
|
## Inputs
|
|
25
30
|
- Raw notes: pasted text or a file path. Any language, any quality.
|
|
@@ -43,7 +48,9 @@ continue with the procedure below.
|
|
|
43
48
|
with a clear message until it is filled), never into `profiles.yaml`.
|
|
44
49
|
5. **Ask before writing.** Show the user a short plan and the open questions in one message: the proposed split
|
|
45
50
|
into cases, the classification (which steps write, which send email), and the gaps with your suggested default
|
|
46
|
-
for each. Wait for answers. Do not write case files before that.
|
|
51
|
+
for each. Wait for answers. Do not write case files before that. **Never ask a project-wide question:** §2 and
|
|
52
|
+
`pnpm asrun rules` settle project policy (the test-data prefix, the language, a wait); apply them. Ask only about
|
|
53
|
+
this case's unknowns.
|
|
47
54
|
6. **Split** (`references/splitting.md`) and **write** `cases/<id>.md` for each case from `cases/_template.md`.
|
|
48
55
|
- Group folder: when the notes (or the user) name an area (documents, search, account...), put the cases in
|
|
49
56
|
`cases/<group>/<name>.md` and set `id: <group>/<name>` in the front matter. Use one folder level and
|
|
@@ -14,9 +14,12 @@ dialog (read-only) but sending the share is `[writes]` and probably `[email]`. W
|
|
|
14
14
|
## What a case with writes must look like
|
|
15
15
|
- Front matter: `writes: [<env>, ...]`, the environments the finished spec may write in (`[staging]`; add `production`
|
|
16
16
|
only when the notes or the user say so), and a `writes-note` that says what is created, changed and removed, in one
|
|
17
|
-
sentence ("Creates one draft document named '[asrun] ...', publishes it, deletes it at the end"). The `writes-note`
|
|
17
|
+
sentence ("Creates one draft document named '[asrun] ... (<run stamp>)', publishes it, deletes it at the end"). The `writes-note`
|
|
18
18
|
is the only summary; there is no separate Data section, and the removal itself is a step in `## Steps`.
|
|
19
|
-
- Everything created is named
|
|
19
|
+
- Everything created is named `<prefix> <something> (<run stamp>)`, where the prefix is the project's
|
|
20
|
+
`asrun.testDataPrefix` (default `[asrun]`; `pnpm asrun rules` prints it) and the stamp is the UTC time of the run
|
|
21
|
+
(`testDataName()` builds it). The case's `## Values` holds the base without the stamp; the spec composes it, so two
|
|
22
|
+
runs never collide and a person can find one run's data in the product.
|
|
20
23
|
- The case ends by deleting what it created. If the notes do not say how to delete it, that is a gap: ask, and
|
|
21
24
|
never guess a delete path. Until it is known, write the likeliest delete step ("Select the 'Delete' option of the
|
|
22
25
|
item's menu on the list page.") and put a `TODO [blocking]: How is <thing> deleted?` block under that step (see
|
package/AGENTS.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
This file is the binding baseline for every project that uses the AsRun framework. It describes no particular
|
|
4
4
|
product: a project adds its own rules in its own `AGENTS.md` §2, and they win where they differ (see "Precedence").
|
|
5
|
-
Its hard rules carry ids (`F1`..`
|
|
5
|
+
Its hard rules carry ids (`F1`..`F11`, `B1`..`B4`, `A1`..`A6`, `R1`..`R4`).
|
|
6
6
|
|
|
7
7
|
Paths are relative to the **workspace root**: the project that owns `profiles.yaml`, `cases/` and `tests/`. In a
|
|
8
8
|
project that uses asrun, the framework files are in `node_modules/@asrunjs/asrun` (an installed dependency: never edit it).
|
|
@@ -19,9 +19,16 @@ Rules are applied in this order; a later layer wins where they conflict.
|
|
|
19
19
|
own procedure.
|
|
20
20
|
5. **The project's `AGENTS.md` §2 (Project rules).** It states the project's intent and wins over layers 2–4.
|
|
21
21
|
|
|
22
|
-
A project writes §2 only for what differs from the defaults here
|
|
23
|
-
`
|
|
24
|
-
|
|
22
|
+
A project writes §2 only for what differs from the defaults here. A row names a baseline rule and says whether it
|
|
23
|
+
`narrows` (stricter) or `replaces` (a different default) it; a rule with no baseline counterpart gets a project id
|
|
24
|
+
(`P1`, `P2`, ...) and `adds`. `replaces` is refused for F1–F6 and R2; `pnpm asrun doctor` checks the table and
|
|
25
|
+
`pnpm asrun rules` prints the effective set. A rule file never renames the structural tokens under "Fixed tokens".
|
|
26
|
+
|
|
27
|
+
**A project rule is applied, never questioned.** §2 wins over the baseline defaults by definition, and a `P<n>` rule
|
|
28
|
+
stands next to them: a difference between §2 and a framework default (a naming prefix, a language, a wait) is not a
|
|
29
|
+
conflict and never becomes a TODO, a marker or a question in a case. The only refusal is a §2 row that weakens
|
|
30
|
+
F1–F6/R2, and `doctor` reports that before any run. Options the framework needs at run time (the test-data prefix)
|
|
31
|
+
live in `package.json` `"asrun"`, so the document and the code read the same value.
|
|
25
32
|
|
|
26
33
|
Inside a case, `## Rules` is a case-local layer below this file and the project's §2: it says how to run that case,
|
|
27
34
|
never what is true (`R1`), and a rule that conflicts with the rest of the case is a question, not an override.
|
|
@@ -57,6 +64,12 @@ never what is true (`R1`), and a rule that conflicts with the rest of the case i
|
|
|
57
64
|
written in English by default. A project replaces this in §2 of its own `AGENTS.md` (for example Simplified Chinese
|
|
58
65
|
for case prose and reports). Code, comments, identifiers, commit messages and the structural tokens under
|
|
59
66
|
"Fixed tokens" always stay in English.
|
|
67
|
+
- **F11 — Test data is findable per run.** A step that writes names what it creates with the project's test-data
|
|
68
|
+
prefix and a run stamp: `testDataName(base)` in `@asrunjs/asrun/records` builds `<prefix> <base> (<YYYY-MM-DD
|
|
69
|
+
HH-mm-ss>)`, the UTC time of the run id (so it matches `reports/<runId>` and the ledger). The prefix is
|
|
70
|
+
`asrun.testDataPrefix` in the project's `package.json` (default `[asrun]`). Two runs never share a name, and a
|
|
71
|
+
person can search the product for one run's data to edit or delete it by hand. The case's `## Values` holds the base
|
|
72
|
+
literal without the stamp; the spec composes it, and a check on the created item uses the composed name.
|
|
60
73
|
|
|
61
74
|
## Browser exploration rules (`B`)
|
|
62
75
|
These apply to every agent that drives a browser to explore pages for cases or specs (`making-cases`,
|
|
@@ -160,7 +173,7 @@ This file holds the rules. The exact shape those rules apply to is documented on
|
|
|
160
173
|
- `.claude/skills/exploring-cases/references/tools.md`: which browser tools the `playwright-test` MCP can run, the
|
|
161
174
|
substitute for each one it only lists, and the arguments every call has to carry. Read it before the first browser
|
|
162
175
|
call of a run.
|
|
163
|
-
- `docs/commands.md`: every command and flag
|
|
176
|
+
- `docs/commands.md`: every command and flag, including `pnpm asrun rules`.
|
|
164
177
|
- `docs/cases.md`: the same case format for the people who write cases by hand.
|
|
165
178
|
- `docs/agents/framework-dev.md`: the framework's own layout and how to change it.
|
|
166
179
|
The project's own `AGENTS.md` §2 (project rules) and §3 (project facts) still win over the documentation where they
|
|
@@ -173,6 +186,8 @@ differ, and `F1`–`F6` and `R2` win over everything.
|
|
|
173
186
|
- `tests/**/*.spec.ts` generated Playwright specs (output of the AI workflow), one per case, same folder as the case,
|
|
174
187
|
with their `*-snapshots/` baselines next to them
|
|
175
188
|
- `profiles.yaml` one entry per environment: URL, account label, tags, environment (connection settings only)
|
|
189
|
+
- `package.json` npm scripts and the framework dependency; its `"asrun"` object holds project settings
|
|
190
|
+
(for example `testDataPrefix`)
|
|
176
191
|
- `AGENTS.md` the project's own rules: they point at this baseline, and §2 holds what differs
|
|
177
192
|
- `.auth/` `reports/` `.tmp/` `.explore/` local state, never committed
|
|
178
193
|
|
|
@@ -184,15 +199,16 @@ Three steps, each learns one thing: the intent, the facts and inputs, the implem
|
|
|
184
199
|
1. `making-cases` (or `/make-case <notes>`) turns raw notes into case documents. It writes `cases/*.md` only and
|
|
185
200
|
never opens a browser. Check them with `pnpm case:check`.
|
|
186
201
|
2. `exploring-cases` (or `/explore-case <case-id> <env> [--attended] [--profiles a,b]`) explores the real pages for one
|
|
187
|
-
case. It edits that case file
|
|
188
|
-
of the steps that write or need a choice
|
|
189
|
-
can be explored in
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
202
|
+
case. It edits that case file: wording, observations, TODO blocks, and, with `--attended`, the `- Recipe:` lines
|
|
203
|
+
of the steps that write or need a choice, and it writes a local note under `.explore/notes/` with the locators and
|
|
204
|
+
data-call paths it saw. `--profiles` narrows the run to those profiles; runs add up, so the case can be explored in
|
|
205
|
+
passes.
|
|
206
|
+
3. `generating-specs` (or `/gen-spec <case-id> <env>`) turns the case into a spec. It replays the case, its
|
|
207
|
+
step recipes and the exploration note, and `## Values`, writes the spec (values stay in the case document), but never
|
|
208
|
+
runs the tests, never writes data and never creates baselines. There is no profile selection here: it covers every
|
|
209
|
+
profile the case names, which is why a case explored in passes generates one spec for all of them. A tagged step with
|
|
210
|
+
no `- Recipe:` line (an old `## Inputs` line counts until it is moved) stops with `needs-input`: run
|
|
211
|
+
`/explore-case` first. Running specs is a separate step (`pnpm asrun`, or the Run page).
|
|
196
212
|
Workspace maintenance is separate from the AI steps: `pnpm asrun upgrade` installs the newest framework, refreshes the
|
|
197
213
|
generated files and applies the release's workspace migrations (it checks the target release for breaking changes
|
|
198
214
|
before installing), `pnpm asrun init [--update] [--force]` writes or refreshes the workspace files, `pnpm asrun
|
package/README.md
CHANGED
|
@@ -99,11 +99,12 @@ exempts `@asrunjs/asrun` from pnpm's 24-hour release age, so a fresh release ins
|
|
|
99
99
|
3. **Write a case.** `pnpm case:new header-logo` creates `cases/header-logo.md` from the template; or start from raw
|
|
100
100
|
notes with `/make-case notes.md`. Fill in the steps, what is expected and the exact values.
|
|
101
101
|
4. **Check it against the real pages.** `/explore-case header-logo staging` opens the pages (with your saved session),
|
|
102
|
-
writes what they show into the case and turns what it cannot know into questions.
|
|
103
|
-
|
|
104
|
-
with the others adds theirs, and the case
|
|
105
|
-
|
|
106
|
-
|
|
102
|
+
writes what they show into the case and turns what it cannot know into questions. It also saves a local note of the
|
|
103
|
+
locators and data calls it saw. Answer the questions in the case file, or in the UI's **Answer** sheet, and run it
|
|
104
|
+
again. With many profiles, `--profiles a,b` opens only those; a later run with the others adds theirs, and the case
|
|
105
|
+
keeps both.
|
|
106
|
+
5. **Generate, run, read.** `/gen-spec header-logo staging` reads those notes, writes `tests/header-logo.spec.ts` and
|
|
107
|
+
fills the case's `## Values`. Then:
|
|
107
108
|
|
|
108
109
|
```bash
|
|
109
110
|
pnpm asrun --env staging --grep header-logo
|
|
@@ -137,11 +138,14 @@ Rules combine in this order, and a later layer wins where they conflict:
|
|
|
137
138
|
2. The framework baseline, `node_modules/@asrunjs/asrun/AGENTS.md`.
|
|
138
139
|
3. The framework skill.
|
|
139
140
|
4. Your override, `.claude/skills/<skill>.md`.
|
|
140
|
-
5. Your own `AGENTS.md` §2: a table of rule ids (`F7`–`
|
|
141
|
-
the artifact language
|
|
141
|
+
5. Your own `AGENTS.md` §2: a table of rule ids (`F7`–`F11`, `R1`, `R3`, `R4`) with `narrows` or `replaces`, and
|
|
142
|
+
project ids (`P1`, ...) with `adds` for a rule the baseline does not have, for example the artifact language or the
|
|
143
|
+
test-data prefix (`package.json` `"asrun"`). A difference from the baseline is applied, never turned into a
|
|
144
|
+
question.
|
|
142
145
|
6. A case's `## Rules`: how to run that one case, never what is true.
|
|
143
146
|
|
|
144
|
-
`pnpm asrun doctor` validates the table and
|
|
147
|
+
`pnpm asrun doctor` validates the table and `pnpm asrun rules` prints the effective set (the AI steps read it before
|
|
148
|
+
they apply anything). An update never rewrites your rules:
|
|
145
149
|
`init --force` refuses `AGENTS.md`, `CLAUDE.md` and `.claude/skills/**` without `--reset-rules`. Never edit the
|
|
146
150
|
framework package itself: an update replaces it, and only the override files are yours. The AI tool stays configurable
|
|
147
151
|
too: `.mcp.json` (Claude Code), `opencode.jsonc` (OpenCode and its model), `.claude/settings.json` (sandbox
|
|
@@ -192,7 +196,8 @@ the top bar. The check never blocks a command and stays quiet when the registry
|
|
|
192
196
|
Tests run against real sites, so the defaults are strict:
|
|
193
197
|
|
|
194
198
|
- **Read-only unless the case says otherwise.** A case may create, edit or delete data only in the environments it
|
|
195
|
-
lists under `writes`, and only data of the test accounts, named
|
|
199
|
+
lists under `writes`, and only data of the test accounts, named with the project's prefix (default `[asrun]`) and a
|
|
200
|
+
run stamp such as `[autotest] Draft CMA (2026-11-09 12-34-56)`, which it must clean up. Elsewhere the
|
|
196
201
|
case is skipped with a notice, not failed.
|
|
197
202
|
- **A person confirms non-staging writes on every run** by typing the environment name, or in a dialog in the UI.
|
|
198
203
|
`--read-only` leaves those cases out; `--allow-writes=<env>` is for unattended runs, and AI agents never use it.
|
|
@@ -201,8 +206,8 @@ Tests run against real sites, so the defaults are strict:
|
|
|
201
206
|
the browser, in an attended exploration. Logins are always a person's job, 2FA included.
|
|
202
207
|
- **A failing baseline is a person's decision.** An agent never updates a baseline to make a difference disappear.
|
|
203
208
|
|
|
204
|
-
The rules that enforce this are `F1`–`
|
|
205
|
-
|
|
209
|
+
The rules that enforce this are `F1`–`F11` in [AGENTS.md](./AGENTS.md); a project can tighten or replace the last
|
|
210
|
+
five, never the first six.
|
|
206
211
|
|
|
207
212
|
## Documentation
|
|
208
213
|
|
package/bin/asrun.mjs
CHANGED
package/dist/lib/case-doc.d.ts
CHANGED
|
@@ -131,7 +131,7 @@ export interface CaseReadySignal {
|
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
133
|
* One `- Recipe (phase): ...` bullet under a step: what a person did for that step during `/explore-case --attended`,
|
|
134
|
-
* as a general recipe (positions,
|
|
134
|
+
* as a general recipe (positions, created names, control labels). It is not test data: the asserted values live in
|
|
135
135
|
* `## Values`.
|
|
136
136
|
*/
|
|
137
137
|
export interface CaseRecipe {
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Project-wide settings that the framework needs at run time, read from the `asrun` object of the workspace's
|
|
3
|
+
* `package.json`:
|
|
4
|
+
*
|
|
5
|
+
* ```json
|
|
6
|
+
* "asrun": { "testDataPrefix": "[autotest]" }
|
|
7
|
+
* ```
|
|
8
|
+
*
|
|
9
|
+
* The file is the project's own: `pnpm asrun init` merges it and never replaces it. A project that does not set an
|
|
10
|
+
* option gets the default below, so an existing workspace keeps working unchanged.
|
|
11
|
+
*/
|
|
12
|
+
export interface WorkspaceConfig {
|
|
13
|
+
/** The prefix every piece of data a case creates carries. Default `[asrun]`. */
|
|
14
|
+
testDataPrefix: string;
|
|
15
|
+
}
|
|
16
|
+
/** The frame's prefix: unmistakably test data, so cleanup can never touch real data. */
|
|
17
|
+
export declare const DEFAULT_TEST_DATA_PREFIX = "[asrun]";
|
|
18
|
+
/**
|
|
19
|
+
* Reads and validates the project settings. Throws an Error whose message says what to fix, so `doctor` and `rules`
|
|
20
|
+
* can report it and a spec that imports it fails with the same sentence instead of a confusing one.
|
|
21
|
+
*/
|
|
22
|
+
export declare function readWorkspaceConfig(): WorkspaceConfig;
|
|
23
|
+
/** The prefix every name a case creates must start with. Reads `package.json`; default `[asrun]`. */
|
|
24
|
+
export declare function testDataPrefix(): string;
|
|
25
|
+
/** Forget the cached settings (a tool that edits `package.json` in the same process). */
|
|
26
|
+
export declare function resetWorkspaceConfig(): void;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { ROOT } from './paths.js';
|
|
4
|
+
/** The frame's prefix: unmistakably test data, so cleanup can never touch real data. */
|
|
5
|
+
export const DEFAULT_TEST_DATA_PREFIX = '[asrun]';
|
|
6
|
+
/** A bracketed lowercase word: `[asrun]`, `[autotest]`, `[qa_team]`. Easy to search for in the product. */
|
|
7
|
+
const PREFIX = /^\[[a-z0-9][a-z0-9_-]*\]$/i;
|
|
8
|
+
let cached = null;
|
|
9
|
+
function readPackageJson() {
|
|
10
|
+
try {
|
|
11
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
|
|
12
|
+
return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
13
|
+
? parsed
|
|
14
|
+
: null;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Reads and validates the project settings. Throws an Error whose message says what to fix, so `doctor` and `rules`
|
|
22
|
+
* can report it and a spec that imports it fails with the same sentence instead of a confusing one.
|
|
23
|
+
*/
|
|
24
|
+
export function readWorkspaceConfig() {
|
|
25
|
+
if (cached !== null)
|
|
26
|
+
return cached;
|
|
27
|
+
const pkg = readPackageJson();
|
|
28
|
+
const section = pkg?.asrun;
|
|
29
|
+
if (section !== undefined && (section === null || typeof section !== 'object' || Array.isArray(section))) {
|
|
30
|
+
throw new Error('package.json: "asrun" must be an object, for example "asrun": { "testDataPrefix": "[autotest]" }.');
|
|
31
|
+
}
|
|
32
|
+
const raw = section?.testDataPrefix;
|
|
33
|
+
if (raw !== undefined && typeof raw !== 'string') {
|
|
34
|
+
throw new Error('package.json: "asrun.testDataPrefix" must be a string like "[asrun]".');
|
|
35
|
+
}
|
|
36
|
+
const prefix = (raw ?? DEFAULT_TEST_DATA_PREFIX).trim();
|
|
37
|
+
if (!PREFIX.test(prefix)) {
|
|
38
|
+
throw new Error(`package.json: "asrun.testDataPrefix" is "${prefix}"; use a bracketed word such as "[asrun]" or "[autotest]" ` +
|
|
39
|
+
'(letters, digits, "-" and "_" inside the brackets).');
|
|
40
|
+
}
|
|
41
|
+
cached = { testDataPrefix: prefix };
|
|
42
|
+
return cached;
|
|
43
|
+
}
|
|
44
|
+
/** The prefix every name a case creates must start with. Reads `package.json`; default `[asrun]`. */
|
|
45
|
+
export function testDataPrefix() {
|
|
46
|
+
return readWorkspaceConfig().testDataPrefix;
|
|
47
|
+
}
|
|
48
|
+
/** Forget the cached settings (a tool that edits `package.json` in the same process). */
|
|
49
|
+
export function resetWorkspaceConfig() {
|
|
50
|
+
cached = null;
|
|
51
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local exploration note: what `/explore-case` saw on the page, in a form `/gen-spec` can read.
|
|
3
|
+
*
|
|
4
|
+
* It lives under `.explore/notes/<case-id>/<profile-id>.json`, one file per profile, so a run with `--profiles` never
|
|
5
|
+
* has to merge with an earlier one. `.explore/` is git-ignored local state: a note is optional, may be deleted at any
|
|
6
|
+
* time, and a fresh checkout simply has none (generation then replays the case as before).
|
|
7
|
+
*
|
|
8
|
+
* A note holds only what the case does not: locators the agent confirmed on the page, the data-call paths of the
|
|
9
|
+
* actions that load something, and the match count of a target card's element. Expected values stay in the case's
|
|
10
|
+
* `## Values`, ready signals in the case's `Ready when:` lines, and assertions nowhere near this file.
|
|
11
|
+
*/
|
|
12
|
+
export declare const NOTES_VERSION = 1;
|
|
13
|
+
export interface ExploreLocator {
|
|
14
|
+
/** What the locator is for, in words ("the header navigation link"). */
|
|
15
|
+
purpose: string;
|
|
16
|
+
/** A Playwright locator as `browser_generate_locator` returned it, for example `page.getByRole('link', {name: 'Home'})`. */
|
|
17
|
+
locator: string;
|
|
18
|
+
}
|
|
19
|
+
export interface ExploreNoteStep {
|
|
20
|
+
/** The path the step runs on, relative to `baseURL` (for example `/documents/`). */
|
|
21
|
+
url?: string;
|
|
22
|
+
locators?: ExploreLocator[];
|
|
23
|
+
/** Data-call paths the step's action starts, path only (`/api/articles`). */
|
|
24
|
+
requests?: string[];
|
|
25
|
+
}
|
|
26
|
+
export interface ExploreNoteTarget {
|
|
27
|
+
locator: string;
|
|
28
|
+
/** How many elements the locator matched when it was confirmed (1 for a target that is not a question). */
|
|
29
|
+
matches?: number;
|
|
30
|
+
}
|
|
31
|
+
export interface ExploreNote {
|
|
32
|
+
version: typeof NOTES_VERSION;
|
|
33
|
+
caseId: string;
|
|
34
|
+
profile: string;
|
|
35
|
+
env?: string;
|
|
36
|
+
/** sha256 of `cases/<case-id>.md` when the note was written; a different one makes the note stale. */
|
|
37
|
+
caseSha256: string;
|
|
38
|
+
capturedAt: string;
|
|
39
|
+
/** Set when a later `/gen-spec` refreshed an entry it confirmed on the page. */
|
|
40
|
+
updatedAt?: string;
|
|
41
|
+
steps: Record<string, ExploreNoteStep>;
|
|
42
|
+
targets?: Record<string, ExploreNoteTarget>;
|
|
43
|
+
}
|
|
44
|
+
export interface ExploreNoteError {
|
|
45
|
+
file: string;
|
|
46
|
+
problems: string[];
|
|
47
|
+
}
|
|
48
|
+
/** The folder that holds one case's notes; `<case-id>` may contain its group folder (`documents/landing-page`). */
|
|
49
|
+
export declare function exploreNotesDir(caseId: string): string;
|
|
50
|
+
/** The note of one profile: `.explore/notes/<case-id>/<profile-id>.json`. */
|
|
51
|
+
export declare function exploreNotesFile(caseId: string, profileId: string): string;
|
|
52
|
+
/** sha256 of the case document, or null when it does not exist. */
|
|
53
|
+
export declare function caseSha256(caseId: string): string | null;
|
|
54
|
+
export type NoteFreshness = 'fresh' | 'stale' | 'no-case';
|
|
55
|
+
/**
|
|
56
|
+
* Whether a note still describes the case. A stale note is not an error: `/gen-spec` replays the case and ignores it.
|
|
57
|
+
*/
|
|
58
|
+
export declare function noteFreshness(note: ExploreNote): {
|
|
59
|
+
state: NoteFreshness;
|
|
60
|
+
reason?: string;
|
|
61
|
+
};
|
|
62
|
+
/** Every reason the note is not usable, in plain words. An empty list means it is valid. */
|
|
63
|
+
export declare function validateExploreNote(value: unknown, expected: {
|
|
64
|
+
caseId: string;
|
|
65
|
+
profile: string;
|
|
66
|
+
}): string[];
|
|
67
|
+
/**
|
|
68
|
+
* Reads every note of a case (or one profile's), parses and validates it. A malformed file is reported, never thrown:
|
|
69
|
+
* the caller decides whether it matters, and generation treats an unreadable note as a missing one.
|
|
70
|
+
*/
|
|
71
|
+
export declare function readExploreNotes(caseId: string, profileId?: string): {
|
|
72
|
+
notes: ExploreNote[];
|
|
73
|
+
errors: ExploreNoteError[];
|
|
74
|
+
};
|