@asrunjs/asrun 0.2.0 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/explore-case.md +18 -15
- package/.claude/commands/gen-spec.md +16 -13
- package/.claude/commands/heal-spec.md +26 -23
- package/.claude/commands/make-case.md +11 -8
- package/.claude/skills/exploring-cases/SKILL.md +146 -132
- package/.claude/skills/exploring-cases/references/open-questions.md +116 -112
- package/.claude/skills/exploring-cases/references/readiness.md +146 -137
- package/.claude/skills/exploring-cases/references/user-handoff.md +157 -147
- package/.claude/skills/generating-specs/SKILL.md +219 -207
- package/.claude/skills/generating-specs/references/conventions.md +148 -148
- package/.claude/skills/making-cases/SKILL.md +72 -64
- package/.claude/skills/making-cases/references/normalizing.md +39 -38
- package/.claude/skills/making-cases/references/writes-and-email.md +37 -36
- package/AGENTS.md +169 -154
- package/CLAUDE.md +2 -1
- package/README.md +221 -410
- package/dist/lib/build-artifacts.d.ts +8 -0
- package/dist/lib/build-artifacts.js +10 -0
- package/dist/lib/case-doc.d.ts +311 -0
- package/dist/lib/case-doc.js +1117 -0
- package/dist/lib/cases.d.ts +2 -15
- package/dist/lib/cases.js +5 -78
- package/dist/lib/command.d.ts +24 -0
- package/dist/lib/command.js +90 -0
- package/dist/lib/fixtures.js +2 -2
- package/dist/lib/login-card.d.ts +22 -0
- package/dist/lib/login-card.js +183 -0
- package/dist/lib/node-cli.d.ts +6 -2
- package/dist/lib/node-cli.js +16 -6
- package/dist/lib/profiles.d.ts +25 -1
- package/dist/lib/profiles.js +54 -1
- package/dist/lib/run-filters.js +10 -2
- package/dist/lib/session.d.ts +13 -1
- package/dist/lib/session.js +73 -7
- package/dist/lib/targets.d.ts +5 -13
- package/dist/lib/targets.js +4 -29
- package/dist/lib/values.js +2 -2
- package/dist/scripts/build.js +2 -1
- package/dist/scripts/check-cases.js +74 -6
- package/dist/scripts/doctor.js +115 -4
- package/dist/scripts/explore.js +22 -1
- package/dist/scripts/go.mjs +188 -188
- package/dist/scripts/init.d.ts +7 -1
- package/dist/scripts/init.js +62 -14
- package/dist/scripts/lint-specs.js +4 -1
- package/dist/scripts/login.js +30 -2
- package/dist/ui/server/actions.d.ts +4 -0
- package/dist/ui/server/actions.js +25 -12
- package/dist/ui/server/ai-models.d.ts +3 -0
- package/dist/ui/server/ai-models.js +347 -0
- package/dist/ui/server/ai-stream.d.ts +34 -9
- package/dist/ui/server/ai-stream.js +283 -77
- package/dist/ui/server/app.js +74 -12
- package/dist/ui/server/jobs.d.ts +7 -1
- package/dist/ui/server/jobs.js +69 -15
- package/dist/ui/server/profiles.d.ts +20 -1
- package/dist/ui/server/profiles.js +214 -38
- package/dist/ui/shared/ai-commands.d.ts +16 -6
- package/dist/ui/shared/ai-commands.js +35 -12
- package/dist/ui/shared/types.d.ts +114 -0
- package/docs/agents/case-format.md +193 -0
- package/docs/agents/framework-dev.md +167 -0
- package/docs/ai-workflow.md +235 -0
- package/docs/cases.md +189 -0
- package/docs/commands.md +175 -0
- package/docs/install.md +174 -0
- package/docs/profiles.md +139 -0
- package/docs/theme-guide.html +922 -692
- package/docs/troubleshooting.md +101 -0
- package/docs/web-ui.md +139 -0
- package/package.json +20 -1
- package/templates/.claude/commands/explore-case.md +22 -21
- package/templates/.claude/commands/gen-spec.md +21 -20
- package/templates/.claude/commands/heal-spec.md +31 -30
- package/templates/.claude/commands/make-case.md +13 -12
- package/templates/.mcp.json +8 -8
- package/templates/AGENTS.md +66 -45
- package/templates/cases/_template.md +57 -63
- package/templates/opencode.jsonc +16 -11
- package/templates/profiles.yaml +42 -29
- package/ui/dist/assets/{dist-D_I45xgJ.js → dist-BudNAjOv.js} +1 -1
- package/ui/dist/assets/{dist-CBV7_vDN.js → dist-C-eCSFja.js} +1 -1
- package/ui/dist/assets/{dist-gF2bFn-5.js → dist-CO4heSzh.js} +1 -1
- package/ui/dist/assets/{dist-DX8knLVr.js → dist-CPLhRzom.js} +1 -1
- package/ui/dist/assets/{dist-BqAROxzI.js → dist-DEPJZ3Me.js} +1 -1
- package/ui/dist/assets/{dist-BwYRbe5s.js → dist-DZ7RiqmA.js} +1 -1
- package/ui/dist/assets/{dist-BHaIOZdW.js → dist-LixzGniF.js} +1 -1
- package/ui/dist/assets/{dist-DHCY9KpV.js → dist-O_RVi4tP.js} +1 -1
- package/ui/dist/assets/{dist-BiE4suo9.js → dist-jPS6HCJ5.js} +1 -1
- package/ui/dist/assets/{dist-BSJMM_jp.js → dist-vmf3fnkM.js} +1 -1
- package/ui/dist/assets/index-BIGt8Hfz.js +338 -0
- package/ui/dist/assets/index-CTmnF0Wp.css +2 -0
- package/ui/dist/help/ai-tools.png +0 -0
- package/ui/dist/help/case-explore.png +0 -0
- package/ui/dist/help/case-generate.png +0 -0
- package/ui/dist/help/cases-memo.png +0 -0
- package/ui/dist/help/profile-form.png +0 -0
- package/ui/dist/help/profiles-login.png +0 -0
- package/ui/dist/help/reports.png +0 -0
- package/ui/dist/help/run-tests.png +0 -0
- package/ui/dist/index.html +22 -16
- package/ui/dist/assets/index-DmasQ6Rq.css +0 -2
- package/ui/dist/assets/index-I7uU93kk.js +0 -294
|
@@ -1,148 +1,148 @@
|
|
|
1
|
-
# Spec conventions
|
|
2
|
-
|
|
3
|
-
## Shape
|
|
4
|
-
A case id is its path below `cases/` without `.md`: `hub-landing`, or `documents/landing-page` when the case is in a
|
|
5
|
-
folder (one level only). The spec is at the same path below `tests/` (`tests/documents/landing-page.spec.ts`).
|
|
6
|
-
|
|
7
|
-
```ts
|
|
8
|
-
// case: cases/<id>.md
|
|
9
|
-
import {caseTags} from '@asrunjs/asrun/cases';
|
|
10
|
-
import {configureCase, expect, test} from '@asrunjs/asrun/fixtures';
|
|
11
|
-
import {requireValues} from '@asrunjs/asrun/values';
|
|
12
|
-
|
|
13
|
-
test.use({mustLogin: false}); // only for login: none cases (public pages)
|
|
14
|
-
|
|
15
|
-
test.describe('<Case title>', {tag: caseTags('<id>')}, () => {
|
|
16
|
-
configureCase('<id>'); // from asrun/fixtures; no retries and a longer timeout for cases that write data
|
|
17
|
-
test('<single outcome>', async ({page, profile}) => {
|
|
18
|
-
const expected = requireValues<{...}>('<id>', profile); // the case's "## Values": default + this profile
|
|
19
|
-
// Locators are lazy: declare them here when several steps share them.
|
|
20
|
-
|
|
21
|
-
await test.step('1. <step text from the case>', async () => {
|
|
22
|
-
...
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
await test.step('Visual check: <what is compared>', async () => {
|
|
26
|
-
await expect(locator).toHaveScreenshot('<name>.png');
|
|
27
|
-
});
|
|
28
|
-
});
|
|
29
|
-
});
|
|
30
|
-
```
|
|
31
|
-
- Import `test`/`expect` from `@asrunjs/asrun/fixtures`, never from `@playwright/test`. The package name is the import path
|
|
32
|
-
(the package's `exports` map points at its `dist/` build), so a spec in a group folder uses the same imports as one
|
|
33
|
-
at the top of `tests/`.
|
|
34
|
-
- `profile` fixture = the current project's profile (`profile.id`, `profile.env`, `profile.baseURL`, `profile.account`,
|
|
35
|
-
`profile.tags`, `profile.email`). Never branch on `profile.id` or `profile.env` inside a spec: profile differences go
|
|
36
|
-
into the case's `## Values`, and the selectors come from `## Targets`.
|
|
37
|
-
- Paths are relative (`page.goto('/account/')`); the project sets `baseURL` per profile.
|
|
38
|
-
|
|
39
|
-
## Tags
|
|
40
|
-
The case's front matter (`suites: [...]`, `writes`) is the single source of truth. The spec never hard-codes
|
|
41
|
-
tags: it passes `{tag: caseTags('<id>')}` (from `@asrunjs/asrun/cases`) to the top-level `test.describe`.
|
|
42
|
-
`caseTags` returns one `@<suite>` per suite (shown as badges in the HTML report, drives `pnpm asrun --suite <name>`)
|
|
43
|
-
and `@writes` for cases that write data (drives `--read-only`).
|
|
44
|
-
- Do not copy suites into the describe title.
|
|
45
|
-
- If the front matter is invalid, `caseTags` throws; ask the user to fix the case.
|
|
46
|
-
- Write permission and `envs` limits are enforced by the `caseGate` fixture in `@asrunjs/asrun/fixtures`; specs do nothing.
|
|
47
|
-
A case that writes only in some environments is skipped (not failed) in the others, with a notice.
|
|
48
|
-
|
|
49
|
-
## Cases that write data (`writes: [<env>, ...]`)
|
|
50
|
-
- Name everything the case creates `[asrun] <something>` (`TEST_DATA_PREFIX` in `@asrunjs/asrun/records`). `records.track`
|
|
51
|
-
refuses other names, so cleanup can never touch real data.
|
|
52
|
-
- Register data with the `records` fixture right after creating it, together with the function that removes it:
|
|
53
|
-
`await records.track({kind: 'Document draft', name, where: '/app/documents'}, async () => {...delete it from the documents list...});`
|
|
54
|
-
Everything tracked is removed after the test, newest first, also when the test failed. The remove function opens
|
|
55
|
-
its own page (it cannot know where the test stopped) and must not fail if the data is already gone.
|
|
56
|
-
- The case's last step is normally the deletion itself; `track` is the safety net. Do not rely on it alone.
|
|
57
|
-
- If a cleanup fails, the test fails, the data is listed in the run summary (web UI banner and terminal) and
|
|
58
|
-
`pnpm cleanup --env <env>` lists it until a person deleted it and ran `--forget`.
|
|
59
|
-
- `configureCase('<id>')` turns retries off (a retry would create the data twice) and raises the timeout.
|
|
60
|
-
Cases that write run one at a time per profile, whatever the number of workers.
|
|
61
|
-
- Mark steps in the case text with `[writes]`; keep their titles unchanged in `test.step`.
|
|
62
|
-
- Email: wrap steps that send email in `if (!emailStepAllowed(profile, '<step title>')) return;` (`@asrunjs/asrun/email`).
|
|
63
|
-
It is false in production and for profiles with `email: false`; the skip is written to the report annotations.
|
|
64
|
-
|
|
65
|
-
## Steps (`test.step`)
|
|
66
|
-
Every numbered step of the case is wrapped in `await test.step('<n>. <step text>', async () => {...})`.
|
|
67
|
-
The report and the trace then show the case's own steps, so a reviewer can check what was executed
|
|
68
|
-
without reading the spec.
|
|
69
|
-
- Step title = the case's step text, verbatim, with its number. Do not paraphrase.
|
|
70
|
-
- Put the assertions that belong to a step inside that step.
|
|
71
|
-
- One extra step, titled `Visual check: <what is compared>`, per visual check from the case.
|
|
72
|
-
- A value needed by a later step (for example a page opened in a new tab) is returned from the step:
|
|
73
|
-
`const previewPage = await test.step('3. ...', async () => {...; return opened;});`
|
|
74
|
-
- Do not put `test.step` inside helpers in the framework (`@asrunjs/asrun/*`), and do not nest steps.
|
|
75
|
-
- Use step comments only for things the title cannot say; the title replaces the old `// 1. ...` comments.
|
|
76
|
-
|
|
77
|
-
## Assertion layers (use all that apply)
|
|
78
|
-
1. Text: `toHaveText`, `toHaveTitle`, `toContainText`, with values from `requireValues('<id>', profile)`.
|
|
79
|
-
2. DOM of images: `expectLogo(container, expected.logo)` / `expectImageLoaded`
|
|
80
|
-
(alt, src, loaded). Use `expectImageSha256` when the exact file must match.
|
|
81
|
-
3. Pixels: `await expect(locator).toHaveScreenshot('<name>.png')`. Baselines are stored per
|
|
82
|
-
profile automatically. Screenshot a small element, not the full page (full pages are flaky).
|
|
83
|
-
|
|
84
|
-
## Readable failures
|
|
85
|
-
The UI shows the first line of the error to QA and non-technical people, and hides the call log and stack
|
|
86
|
-
trace behind a "Technical details" fold. Write errors so that first line makes sense on its own.
|
|
87
|
-
- Read per-profile data with `requireValues<T>('<id>', profile)` (from `@asrunjs/asrun/values`), never by reading the case
|
|
88
|
-
yaml yourself. A case with no entry for the profile, or with `TODO`/empty values, then fails with
|
|
89
|
-
"Missing values: cases/<id>.md has no \"## Values\" for profile ..." instead of a cryptic crash.
|
|
90
|
-
- Give every `expect` a message in plain words about the product, not the code:
|
|
91
|
-
`expect(locator, 'The documents page did not finish loading').toBeVisible()`.
|
|
92
|
-
Say what is wrong ("The product logo has the wrong name"), not what the matcher is ("expected true").
|
|
93
|
-
- Guard preconditions that need manual data (for example "at least one published document exists") with an
|
|
94
|
-
`expect(..., 'No published document found. Create one manually; this test never writes data.')` in the step,
|
|
95
|
-
so the failure says what the human has to do.
|
|
96
|
-
- Messages
|
|
97
|
-
|
|
98
|
-
## Ready, targets and downloads (`@asrunjs/asrun/ready`, `@asrunjs/asrun/downloads`)
|
|
99
|
-
Binding rules are in `../exploring-cases/references/readiness.md` and `.../assertion-targets.md`. In short:
|
|
100
|
-
- An action that loads data (opens a dialog, a composer, a tab, a list, a preview) goes through
|
|
101
|
-
`actAndWait(page, () => click(), {responses: [/api-url/], ready: [locator]})`: the response waiters exist before the
|
|
102
|
-
action, then the content signal must be visible. Read the result only afterwards. `expectNotBusy(scope)` waits until
|
|
103
|
-
no spinner, skeleton or `aria-busy` is left.
|
|
104
|
-
- A confirmation that may or may not appear after an action (the "Save as Draft?" dialog when closing a composer) is
|
|
105
|
-
handled with `clickOptionalUntilHidden(confirmButton, target, {message})`: one retry loop that clicks the confirmation
|
|
106
|
-
when it is visible and checks that the target is hidden. Never `if (await x.isVisible(...)) await x.click()`; see
|
|
107
|
-
`readiness.md`, "Optional confirmations".
|
|
108
|
-
- A check on a logo, image, file or screenshot has `// target: T<n>` on the line before it (a card in the case's
|
|
109
|
-
`## Targets`). The element is found by the card's `Identified by` (role and name, test id, alt/title), never `.first()` or
|
|
110
|
-
a bare `img`/`svg`. Semantic assertion first (name/alt/src, where it is), pixels second, on that element only.
|
|
111
|
-
- A check for absence ("no", "missing", "empty", "hidden") has `// ready: <signal>` in front of it and a positive wait
|
|
112
|
-
before it. The page must have loaded before "nothing" means anything.
|
|
113
|
-
- A download is `await verifyDownload(page, () => option.click(), {kind: 'pdf'})`. It checks, and fails the step on
|
|
114
|
-
any failure or timeout: the download starts, finishes, the file exists, the file is valid for its kind (PDF: `%PDF-`
|
|
115
|
-
start, `%%EOF` end, opens with pages). Add `validate` for the case's own check of the content. Never
|
|
116
|
-
`waitForEvent('download')` or a magic-number check on its own.
|
|
117
|
-
- A PDF's content is checked with `expectPdf(file.pdf!, {...})` (`@asrunjs/asrun/pdf`): pages rendered to images, regions that must not
|
|
118
|
-
be blank and are compared with the profile's baseline (2%), extracted text. Pages follow the case: every page by default
|
|
119
|
-
(`pages: 'all'`), only the named ones when the case says ONLY (a strong requirement) with `// pages: <the case's words>`.
|
|
120
|
-
The call has `// target: T<n>`. Take names and texts to avoid from `requireValues('<id>', profile)`, never literals.
|
|
121
|
-
```ts
|
|
122
|
-
const file = await verifyDownload(page, () => option.click(), {kind: 'pdf'});
|
|
123
|
-
// target: T3
|
|
124
|
-
await expectPdf(file.pdf!, {
|
|
125
|
-
regions: [{name: 'profile-logo', pages: 'all', region: corner('top-left')}],
|
|
126
|
-
textNotContains: expected.otherProfileNames,
|
|
127
|
-
});
|
|
128
|
-
```
|
|
129
|
-
- `pnpm spec:lint <id>` (static) must have no `FAIL`. `pnpm baselines:review <id>` lists every baseline with the card
|
|
130
|
-
that says what it must show.
|
|
131
|
-
|
|
132
|
-
## Do
|
|
133
|
-
- Locators: `getByTestId` (data-tn) > `getByRole` > `getByLabel` > text.
|
|
134
|
-
- Web-first assertions only; they auto-wait. Never `waitForTimeout`, never `networkidle`, never `if (await x.isVisible())`
|
|
135
|
-
to decide something that may not have loaded.
|
|
136
|
-
- Keep every test read-only unless the case lists environments under `writes`.
|
|
137
|
-
|
|
138
|
-
## Don't
|
|
139
|
-
- No login code in specs, and no cases that test the login page or the login flow. Sessions come from
|
|
140
|
-
`.auth/<profile-id>.json` via the project config; a profile without a valid session fails fast (see
|
|
141
|
-
`@asrunjs/asrun/fixtures`).
|
|
142
|
-
- No hard-coded profile names or expected values in specs; they come from the case's `## Values` through
|
|
143
|
-
`requireValues('<id>', profile)`. Site addresses come from `profile.baseURL`.
|
|
144
|
-
- No screenshots of areas with dynamic data (rows, prices, dates, avatars).
|
|
145
|
-
- No `.first()` / `.nth()` / bare `img`, `svg` or `img, svg` to find a logo or image.
|
|
146
|
-
- No hard assertion about something that was never seen on a page.
|
|
147
|
-
- No `isVisible({timeout})` / `isHidden({timeout})` (the timeout is ignored, nothing is waited for), no
|
|
148
|
-
`if (await x.isVisible()) await x.click()` outside a retry loop, and no `.catch(() => false)` to fake a wait.
|
|
1
|
+
# Spec conventions
|
|
2
|
+
|
|
3
|
+
## Shape
|
|
4
|
+
A case id is its path below `cases/` without `.md`: `hub-landing`, or `documents/landing-page` when the case is in a
|
|
5
|
+
folder (one level only). The spec is at the same path below `tests/` (`tests/documents/landing-page.spec.ts`).
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
// case: cases/<id>.md
|
|
9
|
+
import {caseTags} from '@asrunjs/asrun/cases';
|
|
10
|
+
import {configureCase, expect, test} from '@asrunjs/asrun/fixtures';
|
|
11
|
+
import {requireValues} from '@asrunjs/asrun/values';
|
|
12
|
+
|
|
13
|
+
test.use({mustLogin: false}); // only for login: none cases (public pages)
|
|
14
|
+
|
|
15
|
+
test.describe('<Case title>', {tag: caseTags('<id>')}, () => {
|
|
16
|
+
configureCase('<id>'); // from asrun/fixtures; no retries and a longer timeout for cases that write data
|
|
17
|
+
test('<single outcome>', async ({page, profile}) => {
|
|
18
|
+
const expected = requireValues<{...}>('<id>', profile); // the case's "## Values": default + this profile
|
|
19
|
+
// Locators are lazy: declare them here when several steps share them.
|
|
20
|
+
|
|
21
|
+
await test.step('1. <step text from the case>', async () => {
|
|
22
|
+
...
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
await test.step('Visual check: <what is compared>', async () => {
|
|
26
|
+
await expect(locator).toHaveScreenshot('<name>.png');
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
- Import `test`/`expect` from `@asrunjs/asrun/fixtures`, never from `@playwright/test`. The package name is the import path
|
|
32
|
+
(the package's `exports` map points at its `dist/` build), so a spec in a group folder uses the same imports as one
|
|
33
|
+
at the top of `tests/`.
|
|
34
|
+
- `profile` fixture = the current project's profile (`profile.id`, `profile.env`, `profile.baseURL`, `profile.account`,
|
|
35
|
+
`profile.tags`, `profile.email`). Never branch on `profile.id` or `profile.env` inside a spec: profile differences go
|
|
36
|
+
into the case's `## Values`, and the selectors come from `## Targets`.
|
|
37
|
+
- Paths are relative (`page.goto('/account/')`); the project sets `baseURL` per profile.
|
|
38
|
+
|
|
39
|
+
## Tags
|
|
40
|
+
The case's front matter (`suites: [...]`, `writes`) is the single source of truth. The spec never hard-codes
|
|
41
|
+
tags: it passes `{tag: caseTags('<id>')}` (from `@asrunjs/asrun/cases`) to the top-level `test.describe`.
|
|
42
|
+
`caseTags` returns one `@<suite>` per suite (shown as badges in the HTML report, drives `pnpm asrun --suite <name>`)
|
|
43
|
+
and `@writes` for cases that write data (drives `--read-only`).
|
|
44
|
+
- Do not copy suites into the describe title.
|
|
45
|
+
- If the front matter is invalid, `caseTags` throws; ask the user to fix the case.
|
|
46
|
+
- Write permission and `envs` limits are enforced by the `caseGate` fixture in `@asrunjs/asrun/fixtures`; specs do nothing.
|
|
47
|
+
A case that writes only in some environments is skipped (not failed) in the others, with a notice.
|
|
48
|
+
|
|
49
|
+
## Cases that write data (`writes: [<env>, ...]`)
|
|
50
|
+
- Name everything the case creates `[asrun] <something>` (`TEST_DATA_PREFIX` in `@asrunjs/asrun/records`). `records.track`
|
|
51
|
+
refuses other names, so cleanup can never touch real data.
|
|
52
|
+
- Register data with the `records` fixture right after creating it, together with the function that removes it:
|
|
53
|
+
`await records.track({kind: 'Document draft', name, where: '/app/documents'}, async () => {...delete it from the documents list...});`
|
|
54
|
+
Everything tracked is removed after the test, newest first, also when the test failed. The remove function opens
|
|
55
|
+
its own page (it cannot know where the test stopped) and must not fail if the data is already gone.
|
|
56
|
+
- The case's last step is normally the deletion itself; `track` is the safety net. Do not rely on it alone.
|
|
57
|
+
- If a cleanup fails, the test fails, the data is listed in the run summary (web UI banner and terminal) and
|
|
58
|
+
`pnpm cleanup --env <env>` lists it until a person deleted it and ran `--forget`.
|
|
59
|
+
- `configureCase('<id>')` turns retries off (a retry would create the data twice) and raises the timeout.
|
|
60
|
+
Cases that write run one at a time per profile, whatever the number of workers.
|
|
61
|
+
- Mark steps in the case text with `[writes]`; keep their titles unchanged in `test.step`.
|
|
62
|
+
- Email: wrap steps that send email in `if (!emailStepAllowed(profile, '<step title>')) return;` (`@asrunjs/asrun/email`).
|
|
63
|
+
It is false in production and for profiles with `email: false`; the skip is written to the report annotations.
|
|
64
|
+
|
|
65
|
+
## Steps (`test.step`)
|
|
66
|
+
Every numbered step of the case is wrapped in `await test.step('<n>. <step text>', async () => {...})`.
|
|
67
|
+
The report and the trace then show the case's own steps, so a reviewer can check what was executed
|
|
68
|
+
without reading the spec.
|
|
69
|
+
- Step title = the case's step text, verbatim, with its number. Do not paraphrase.
|
|
70
|
+
- Put the assertions that belong to a step inside that step.
|
|
71
|
+
- One extra step, titled `Visual check: <what is compared>`, per visual check from the case.
|
|
72
|
+
- A value needed by a later step (for example a page opened in a new tab) is returned from the step:
|
|
73
|
+
`const previewPage = await test.step('3. ...', async () => {...; return opened;});`
|
|
74
|
+
- Do not put `test.step` inside helpers in the framework (`@asrunjs/asrun/*`), and do not nest steps.
|
|
75
|
+
- Use step comments only for things the title cannot say; the title replaces the old `// 1. ...` comments.
|
|
76
|
+
|
|
77
|
+
## Assertion layers (use all that apply)
|
|
78
|
+
1. Text: `toHaveText`, `toHaveTitle`, `toContainText`, with values from `requireValues('<id>', profile)`.
|
|
79
|
+
2. DOM of images: `expectLogo(container, expected.logo)` / `expectImageLoaded`
|
|
80
|
+
(alt, src, loaded). Use `expectImageSha256` when the exact file must match.
|
|
81
|
+
3. Pixels: `await expect(locator).toHaveScreenshot('<name>.png')`. Baselines are stored per
|
|
82
|
+
profile automatically. Screenshot a small element, not the full page (full pages are flaky).
|
|
83
|
+
|
|
84
|
+
## Readable failures
|
|
85
|
+
The UI shows the first line of the error to QA and non-technical people, and hides the call log and stack
|
|
86
|
+
trace behind a "Technical details" fold. Write errors so that first line makes sense on its own.
|
|
87
|
+
- Read per-profile data with `requireValues<T>('<id>', profile)` (from `@asrunjs/asrun/values`), never by reading the case
|
|
88
|
+
yaml yourself. A case with no entry for the profile, or with `TODO`/empty values, then fails with
|
|
89
|
+
"Missing values: cases/<id>.md has no \"## Values\" for profile ..." instead of a cryptic crash.
|
|
90
|
+
- Give every `expect` a message in plain words about the product, not the code:
|
|
91
|
+
`expect(locator, 'The documents page did not finish loading').toBeVisible()`.
|
|
92
|
+
Say what is wrong ("The product logo has the wrong name"), not what the matcher is ("expected true").
|
|
93
|
+
- Guard preconditions that need manual data (for example "at least one published document exists") with an
|
|
94
|
+
`expect(..., 'No published document found. Create one manually; this test never writes data.')` in the step,
|
|
95
|
+
so the failure says what the human has to do.
|
|
96
|
+
- Messages follow the artifact language of `AGENTS.md` §2 (English by default), one sentence, no selectors or test ids.
|
|
97
|
+
|
|
98
|
+
## Ready, targets and downloads (`@asrunjs/asrun/ready`, `@asrunjs/asrun/downloads`)
|
|
99
|
+
Binding rules are in `../exploring-cases/references/readiness.md` and `.../assertion-targets.md`. In short:
|
|
100
|
+
- An action that loads data (opens a dialog, a composer, a tab, a list, a preview) goes through
|
|
101
|
+
`actAndWait(page, () => click(), {responses: [/api-url/], ready: [locator]})`: the response waiters exist before the
|
|
102
|
+
action, then the content signal must be visible. Read the result only afterwards. `expectNotBusy(scope)` waits until
|
|
103
|
+
no spinner, skeleton or `aria-busy` is left.
|
|
104
|
+
- A confirmation that may or may not appear after an action (the "Save as Draft?" dialog when closing a composer) is
|
|
105
|
+
handled with `clickOptionalUntilHidden(confirmButton, target, {message})`: one retry loop that clicks the confirmation
|
|
106
|
+
when it is visible and checks that the target is hidden. Never `if (await x.isVisible(...)) await x.click()`; see
|
|
107
|
+
`readiness.md`, "Optional confirmations".
|
|
108
|
+
- A check on a logo, image, file or screenshot has `// target: T<n>` on the line before it (a card in the case's
|
|
109
|
+
`## Targets`). The element is found by the card's `Identified by` (role and name, test id, alt/title), never `.first()` or
|
|
110
|
+
a bare `img`/`svg`. Semantic assertion first (name/alt/src, where it is), pixels second, on that element only.
|
|
111
|
+
- A check for absence ("no", "missing", "empty", "hidden") has `// ready: <signal>` in front of it and a positive wait
|
|
112
|
+
before it. The page must have loaded before "nothing" means anything.
|
|
113
|
+
- A download is `await verifyDownload(page, () => option.click(), {kind: 'pdf'})`. It checks, and fails the step on
|
|
114
|
+
any failure or timeout: the download starts, finishes, the file exists, the file is valid for its kind (PDF: `%PDF-`
|
|
115
|
+
start, `%%EOF` end, opens with pages). Add `validate` for the case's own check of the content. Never
|
|
116
|
+
`waitForEvent('download')` or a magic-number check on its own.
|
|
117
|
+
- A PDF's content is checked with `expectPdf(file.pdf!, {...})` (`@asrunjs/asrun/pdf`): pages rendered to images, regions that must not
|
|
118
|
+
be blank and are compared with the profile's baseline (2%), extracted text. Pages follow the case: every page by default
|
|
119
|
+
(`pages: 'all'`), only the named ones when the case says ONLY (a strong requirement) with `// pages: <the case's words>`.
|
|
120
|
+
The call has `// target: T<n>`. Take names and texts to avoid from `requireValues('<id>', profile)`, never literals.
|
|
121
|
+
```ts
|
|
122
|
+
const file = await verifyDownload(page, () => option.click(), {kind: 'pdf'});
|
|
123
|
+
// target: T3
|
|
124
|
+
await expectPdf(file.pdf!, {
|
|
125
|
+
regions: [{name: 'profile-logo', pages: 'all', region: corner('top-left')}],
|
|
126
|
+
textNotContains: expected.otherProfileNames,
|
|
127
|
+
});
|
|
128
|
+
```
|
|
129
|
+
- `pnpm spec:lint <id>` (static) must have no `FAIL`. `pnpm baselines:review <id>` lists every baseline with the card
|
|
130
|
+
that says what it must show.
|
|
131
|
+
|
|
132
|
+
## Do
|
|
133
|
+
- Locators: `getByTestId` (data-tn) > `getByRole` > `getByLabel` > text.
|
|
134
|
+
- Web-first assertions only; they auto-wait. Never `waitForTimeout`, never `networkidle`, never `if (await x.isVisible())`
|
|
135
|
+
to decide something that may not have loaded.
|
|
136
|
+
- Keep every test read-only unless the case lists environments under `writes`.
|
|
137
|
+
|
|
138
|
+
## Don't
|
|
139
|
+
- No login code in specs, and no cases that test the login page or the login flow. Sessions come from
|
|
140
|
+
`.auth/<profile-id>.json` via the project config; a profile without a valid session fails fast (see
|
|
141
|
+
`@asrunjs/asrun/fixtures`).
|
|
142
|
+
- No hard-coded profile names or expected values in specs; they come from the case's `## Values` through
|
|
143
|
+
`requireValues('<id>', profile)`. Site addresses come from `profile.baseURL`.
|
|
144
|
+
- No screenshots of areas with dynamic data (rows, prices, dates, avatars).
|
|
145
|
+
- No `.first()` / `.nth()` / bare `img`, `svg` or `img, svg` to find a logo or image.
|
|
146
|
+
- No hard assertion about something that was never seen on a page.
|
|
147
|
+
- No `isVisible({timeout})` / `isHidden({timeout})` (the timeout is ignored, nothing is waited for), no
|
|
148
|
+
`if (await x.isVisible()) await x.click()` outside a retry loop, and no `.catch(() => false)` to fake a wait.
|
|
@@ -1,64 +1,72 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: making-cases
|
|
3
|
-
description: Turns raw, informal test notes from a human (a QA memo, a list of buttons to check, a pasted chat) into well-formed case documents (cases/<id>.md) for the AsRun suite. Normalizes the text, classifies steps as read-only, writing or email, finds gaps and asks about them, never opens a browser, and splits the notes into cases. Use when asked to make, write or normalize cases from notes, or for /make-case. The next step after it is exploring-cases, then generating-specs.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Making case documents from raw notes
|
|
7
|
-
|
|
8
|
-
Read `AGENTS.md` (hard rules) and `cases/_template.md` (the target format) first. Then read the references:
|
|
9
|
-
- `references/normalizing.md`: how to clean up the text and keep the original traceable.
|
|
10
|
-
- `references/writes-and-email.md`: how to classify steps that write data or send email, and the data rules.
|
|
11
|
-
- `references/gaps.md`: what to ask, and the default to suggest for each question.
|
|
12
|
-
- `references/splitting.md`: how many cases to make.
|
|
13
|
-
|
|
14
|
-
This skill writes `cases/*.md` only. It never writes specs, `profiles.yaml`, or anything under `tests/`.
|
|
15
|
-
|
|
16
|
-
##
|
|
17
|
-
|
|
18
|
-
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
1
|
+
---
|
|
2
|
+
name: making-cases
|
|
3
|
+
description: Turns raw, informal test notes from a human (a QA memo, a list of buttons to check, a pasted chat) into well-formed case documents (cases/<id>.md) for the AsRun suite. Normalizes the text, classifies steps as read-only, writing or email, finds gaps and asks about them, never opens a browser, and splits the notes into cases. Use when asked to make, write or normalize cases from notes, or for /make-case. The next step after it is exploring-cases, then generating-specs.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Making case documents from raw notes
|
|
7
|
+
|
|
8
|
+
Read the workspace `AGENTS.md` (hard rules and project rules) and `cases/_template.md` (the target format) first. Then read the references:
|
|
9
|
+
- `references/normalizing.md`: how to clean up the text and keep the original traceable.
|
|
10
|
+
- `references/writes-and-email.md`: how to classify steps that write data or send email, and the data rules.
|
|
11
|
+
- `references/gaps.md`: what to ask, and the default to suggest for each question.
|
|
12
|
+
- `references/splitting.md`: how many cases to make.
|
|
13
|
+
|
|
14
|
+
This skill writes `cases/*.md` only. It never writes specs, `profiles.yaml`, or anything under `tests/`.
|
|
15
|
+
|
|
16
|
+
## Before you start
|
|
17
|
+
Read the workspace `AGENTS.md`: its §2 holds the project rules, and it points at the framework baseline. If
|
|
18
|
+
`.claude/skills/making-cases.md` exists in the workspace, read it and apply it after this skill. Then apply `AGENTS.md`
|
|
19
|
+
§2, which wins over both (the baseline, "Precedence"). F1–F6 always win. This is not a stop condition: continue with
|
|
20
|
+
the procedure below.
|
|
21
|
+
|
|
22
|
+
## Inputs
|
|
23
|
+
- Raw notes: pasted text or a file path. Any language, any quality.
|
|
24
|
+
- No environment: this skill never opens a browser. What the pages really show is found by `/explore-case`.
|
|
25
|
+
|
|
26
|
+
## Procedure
|
|
27
|
+
1. **Read everything and keep the original.** Number every distinct statement of the notes (N1, N2, ...). The
|
|
28
|
+
original wording is stored verbatim in each case's `## Source` table, so nothing is lost and a reviewer can check
|
|
29
|
+
what became of each line. Do not edit the input file.
|
|
30
|
+
2. **Normalize.** Follow `references/normalizing.md`: the artifact language of `AGENTS.md` §2 (English by default),
|
|
31
|
+
one action or check per step, concrete UI wording,
|
|
32
|
+
quoted UI text kept exactly. Do not add checks the notes did not ask for, and do not drop any.
|
|
33
|
+
3. **Classify each step**: read-only, `[writes]`, or `[email]` (`references/writes-and-email.md`). A step that
|
|
34
|
+
creates, edits, publishes, shares, deletes or sends is not read-only, even if the notes do not say so.
|
|
35
|
+
4. **Find the gaps** (`references/gaps.md`): missing expected results, unknown values a spec must assert, unclear starting
|
|
36
|
+
state, steps that need data that may not exist, ordering that is not linear. Never invent an expected value.
|
|
37
|
+
Unknown values are written as the suggested default sentence with a `❓ TODO` block under it (see `gaps.md`), or as an
|
|
38
|
+
`Observed, 🔍 to be confirmed by QA` bullet. Every marker starts with its emoji. A literal nobody knows yet goes into
|
|
39
|
+
the case's `## Values` as `TODO` (the spec then fails with a clear message until it is filled), never into
|
|
40
|
+
`profiles.yaml`.
|
|
41
|
+
5. **Ask before writing.** Show the user a short plan and the open questions in one message: the proposed split
|
|
42
|
+
into cases, the classification (which steps write, which send email), and the gaps with your suggested default
|
|
43
|
+
for each. Wait for answers. Do not write case files before that.
|
|
44
|
+
6. **Split** (`references/splitting.md`) and **write** `cases/<id>.md` for each case from `cases/_template.md`.
|
|
45
|
+
- Group folder: when the notes (or the user) name an area (documents, search, account...), put the cases in
|
|
46
|
+
`cases/<group>/<name>.md` and set `id: <group>/<name>` in the front matter. Use one folder level and
|
|
47
|
+
lowercase kebab-case. Reuse an existing folder in `cases/` when the area matches. If the area is unclear,
|
|
48
|
+
ask in step 5. The group does not repeat in the name: `documents/landing-page`, not `documents/documents-landing-page`.
|
|
49
|
+
- Front matter: `id`, `title`, `login`, `suites`, `writes`, `writes-note` (required when `writes` is not `none`),
|
|
50
|
+
`envs` when a case must not run everywhere. When unsure: `suites: [regression]`, no `writes` (read-only), or `writes: [staging]` when a step writes.
|
|
51
|
+
- Fill the sections that apply (Goal, Steps, Expected, Values, Visual checks, Source, Notes).
|
|
52
|
+
Remove template placeholders you do not use. Use `## Values` only for the literals a spec asserts; `default`
|
|
53
|
+
applies to every profile, and a profile id entry holds only what differs. Do not add `- Recipe:` lines: a person
|
|
54
|
+
records them during `/explore-case --attended`.
|
|
55
|
+
- Case ids are lowercase kebab-case and name the thing, not the ticket: `landing-page`.
|
|
56
|
+
- Existing case files: do not overwrite. If a new case overlaps one (for example another case about the same page), say so,
|
|
57
|
+
propose "keep", "merge" or "replace", and wait for the user.
|
|
58
|
+
7. **Check.** Run `pnpm case:check`. Fix every FAIL. Re-read each written case against the numbered statements: every
|
|
59
|
+
N must appear in exactly one `## Source` table, or in the report as "not used, because ...".
|
|
60
|
+
8. **Report** in the user's language: the cases written (id, suites, writes, envs), the traceability summary, the
|
|
61
|
+
questions that are still open, and the next steps per case: `/explore-case <id> <env> --attended` (after `pnpm signin`
|
|
62
|
+
for the profiles and environment; a person does the steps that write data), then `/gen-spec <id> <env>`.
|
|
63
|
+
|
|
64
|
+
## Hard rules
|
|
65
|
+
- Never open a browser and never write data. A step that writes is only described and marked; the case lists the
|
|
66
|
+
environments the spec may write in under `writes`. Never add `--allow-writes`, never set
|
|
67
|
+
`ASRUN_ALLOW_WRITES`; this skill does not run write cases at all.
|
|
68
|
+
- Never send email in production. In other environments only for profiles with `email: true`. New profiles and AI-written
|
|
69
|
+
entries in `profiles.yaml` get `email: false`; do not change an existing `email: true`.
|
|
70
|
+
- Never write a case for the login page or the login flow.
|
|
71
|
+
- Never invent expected values, profile names, counts, or UI texts.
|
|
72
|
+
- Prose follows the artifact language of the workspace `AGENTS.md` §2 (English by default); structural tokens stay English.
|
|
@@ -1,38 +1,39 @@
|
|
|
1
|
-
# Normalizing raw notes
|
|
2
|
-
|
|
3
|
-
## Goal
|
|
4
|
-
Turn informal notes into steps that someone who never saw the notes can follow, without changing what was asked.
|
|
5
|
-
|
|
6
|
-
## Rules
|
|
7
|
-
1. **
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
the
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
-
|
|
37
|
-
-
|
|
38
|
-
- A statement
|
|
1
|
+
# Normalizing raw notes
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
Turn informal notes into steps that someone who never saw the notes can follow, without changing what was asked.
|
|
5
|
+
|
|
6
|
+
## Rules
|
|
7
|
+
1. **Use the artifact language.** Translate the notes into the language `AGENTS.md` §2 sets (English by default).
|
|
8
|
+
Keep text the page shows exactly as written in the notes (button labels such as
|
|
9
|
+
"Create New", pill names such as "My documents", "Shared with me"). Do not translate or "fix" those.
|
|
10
|
+
2. **One action or one check per step.** Split "create, publish and preview it" into three steps. A check that
|
|
11
|
+
belongs to an action stays in the same step as its action ("Open Share and check the logo").
|
|
12
|
+
3. **Make the wording concrete.** Say where (page, path, card, corner), what (exact label), and what happens.
|
|
13
|
+
"Check the logo" is not a step by itself: it needs an expected result (see `gaps.md`). Until it has one, write
|
|
14
|
+
the default as a sentence under "Expected" ("The <thing> is visible and not empty.") with a
|
|
15
|
+
`TODO [non-blocking]` block under it (see `gaps.md`).
|
|
16
|
+
4. **Keep the order of the notes** unless the notes say otherwise, then fix the order and say so in the Source table.
|
|
17
|
+
5. **Do not add checks** the notes did not ask for (no extra assertions "while we are at it"), and do not drop any.
|
|
18
|
+
When a line is a heading or a list of things on one page ("Landing: Create New, All, My documents, Shared with me,
|
|
19
|
+
search bar"), each item becomes a check: "is visible" is the only meaning you may assume, and you say so.
|
|
20
|
+
6. **Mark steps** with `[writes]` or `[email]` at the end of the step text (see `writes-and-email.md`).
|
|
21
|
+
7. **Use paths relative to the app** (`/app/documents`), never a full domain. Domains differ per profile and environment.
|
|
22
|
+
8. **No profile-specific values in steps.** The exact literals go in `## Values`; a sentence about behaviour goes in `Expected`.
|
|
23
|
+
|
|
24
|
+
## Traceability (`## Source` section in every case)
|
|
25
|
+
Number every statement of the raw notes once (N1, N2, ...). Each case ends with:
|
|
26
|
+
|
|
27
|
+
```markdown
|
|
28
|
+
## Source
|
|
29
|
+
Original notes, verbatim. Do not edit this table; it only exists so a reviewer can check what became of each line.
|
|
30
|
+
|
|
31
|
+
| # | Original text | Became |
|
|
32
|
+
|---|---------------|--------|
|
|
33
|
+
| N3 | `Published document: publish (a follow-up modal opens by itself)` | Steps 6-7 |
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
- The original text is copied character by character, in a code span, in its original language.
|
|
37
|
+
- "Became" points at step numbers, or says "Expected", "Visual checks", "Not used: <reason>".
|
|
38
|
+
- A statement used by two cases appears in both tables.
|
|
39
|
+
- A statement that is not used anywhere is a gap: ask about it, do not silently drop it.
|