@asrunjs/asrun 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/explore-case.md +15 -0
- package/.claude/commands/gen-spec.md +13 -0
- package/.claude/commands/heal-spec.md +23 -0
- package/.claude/commands/make-case.md +8 -0
- package/.claude/settings.json +5 -0
- package/.claude/skills/exploring-cases/SKILL.md +132 -0
- package/.claude/skills/exploring-cases/references/assertion-targets.md +103 -0
- package/.claude/skills/exploring-cases/references/handoff-card.js +128 -0
- package/.claude/skills/exploring-cases/references/open-questions.md +112 -0
- package/.claude/skills/exploring-cases/references/readiness.md +137 -0
- package/.claude/skills/exploring-cases/references/user-handoff.md +147 -0
- package/.claude/skills/generating-specs/SKILL.md +207 -0
- package/.claude/skills/generating-specs/references/conventions.md +148 -0
- package/.claude/skills/making-cases/SKILL.md +64 -0
- package/.claude/skills/making-cases/references/gaps.md +36 -0
- package/.claude/skills/making-cases/references/normalizing.md +38 -0
- package/.claude/skills/making-cases/references/splitting.md +28 -0
- package/.claude/skills/making-cases/references/writes-and-email.md +36 -0
- package/AGENTS.md +154 -0
- package/CLAUDE.md +1 -0
- package/LICENSE +21 -0
- package/README.md +410 -3
- package/bin/asrun.mjs +49 -0
- package/dist/lib/account-lock.d.ts +6 -0
- package/dist/lib/account-lock.js +52 -0
- package/dist/lib/assertions.d.ts +14 -0
- package/dist/lib/assertions.js +39 -0
- package/dist/lib/baselines.d.ts +43 -0
- package/dist/lib/baselines.js +108 -0
- package/dist/lib/case-id.d.ts +11 -0
- package/dist/lib/case-id.js +17 -0
- package/dist/lib/case-ops.d.ts +26 -0
- package/dist/lib/case-ops.js +110 -0
- package/dist/lib/cases.d.ts +29 -0
- package/dist/lib/cases.js +124 -0
- package/dist/lib/cli.d.ts +10 -0
- package/dist/lib/cli.js +18 -0
- package/dist/lib/downloads.d.ts +42 -0
- package/dist/lib/downloads.js +84 -0
- package/dist/lib/email.d.ts +10 -0
- package/dist/lib/email.js +19 -0
- package/dist/lib/ffmpeg.d.ts +8 -0
- package/dist/lib/ffmpeg.js +50 -0
- package/dist/lib/fixtures.d.ts +40 -0
- package/dist/lib/fixtures.js +149 -0
- package/dist/lib/node-cli.d.ts +11 -0
- package/dist/lib/node-cli.js +26 -0
- package/dist/lib/paths.d.ts +25 -0
- package/dist/lib/paths.js +88 -0
- package/dist/lib/pdf.d.ts +71 -0
- package/dist/lib/pdf.js +140 -0
- package/dist/lib/profiles.d.ts +38 -0
- package/dist/lib/profiles.js +121 -0
- package/dist/lib/ready.d.ts +36 -0
- package/dist/lib/ready.js +53 -0
- package/dist/lib/records.d.ts +33 -0
- package/dist/lib/records.js +61 -0
- package/dist/lib/run-filters.d.ts +19 -0
- package/dist/lib/run-filters.js +30 -0
- package/dist/lib/run-summary.d.ts +90 -0
- package/dist/lib/run-summary.js +35 -0
- package/dist/lib/session.d.ts +18 -0
- package/dist/lib/session.js +45 -0
- package/dist/lib/summary-reporter.d.ts +16 -0
- package/dist/lib/summary-reporter.js +146 -0
- package/dist/lib/targets.d.ts +27 -0
- package/dist/lib/targets.js +51 -0
- package/dist/lib/ui-build.d.ts +10 -0
- package/dist/lib/ui-build.js +30 -0
- package/dist/lib/values.d.ts +29 -0
- package/dist/lib/values.js +85 -0
- package/dist/lib/write-gate.d.ts +41 -0
- package/dist/lib/write-gate.js +129 -0
- package/dist/playwright.config.d.ts +2 -0
- package/dist/playwright.config.js +92 -0
- package/dist/scripts/build.d.ts +1 -0
- package/dist/scripts/build.js +20 -0
- package/dist/scripts/check-cases.d.ts +1 -0
- package/dist/scripts/check-cases.js +81 -0
- package/dist/scripts/cleanup.d.ts +1 -0
- package/dist/scripts/cleanup.js +25 -0
- package/dist/scripts/cli.d.ts +1 -0
- package/dist/scripts/cli.js +28 -0
- package/dist/scripts/doctor.d.ts +1 -0
- package/dist/scripts/doctor.js +185 -0
- package/dist/scripts/explore.d.ts +1 -0
- package/dist/scripts/explore.js +54 -0
- package/dist/scripts/go.mjs +188 -0
- package/dist/scripts/heal-evidence.mjs +143 -0
- package/dist/scripts/init.d.ts +16 -0
- package/dist/scripts/init.js +284 -0
- package/dist/scripts/lint-specs.d.ts +1 -0
- package/dist/scripts/lint-specs.js +138 -0
- package/dist/scripts/login.d.ts +1 -0
- package/dist/scripts/login.js +47 -0
- package/dist/scripts/new-case.d.ts +1 -0
- package/dist/scripts/new-case.js +19 -0
- package/dist/scripts/render-pdf.d.ts +1 -0
- package/dist/scripts/render-pdf.js +44 -0
- package/dist/scripts/report.d.ts +1 -0
- package/dist/scripts/report.js +14 -0
- package/dist/scripts/review-baselines.d.ts +1 -0
- package/dist/scripts/review-baselines.js +30 -0
- package/dist/scripts/run.d.ts +1 -0
- package/dist/scripts/run.js +423 -0
- package/dist/scripts/status.d.ts +1 -0
- package/dist/scripts/status.js +31 -0
- package/dist/scripts/ui-build.d.ts +1 -0
- package/dist/scripts/ui-build.js +7 -0
- package/dist/scripts/ui-dev.d.ts +1 -0
- package/dist/scripts/ui-dev.js +3 -0
- package/dist/scripts/ui.d.ts +1 -0
- package/dist/scripts/ui.js +13 -0
- package/dist/ui/dev.d.ts +1 -0
- package/dist/ui/dev.js +36 -0
- package/dist/ui/server/actions.d.ts +91 -0
- package/dist/ui/server/actions.js +352 -0
- package/dist/ui/server/ai-stream.d.ts +41 -0
- package/dist/ui/server/ai-stream.js +202 -0
- package/dist/ui/server/app.d.ts +4 -0
- package/dist/ui/server/app.js +333 -0
- package/dist/ui/server/cases.d.ts +23 -0
- package/dist/ui/server/cases.js +170 -0
- package/dist/ui/server/files.d.ts +5 -0
- package/dist/ui/server/files.js +67 -0
- package/dist/ui/server/index.d.ts +1 -0
- package/dist/ui/server/index.js +33 -0
- package/dist/ui/server/jobs.d.ts +74 -0
- package/dist/ui/server/jobs.js +263 -0
- package/dist/ui/server/profiles.d.ts +16 -0
- package/dist/ui/server/profiles.js +195 -0
- package/dist/ui/server/watch-files.d.ts +4 -0
- package/dist/ui/server/watch-files.js +36 -0
- package/dist/ui/shared/ai-commands.d.ts +78 -0
- package/dist/ui/shared/ai-commands.js +154 -0
- package/dist/ui/shared/case-id.d.ts +1 -0
- package/dist/ui/shared/case-id.js +1 -0
- package/dist/ui/shared/jobs.d.ts +6 -0
- package/dist/ui/shared/jobs.js +7 -0
- package/dist/ui/shared/product.d.ts +3 -0
- package/dist/ui/shared/product.js +3 -0
- package/dist/ui/shared/run-filters.d.ts +1 -0
- package/dist/ui/shared/run-filters.js +1 -0
- package/dist/ui/shared/site-address.d.ts +6 -0
- package/dist/ui/shared/site-address.js +11 -0
- package/dist/ui/shared/types.d.ts +257 -0
- package/dist/ui/shared/types.js +1 -0
- package/docs/theme-guide.html +692 -0
- package/package.json +98 -3
- package/templates/.claude/commands/explore-case.md +21 -0
- package/templates/.claude/commands/gen-spec.md +20 -0
- package/templates/.claude/commands/heal-spec.md +30 -0
- package/templates/.claude/commands/make-case.md +12 -0
- package/templates/.claude/settings.json +5 -0
- package/templates/.mcp.json +8 -0
- package/templates/AGENTS.md +45 -0
- package/templates/CLAUDE.md +1 -0
- package/templates/cases/_template.md +63 -0
- package/templates/gitignore +13 -0
- package/templates/opencode.jsonc +11 -0
- package/templates/package.json +31 -0
- package/templates/playwright.config.ts +3 -0
- package/templates/pnpm-workspace.yaml +5 -0
- package/templates/profiles.yaml +29 -0
- package/templates/seed.spec.ts +9 -0
- package/templates/tsconfig.json +14 -0
- package/ui/dist/apple-touch-icon.png +0 -0
- package/ui/dist/assets/apl-gfMMiTNf.js +1 -0
- package/ui/dist/assets/asciiarmor-C_eisQCp.js +1 -0
- package/ui/dist/assets/asn1-DCl_8MCW.js +1 -0
- package/ui/dist/assets/asterisk-BEn0Cwpx.js +1 -0
- package/ui/dist/assets/brainfuck-D5EjA2JK.js +1 -0
- package/ui/dist/assets/clike-xvONpD0Z.js +1 -0
- package/ui/dist/assets/clojure-BflJGmDX.js +1 -0
- package/ui/dist/assets/cmake-84nbDH0I.js +1 -0
- package/ui/dist/assets/cobol-DHLQUbN7.js +1 -0
- package/ui/dist/assets/coffeescript-De_zI4J0.js +1 -0
- package/ui/dist/assets/commonlisp-DgAKBe0r.js +1 -0
- package/ui/dist/assets/crystal-Bl6YKCG_.js +1 -0
- package/ui/dist/assets/css-CnaYoQib.js +1 -0
- package/ui/dist/assets/cypher-p9eYesGn.js +1 -0
- package/ui/dist/assets/d-DWkQyNnU.js +1 -0
- package/ui/dist/assets/diff-ChtP43wD.js +1 -0
- package/ui/dist/assets/dist-BHaIOZdW.js +1 -0
- package/ui/dist/assets/dist-BSJMM_jp.js +1 -0
- package/ui/dist/assets/dist-BiE4suo9.js +1 -0
- package/ui/dist/assets/dist-BqAROxzI.js +1 -0
- package/ui/dist/assets/dist-Bw5-uxs9.js +1 -0
- package/ui/dist/assets/dist-BwYRbe5s.js +9 -0
- package/ui/dist/assets/dist-BzulKuec.js +1 -0
- package/ui/dist/assets/dist-CBV7_vDN.js +2 -0
- package/ui/dist/assets/dist-CgIQYY1o.js +1 -0
- package/ui/dist/assets/dist-DDJNvivM.js +1 -0
- package/ui/dist/assets/dist-DF3F_I8N.js +1 -0
- package/ui/dist/assets/dist-DHCY9KpV.js +1 -0
- package/ui/dist/assets/dist-DX8knLVr.js +1 -0
- package/ui/dist/assets/dist-D_I45xgJ.js +11 -0
- package/ui/dist/assets/dist-Dlorc3Sx.js +1 -0
- package/ui/dist/assets/dist-Dr9crbBz.js +13 -0
- package/ui/dist/assets/dist-gF2bFn-5.js +1 -0
- package/ui/dist/assets/dockerfile-DJkHQgkl.js +1 -0
- package/ui/dist/assets/dtd-DTF72YNO.js +1 -0
- package/ui/dist/assets/dylan-S7ucR4pR.js +1 -0
- package/ui/dist/assets/ebnf-D4c0_ac3.js +1 -0
- package/ui/dist/assets/ecl-BVIeWmwY.js +1 -0
- package/ui/dist/assets/eiffel-BEjRio4Q.js +1 -0
- package/ui/dist/assets/elm-Cshzl8qu.js +1 -0
- package/ui/dist/assets/erlang-BV0c4Hdn.js +1 -0
- package/ui/dist/assets/factor-LKH5gN8L.js +1 -0
- package/ui/dist/assets/fcl-ClOhbFR7.js +1 -0
- package/ui/dist/assets/forth-D6RzAl64.js +1 -0
- package/ui/dist/assets/fortran-Bt6PBEDR.js +1 -0
- package/ui/dist/assets/gas-BHEdbvp9.js +1 -0
- package/ui/dist/assets/gherkin-oBAE_ms0.js +1 -0
- package/ui/dist/assets/groovy-CUrWKPia.js +1 -0
- package/ui/dist/assets/haskell-BRsoo5mP.js +1 -0
- package/ui/dist/assets/haxe-DjVOVvNP.js +1 -0
- package/ui/dist/assets/http-CLVgA2GD.js +1 -0
- package/ui/dist/assets/idl-BUZw3wgd.js +1 -0
- package/ui/dist/assets/index-DmasQ6Rq.css +2 -0
- package/ui/dist/assets/index-I7uU93kk.js +294 -0
- package/ui/dist/assets/javascript-lS2r2ifg.js +1 -0
- package/ui/dist/assets/julia-CQfgDRfP.js +1 -0
- package/ui/dist/assets/livescript-BXxG0Yva.js +1 -0
- package/ui/dist/assets/lua-DPCEkfD_.js +1 -0
- package/ui/dist/assets/mathematica-u8YMmbU0.js +1 -0
- package/ui/dist/assets/mbox-BEMVcqjs.js +1 -0
- package/ui/dist/assets/mirc-C9zpzAZU.js +1 -0
- package/ui/dist/assets/mllike-B45xgp2S.js +1 -0
- package/ui/dist/assets/modelica-CqHI_q-D.js +1 -0
- package/ui/dist/assets/mscgen-qgSLujhx.js +1 -0
- package/ui/dist/assets/mumps-BSzj6Xyz.js +1 -0
- package/ui/dist/assets/nginx-C-ORUuei.js +1 -0
- package/ui/dist/assets/nsis-viPvXFiq.js +1 -0
- package/ui/dist/assets/ntriples-CzMBulkp.js +1 -0
- package/ui/dist/assets/octave-D2Q8cUp1.js +1 -0
- package/ui/dist/assets/oz-BwTct_5T.js +1 -0
- package/ui/dist/assets/pascal-Dp_KdFgX.js +1 -0
- package/ui/dist/assets/perl-BkSkiI9m.js +1 -0
- package/ui/dist/assets/pig-AF4Hwhju.js +1 -0
- package/ui/dist/assets/powershell-D8WpQPaX.js +1 -0
- package/ui/dist/assets/properties-C357ku5U.js +1 -0
- package/ui/dist/assets/protobuf-D1ij_kNL.js +1 -0
- package/ui/dist/assets/pug-BqCHZ5xT.js +1 -0
- package/ui/dist/assets/puppet-BZyAS7I4.js +1 -0
- package/ui/dist/assets/python-B_1Jd8HM.js +1 -0
- package/ui/dist/assets/q-B9NhS7L_.js +1 -0
- package/ui/dist/assets/r-CxDmU1I4.js +1 -0
- package/ui/dist/assets/rpm-gpDK5sbt.js +1 -0
- package/ui/dist/assets/ruby-ChfNwlCY.js +1 -0
- package/ui/dist/assets/sas-C2OyD2Y6.js +1 -0
- package/ui/dist/assets/scheme-D9qYN0V5.js +1 -0
- package/ui/dist/assets/shell-DwuoZtxw.js +1 -0
- package/ui/dist/assets/sieve-tLWDfEkZ.js +1 -0
- package/ui/dist/assets/simple-mode-DXiLM1-G.js +1 -0
- package/ui/dist/assets/smalltalk-BLp5-jwC.js +1 -0
- package/ui/dist/assets/solr-B78wSQJQ.js +1 -0
- package/ui/dist/assets/sparql-Dz-mlCAC.js +1 -0
- package/ui/dist/assets/spreadsheet-CTIncYxj.js +1 -0
- package/ui/dist/assets/sql-DuQMFDvy.js +1 -0
- package/ui/dist/assets/stex-DNzd62Q_.js +1 -0
- package/ui/dist/assets/stylus-BOVgKnng.js +1 -0
- package/ui/dist/assets/swift-DKB6_1j6.js +1 -0
- package/ui/dist/assets/tcl-BGfruO1u.js +1 -0
- package/ui/dist/assets/textile-C4jVoHG1.js +1 -0
- package/ui/dist/assets/tiddlywiki-VlUiK86J.js +1 -0
- package/ui/dist/assets/tiki-CvUEFusT.js +1 -0
- package/ui/dist/assets/toml-BPTmHmyx.js +1 -0
- package/ui/dist/assets/troff-C2dAgh7P.js +1 -0
- package/ui/dist/assets/ttcn-PPSTk1ow.js +1 -0
- package/ui/dist/assets/ttcn-cfg-2b4Gb5sE.js +1 -0
- package/ui/dist/assets/turtle-Cx3rYX2g.js +1 -0
- package/ui/dist/assets/vb-BthdM_4N.js +1 -0
- package/ui/dist/assets/vbscript-D9f6rSsU.js +1 -0
- package/ui/dist/assets/velocity-CqftRsjg.js +1 -0
- package/ui/dist/assets/verilog-CjI9Y6RI.js +1 -0
- package/ui/dist/assets/vhdl-kS471soi.js +1 -0
- package/ui/dist/assets/webidl-D5jiJvOU.js +1 -0
- package/ui/dist/assets/xquery-BKJx4mf9.js +1 -0
- package/ui/dist/assets/yacas-CXXgtw0p.js +1 -0
- package/ui/dist/assets/z80-Co3PbHdE.js +1 -0
- package/ui/dist/favicon.ico +0 -0
- package/ui/dist/favicon.svg +11 -0
- package/ui/dist/index.html +17 -0
- package/ui/dist/logos/claude.svg +1 -0
- package/ui/dist/logos/opencode.svg +1 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# Spec conventions
|
|
2
|
+
|
|
3
|
+
## Shape
|
|
4
|
+
A case id is its path below `cases/` without `.md`: `hub-landing`, or `documents/landing-page` when the case is in a
|
|
5
|
+
folder (one level only). The spec is at the same path below `tests/` (`tests/documents/landing-page.spec.ts`).
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
// case: cases/<id>.md
|
|
9
|
+
import {caseTags} from '@asrunjs/asrun/cases';
|
|
10
|
+
import {configureCase, expect, test} from '@asrunjs/asrun/fixtures';
|
|
11
|
+
import {requireValues} from '@asrunjs/asrun/values';
|
|
12
|
+
|
|
13
|
+
test.use({mustLogin: false}); // only for login: none cases (public pages)
|
|
14
|
+
|
|
15
|
+
test.describe('<Case title>', {tag: caseTags('<id>')}, () => {
|
|
16
|
+
configureCase('<id>'); // from asrun/fixtures; no retries and a longer timeout for cases that write data
|
|
17
|
+
test('<single outcome>', async ({page, profile}) => {
|
|
18
|
+
const expected = requireValues<{...}>('<id>', profile); // the case's "## Values": default + this profile
|
|
19
|
+
// Locators are lazy: declare them here when several steps share them.
|
|
20
|
+
|
|
21
|
+
await test.step('1. <step text from the case>', async () => {
|
|
22
|
+
...
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
await test.step('Visual check: <what is compared>', async () => {
|
|
26
|
+
await expect(locator).toHaveScreenshot('<name>.png');
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
- Import `test`/`expect` from `@asrunjs/asrun/fixtures`, never from `@playwright/test`. The package name is the import path
|
|
32
|
+
(the package's `exports` map points at its `dist/` build), so a spec in a group folder uses the same imports as one
|
|
33
|
+
at the top of `tests/`.
|
|
34
|
+
- `profile` fixture = the current project's profile (`profile.id`, `profile.env`, `profile.baseURL`, `profile.account`,
|
|
35
|
+
`profile.tags`, `profile.email`). Never branch on `profile.id` or `profile.env` inside a spec: profile differences go
|
|
36
|
+
into the case's `## Values`, and the selectors come from `## Targets`.
|
|
37
|
+
- Paths are relative (`page.goto('/account/')`); the project sets `baseURL` per profile.
|
|
38
|
+
|
|
39
|
+
## Tags
|
|
40
|
+
The case's front matter (`suites: [...]`, `writes`) is the single source of truth. The spec never hard-codes
|
|
41
|
+
tags: it passes `{tag: caseTags('<id>')}` (from `@asrunjs/asrun/cases`) to the top-level `test.describe`.
|
|
42
|
+
`caseTags` returns one `@<suite>` per suite (shown as badges in the HTML report, drives `pnpm asrun --suite <name>`)
|
|
43
|
+
and `@writes` for cases that write data (drives `--read-only`).
|
|
44
|
+
- Do not copy suites into the describe title.
|
|
45
|
+
- If the front matter is invalid, `caseTags` throws; ask the user to fix the case.
|
|
46
|
+
- Write permission and `envs` limits are enforced by the `caseGate` fixture in `@asrunjs/asrun/fixtures`; specs do nothing.
|
|
47
|
+
A case that writes only in some environments is skipped (not failed) in the others, with a notice.
|
|
48
|
+
|
|
49
|
+
## Cases that write data (`writes: [<env>, ...]`)
|
|
50
|
+
- Name everything the case creates `[asrun] <something>` (`TEST_DATA_PREFIX` in `@asrunjs/asrun/records`). `records.track`
|
|
51
|
+
refuses other names, so cleanup can never touch real data.
|
|
52
|
+
- Register data with the `records` fixture right after creating it, together with the function that removes it:
|
|
53
|
+
`await records.track({kind: 'Document draft', name, where: '/app/documents'}, async () => {...delete it from the documents list...});`
|
|
54
|
+
Everything tracked is removed after the test, newest first, also when the test failed. The remove function opens
|
|
55
|
+
its own page (it cannot know where the test stopped) and must not fail if the data is already gone.
|
|
56
|
+
- The case's last step is normally the deletion itself; `track` is the safety net. Do not rely on it alone.
|
|
57
|
+
- If a cleanup fails, the test fails, the data is listed in the run summary (web UI banner and terminal) and
|
|
58
|
+
`pnpm cleanup --env <env>` lists it until a person deleted it and ran `--forget`.
|
|
59
|
+
- `configureCase('<id>')` turns retries off (a retry would create the data twice) and raises the timeout.
|
|
60
|
+
Cases that write run one at a time per profile, whatever the number of workers.
|
|
61
|
+
- Mark steps in the case text with `[writes]`; keep their titles unchanged in `test.step`.
|
|
62
|
+
- Email: wrap steps that send email in `if (!emailStepAllowed(profile, '<step title>')) return;` (`@asrunjs/asrun/email`).
|
|
63
|
+
It is false in production and for profiles with `email: false`; the skip is written to the report annotations.
|
|
64
|
+
|
|
65
|
+
## Steps (`test.step`)
|
|
66
|
+
Every numbered step of the case is wrapped in `await test.step('<n>. <step text>', async () => {...})`.
|
|
67
|
+
The report and the trace then show the case's own steps, so a reviewer can check what was executed
|
|
68
|
+
without reading the spec.
|
|
69
|
+
- Step title = the case's step text, verbatim, with its number. Do not paraphrase.
|
|
70
|
+
- Put the assertions that belong to a step inside that step.
|
|
71
|
+
- One extra step, titled `Visual check: <what is compared>`, per visual check from the case.
|
|
72
|
+
- A value needed by a later step (for example a page opened in a new tab) is returned from the step:
|
|
73
|
+
`const previewPage = await test.step('3. ...', async () => {...; return opened;});`
|
|
74
|
+
- Do not put `test.step` inside helpers in the framework (`@asrunjs/asrun/*`), and do not nest steps.
|
|
75
|
+
- Use step comments only for things the title cannot say; the title replaces the old `// 1. ...` comments.
|
|
76
|
+
|
|
77
|
+
## Assertion layers (use all that apply)
|
|
78
|
+
1. Text: `toHaveText`, `toHaveTitle`, `toContainText`, with values from `requireValues('<id>', profile)`.
|
|
79
|
+
2. DOM of images: `expectLogo(container, expected.logo)` / `expectImageLoaded`
|
|
80
|
+
(alt, src, loaded). Use `expectImageSha256` when the exact file must match.
|
|
81
|
+
3. Pixels: `await expect(locator).toHaveScreenshot('<name>.png')`. Baselines are stored per
|
|
82
|
+
profile automatically. Screenshot a small element, not the full page (full pages are flaky).
|
|
83
|
+
|
|
84
|
+
## Readable failures
|
|
85
|
+
The UI shows the first line of the error to QA and non-technical people, and hides the call log and stack
|
|
86
|
+
trace behind a "Technical details" fold. Write errors so that first line makes sense on its own.
|
|
87
|
+
- Read per-profile data with `requireValues<T>('<id>', profile)` (from `@asrunjs/asrun/values`), never by reading the case
|
|
88
|
+
yaml yourself. A case with no entry for the profile, or with `TODO`/empty values, then fails with
|
|
89
|
+
"Missing values: cases/<id>.md has no \"## Values\" for profile ..." instead of a cryptic crash.
|
|
90
|
+
- Give every `expect` a message in plain words about the product, not the code:
|
|
91
|
+
`expect(locator, 'The documents page did not finish loading').toBeVisible()`.
|
|
92
|
+
Say what is wrong ("The product logo has the wrong name"), not what the matcher is ("expected true").
|
|
93
|
+
- Guard preconditions that need manual data (for example "at least one published document exists") with an
|
|
94
|
+
`expect(..., 'No published document found. Create one manually; this test never writes data.')` in the step,
|
|
95
|
+
so the failure says what the human has to do.
|
|
96
|
+
- Messages are English, one sentence, no selectors or test ids.
|
|
97
|
+
|
|
98
|
+
## Ready, targets and downloads (`@asrunjs/asrun/ready`, `@asrunjs/asrun/downloads`)
|
|
99
|
+
Binding rules are in `../exploring-cases/references/readiness.md` and `.../assertion-targets.md`. In short:
|
|
100
|
+
- An action that loads data (opens a dialog, a composer, a tab, a list, a preview) goes through
|
|
101
|
+
`actAndWait(page, () => click(), {responses: [/api-url/], ready: [locator]})`: the response waiters exist before the
|
|
102
|
+
action, then the content signal must be visible. Read the result only afterwards. `expectNotBusy(scope)` waits until
|
|
103
|
+
no spinner, skeleton or `aria-busy` is left.
|
|
104
|
+
- A confirmation that may or may not appear after an action (the "Save as Draft?" dialog when closing a composer) is
|
|
105
|
+
handled with `clickOptionalUntilHidden(confirmButton, target, {message})`: one retry loop that clicks the confirmation
|
|
106
|
+
when it is visible and checks that the target is hidden. Never `if (await x.isVisible(...)) await x.click()`; see
|
|
107
|
+
`readiness.md`, "Optional confirmations".
|
|
108
|
+
- A check on a logo, image, file or screenshot has `// target: T<n>` on the line before it (a card in the case's
|
|
109
|
+
`## Targets`). The element is found by the card's `Identified by` (role and name, test id, alt/title), never `.first()` or
|
|
110
|
+
a bare `img`/`svg`. Semantic assertion first (name/alt/src, where it is), pixels second, on that element only.
|
|
111
|
+
- A check for absence ("no", "missing", "empty", "hidden") has `// ready: <signal>` in front of it and a positive wait
|
|
112
|
+
before it. The page must have loaded before "nothing" means anything.
|
|
113
|
+
- A download is `await verifyDownload(page, () => option.click(), {kind: 'pdf'})`. It checks, and fails the step on
|
|
114
|
+
any failure or timeout: the download starts, finishes, the file exists, the file is valid for its kind (PDF: `%PDF-`
|
|
115
|
+
start, `%%EOF` end, opens with pages). Add `validate` for the case's own check of the content. Never
|
|
116
|
+
`waitForEvent('download')` or a magic-number check on its own.
|
|
117
|
+
- A PDF's content is checked with `expectPdf(file.pdf!, {...})` (`@asrunjs/asrun/pdf`): pages rendered to images, regions that must not
|
|
118
|
+
be blank and are compared with the profile's baseline (2%), extracted text. Pages follow the case: every page by default
|
|
119
|
+
(`pages: 'all'`), only the named ones when the case says ONLY (a strong requirement) with `// pages: <the case's words>`.
|
|
120
|
+
The call has `// target: T<n>`. Take names and texts to avoid from `requireValues('<id>', profile)`, never literals.
|
|
121
|
+
```ts
|
|
122
|
+
const file = await verifyDownload(page, () => option.click(), {kind: 'pdf'});
|
|
123
|
+
// target: T3
|
|
124
|
+
await expectPdf(file.pdf!, {
|
|
125
|
+
regions: [{name: 'profile-logo', pages: 'all', region: corner('top-left')}],
|
|
126
|
+
textNotContains: expected.otherProfileNames,
|
|
127
|
+
});
|
|
128
|
+
```
|
|
129
|
+
- `pnpm spec:lint <id>` (static) must have no `FAIL`. `pnpm baselines:review <id>` lists every baseline with the card
|
|
130
|
+
that says what it must show.
|
|
131
|
+
|
|
132
|
+
## Do
|
|
133
|
+
- Locators: `getByTestId` (data-tn) > `getByRole` > `getByLabel` > text.
|
|
134
|
+
- Web-first assertions only; they auto-wait. Never `waitForTimeout`, never `networkidle`, never `if (await x.isVisible())`
|
|
135
|
+
to decide something that may not have loaded.
|
|
136
|
+
- Keep every test read-only unless the case lists environments under `writes`.
|
|
137
|
+
|
|
138
|
+
## Don't
|
|
139
|
+
- No login code in specs, and no cases that test the login page or the login flow. Sessions come from
|
|
140
|
+
`.auth/<profile-id>.json` via the project config; a profile without a valid session fails fast (see
|
|
141
|
+
`@asrunjs/asrun/fixtures`).
|
|
142
|
+
- No hard-coded profile names or expected values in specs; they come from the case's `## Values` through
|
|
143
|
+
`requireValues('<id>', profile)`. Site addresses come from `profile.baseURL`.
|
|
144
|
+
- No screenshots of areas with dynamic data (rows, prices, dates, avatars).
|
|
145
|
+
- No `.first()` / `.nth()` / bare `img`, `svg` or `img, svg` to find a logo or image.
|
|
146
|
+
- No hard assertion about something that was never seen on a page.
|
|
147
|
+
- No `isVisible({timeout})` / `isHidden({timeout})` (the timeout is ignored, nothing is waited for), no
|
|
148
|
+
`if (await x.isVisible()) await x.click()` outside a retry loop, and no `.catch(() => false)` to fake a wait.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: making-cases
|
|
3
|
+
description: Turns raw, informal test notes from a human (a QA memo, a list of buttons to check, a pasted chat) into well-formed case documents (cases/<id>.md) for the AsRun suite. Normalizes the text, classifies steps as read-only, writing or email, finds gaps and asks about them, never opens a browser, and splits the notes into cases. Use when asked to make, write or normalize cases from notes, or for /make-case. The next step after it is exploring-cases, then generating-specs.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Making case documents from raw notes
|
|
7
|
+
|
|
8
|
+
Read `AGENTS.md` (hard rules) and `cases/_template.md` (the target format) first. Then read the references:
|
|
9
|
+
- `references/normalizing.md`: how to clean up the text and keep the original traceable.
|
|
10
|
+
- `references/writes-and-email.md`: how to classify steps that write data or send email, and the data rules.
|
|
11
|
+
- `references/gaps.md`: what to ask, and the default to suggest for each question.
|
|
12
|
+
- `references/splitting.md`: how many cases to make.
|
|
13
|
+
|
|
14
|
+
This skill writes `cases/*.md` only. It never writes specs, `profiles.yaml`, or anything under `tests/`.
|
|
15
|
+
|
|
16
|
+
## Inputs
|
|
17
|
+
- Raw notes: pasted text or a file path. Any language, any quality.
|
|
18
|
+
- No environment: this skill never opens a browser. What the pages really show is found by `/explore-case`.
|
|
19
|
+
|
|
20
|
+
## Procedure
|
|
21
|
+
1. **Read everything and keep the original.** Number every distinct statement of the notes (N1, N2, ...). The
|
|
22
|
+
original wording is stored verbatim in each case's `## Source` table, so nothing is lost and a reviewer can check
|
|
23
|
+
what became of each line. Do not edit the input file.
|
|
24
|
+
2. **Normalize.** Follow `references/normalizing.md`: English, one action or check per step, concrete UI wording,
|
|
25
|
+
quoted UI text kept exactly. Do not add checks the notes did not ask for, and do not drop any.
|
|
26
|
+
3. **Classify each step**: read-only, `[writes]`, or `[email]` (`references/writes-and-email.md`). A step that
|
|
27
|
+
creates, edits, publishes, shares, deletes or sends is not read-only, even if the notes do not say so.
|
|
28
|
+
4. **Find the gaps** (`references/gaps.md`): missing expected results, unknown values a spec must assert, unclear starting
|
|
29
|
+
state, steps that need data that may not exist, ordering that is not linear. Never invent an expected value.
|
|
30
|
+
Unknown values are written as the suggested default sentence with a `❓ TODO` block under it (see `gaps.md`), or as an
|
|
31
|
+
`Observed, 🔍 to be confirmed by QA` bullet. Every marker starts with its emoji. A literal nobody knows yet goes into
|
|
32
|
+
the case's `## Values` as `TODO` (the spec then fails with a clear message until it is filled), never into
|
|
33
|
+
`profiles.yaml`.
|
|
34
|
+
5. **Ask before writing.** Show the user a short plan and the open questions in one message: the proposed split
|
|
35
|
+
into cases, the classification (which steps write, which send email), and the gaps with your suggested default
|
|
36
|
+
for each. Wait for answers. Do not write case files before that.
|
|
37
|
+
6. **Split** (`references/splitting.md`) and **write** `cases/<id>.md` for each case from `cases/_template.md`.
|
|
38
|
+
- Group folder: when the notes (or the user) name an area (documents, search, account...), put the cases in
|
|
39
|
+
`cases/<group>/<name>.md` and set `id: <group>/<name>` in the front matter. Use one folder level and
|
|
40
|
+
lowercase kebab-case. Reuse an existing folder in `cases/` when the area matches. If the area is unclear,
|
|
41
|
+
ask in step 5. The group does not repeat in the name: `documents/landing-page`, not `documents/documents-landing-page`.
|
|
42
|
+
- Front matter: `id`, `title`, `login`, `suites`, `writes`, `writes-note` (required when `writes` is not `none`),
|
|
43
|
+
`envs` when a case must not run everywhere. When unsure: `suites: [regression]`, no `writes` (read-only), or `writes: [staging]` when a step writes.
|
|
44
|
+
- Fill the sections that apply (Goal, Steps, Expected, Values, Visual checks, Data, Source, Notes).
|
|
45
|
+
Remove template placeholders you do not use. Use `## Values` only for the literals a spec asserts; `default`
|
|
46
|
+
applies to every profile, and a profile id entry holds only what differs.
|
|
47
|
+
- Case ids are lowercase kebab-case and name the thing, not the ticket: `landing-page`.
|
|
48
|
+
- Existing case files: do not overwrite. If a new case overlaps one (for example another case about the same page), say so,
|
|
49
|
+
propose "keep", "merge" or "replace", and wait for the user.
|
|
50
|
+
7. **Check.** Run `pnpm case:check`. Fix every FAIL. Re-read each written case against the numbered statements: every
|
|
51
|
+
N must appear in exactly one `## Source` table, or in the report as "not used, because ...".
|
|
52
|
+
8. **Report** in the user's language: the cases written (id, suites, writes, envs), the traceability summary, the
|
|
53
|
+
questions that are still open, and the next steps per case: `/explore-case <id> <env> --attended` (after `pnpm signin`
|
|
54
|
+
for the profiles and environment; a person does the steps that write data), then `/gen-spec <id> <env>`.
|
|
55
|
+
|
|
56
|
+
## Hard rules
|
|
57
|
+
- Never open a browser and never write data. A step that writes is only described and marked; the case lists the
|
|
58
|
+
environments the spec may write in under `writes`. Never add `--allow-writes`, never set
|
|
59
|
+
`ASRUN_ALLOW_WRITES`; this skill does not run write cases at all.
|
|
60
|
+
- Never send email in production. In other environments only for profiles with `email: true`. New profiles and AI-written
|
|
61
|
+
entries in `profiles.yaml` get `email: false`; do not change an existing `email: true`.
|
|
62
|
+
- Never write a case for the login page or the login flow.
|
|
63
|
+
- Never invent expected values, profile names, counts, or UI texts.
|
|
64
|
+
- All files are written in English.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Gap analysis
|
|
2
|
+
|
|
3
|
+
Ask every question in one message, each with a suggested default, so the user can answer "yes to all" or correct
|
|
4
|
+
single lines. Never ask a question whose answer is already in the notes, `AGENTS.md`, `profiles.yaml` or the existing cases.
|
|
5
|
+
|
|
6
|
+
## What to look for
|
|
7
|
+
| Gap | Typical question | Suggested default |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| "check X" without an expected result | What exactly must be true for X (text, logo, count, color)? | Visible and not empty; the value per profile goes in `## Values` |
|
|
10
|
+
| Logo checks | Which logo element and which value per profile? | The product logo is visible and not empty; the value per profile goes in `## Values`, and name the element as a target to settle in `/explore-case` (`[T<n>]`, see `exploring-cases/references/assertion-targets.md`). Ask the user for the exact image/text/place when the notes say only "the logo" and the page may hold several. |
|
|
11
|
+
| A check on an image, logo, file or PDF | Which exact image, text, place or UI element, and where? | Write the check with a `[T<n>]` reference and leave the card to `/explore-case`; never describe it as "the first image" |
|
|
12
|
+
| Page loads data after an action | What shows that the content is really there? | Add `Ready when: <signal>` if the notes name one; else leave it to `/explore-case` |
|
|
13
|
+
| Starting state | Does the case need existing data (a published document, a shared document)? | The case creates what it needs; never depends on pre-existing data |
|
|
14
|
+
| Delete path | How is the created data removed, and from where? | From the list page, with the item's menu; keep a `[blocking]` TODO block until confirmed |
|
|
15
|
+
| Write level | Does this step write? | Yes if it creates, edits, publishes, shares or deletes |
|
|
16
|
+
| Email | Does this step send email? | Yes for share/send/invite; skipped in production and for `email: false` profiles |
|
|
17
|
+
| Environments | Must this case run in production, or only staging? | All environments; production writes need the usual confirmation |
|
|
18
|
+
| Suites | Smoke, regression, or both? | A read-only landing check: `[smoke, regression]`; a flow that writes: `[regression]` |
|
|
19
|
+
| Profiles | Do all profiles see the same UI? | Yes; the exact literals that differ go in `## Values`, never in the steps |
|
|
20
|
+
| Unclear words | What does "collaborator button" / "..." button open? | Describe only what is observed; mark `(observed, 🔍 to be confirmed by QA)` |
|
|
21
|
+
| Unused lines | A statement that fits no case | Ask where it belongs; never drop it silently |
|
|
22
|
+
|
|
23
|
+
## Marking unknowns in a case
|
|
24
|
+
- Unknown expected value: write the suggested default as a normal sentence and hang a TODO block under it, in the
|
|
25
|
+
shape of `<framework>/.claude/skills/exploring-cases/references/open-questions.md`:
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
- The preview header shows the product logo, visible and not empty.
|
|
29
|
+
- ❓ TODO [non-blocking]: Which logo must it be? (👁️ not observed)
|
|
30
|
+
- ✍️ Answer:
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The user accepts the default by deleting the block, or types an answer. Never write a bare `TODO: ...` line, a
|
|
34
|
+
question as the only text, or `Suggested:`.
|
|
35
|
+
- Seen on a page during exploration but not stated in the notes: `- <observation> (observed, 🔍 to be confirmed by QA)`.
|
|
36
|
+
- Never turn an observation into a hard requirement on your own.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Normalizing raw notes
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
Turn informal notes into steps that someone who never saw the notes can follow, without changing what was asked.
|
|
5
|
+
|
|
6
|
+
## Rules
|
|
7
|
+
1. **Translate to English.** Keep text the page shows exactly as written in the notes (button labels such as
|
|
8
|
+
"Create New", pill names such as "My documents", "Shared with me"). Do not translate or "fix" those.
|
|
9
|
+
2. **One action or one check per step.** Split "create, publish and preview it" into three steps. A check that
|
|
10
|
+
belongs to an action stays in the same step as its action ("Open Share and check the logo").
|
|
11
|
+
3. **Make the wording concrete.** Say where (page, path, card, corner), what (exact label), and what happens.
|
|
12
|
+
"Check the logo" is not a step by itself: it needs an expected result (see `gaps.md`). Until it has one, write
|
|
13
|
+
the default as a sentence under "Expected" ("The <thing> is visible and not empty.") with a
|
|
14
|
+
`TODO [non-blocking]` block under it (see `gaps.md`).
|
|
15
|
+
4. **Keep the order of the notes** unless the notes say otherwise, then fix the order and say so in the Source table.
|
|
16
|
+
5. **Do not add checks** the notes did not ask for (no extra assertions "while we are at it"), and do not drop any.
|
|
17
|
+
When a line is a heading or a list of things on one page ("Landing: Create New, All, My documents, Shared with me,
|
|
18
|
+
search bar"), each item becomes a check: "is visible" is the only meaning you may assume, and you say so.
|
|
19
|
+
6. **Mark steps** with `[writes]` or `[email]` at the end of the step text (see `writes-and-email.md`).
|
|
20
|
+
7. **Use paths relative to the app** (`/app/documents`), never a full domain. Domains differ per profile and environment.
|
|
21
|
+
8. **No profile-specific values in steps.** The exact literals go in `## Values`; a sentence about behaviour goes in `Expected`.
|
|
22
|
+
|
|
23
|
+
## Traceability (`## Source` section in every case)
|
|
24
|
+
Number every statement of the raw notes once (N1, N2, ...). Each case ends with:
|
|
25
|
+
|
|
26
|
+
```markdown
|
|
27
|
+
## Source
|
|
28
|
+
Original notes, verbatim. Do not edit this table; it only exists so a reviewer can check what became of each line.
|
|
29
|
+
|
|
30
|
+
| # | Original text | Became |
|
|
31
|
+
|---|---------------|--------|
|
|
32
|
+
| N3 | `Published document: publish (a follow-up modal opens by itself)` | Steps 6-7 |
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- The original text is copied character by character, in a code span, in its original language.
|
|
36
|
+
- "Became" points at step numbers, or says "Expected", "Visual checks", "Not used: <reason>".
|
|
37
|
+
- A statement used by two cases appears in both tables.
|
|
38
|
+
- A statement that is not used anywhere is a gap: ask about it, do not silently drop it.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Splitting notes into cases
|
|
2
|
+
|
|
3
|
+
## Principle
|
|
4
|
+
One case = one real user path. Split only when something real differs:
|
|
5
|
+
1. **Starting state** differs (a landing page you simply open, versus a flow that needs data it creates itself).
|
|
6
|
+
2. **Write level** differs (read-only versus `writes: [<env>, ...]`). Keep read-only checks out of write cases: they can
|
|
7
|
+
run anywhere, without confirmation, in the smoke suite.
|
|
8
|
+
3. **Independence** differs (the checks can pass or fail without the other part, and a failure in one should not hide
|
|
9
|
+
the other).
|
|
10
|
+
|
|
11
|
+
Do not split because a case is long, or because the notes list things under separate headings.
|
|
12
|
+
|
|
13
|
+
## Typical result
|
|
14
|
+
- A read-only landing or overview check (what is on the page): small, fast, `suites: [smoke, regression]`, no `writes` (read-only).
|
|
15
|
+
- A user flow that creates data and cleans it up: `writes: [staging]`, one case, steps in the real order (even if the
|
|
16
|
+
flow is not linear), ending with the deletion.
|
|
17
|
+
- A logo or visual check of data that already exists, kept read-only: its own case when it does not need the flow.
|
|
18
|
+
|
|
19
|
+
## Existing cases
|
|
20
|
+
Look at `cases/` before writing. If a new case overlaps an existing one, do not overwrite it: propose keep, merge or
|
|
21
|
+
replace, with the reason, and wait for the user. Never rename or move an existing case yourself; the Cases page of the web UI does that and keeps the spec in step.
|
|
22
|
+
|
|
23
|
+
## Naming and folders
|
|
24
|
+
Cases of one area live in a folder: `cases/documents/landing-page.md` has the id `documents/landing-page`. The name is
|
|
25
|
+
`<what>` in kebab-case (`hub-landing`, `draft-to-publish`), the folder is the area (`documents`). One folder level only.
|
|
26
|
+
Look at the folders that already exist in `cases/` and reuse them. A case that fits no area stays directly in
|
|
27
|
+
`cases/`. The `title` says what the case proves. Two ids that differ only in `-` versus `/` (`documents-x` and
|
|
28
|
+
`documents/x`) are not allowed: one group level only, so ids stay unambiguous.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Steps that write data or send email
|
|
2
|
+
|
|
3
|
+
## Classify every step
|
|
4
|
+
| Class | Looks like | Marker |
|
|
5
|
+
|---|---|---|
|
|
6
|
+
| read-only | open, look, read, hover, switch tab or filter, open a preview, close a modal | none |
|
|
7
|
+
| writes | create, save, edit, publish, share, delete, upload, accept, assign a label, change a setting | `[writes]` |
|
|
8
|
+
| input | a step that needs a person's choice or typed value (an existing document, items, a contact name), writing or not | `[input]` |
|
|
9
|
+
| email | any step that makes the product send an email or notification (share by email, send to a recipient, invite a collaborator) | `[email]` |
|
|
10
|
+
|
|
11
|
+
A step can be several of these (a step that picks items and saves is `[input] [writes]`). A step is not read-only because the notes did not say it changes something: "Share" opens a
|
|
12
|
+
dialog (read-only) but sending the share is `[writes]` and probably `[email]`. When in doubt, mark it and ask.
|
|
13
|
+
|
|
14
|
+
## What a case with writes must look like
|
|
15
|
+
- Front matter: `writes: [<env>, ...]`, the environments the finished spec may write in (`[staging]`; add `production`
|
|
16
|
+
only when the notes or the user say so), and a `writes-note` that says what is created, changed and removed, in one
|
|
17
|
+
sentence ("Creates one draft document named '[asrun] ...', publishes it, deletes it at the end").
|
|
18
|
+
- A `## Data` section: what is created, its name, and how it is removed.
|
|
19
|
+
- Everything created is named `[asrun] <something>` (the spec enforces this).
|
|
20
|
+
- The case ends by deleting what it created. If the notes do not say how to delete it, that is a gap: ask, and
|
|
21
|
+
never guess a delete path. Until it is known, write the likeliest path as a sentence in Data ("Removal: the
|
|
22
|
+
"Delete" option of the item's menu on the list page.") with a `TODO [blocking]: How is <thing> deleted?` block
|
|
23
|
+
under it (see `gaps.md`).
|
|
24
|
+
- The case needs only data of the test account of the profile. It never touches data that existed before.
|
|
25
|
+
- Do not set `envs` just because a case writes. A case that does not list an environment under `writes` is skipped
|
|
26
|
+
there with a notice (not failed), and a human confirms a run in any environment except staging. When unsure, write
|
|
27
|
+
`writes: [staging]`.
|
|
28
|
+
- The agent never performs a write step while making, exploring or generating: a person does it in
|
|
29
|
+
`/explore-case --attended`, which records a general recipe under `## Inputs`. Here, only mark the step.
|
|
30
|
+
- Email: steps marked `[email]` are skipped by the runner in production and for profiles whose `email` is false.
|
|
31
|
+
Write the case so that skipping them does not break the following steps, or say in Notes that it does.
|
|
32
|
+
|
|
33
|
+
## Non-linear flows
|
|
34
|
+
Real flows often leave and come back (create a draft, stop before Publish, return to the list, check the draft,
|
|
35
|
+
edit it again, publish). Write them as one case with the steps in the real order, including the "return to the
|
|
36
|
+
list" steps. Do not split a flow only because it is long.
|
package/AGENTS.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# AsRun — framework baseline
|
|
2
|
+
|
|
3
|
+
This file is the binding baseline for every project that uses the AsRun framework, and for the framework itself.
|
|
4
|
+
Its hard rules carry ids (`F1`..`F10`, `B1`..`B4`, `A1`..`A6`). A project may narrow one of them in its own
|
|
5
|
+
`AGENTS.md` by naming the id; rules tagged `[non-negotiable]` may not be narrowed (`pnpm asrun doctor` checks it).
|
|
6
|
+
|
|
7
|
+
Paths are relative to the **workspace root**: the project that owns `profiles.yaml`, `cases/` and `tests/`. In a
|
|
8
|
+
project that uses asrun, the framework files are in `node_modules/@asrunjs/asrun` (an installed dependency: never edit it).
|
|
9
|
+
`<framework>` below means that folder (`node_modules/@asrunjs/asrun` in a project, `.` in the framework repository).
|
|
10
|
+
|
|
11
|
+
## Hard rules
|
|
12
|
+
- **F1 — Read-only by default.** [non-negotiable] Tests never create, edit, delete, submit forms that change data,
|
|
13
|
+
send messages, or accept terms. Only a case that lists environments under `writes` in its front matter
|
|
14
|
+
(`writes: [staging]`, `writes: [staging, production]`; with a `writes-note`) may write, only in those environments,
|
|
15
|
+
and only data of the test accounts, which it must clean up. In any other environment the case is skipped with a
|
|
16
|
+
notice, not failed. The `writes` environments must be ones that `envs` lets the case run in. The permission is
|
|
17
|
+
checked at run time.
|
|
18
|
+
- **F2 — No bypassing the write confirmation.** [non-negotiable] A case that writes in any environment except staging
|
|
19
|
+
needs a human's confirmation on every run (typing the environment name in the CLI, or the confirm dialog in the web
|
|
20
|
+
UI). AI agents must NEVER add `--allow-writes`, set `ASRUN_ALLOW_WRITES`, or bypass the confirmation. Use
|
|
21
|
+
staging, or `--read-only`, instead.
|
|
22
|
+
- **F3 — Agents never write while exploring.** [non-negotiable] A step that creates, edits, publishes, shares or
|
|
23
|
+
deletes, and a step that needs a person's choice, is done by the person in the browser window
|
|
24
|
+
(`/explore-case --attended`, see `<framework>/.claude/skills/exploring-cases/references/user-handoff.md`). The
|
|
25
|
+
agent only reads. Without `--attended` an agent never waits for a person.
|
|
26
|
+
- **F4 — No email in production.** [non-negotiable] Never send email from a test in production. In other
|
|
27
|
+
environments only if the profile has `email: true`; new and AI-generated profile entries always get `email: false`.
|
|
28
|
+
- **F5 — Saved logins are passwords.** [non-negotiable] Never read, print, copy or commit anything under `.auth/`.
|
|
29
|
+
- **F6 — No 2FA automation.** [non-negotiable] Login is always done by a human via `pnpm signin <profile-id>`.
|
|
30
|
+
- **F7 — No default environment.** `--env <env>` is required by `pnpm asrun` and by every AI command.
|
|
31
|
+
`pnpm signin <profile-id>` and `pnpm explore <profile-id>` need no `--env`: the profile already names its
|
|
32
|
+
environment. Sessions are saved per profile in `.auth/<profile-id>.json`.
|
|
33
|
+
- **F8 — Login is not a test case.** `pnpm asrun` checks the session of every profile first; a profile that is not
|
|
34
|
+
logged in (missing or expired token) fails right away with a hint to run `pnpm signin <profile-id>` or to skip it
|
|
35
|
+
with `--skip-profile <profile-id>`. Never write specs for the login page or login flow.
|
|
36
|
+
- **F9 — Expected values live in the case document.** They go in its `## Values` block, never in `profiles.yaml`. A
|
|
37
|
+
profile holds connection settings only; adding or changing a profile must not touch any case value.
|
|
38
|
+
- **F10 — Write in English.** Code, comments, docs, case files and commit messages. A Chinese version is only written
|
|
39
|
+
when the user explicitly asks for one.
|
|
40
|
+
|
|
41
|
+
## Browser exploration rules (`B`)
|
|
42
|
+
These apply to every agent that drives a browser to explore pages for cases or specs (`making-cases`,
|
|
43
|
+
`exploring-cases`, `generating-specs`, the `playwright-test` MCP).
|
|
44
|
+
- **B1 — Read-only for the agent.** Open, look, read, hover, switch tabs, open and close dialogs and menus.
|
|
45
|
+
Everything that writes is the person's (see F3).
|
|
46
|
+
- **B2 — Waiting for a person.** Only in an attended run. Announce the task in the browser window (a small draggable
|
|
47
|
+
card with a Done button and click-to-copy for texts to type), name the finish condition, wait, and tell the person
|
|
48
|
+
when control is back. Record what they did as a general recipe in the case's `## Inputs` section: positions instead
|
|
49
|
+
of names of data that can change.
|
|
50
|
+
- **B3 — Closing an email composer.** When closing an email composer makes a confirm dialog appear (for example
|
|
51
|
+
"Discard draft?"), choose "Discard" and continue. Never choose "Save" or "Send"; closing the composer must not keep
|
|
52
|
+
or send anything. The dialog can show up late: in a spec, clicking "Discard" and checking that the composer is
|
|
53
|
+
closed happen in one retry loop (`clickOptionalUntilHidden` in `@asrunjs/asrun/ready`), never after a single
|
|
54
|
+
`isVisible()` look.
|
|
55
|
+
- **B4 — Record it in the case.** If the case file does not already contain this step, do not edit the flow silently.
|
|
56
|
+
Add it to the case as a step with a `❓ TODO [non-blocking]` block for the user's confirmation, in the format of
|
|
57
|
+
`<framework>/.claude/skills/exploring-cases/references/open-questions.md`, for example:
|
|
58
|
+
```
|
|
59
|
+
4. Close the email composer; if a confirm dialog appears, choose "Discard".
|
|
60
|
+
- ❓ TODO [non-blocking]: Is choosing "Discard" in the close-composer confirm dialog correct for this case? (seen while exploring <profile-id>)
|
|
61
|
+
- ✍️ Answer:
|
|
62
|
+
```
|
|
63
|
+
Put the step right after the step that opens the composer (or after the last step that uses it). Mention it in the
|
|
64
|
+
final report so the user can confirm it. If the case already has the step, just follow it and add nothing.
|
|
65
|
+
|
|
66
|
+
## Assertions and readiness (`A`, binding for exploring, generating and healing)
|
|
67
|
+
Details: `<framework>/.claude/skills/exploring-cases/references/assertion-targets.md` and `readiness.md`.
|
|
68
|
+
- **A1 — Never guess what a check looks at.** A check on a logo, image, icon, file, PDF, screenshot or repeated text
|
|
69
|
+
names its exact target in a card in the case (`## Targets`: What, Where, Identified by, Not, Ready when, a
|
|
70
|
+
highlighted reference screenshot, referenced as `[T<n>]`). When in doubt (several candidates, only a position or a
|
|
71
|
+
generic locator, dynamic data in the area, no stable identity, the page differs from the case, it could not be
|
|
72
|
+
seen), write a `🚫 TODO [blocking]` that names the exact image, text, place or UI wanted and lists the candidates.
|
|
73
|
+
Do not pick. Look at the screenshot you saved: it must show what the card says.
|
|
74
|
+
- **A2 — Wait for the content before you read it.** After every action that loads something, wait for its key
|
|
75
|
+
requests to finish and for a positive content signal, then read. The same in exploration
|
|
76
|
+
(`browser_network_requests`, `browser_wait_for`), in specs (`actAndWait` in `@asrunjs/asrun/ready`) and in healing. A
|
|
77
|
+
"missing", "empty" or "hidden" result is only valid after a positive ready signal. Never `waitForTimeout`, never
|
|
78
|
+
`networkidle`.
|
|
79
|
+
- **A3 — Verify every download in five parts** with `verifyDownload` (`@asrunjs/asrun/downloads`): starts, finishes, file
|
|
80
|
+
exists, file is valid, and any failure or timeout fails the whole step.
|
|
81
|
+
- **A4 — A PDF is read like a person reads it.** Its pages are rendered to images (logo regions must not be blank and
|
|
82
|
+
match the profile's baseline, 2%) and its text is extracted (`expectPdf` in `@asrunjs/asrun/pdf`; `pnpm pdf:render`
|
|
83
|
+
while exploring). Every page is checked unless the case limits the pages in strong words ("only", "exactly",
|
|
84
|
+
"must"); "should", "always", "usually" and no wording at all mean every page.
|
|
85
|
+
- **A5 — No hard assertion on what nobody saw on a page.** Specs are checked with `pnpm spec:lint`; baselines with
|
|
86
|
+
`pnpm baselines:review` (open every new image and compare it with its card).
|
|
87
|
+
- **A6 — Observe first, delete last.** Everything that needs the data (preview, share menus, composers, downloads)
|
|
88
|
+
is observed before the person deletes it.
|
|
89
|
+
|
|
90
|
+
## Layout
|
|
91
|
+
|
|
92
|
+
**The workspace** (committed by the project; the framework never holds these):
|
|
93
|
+
- `cases/**/*.md` human-written test cases (input of the AI workflow); `cases/<group>/<name>.md` groups an area
|
|
94
|
+
- `tests/**/*.spec.ts` generated Playwright specs (output of the AI workflow), one per case, same folder as the case,
|
|
95
|
+
with their `*-snapshots/` baselines next to them
|
|
96
|
+
- `profiles.yaml` one entry per environment: URL, account label, tags, environment (connection settings only)
|
|
97
|
+
- `.auth/` `reports/` `.tmp/` `.explore/` local state, never committed
|
|
98
|
+
|
|
99
|
+
**The framework** (this folder; read-only in a project that uses it, as `node_modules/@asrunjs/asrun/`):
|
|
100
|
+
- `lib/` fixtures (`profile`, session guard) and assertion helpers; specs import them as
|
|
101
|
+
`@asrunjs/asrun/<module>` (for example `@asrunjs/asrun/fixtures`, `@asrunjs/asrun/values`)
|
|
102
|
+
- `scripts/` user-facing commands, see README.md
|
|
103
|
+
- `templates/` the files `pnpm asrun init` writes into a workspace
|
|
104
|
+
- `ui/` local web UI (`pnpm ui`). It may only run the fixed actions in `ui/server/actions.ts`
|
|
105
|
+
and must never serve anything under `.auth/`.
|
|
106
|
+
- `docs/theme-guide.html` the Inkstone theme guide of the web UI: color tokens, buttons, badges, icons, surfaces
|
|
107
|
+
- `examples/demo/` a local workspace for developing the framework itself (not test content)
|
|
108
|
+
|
|
109
|
+
## Web UI (`ui/web`)
|
|
110
|
+
Before adding or changing any UI element, read `docs/theme-guide.html` (open the file; it is static HTML) and follow it.
|
|
111
|
+
Its rules are binding, in particular:
|
|
112
|
+
- Icons: Feather via the `react-feather` package only (`import {ChevronDown} from 'react-feather'`). Never draw, paste
|
|
113
|
+
or hand-write SVGs, and never add another icon library. Size 14 in buttons, 16-20 elsewhere; decorative icons get
|
|
114
|
+
`aria-hidden="true"`, icon-only buttons get an `aria-label`. Do not use Unicode symbols (`▲ ▼ ◀ ▶ ⤢`) as icons.
|
|
115
|
+
- Colors: only the tokens in `ui/web/src/index.css` (the default Tailwind palette is removed). Pick the color by impact.
|
|
116
|
+
- Buttons and badges: use `buttonClass()` and `Badge` from `ui/web/src/components/status.tsx`, never hand-style them.
|
|
117
|
+
- Dropdowns: use `Select` from the same file, never a bare `<select>` or hand-copied classes (native arrow and height
|
|
118
|
+
differ). A required dropdown that is empty gets `invalid` (red). See the "Dropdowns" section of the theme guide.
|
|
119
|
+
- If a change adds or alters a token, update `ui/web/src/index.css` and `docs/theme-guide.html` together.
|
|
120
|
+
Before reaching for a new dependency or writing a helper component, grep `package.json` and `ui/web/src` for what the
|
|
121
|
+
project already uses.
|
|
122
|
+
|
|
123
|
+
## Workflow
|
|
124
|
+
Three steps, each learns one thing: the intent, the facts and inputs, the implementation.
|
|
125
|
+
1. `making-cases` (or `/make-case <notes>`) turns raw notes into case documents. It writes `cases/*.md` only and
|
|
126
|
+
never opens a browser. Check them with `pnpm case:check`.
|
|
127
|
+
2. `exploring-cases` (or `/explore-case <case-id> <env> [--attended]`) explores the real pages for one case. It
|
|
128
|
+
edits that case file only: wording, observations, TODO blocks, and, with `--attended`, a `## Inputs` recipe of
|
|
129
|
+
what the person did for the steps that write or need a choice.
|
|
130
|
+
3. `generating-specs` (or `/gen-spec <case-id> <env>`) turns the case into a spec. It replays the case and its
|
|
131
|
+
`## Inputs` and `## Values`, writes the spec (values stay in the case document), but never runs the tests, never
|
|
132
|
+
writes data and never creates baselines. A case with an `[input]` step that has no `## Inputs` recipe stops with
|
|
133
|
+
`needs-input`: run `/explore-case` first. Running specs is a separate step (`pnpm asrun`, or the Run page).
|
|
134
|
+
Do not hand-write specs unless asked. Do not edit `lib/` to make a single spec pass.
|
|
135
|
+
Every case step is wrapped in `test.step('<n>. <case step text>', ...)` so reports and traces show the
|
|
136
|
+
case's own steps (see the skill's `references/conventions.md`).
|
|
137
|
+
Every AI command needs an environment (`--env`/`<env>`); there is no default and no "no environment" mode.
|
|
138
|
+
`/heal-spec <case-id> <env>` reads the existing report first (`pnpm heal:evidence --case <id> --env <env>`), reads the
|
|
139
|
+
user's optional clues (`--hints <file>`), runs only the profiles that are logged in (`--logged-in`) and runs the case
|
|
140
|
+
again only when the report is missing, stale or not enough.
|
|
141
|
+
Run `pnpm typecheck`, `pnpm spec:lint <case-id>` and `pnpm case:check` before reporting that a spec was generated. Run
|
|
142
|
+
`pnpm asrun --env <env> --profile <id> --grep <case-id>` only when healing a spec or when asked to.
|
|
143
|
+
Pick `--env staging` while developing. A case that does not list `production` under `writes` is skipped in
|
|
144
|
+
production with a notice; add `--read-only` to leave all write cases out.
|
|
145
|
+
|
|
146
|
+
## Working on the framework itself
|
|
147
|
+
- The framework repository holds no workspace content: no `profiles.yaml`, `cases/`, `tests/` or reports at the root.
|
|
148
|
+
Develop against `examples/demo`: `pnpm asrun init --root examples/demo` (it also writes the framework's `.env`
|
|
149
|
+
with `ASRUN_ROOT=examples/demo`), then `pnpm asrun --env staging --read-only`.
|
|
150
|
+
- Everything the framework reads for a project goes through `ROOT` in `lib/paths.ts` (the workspace root); its own
|
|
151
|
+
assets (templates, docs, skills, its `node_modules`) go through `FRAMEWORK_ROOT`. Never write into
|
|
152
|
+
`FRAMEWORK_ROOT` at run time.
|
|
153
|
+
- `templates/` is the source of the files a workspace gets. When a template changes, bump `version` in
|
|
154
|
+
`package.json` so `pnpm asrun doctor` in a project shows that it is behind.
|
package/CLAUDE.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@AGENTS.md
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zoro Chen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|