@asrunjs/asrun 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/.claude/commands/explore-case.md +11 -6
  2. package/.claude/commands/gen-spec.md +10 -6
  3. package/.claude/commands/heal-spec.md +13 -9
  4. package/.claude/commands/make-case.md +1 -1
  5. package/.claude/skills/exploring-cases/SKILL.md +65 -31
  6. package/.claude/skills/exploring-cases/references/assertion-targets.md +8 -4
  7. package/.claude/skills/exploring-cases/references/case-rules.md +110 -0
  8. package/.claude/skills/exploring-cases/references/handoff-card.js +73 -11
  9. package/.claude/skills/exploring-cases/references/open-questions.md +58 -6
  10. package/.claude/skills/exploring-cases/references/readiness.md +4 -3
  11. package/.claude/skills/exploring-cases/references/tools.md +117 -0
  12. package/.claude/skills/exploring-cases/references/user-handoff.md +46 -17
  13. package/.claude/skills/generating-specs/SKILL.md +71 -33
  14. package/.claude/skills/generating-specs/references/conventions.md +4 -0
  15. package/.claude/skills/making-cases/SKILL.md +14 -7
  16. package/.claude/skills/making-cases/references/gaps.md +7 -4
  17. package/.claude/skills/making-cases/references/normalizing.md +1 -1
  18. package/AGENTS.md +51 -11
  19. package/README.md +4 -3
  20. package/bin/asrun.mjs +1 -0
  21. package/dist/lib/case-doc.d.ts +41 -4
  22. package/dist/lib/case-doc.js +109 -17
  23. package/dist/scripts/check-cases.js +89 -1
  24. package/dist/scripts/doctor.js +50 -3
  25. package/dist/scripts/markers-confirm.d.ts +1 -0
  26. package/dist/scripts/markers-confirm.js +139 -0
  27. package/dist/ui/server/actions.js +50 -16
  28. package/dist/ui/server/app.js +7 -2
  29. package/dist/ui/server/job-store.d.ts +60 -0
  30. package/dist/ui/server/job-store.js +198 -0
  31. package/dist/ui/server/jobs.d.ts +46 -4
  32. package/dist/ui/server/jobs.js +361 -59
  33. package/dist/ui/server/process-info.d.ts +17 -0
  34. package/dist/ui/server/process-info.js +126 -0
  35. package/dist/ui/server/result-state.d.ts +10 -0
  36. package/dist/ui/server/result-state.js +106 -0
  37. package/dist/ui/shared/ai-commands.d.ts +4 -1
  38. package/dist/ui/shared/ai-commands.js +14 -4
  39. package/dist/ui/shared/types.d.ts +17 -2
  40. package/docs/agents/case-format.md +39 -5
  41. package/docs/ai-workflow.md +47 -15
  42. package/docs/cases.md +28 -4
  43. package/docs/commands.md +6 -1
  44. package/docs/troubleshooting.md +40 -0
  45. package/docs/web-ui.md +29 -6
  46. package/package.json +2 -1
  47. package/templates/.claude/commands/explore-case.md +11 -6
  48. package/templates/.claude/commands/gen-spec.md +10 -5
  49. package/templates/.claude/commands/heal-spec.md +11 -7
  50. package/templates/.claude/commands/make-case.md +1 -1
  51. package/templates/AGENTS.md +9 -6
  52. package/templates/cases/_template.md +7 -0
  53. package/templates/package.json +1 -0
  54. package/ui/dist/assets/{dist-B7HeglDd.js → dist-2msWI-5M.js} +1 -1
  55. package/ui/dist/assets/{dist-DfBpjgZT.js → dist-4U_gPHXf.js} +1 -1
  56. package/ui/dist/assets/{dist-B9_GXR2G.js → dist-BMfJatHo.js} +1 -1
  57. package/ui/dist/assets/{dist-Bb36rNUw.js → dist-BVvHVJ3N.js} +1 -1
  58. package/ui/dist/assets/{dist-WvvCZeeu.js → dist-BmWaNjqO.js} +1 -1
  59. package/ui/dist/assets/{dist-G9KNeh6u.js → dist-BwNz_xW2.js} +1 -1
  60. package/ui/dist/assets/{dist-GvRUDL-2.js → dist-J36xayOv.js} +1 -1
  61. package/ui/dist/assets/{dist-Dqc_tfzC.js → dist-e-KhTWHJ.js} +1 -1
  62. package/ui/dist/assets/{dist-DevASegc.js → dist-sfZKBSMm.js} +1 -1
  63. package/ui/dist/assets/{dist-Dq4Vnw5N.js → dist-wjB7g6OJ.js} +1 -1
  64. package/ui/dist/assets/{index-BJ5D2mFK.js → index-CGbRJT3b.js} +134 -132
  65. package/ui/dist/assets/index-CI0i0LWc.css +2 -0
  66. package/ui/dist/index.html +2 -2
  67. package/ui/dist/assets/index-BXY-eKBn.css +0 -2
@@ -10,14 +10,16 @@ Read the workspace `AGENTS.md` (hard rules and project rules) and `cases/_templa
10
10
  - `references/writes-and-email.md`: how to classify steps that write data or send email, and the data rules.
11
11
  - `references/gaps.md`: what to ask, and the default to suggest for each question.
12
12
  - `references/splitting.md`: how many cases to make.
13
+ - `../exploring-cases/references/case-rules.md`: what belongs in `## Rules` (a method constraint), and what stays in
14
+ Steps, Expected, Values or Targets.
13
15
 
14
16
  This skill writes `cases/*.md` only. It never writes specs, `profiles.yaml`, or anything under `tests/`.
15
17
 
16
18
  ## Before you start
17
19
  Read the workspace `AGENTS.md`: its §2 holds the project rules, and it points at the framework baseline. If
18
20
  `.claude/skills/making-cases.md` exists in the workspace, read it and apply it after this skill. Then apply `AGENTS.md`
19
- §2, which wins over both (the baseline, "Precedence"). F1–F6 always win. This is not a stop condition: continue with
20
- the procedure below.
21
+ §2, which wins over both (the baseline, "Precedence"). F1–F6 and R2 always win. This is not a stop condition:
22
+ continue with the procedure below.
21
23
 
22
24
  ## Inputs
23
25
  - Raw notes: pasted text or a file path. Any language, any quality.
@@ -34,10 +36,11 @@ the procedure below.
34
36
  creates, edits, publishes, shares, deletes or sends is not read-only, even if the notes do not say so.
35
37
  4. **Find the gaps** (`references/gaps.md`): missing expected results, unknown values a spec must assert, unclear starting
36
38
  state, steps that need data that may not exist, ordering that is not linear. Never invent an expected value.
37
- Unknown values are written as the suggested default sentence with a `❓ TODO` block under it (see `gaps.md`), or as an
38
- `Observed, 🔍 to be confirmed by QA` bullet. Every marker starts with its emoji. A literal nobody knows yet goes into
39
- the case's `## Values` as `TODO` (the spec then fails with a clear message until it is filled), never into
40
- `profiles.yaml`.
39
+ Unknown values are written as the suggested default sentence with a `❓ TODO` block under it (see `gaps.md`). A fact
40
+ from the notes that nobody questions is a plain note `(seen on <profile>)`; only a literal the spec will assert and
41
+ that would pass while wrong gets `(observed on <profile>, 🔍 to be confirmed: <what to decide>)`. A marker never
42
+ goes inside a TODO block. A literal nobody knows yet goes into the case's `## Values` as `TODO` (the spec then fails
43
+ with a clear message until it is filled), never into `profiles.yaml`.
41
44
  5. **Ask before writing.** Show the user a short plan and the open questions in one message: the proposed split
42
45
  into cases, the classification (which steps write, which send email), and the gaps with your suggested default
43
46
  for each. Wait for answers. Do not write case files before that.
@@ -48,10 +51,14 @@ the procedure below.
48
51
  ask in step 5. The group does not repeat in the name: `documents/landing-page`, not `documents/documents-landing-page`.
49
52
  - Front matter: `id`, `title`, `login`, `suites`, `writes`, `writes-note` (required when `writes` is not `none`),
50
53
  `envs` when a case must not run everywhere. When unsure: `suites: [regression]`, no `writes` (read-only), or `writes: [staging]` when a step writes.
51
- - Fill the sections that apply (Goal, Steps, Expected, Values, Visual checks, Source, Notes).
54
+ - Fill the sections that apply (Goal, Steps, Expected, Rules, Values, Visual checks, Source, Notes).
52
55
  Remove template placeholders you do not use. Use `## Values` only for the literals a spec asserts; `default`
53
56
  applies to every profile, and a profile id entry holds only what differs. Do not add `- Recipe:` lines: a person
54
57
  records them during `/explore-case --attended`.
58
+ - Put a note in `## Rules` only when it states a **method** constraint (dismiss a banner before step 1, never use
59
+ an entry that needs 2FA). A constraint that is part of the tested flow is a step; one that creates, changes or
60
+ deletes data is a tagged step with a later recipe. Never invent a rule, and never put an expectation or a literal
61
+ there: those go to `## Expected` or `## Values`. The note that became a rule is listed in its `## Source` row.
55
62
  - Case ids are lowercase kebab-case and name the thing, not the ticket: `landing-page`.
56
63
  - Existing case files: do not overwrite. If a new case overlaps one (for example another case about the same page), say so,
57
64
  propose "keep", "merge" or "replace", and wait for the user.
@@ -17,7 +17,7 @@ single lines. Never ask a question whose answer is already in the notes, `AGENTS
17
17
  | Environments | Must this case run in production, or only staging? | All environments; production writes need the usual confirmation |
18
18
  | Suites | Smoke, regression, or both? | A read-only landing check: `[smoke, regression]`; a flow that writes: `[regression]` |
19
19
  | Profiles | Do all profiles see the same UI? | Yes; the exact literals that differ go in `## Values`, never in the steps |
20
- | Unclear words | What does "collaborator button" / "..." button open? | Describe only what is observed; mark `(observed, 🔍 to be confirmed by QA)` |
20
+ | Unclear words | What does "collaborator button" / "..." button open? | Describe the safest reading and hang a TODO under it; a detail nobody questions is a plain note `(seen on <profile>)` |
21
21
  | Unused lines | A statement that fits no case | Ask where it belongs; never drop it silently |
22
22
 
23
23
  ## Marking unknowns in a case
@@ -26,11 +26,14 @@ single lines. Never ask a question whose answer is already in the notes, `AGENTS
26
26
 
27
27
  ```
28
28
  - The preview header shows the product logo, visible and not empty.
29
- - ❓ TODO [non-blocking]: Which logo must it be? (👁️ not observed)
29
+ - ❓ TODO [non-blocking]: Which logo must it be? (the notes say only "the logo")
30
30
  - ✍️ Answer:
31
31
  ```
32
32
 
33
33
  The user accepts the default by deleting the block, or types an answer. Never write a bare `TODO: ...` line, a
34
- question as the only text, or `Suggested:`.
35
- - Seen on a page during exploration but not stated in the notes: `- <observation> (observed, 🔍 to be confirmed by QA)`.
34
+ question as the only text, or `Suggested:`. Never put a marker word inside the block: one claim carries one signal.
35
+ - Seen on a page during exploration but not stated in the notes: `- <observation> (seen on <profile>)`. A literal the
36
+ spec will assert is different: `(observed on <profile>, 🔍 to be confirmed: <what a person must decide>)`, and only
37
+ when a wrong reading would still pass. The gate is "Which sentences carry a marker" in
38
+ `exploring-cases/references/open-questions.md`.
36
39
  - Never turn an observation into a hard requirement on your own.
@@ -34,6 +34,6 @@ Original notes, verbatim. Do not edit this table; it only exists so a reviewer c
34
34
  ```
35
35
 
36
36
  - The original text is copied character by character, in a code span, in its original language.
37
- - "Became" points at step numbers, or says "Expected", "Visual checks", "Not used: <reason>".
37
+ - "Became" points at step numbers, or says "Expected", "Rules", "Visual checks", "Not used: <reason>".
38
38
  - A statement used by two cases appears in both tables.
39
39
  - A statement that is not used anywhere is a gap: ask about it, do not silently drop it.
package/AGENTS.md CHANGED
@@ -11,7 +11,7 @@ project that uses asrun, the framework files are in `node_modules/@asrunjs/asrun
11
11
  ## Precedence
12
12
 
13
13
  Rules are applied in this order; a later layer wins where they conflict.
14
- 1. **F1–F6: safety.** Nothing overrides them, in any project. They are enforced in code as well
14
+ 1. **F1–F6 and R2: safety.** Nothing overrides them, in any project. They are enforced in code as well
15
15
  (`lib/write-gate.ts`, `lib/session.ts`, the run filters), so a rule file cannot weaken them.
16
16
  2. **This file.** The generic facts and defaults.
17
17
  3. **The framework skills** (`<framework>/.claude/skills/**`).
@@ -20,8 +20,11 @@ Rules are applied in this order; a later layer wins where they conflict.
20
20
  5. **The project's `AGENTS.md` §2 (Project rules).** It states the project's intent and wins over layers 2–4.
21
21
 
22
22
  A project writes §2 only for what differs from the defaults here, names the rule id, and says whether it `narrows` or
23
- `replaces` the rule. `replaces` is refused for F1–F6; `pnpm asrun doctor` checks the table. A rule file never renames
24
- the structural tokens under "Fixed tokens".
23
+ `replaces` the rule. `replaces` is refused for F1–F6 and R2; `pnpm asrun doctor` checks the table. A rule file never
24
+ renames the structural tokens under "Fixed tokens".
25
+
26
+ Inside a case, `## Rules` is a case-local layer below this file and the project's §2: it says how to run that case,
27
+ never what is true (`R1`), and a rule that conflicts with the rest of the case is a question, not an override.
25
28
 
26
29
  ## Hard rules
27
30
  - **F1 — Read-only by default.** [non-negotiable] Tests never create, edit, delete, submit forms that change data,
@@ -62,8 +65,12 @@ These apply to every agent that drives a browser to explore pages for cases or s
62
65
  Everything that writes is the person's (see F3).
63
66
  - **B2 — Waiting for a person.** Only in an attended run. Announce the task in the browser window (a small draggable
64
67
  card with a Done button and click-to-copy for texts to type), name the finish condition, wait, and tell the person
65
- when control is back. Record what they did as a general recipe under the step (`- Recipe (phase): ...`): positions
66
- instead of names of data that can change.
68
+ when control is back. The card starts and stops the recording itself and returns `recorded`; never call
69
+ `browser_start_recording` or `browser_stop_recording`: the `playwright-test` MCP lists many browser tools it cannot
70
+ run (`devtools`, `storage`, `network`, `pdf`, `vision`, and the `skillOnly` ones such as `browser_network_clear`),
71
+ and the substitutes for each are in `<framework>/.claude/skills/exploring-cases/references/tools.md`. Record what
72
+ they did as a general recipe under the step (`- Recipe (phase): ...`): positions instead of names of data that can
73
+ change.
67
74
  - **B3 — Closing an email composer.** When closing an email composer makes a confirm dialog appear (for example
68
75
  "Discard draft?"), choose "Discard" and continue. Never choose "Save" or "Send"; closing the composer must not keep
69
76
  or send anything. The dialog can show up late: in a spec, clicking "Discard" and checking that the composer is
@@ -104,27 +111,60 @@ Details: `<framework>/.claude/skills/exploring-cases/references/assertion-target
104
111
  - **A6 — Observe first, delete last.** Everything that needs the data (preview, share menus, composers, downloads) is
105
112
  observed before the person deletes it.
106
113
 
114
+ ## Case rules (`R`, binding for exploring, generating, healing and making cases)
115
+ A case document may hold a `## Rules` section: case-local instructions a person writes for the agents. They say **how**
116
+ to run that case, never **what is true**. Details: `<framework>/.claude/skills/exploring-cases/references/case-rules.md`.
117
+ - **R1 — Rules describe the method, not the outcome.** `## Steps`, `## Expected`, `## Values` and `## Targets` stay the
118
+ only place for behaviour, expectations and literals. A rule that contradicts them is a `🚫 TODO [blocking]` question,
119
+ never a silent override. A sentence that says something must be true belongs in `## Expected` (with a target card when
120
+ it looks at an image, file, PDF, screenshot or repeated text).
121
+ - **R2 — Rules never authorize what F1–F6 forbid.** [non-negotiable] A rule cannot allow creating, changing or
122
+ deleting data, sending email, accepting terms, or doing a person's step. Those stay tagged steps (`[writes]`,
123
+ `[email]`, `[input]`) with their `- Recipe:` lines and the front-matter `writes`/`writes-note` the case needs. The
124
+ agent never performs them while exploring or generating, whatever a rule says.
125
+ - **R3 — Every AI step reads every rule and accounts for it.** `/explore-case`, `/gen-spec` and `/heal-spec` (and
126
+ `/make-case` when it edits an existing case) read the whole `## Rules` section before they start, apply it, and list
127
+ each rule in the final report with where it was applied or why it was not. Exploration re-checks each rule against
128
+ the page: a rule that no longer holds gets a TODO with evidence, never silent deletion or rewriting. The agent may fix
129
+ a typo and may add a question under a rule; it never changes what a rule says.
130
+ - **R4 — Only reusable method is written back.** When a person supplies instructions before or during a run (`--hints`,
131
+ free text, an answer in the case), the agent appends to `## Rules` only what a future run of the same case needs,
132
+ stated in general terms (no account, date, run id or one-off failure), and only when it is method rather than an
133
+ expectation. A rule the agent inferred from a page carries a marker only when the rule changes what is asserted
134
+ (`(observed on <profile>, 🔍 to be confirmed: <what a person must decide>)`); a method rule whose wrong version would
135
+ fail the next run is a plain note `(seen on <profile>)`, and a rule the person stated carries nothing. Run-specific
136
+ information stays in the report. A marker inside `## Rules` does not stop spec generation the way an open question
137
+ about an assertion does.
138
+
107
139
  ## Fixed tokens
108
140
  `lib/case-doc.ts`, the scripts and the web panel parse these literals. A project language never translates them, and
109
141
  a project rule never renames them.
110
- - Section headings: `## Goal`, `## Steps`, `## Expected`, `## Values`, `## Visual checks`, `## Targets`, `## Notes`,
111
- `## Source` (only when the case was made from notes).
142
+ - Section headings: `## Goal`, `## Rules`, `## Steps`, `## Expected`, `## Values`, `## Visual checks`, `## Targets`,
143
+ `## Notes`, `## Source` (only when the case was made from notes).
112
144
  - A step's recipe lines: `- Recipe (phase): ...` under the step it belongs to.
113
145
  - `### T<n>` target cards and the `[T<n>]` references to them.
114
146
  - Question blocks: `🚫 TODO [blocking]`, `❓ TODO [non-blocking]`, `✍️ Answer:` (with its emoji), and the markers
115
- `🔍 to be confirmed by QA` / `👁️ not observed`.
147
+ `🔍 to be confirmed` and `👁️ not observed`. A marker may carry its reason and a screenshot in the same
148
+ parentheses: `(observed on <profile>, 🔍 to be confirmed: <what a person must decide>, evidence: <path>)`.
149
+ `(seen on <profile>)` is not a token: it is an ordinary note.
116
150
  - Step tags `[input]`, `[writes]`, `[email]`, and the last line `RESULT: ready` / `RESULT: needs-input`.
151
+ The RESULT line stands on its own: nothing in front of it on that line (no label, in any language), nothing after
152
+ it but closing punctuation, and plain text: never inside a code fence or in emphasis.
117
153
  - Front-matter keys: `id`, `title`, `login`, `suites`, `writes`, `writes-note`, `envs`.
118
154
 
119
155
  ## Where the details are
120
156
  This file holds the rules. The exact shape those rules apply to is documented once, and not repeated here:
121
- - `docs/agents/case-format.md`: the case document contract (front matter, steps, `## Values`, target cards, the
122
- spec's consumption of them, the write gate). Read it before editing a case document or a generated spec.
157
+ - `docs/agents/case-format.md`: the case document contract (front matter, steps, `## Rules`, `## Values`, target cards,
158
+ the spec's consumption of them, the write gate). Read it before editing a case document or a generated spec.
159
+ - `.claude/skills/exploring-cases/references/case-rules.md`: how a case's `## Rules` are read, applied and maintained.
160
+ - `.claude/skills/exploring-cases/references/tools.md`: which browser tools the `playwright-test` MCP can run, the
161
+ substitute for each one it only lists, and the arguments every call has to carry. Read it before the first browser
162
+ call of a run.
123
163
  - `docs/commands.md`: every command and flag.
124
164
  - `docs/cases.md`: the same case format for the people who write cases by hand.
125
165
  - `docs/agents/framework-dev.md`: the framework's own layout and how to change it.
126
166
  The project's own `AGENTS.md` §2 (project rules) and §3 (project facts) still win over the documentation where they
127
- differ, and `F1`–`F6` win over everything.
167
+ differ, and `F1`–`F6` and `R2` win over everything.
128
168
 
129
169
  ## Layout
130
170
 
package/README.md CHANGED
@@ -133,12 +133,13 @@ It is read after the framework skill and before your project rules. Write only w
133
133
 
134
134
  Rules combine in this order, and a later layer wins where they conflict:
135
135
 
136
- 1. `F1`–`F6` of the framework baseline (safety, in every project). They cannot be changed.
136
+ 1. `F1`–`F6` and `R2` of the framework baseline (safety, in every project). They cannot be changed.
137
137
  2. The framework baseline, `node_modules/@asrunjs/asrun/AGENTS.md`.
138
138
  3. The framework skill.
139
139
  4. Your override, `.claude/skills/<skill>.md`.
140
- 5. Your own `AGENTS.md` §2: a table of rule ids (`F7`–`F10`) with `narrows` or `replaces`, for example the artifact
141
- language.
140
+ 5. Your own `AGENTS.md` §2: a table of rule ids (`F7`–`F10`, `R1`, `R3`, `R4`) with `narrows` or `replaces`, for example
141
+ the artifact language.
142
+ 6. A case's `## Rules`: how to run that one case, never what is true.
142
143
 
143
144
  `pnpm asrun doctor` validates the table and lists the overrides it sees. An update never rewrites your rules:
144
145
  `init --force` refuses `AGENTS.md`, `CLAUDE.md` and `.claude/skills/**` without `--reset-rules`. Never edit the
package/bin/asrun.mjs CHANGED
@@ -26,6 +26,7 @@ const SCRIPTS = {
26
26
  cleanup: 'cleanup.js',
27
27
  'case:new': 'new-case.js',
28
28
  'case:check': 'check-cases.js',
29
+ 'markers:confirm': 'markers-confirm.js',
29
30
  'spec:lint': 'lint-specs.js',
30
31
  'baselines:review': 'review-baselines.js',
31
32
  'pdf:render': 'render-pdf.js',
@@ -183,7 +183,7 @@ export interface CaseQuestion {
183
183
  region: TextRange;
184
184
  }
185
185
  export type MarkerKind = 'confirm' | 'unseen';
186
- /** `🔍 to be confirmed by QA` and `👁️ not observed`: markers of what was seen but not checked by a person. */
186
+ /** `🔍 to be confirmed` and `👁️ not observed`: markers of what was seen but not checked by a person. */
187
187
  export interface CaseMarker {
188
188
  kind: MarkerKind;
189
189
  /** The marker as written, including its emoji. */
@@ -193,6 +193,21 @@ export interface CaseMarker {
193
193
  sentence: string;
194
194
  section: string;
195
195
  line: number;
196
+ /**
197
+ * What a person has to decide, written after the marker's colon inside the same parentheses
198
+ * (`🔍 to be confirmed: is this the label users see?`). Null in the old form, which only says "shown by QA".
199
+ */
200
+ detail: string | null;
201
+ /** A screenshot a person can look at instead of opening the page again: a `.explore/` image path. */
202
+ evidence: string | null;
203
+ /** The marker plus its `: <reason>` and `evidence: <path>` parts: what deleting the marker removes. */
204
+ removeRange: TextRange;
205
+ }
206
+ /** What a marked line feeds: an assertion, the wording of a step, a method rule, or only context. */
207
+ export interface CaseMarkerFeed {
208
+ kind: 'assertion' | 'step' | 'rule' | 'note';
209
+ /** Short text for the panel, e.g. `## Expected` or `a step (the run checks it)`. */
210
+ label: string;
196
211
  }
197
212
  export type CaseValues = Record<string, Record<string, unknown>>;
198
213
  export interface CaseValuesBlock {
@@ -225,6 +240,12 @@ export interface CaseDocument {
225
240
  frontMatter: CaseFrontMatter | null;
226
241
  sections: CaseSection[];
227
242
  steps: CaseStep[];
243
+ /**
244
+ * The top-level bullets of `## Rules`: case-local instructions on *how* to run the case (a page quirk, a mandatory
245
+ * read-only preparation, a control to avoid). They never define what is true: that stays in Steps, Expected, Values
246
+ * and Targets (framework baseline `R1`).
247
+ */
248
+ rules: CaseBullet[];
228
249
  questions: CaseQuestion[];
229
250
  markers: CaseMarker[];
230
251
  values: CaseValuesBlock | null;
@@ -243,6 +264,18 @@ export declare function endWithBreak(text: string, end: number): number;
243
264
  export declare const afterLine: (text: string, line: CaseLine) => number;
244
265
  /** The front matter block: its parsed data plus the range of every key, so the panel can edit one key at a time. */
245
266
  export declare function parseCaseFrontMatter(text: string): CaseFrontMatter | null;
267
+ /**
268
+ * What a marked line feeds. Only the first two kinds can be wrong and still pass: an assertion, and the wording of an
269
+ * expectation. A step, a rule and a note are checked by the run itself, so a person does not have to re-read them.
270
+ */
271
+ export declare function markerFeed(doc: CaseDocument, marker: CaseMarker): CaseMarkerFeed;
272
+ /** How many markers of a case feed an assertion, i.e. how many are worth a person's attention. */
273
+ export declare function assertionMarkerCount(doc: CaseDocument): number;
274
+ /**
275
+ * The line a marker sits on, without the marker, its reason and its evidence: the sentence a person actually reads.
276
+ * The `(observed on ...)` part stays, because where it was seen is the context they need.
277
+ */
278
+ export declare function markerSentence(text: string, marker: CaseMarker): string;
246
279
  /** A prose section (`## Goal`, `## Notes`): the panel renders it read-only and reveals a line on click. */
247
280
  export declare function parseProse(text: string, name: string): CaseProse | null;
248
281
  /** The top-level bullets of a list section, with the blocks (a wrapped text, a TODO block) that belong to each. */
@@ -263,11 +296,15 @@ export declare function setQuestionAnswer(text: string, question: CaseQuestion,
263
296
  /** Accepts the suggestion: deletes the whole question block, the answer line included. */
264
297
  export declare function acceptQuestion(text: string, question: CaseQuestion): string;
265
298
  /**
266
- * Removes a `🔍 to be confirmed by QA` / `👁️ not observed` marker. When the marker is the only content of its
267
- * parentheses, the parentheses go too (`(👁️ not observed)`); otherwise the marker and one separator leave
268
- * `(observed, 🔍 to be confirmed by QA)` as `(observed)`.
299
+ * What deleting a `🔍 to be confirmed` / `👁️ not observed` marker removes. When the marker is the only content
300
+ * of its parentheses, the parentheses go too (`(👁️ not observed)`); otherwise the marker and one separator leave
301
+ * `(observed, 🔍 to be confirmed)` as `(observed)`.
269
302
  */
303
+ export declare function markerEdits(text: string, marker: CaseMarker): TextEdit;
304
+ /** Removes one marker, its punctuation and, when it was alone, its parentheses. */
270
305
  export declare function removeMarker(text: string, marker: CaseMarker): string;
306
+ /** Removes several markers in one edit, so the panel can confirm a whole list with one undo step. */
307
+ export declare function removeMarkers(text: string, markers: CaseMarker[]): string;
271
308
  /** Sets one value of one profile; `null` removes it (an empty cell in the panel means "not set here"). */
272
309
  export declare function setCaseValue(text: string, profile: string, key: string, value: string | null): string;
273
310
  /** Adds a key to the `default` values of the case. */
@@ -465,8 +465,41 @@ function parseQuestions(text, sections) {
465
465
  }
466
466
  /* Kept in step with `ui/web/src/lib/highlight-keywords.ts` (same words, same emoji prefixes). */
467
467
  const MARKER_RE = /(?:\u{1F50D}\uFE0F?[ \t]*)?(?:\bto be confirmed(?:\s+by\s+QA)?\b|\bTBC\b|\bTBD\b|\bneeds?\s+confirmation\b)|(?:\u{1F441}\uFE0F?[ \t]*)?\bnot observed\b/giu;
468
- function parseMarkers(text, sections) {
468
+ /** A screenshot named in the same parentheses as a marker: the evidence a person looks at instead of the page. */
469
+ const MARKER_EVIDENCE_RE = /\bevidence:[ \t]*([^\s;)]+\.(?:png|jpe?g|webp))/i;
470
+ /**
471
+ * What the parentheses around a marker say beyond the marker itself: the one thing a person has to decide (`: ...`)
472
+ * and the evidence image (`evidence: <path>`). Both are optional, so the old `(observed, 🔍 to be confirmed by QA)`
473
+ * still reads; it just says less. `removeEnd` runs to the end of the reason and the evidence, so deleting the marker
474
+ * leaves the observation alone.
475
+ */
476
+ function markerParts(text, start, end) {
477
+ const open = text.lastIndexOf('(', start);
478
+ const close = text.indexOf(')', end);
479
+ if (open < 0 || close < 0 || close < end || /[\r\n]/.test(text.slice(open, close))) {
480
+ return { detail: null, evidence: null, removeEnd: end };
481
+ }
482
+ const inside = text.slice(open + 1, close);
483
+ const evidenceMatch = MARKER_EVIDENCE_RE.exec(inside);
484
+ const evidence = evidenceMatch?.[1] ?? null;
485
+ const tail = text.slice(end, close);
486
+ const colon = /^[ \t]*:[ \t]*/.exec(tail);
487
+ const raw = colon ? tail.slice(colon[0].length) : '';
488
+ const detail = colon ? raw.replace(/[;,][ \t]*evidence:[\s\S]*$/i, '').trim() : '';
489
+ const removeEnd = evidenceMatch
490
+ ? open + 1 + evidenceMatch.index + evidenceMatch[0].length
491
+ : colon
492
+ ? end + colon[0].length + raw.trimEnd().length
493
+ : end;
494
+ return { detail: detail === '' ? null : detail, evidence, removeEnd };
495
+ }
496
+ /**
497
+ * Markers of `text`. A marker written inside a question block is not one: the question already says that a person has
498
+ * to decide, and counting it twice would put the same claim in "Open questions" and "To confirm".
499
+ */
500
+ function parseMarkers(text, sections, questions) {
469
501
  const markers = [];
502
+ const blocks = questions.map((question) => question.blockRange);
470
503
  let fenced = false;
471
504
  for (const line of splitLines(text)) {
472
505
  if (isFence(line.text)) {
@@ -478,18 +511,59 @@ function parseMarkers(text, sections) {
478
511
  for (const match of line.text.matchAll(MARKER_RE)) {
479
512
  if (match[0] === '')
480
513
  continue;
514
+ const start = line.start + (match.index ?? 0);
515
+ const end = start + match[0].length;
516
+ if (blocks.some((block) => start >= block.start && start <= block.end))
517
+ continue;
518
+ const parts = markerParts(text, start, end);
481
519
  markers.push({
482
520
  kind: /not observed/i.test(match[0]) ? 'unseen' : 'confirm',
483
521
  text: match[0],
484
- range: { start: line.start + (match.index ?? 0), end: line.start + (match.index ?? 0) + match[0].length },
522
+ range: { start, end },
523
+ removeRange: { start, end: parts.removeEnd },
485
524
  sentence: line.text.trim(),
486
525
  section: sectionAt(sections, line.start),
487
526
  line: line.number,
527
+ detail: parts.detail,
528
+ evidence: parts.evidence,
488
529
  });
489
530
  }
490
531
  }
491
532
  return markers;
492
533
  }
534
+ /**
535
+ * What a marked line feeds. Only the first two kinds can be wrong and still pass: an assertion, and the wording of an
536
+ * expectation. A step, a rule and a note are checked by the run itself, so a person does not have to re-read them.
537
+ */
538
+ export function markerFeed(doc, marker) {
539
+ if (marker.section === 'Values' || marker.section === 'Expected' || marker.section === 'Visual checks') {
540
+ return { kind: 'assertion', label: `## ${marker.section}` };
541
+ }
542
+ if (marker.section === 'Steps')
543
+ return { kind: 'step', label: 'a step (the run checks it)' };
544
+ if (marker.section === 'Rules')
545
+ return { kind: 'rule', label: 'a method rule (never an assertion)' };
546
+ return { kind: 'note', label: marker.section === '' ? 'the case' : `## ${marker.section}` };
547
+ }
548
+ /** How many markers of a case feed an assertion, i.e. how many are worth a person's attention. */
549
+ export function assertionMarkerCount(doc) {
550
+ return doc.markers.filter((marker) => markerFeed(doc, marker).kind === 'assertion').length;
551
+ }
552
+ /**
553
+ * The line a marker sits on, without the marker, its reason and its evidence: the sentence a person actually reads.
554
+ * The `(observed on ...)` part stays, because where it was seen is the context they need.
555
+ */
556
+ export function markerSentence(text, marker) {
557
+ const lineStart = text.lastIndexOf('\n', marker.range.start) + 1;
558
+ const lineBreak = text.indexOf('\n', marker.range.start);
559
+ const lineEnd = lineBreak === -1 ? text.length : lineBreak;
560
+ return (text.slice(lineStart, marker.range.start) + text.slice(marker.removeRange.end, lineEnd))
561
+ .replace(/^[ \t]*(?:[-*+]|\d+\.)[ \t]+/, '')
562
+ .replace(/[,;][ \t]*\)/g, ')')
563
+ .replace(/\([ \t]*\)/g, '')
564
+ .replace(/[ \t]{2,}/g, ' ')
565
+ .trim();
566
+ }
493
567
  const VALUES_OPEN_RE = /^[ \t]*```ya?ml[ \t]*$/i;
494
568
  const VALUES_CLOSE_RE = /^[ \t]*```[ \t]*$/;
495
569
  const isPlaceholderValue = (value) => value === null || value === undefined || value === '' || (typeof value === 'string' && /^\s*TODO\b/i.test(value));
@@ -801,13 +875,16 @@ export function caseMetaProblems(id, data, knownEnvs, label = 'case document') {
801
875
  export function parseCaseDocument(text) {
802
876
  const lines = splitLines(text);
803
877
  const sections = sectionsOf(text, lines);
878
+ // Questions first: a marker written inside one belongs to that question, not to "To confirm".
879
+ const questions = parseQuestions(text, sections);
804
880
  return {
805
881
  text,
806
882
  frontMatter: parseCaseFrontMatter(text),
807
883
  sections,
808
884
  steps: parseSteps(text, sections.find((section) => section.name === 'Steps')),
809
- questions: parseQuestions(text, sections),
810
- markers: parseMarkers(text, sections),
885
+ rules: parseBullets(text, 'Rules'),
886
+ questions,
887
+ markers: parseMarkers(text, sections, questions),
811
888
  values: parseValues(text, sections.find((section) => section.name === 'Values')),
812
889
  targets: parseTargetCards(text),
813
890
  inputs: parseInputs(text, sections.find((section) => section.name === 'Inputs')),
@@ -877,27 +954,42 @@ export function acceptQuestion(text, question) {
877
954
  return patchText(text, [{ start: question.blockRange.start, end: endWithBreak(text, question.blockRange.end), text: '' }]);
878
955
  }
879
956
  /**
880
- * Removes a `🔍 to be confirmed by QA` / `👁️ not observed` marker. When the marker is the only content of its
881
- * parentheses, the parentheses go too (`(👁️ not observed)`); otherwise the marker and one separator leave
882
- * `(observed, 🔍 to be confirmed by QA)` as `(observed)`.
957
+ * What deleting a `🔍 to be confirmed` / `👁️ not observed` marker removes. When the marker is the only content
958
+ * of its parentheses, the parentheses go too (`(👁️ not observed)`); otherwise the marker and one separator leave
959
+ * `(observed, 🔍 to be confirmed)` as `(observed)`.
883
960
  */
884
- export function removeMarker(text, marker) {
885
- const open = text.lastIndexOf('(', marker.range.start);
886
- const close = text.indexOf(')', marker.range.end);
961
+ export function markerEdits(text, marker) {
962
+ const start = marker.range.start;
963
+ const markerEnd = marker.removeRange.end;
964
+ const open = text.lastIndexOf('(', start);
965
+ const close = text.indexOf(')', markerEnd);
887
966
  if (open >= 0 && close >= 0 && !/[\r\n]/.test(text.slice(open, close))) {
888
- const rest = text.slice(open + 1, marker.range.start) + text.slice(marker.range.end, close);
967
+ const rest = text.slice(open + 1, start) + text.slice(markerEnd, close);
889
968
  if (/^[\s,;:—-]*$/.test(rest)) {
890
969
  const before = text[open - 1] === ' ' && /[.,;:!?]/.test(text[close + 1] ?? '') ? open - 1 : open;
891
- return patchText(text, [{ start: before, end: close + 1, text: '' }]);
970
+ return { start: before, end: close + 1, text: '' };
892
971
  }
893
972
  }
894
- const lead = /(?:[ \t]*[,;][ \t]*|[ \t]+)$/.exec(text.slice(0, marker.range.start));
973
+ const lead = /(?:[ \t]*[,;][ \t]*|[ \t]+)$/.exec(text.slice(0, start));
895
974
  if (lead) {
896
- return patchText(text, [{ start: marker.range.start - lead[0].length, end: marker.range.end, text: '' }]);
975
+ return { start: start - lead[0].length, end: markerEnd, text: '' };
897
976
  }
898
- const trail = /^[ \t]+/.exec(text.slice(marker.range.end));
899
- const end = trail ? marker.range.end + trail[0].length : marker.range.end;
900
- return patchText(text, [{ start: marker.range.start, end, text: '' }]);
977
+ const trail = /^[ \t]+/.exec(text.slice(markerEnd));
978
+ const end = trail ? markerEnd + trail[0].length : markerEnd;
979
+ return { start, end, text: '' };
980
+ }
981
+ /** Removes one marker, its punctuation and, when it was alone, its parentheses. */
982
+ export function removeMarker(text, marker) {
983
+ return patchText(text, [markerEdits(text, marker)]);
984
+ }
985
+ /** Removes several markers in one edit, so the panel can confirm a whole list with one undo step. */
986
+ export function removeMarkers(text, markers) {
987
+ if (markers.length === 0)
988
+ return text;
989
+ const edits = [...markers]
990
+ .sort((a, b) => b.range.start - a.range.start)
991
+ .map((marker) => markerEdits(text, marker));
992
+ return patchText(text, edits);
901
993
  }
902
994
  function replaceValues(text, block, values) {
903
995
  const yaml = stringifyYaml(values, { lineWidth: 0 }).replace(/\r?\n/g, newlineOf(text));
@@ -1,11 +1,35 @@
1
1
  import fs from 'node:fs';
2
2
  import { CASES_DIR } from '../lib/paths.js';
3
3
  import { listCaseFileIds, readCaseMeta } from '../lib/cases.js';
4
- import { parseCaseDocument } from '../lib/case-doc.js';
4
+ import { parseCaseDocument, splitLines } from '../lib/case-doc.js';
5
5
  import path from 'node:path';
6
6
  import { ROOT } from '../lib/paths.js';
7
7
  import { checksNeedingTarget, parseTargets, readCaseText, REQUIRED_FIELDS, targetRefsIn } from '../lib/targets.js';
8
8
  import { parseCaseValues } from '../lib/values.js';
9
+ const LIST_ITEM = /^[ \t]*(?:[-*+]|\d+\.)[ \t]+\S/;
10
+ const HEADING = /^##[ \t]+\S/;
11
+ /** The words the panel and `case:check` treat as a marker: one claim carries one signal. */
12
+ const MARKER_WORD = /\b(?:to be confirmed(?:\s+by\s+QA)?|TBC|TBD|not observed)\b/i;
13
+ const indentOf = (text) => text.length - text.trimStart().length;
14
+ /**
15
+ * The line a question immediately hangs under: the nearest line above it that is indented less and can carry text.
16
+ * `null` when that line is a paragraph (the model extends a paragraph upwards on purpose) or nothing qualifies.
17
+ */
18
+ function immediateAnchor(lines, questionLine, indent) {
19
+ const at = lines.findIndex((line) => line.number === questionLine);
20
+ for (let index = at - 1; index >= 0; index--) {
21
+ const text = lines[index].text;
22
+ if (text.trim() === '' || HEADING.test(text) || /^[ \t]*```/.test(text))
23
+ return null;
24
+ if (indentOf(text) >= indent)
25
+ continue;
26
+ // Another question at a shallower depth is not the sentence this one stands for.
27
+ if (/^[ \t]*[-*+][ \t]+(?:\u{1F6AB}|\u{2753}|\u270D)/u.test(text))
28
+ continue;
29
+ return LIST_ITEM.test(text) ? lines[index].number : null;
30
+ }
31
+ return null;
32
+ }
9
33
  /** Step numbers of the lines marked `[input]` or `[writes]` in "## Steps" (a person does them), and the numbers that have a `- Recipe:` line under them (or an old "## Inputs" line). */
10
34
  function inputSteps(id) {
11
35
  const doc = parseCaseDocument(fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8'));
@@ -41,6 +65,11 @@ function questionProblems(id) {
41
65
  if (step && question.indent < 4) {
42
66
  warnings.push(`the question on line ${question.line} is indented by ${question.indent} space(s) under step ${step.number}; 4 renders it inside the step`);
43
67
  }
68
+ const anchor = immediateAnchor(splitLines(doc.text), question.line, question.indent);
69
+ if (anchor !== null && anchor !== question.parent.line) {
70
+ const above = splitLines(doc.text).find((line) => line.number === anchor)?.text.trim() ?? '';
71
+ warnings.push(`the question on line ${question.line} hangs under "${above.slice(0, 80)}", but the panel shows "${question.parent.text.slice(0, 80)}" as the text above it; indent it under that sentence or lift that sentence to a top-level bullet`);
72
+ }
44
73
  }
45
74
  for (const step of doc.steps) {
46
75
  if (step.recipes.length === 0)
@@ -62,6 +91,58 @@ function questionProblems(id) {
62
91
  }
63
92
  return warnings;
64
93
  }
94
+ /**
95
+ * Problems with the markers (`🔍 to be confirmed by QA`, `👁️ not observed`): one claim carries one signal, and a
96
+ * marker says what a person has to decide. A marker inside a question block is a failure; the old form without a
97
+ * "why" is a warning, so old cases keep working.
98
+ */
99
+ function markerProblems(id) {
100
+ const text = fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8');
101
+ const doc = parseCaseDocument(text);
102
+ const fail = [];
103
+ const warn = [];
104
+ for (const question of doc.questions) {
105
+ const end = question.hasAnswerLine ? question.answerRange.start : question.blockRange.end;
106
+ if (MARKER_WORD.test(text.slice(question.blockRange.start, end))) {
107
+ fail.push(`the question on line ${question.line} holds a marker word ("to be confirmed" / "not observed"): one claim carries one signal, so keep the question and drop the marker`);
108
+ }
109
+ }
110
+ if (doc.values?.hasBlock && MARKER_WORD.test(text.slice(doc.values.range.start, doc.values.range.end))) {
111
+ fail.push('a marker inside the "## Values" yaml block would become part of the expected value: put it on the prose line above the block');
112
+ }
113
+ const parentLines = new Set(doc.questions.map((question) => question.parent?.line).filter((line) => line !== undefined));
114
+ for (const marker of doc.markers) {
115
+ if (parentLines.has(marker.line)) {
116
+ fail.push(`line ${marker.line} carries a marker and a question under it: keep the question (drop the marker) or the marker (drop the question)`);
117
+ }
118
+ if (marker.detail === null) {
119
+ warn.push(`the marker on line ${marker.line} does not say what a person must confirm; write "🔍 to be confirmed: <the one thing to decide>"` +
120
+ (marker.evidence === null
121
+ ? ', plus "evidence: .explore/observations/<case-id>/<image>.png" when a screenshot says it better'
122
+ : ''));
123
+ }
124
+ }
125
+ return { fail, warn };
126
+ }
127
+ /**
128
+ * Problems with the `## Rules` bullets: a rule may name the step it belongs to (`Step 3:`, `Before step 3:` or
129
+ * `After step 3:`). A reference to a step the case does not have is a failure, because the rule would silently lose its
130
+ * scope. An unprefixed rule applies to the whole case and is always valid.
131
+ */
132
+ function ruleProblems(id) {
133
+ const doc = parseCaseDocument(fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8'));
134
+ const failures = [];
135
+ for (const rule of doc.rules) {
136
+ const match = /^(?:(?:before|after)[ \t]+)?step[ \t]+(\d+)\b/i.exec(rule.text);
137
+ if (!match)
138
+ continue;
139
+ const number = Number(match[1]);
140
+ if (!doc.steps.some((step) => step.number === number)) {
141
+ failures.push(`the rule on line ${rule.line} refers to step ${number}, which this case does not have`);
142
+ }
143
+ }
144
+ return failures;
145
+ }
65
146
  /**
66
147
  * Warnings for the old layout. `## Data` and `## Inputs` are deprecated: the one-sentence summary belongs in
67
148
  * `writes-note`, the cleanup in steps, and a person's recipe under its step. Cases keep working, so this never fails.
@@ -131,6 +212,8 @@ for (const id of ids) {
131
212
  console.log(`warn ${id}: ${message}`);
132
213
  for (const message of targets.fail)
133
214
  throw new Error(message);
215
+ for (const message of ruleProblems(id))
216
+ throw new Error(message);
134
217
  const { marked, recorded } = inputSteps(id);
135
218
  const missing = marked.filter((step) => !recorded.includes(step));
136
219
  if (missing.length > 0) {
@@ -140,6 +223,11 @@ for (const id of ids) {
140
223
  console.log(`warn ${id}: ${message}`);
141
224
  for (const message of questionProblems(id))
142
225
  console.log(`warn ${id}: ${message}`);
226
+ const markers = markerProblems(id);
227
+ for (const message of markers.warn)
228
+ console.log(`warn ${id}: ${message}`);
229
+ for (const message of markers.fail)
230
+ throw new Error(message);
143
231
  }
144
232
  catch (error) {
145
233
  problems++;