@asrunjs/asrun 0.9.0 → 0.11.0

package/dist/scripts/check-cases.js

@@ -1,11 +1,35 @@
 import fs from 'node:fs';
 import { CASES_DIR } from '../lib/paths.js';
 import { listCaseFileIds, readCaseMeta } from '../lib/cases.js';
-import { parseCaseDocument } from '../lib/case-doc.js';
+import { parseCaseDocument, splitLines } from '../lib/case-doc.js';
 import path from 'node:path';
 import { ROOT } from '../lib/paths.js';
 import { checksNeedingTarget, parseTargets, readCaseText, REQUIRED_FIELDS, targetRefsIn } from '../lib/targets.js';
 import { parseCaseValues } from '../lib/values.js';
+const LIST_ITEM = /^[ \t]*(?:[-*+]|\d+\.)[ \t]+\S/;
+const HEADING = /^##[ \t]+\S/;
+/** The words the panel and `case:check` treat as a marker: one claim carries one signal. */
+const MARKER_WORD = /\b(?:to be confirmed(?:\s+by\s+QA)?|TBC|TBD|not observed)\b/i;
+const indentOf = (text) => text.length - text.trimStart().length;
+/**
+ * The line a question immediately hangs under: the nearest line above it that is indented less and can carry text.
+ * `null` when that line is a paragraph (the model extends a paragraph upwards on purpose) or nothing qualifies.
+ */
+function immediateAnchor(lines, questionLine, indent) {
+    const at = lines.findIndex((line) => line.number === questionLine);
+    for (let index = at - 1; index >= 0; index--) {
+        const text = lines[index].text;
+        if (text.trim() === '' || HEADING.test(text) || /^[ \t]*```/.test(text))
+            return null;
+        if (indentOf(text) >= indent)
+            continue;
+        // Another question at a shallower depth is not the sentence this one stands for.
+        if (/^[ \t]*[-*+][ \t]+(?:\u{1F6AB}|\u{2753}|\u270D)/u.test(text))
+            continue;
+        return LIST_ITEM.test(text) ? lines[index].number : null;
+    }
+    return null;
+}
 /** Step numbers of the lines marked `[input]` or `[writes]` in "## Steps" (a person does them), and the numbers that have a `- Recipe:` line under them (or an old "## Inputs" line). */
 function inputSteps(id) {
     const doc = parseCaseDocument(fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8'));
@@ -41,6 +65,11 @@ function questionProblems(id) {
         if (step && question.indent < 4) {
             warnings.push(`the question on line ${question.line} is indented by ${question.indent} space(s) under step ${step.number}; 4 renders it inside the step`);
         }
+        const anchor = immediateAnchor(splitLines(doc.text), question.line, question.indent);
+        if (anchor !== null && anchor !== question.parent.line) {
+            const above = splitLines(doc.text).find((line) => line.number === anchor)?.text.trim() ?? '';
+            warnings.push(`the question on line ${question.line} hangs under "${above.slice(0, 80)}", but the panel shows "${question.parent.text.slice(0, 80)}" as the text above it; indent it under that sentence or lift that sentence to a top-level bullet`);
+        }
     }
     for (const step of doc.steps) {
         if (step.recipes.length === 0)
@@ -62,6 +91,58 @@ function questionProblems(id) {
     }
     return warnings;
 }
+/**
+ * Problems with the markers (`🔍 to be confirmed by QA`, `👁️ not observed`): one claim carries one signal, and a
+ * marker says what a person has to decide. A marker inside a question block is a failure; the old form without a
+ * "why" is a warning, so old cases keep working.
+ */
+function markerProblems(id) {
+    const text = fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8');
+    const doc = parseCaseDocument(text);
+    const fail = [];
+    const warn = [];
+    for (const question of doc.questions) {
+        const end = question.hasAnswerLine ? question.answerRange.start : question.blockRange.end;
+        if (MARKER_WORD.test(text.slice(question.blockRange.start, end))) {
+            fail.push(`the question on line ${question.line} holds a marker word ("to be confirmed" / "not observed"): one claim carries one signal, so keep the question and drop the marker`);
+        }
+    }
+    if (doc.values?.hasBlock && MARKER_WORD.test(text.slice(doc.values.range.start, doc.values.range.end))) {
+        fail.push('a marker inside the "## Values" yaml block would become part of the expected value: put it on the prose line above the block');
+    }
+    const parentLines = new Set(doc.questions.map((question) => question.parent?.line).filter((line) => line !== undefined));
+    for (const marker of doc.markers) {
+        if (parentLines.has(marker.line)) {
+            fail.push(`line ${marker.line} carries a marker and a question under it: keep the question (drop the marker) or the marker (drop the question)`);
+        }
+        if (marker.detail === null) {
+            warn.push(`the marker on line ${marker.line} does not say what a person must confirm; write "🔍 to be confirmed: <the one thing to decide>"` +
+                (marker.evidence === null
+                    ? ', plus "evidence: .explore/observations/<case-id>/<image>.png" when a screenshot says it better'
+                    : ''));
+        }
+    }
+    return { fail, warn };
+}
+/**
+ * Problems with the `## Rules` bullets: a rule may name the step it belongs to (`Step 3:`, `Before step 3:` or
+ * `After step 3:`). A reference to a step the case does not have is a failure, because the rule would silently lose its
+ * scope. An unprefixed rule applies to the whole case and is always valid.
+ */
+function ruleProblems(id) {
+    const doc = parseCaseDocument(fs.readFileSync(path.join(CASES_DIR, `${id}.md`), 'utf8'));
+    const failures = [];
+    for (const rule of doc.rules) {
+        const match = /^(?:(?:before|after)[ \t]+)?step[ \t]+(\d+)\b/i.exec(rule.text);
+        if (!match)
+            continue;
+        const number = Number(match[1]);
+        if (!doc.steps.some((step) => step.number === number)) {
+            failures.push(`the rule on line ${rule.line} refers to step ${number}, which this case does not have`);
+        }
+    }
+    return failures;
+}
 /**
  * Warnings for the old layout. `## Data` and `## Inputs` are deprecated: the one-sentence summary belongs in
  * `writes-note`, the cleanup in steps, and a person's recipe under its step. Cases keep working, so this never fails.
@@ -131,6 +212,8 @@ for (const id of ids) {
             console.log(`warn ${id}: ${message}`);
         for (const message of targets.fail)
             throw new Error(message);
+        for (const message of ruleProblems(id))
+            throw new Error(message);
         const { marked, recorded } = inputSteps(id);
         const missing = marked.filter((step) => !recorded.includes(step));
         if (missing.length > 0) {
@@ -140,6 +223,11 @@ for (const id of ids) {
             console.log(`warn ${id}: ${message}`);
         for (const message of questionProblems(id))
             console.log(`warn ${id}: ${message}`);
+        const markers = markerProblems(id);
+        for (const message of markers.warn)
+            console.log(`warn ${id}: ${message}`);
+        for (const message of markers.fail)
+            throw new Error(message);
     }
     catch (error) {
         problems++;

package/dist/scripts/cli.js

@@ -32,13 +32,19 @@ async function main() {
         process.exitCode = await migrate(rest);
         return;
     }
+    if (command === 'explore:notes') {
+        // The script reads `process.argv` itself; drop the subcommand so it sees only its own arguments.
+        process.argv.splice(2, 1);
+        await import('./explore-notes.js');
+        return;
+    }
     if (command === 'help' || command === '--help-init') {
         console.log(USAGE);
         return;
     }
     await import('./run.js');
 }
-const USAGE = `Usage: pnpm asrun [init|doctor|reset|upgrade|migrate] [options]
+const USAGE = `Usage: pnpm asrun [init|doctor|reset|upgrade|migrate|explore:notes] [options]
 
   (no subcommand)  Run the tests of one environment. See "pnpm asrun --help".
   init             Create the files a workspace needs (never overwrites; --update refreshes generated files,
@@ -50,5 +56,6 @@ const USAGE = `Usage: pnpm asrun [init|doctor|reset|upgrade|migrate] [options]
   upgrade          Install the newest asrun, refresh the workspace files and apply its migrations, after checking the
                    target release for breaking changes (see "pnpm asrun upgrade --help").
   migrate          Apply the workspace migrations of the installed release, or list them (see "pnpm asrun migrate
-                   --help"). "upgrade" runs it; use it after a manual install.`;
+                   --help"). "upgrade" runs it; use it after a manual install.
+  explore:notes    Print or validate the local exploration note of a case (see "pnpm asrun explore:notes --help").`;
 await main();

package/dist/scripts/doctor.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { createRequire } from 'node:module';
 import { loadProfiles, listEnvs, PROFILES_FILE } from '../lib/profiles.js';
 import { FRAMEWORK_ROOT, ROOT, SKILLS_DIR } from '../lib/paths.js';
 import { spawnTarget } from '../lib/command.js';
@@ -25,17 +26,20 @@ function read(file) {
 function nonNegotiableRules() {
     const ids = new Set();
     for (const line of (read(path.join(FRAMEWORK_ROOT, 'AGENTS.md')) ?? '').split('\n')) {
-        const match = /^- \*\*(F\d+)\b/.exec(line.trim());
+        const match = /^- \*\*([A-Z]\d+)\b/.exec(line.trim());
         if (match && line.includes('[non-negotiable]'))
             ids.add(match[1]);
     }
     return ids;
 }
-/** Every rule id the framework baseline defines (`F1`..`F10`), so a project cannot name one that does not exist. */
+/**
+ * Every rule id the framework baseline defines (`F1`..`F10`, `B1`..`B4`, `A1`..`A6`, `R1`..`R4`), so a project cannot
+ * name one that does not exist.
+ */
 function frameworkRuleIds() {
     const ids = new Set();
     for (const line of (read(path.join(FRAMEWORK_ROOT, 'AGENTS.md')) ?? '').split('\n')) {
-        const match = /^- \*\*(F\d+)\b/.exec(line.trim());
+        const match = /^- \*\*([A-Z]\d+)\b/.exec(line.trim());
         if (match)
             ids.add(match[1]);
     }
@@ -70,6 +74,38 @@ function staleDistFiles() {
     walk('scripts');
     return stale;
 }
+/**
+ * The browser tools the workspace's `playwright-test` MCP can actually run, and the Playwright version it comes from.
+ *
+ * `playwright run-test-mcp-server` advertises every browser tool Playwright has, but a call runs in a paused test whose
+ * backend installs only the `core` and `testing` capabilities and skips the `skillOnly` tools; anything else answers
+ * `Tool "<name>" not found`. The skills' `references/tools.md` documents the substitutes, so this tells a person when
+ * an upgrade moved a tool out of that set before an exploration discovers it. `null` when the install cannot be
+ * inspected (then the check says nothing rather than guessing).
+ */
+function runnableBrowserTools() {
+    try {
+        const requireFrom = createRequire(path.join(ROOT, 'package.json'));
+        const testManifest = requireFrom.resolve('@playwright/test/package.json');
+        const playwrightManifest = requireFrom.resolve('playwright/package.json', { paths: [path.dirname(testManifest)] });
+        const coreManifest = requireFrom.resolve('playwright-core/package.json', { paths: [path.dirname(playwrightManifest)] });
+        const bundle = requireFrom(path.join(path.dirname(coreManifest), 'lib', 'coreBundle.js'));
+        const tools = bundle.tools?.filteredTools?.({ capabilities: ['testing'] });
+        if (!tools)
+            return null;
+        const version = JSON.parse(read(playwrightManifest) ?? '{}').version ?? 'unknown';
+        return { version, tools: tools.map((tool) => tool.schema.name) };
+    }
+    catch {
+        return null;
+    }
+}
+/** Every tool the skills' `tools.md` lists under `## Runnable`, i.e. the ones an exploration is told to use. */
+function documentedBrowserTools() {
+    const text = read(path.join(SKILLS_DIR, 'exploring-cases', 'references', 'tools.md')) ?? '';
+    const section = /^## Runnable\n([\s\S]*?)^## /m.exec(text)?.[1] ?? '';
+    return [...new Set(section.match(/browser_[a-z0-9_]+/g) ?? [])];
+}
 export async function main(args) {
     if (args.includes('--help')) {
         console.log('Usage: pnpm asrun doctor');
@@ -326,6 +362,17 @@ export async function main(args) {
             warnings.push(`${file} starts the MCP server with "${command}", which does not exist: run "pnpm asrun init --force ${file}".`);
         }
     }
+    // The browser tools the `playwright-test` MCP can really run: the skills' `tools.md` must not name one it cannot.
+    const runnable = runnableBrowserTools();
+    if (runnable !== null) {
+        facts.push(`playwright-test mcp:  ${runnable.tools.length} runnable browser tools (Playwright ${runnable.version})`);
+        const missing = documentedBrowserTools().filter((tool) => !runnable.tools.includes(tool));
+        if (missing.length > 0) {
+            warnings.push(`Playwright ${runnable.version} cannot run ${missing.join(', ')} in this MCP, but ` +
+                '.claude/skills/exploring-cases/references/tools.md names it as runnable: the test-runner backend changed ' +
+                'its capabilities. Update that file (and the substitute for each tool) before exploring.');
+        }
+    }
     const settings = read(path.join(ROOT, '.claude', 'settings.json'));
     if (settings === null) {
         warnings.push('.claude/settings.json is missing: run "pnpm asrun init" (Claude Code needs its sandbox exclusions for pnpm asrun).');

package/dist/scripts/explore-notes.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/scripts/explore-notes.js

@@ -0,0 +1,131 @@
+import path from 'node:path';
+import { isValidCaseId } from '../lib/case-id.js';
+import { caseSha256, exploreNotesDir, exploreNotesFile, noteFreshness, readExploreNotes, } from '../lib/explore-notes.js';
+import { ROOT } from '../lib/paths.js';
+import { loadProfiles } from '../lib/profiles.js';
+const USAGE = `Usage: pnpm explore:notes <case-id> [--profile <profile-id>] [--json] [--check] [--hash]
+
+Prints the exploration notes /explore-case wrote for a case under .explore/notes/<case-id>/.
+They are local, git-ignored and optional: /gen-spec reads a fresh one to verify instead of rediscovering the page,
+and falls back to a full replay when it is missing or stale.
+
+  --profile <id>  only this profile's note
+  --json          the parsed notes (and any errors) as JSON, for a run that reads them
+  --check         validate every note; fails on a malformed file, warns on a stale one
+  --hash          print the case file's sha256, for "caseSha256" in a note
+
+Paths are relative to the workspace root (${ROOT}).`;
+const args = process.argv.slice(2);
+let caseId;
+let profile;
+let json = false;
+let check = false;
+let hash = false;
+let help = false;
+for (let index = 0; index < args.length; index++) {
+    const arg = args[index];
+    if (arg === '--help' || arg === '-h')
+        help = true;
+    else if (arg === '--json')
+        json = true;
+    else if (arg === '--check')
+        check = true;
+    else if (arg === '--hash')
+        hash = true;
+    else if (arg === '--profile') {
+        const value = args[++index];
+        if (value === undefined || value.startsWith('--')) {
+            console.error(`--profile needs a profile id.\n\n${USAGE}`);
+            process.exit(2);
+        }
+        profile = value;
+    }
+    else if (!arg.startsWith('--') && caseId === undefined)
+        caseId = arg;
+    else {
+        console.error(`Unexpected argument: ${arg}\n\n${USAGE}`);
+        process.exit(2);
+    }
+}
+if (help) {
+    console.log(USAGE);
+    process.exit(0);
+}
+if (caseId === undefined) {
+    console.error(USAGE);
+    process.exit(1);
+}
+if (!isValidCaseId(caseId)) {
+    console.error(`"${caseId}" is not a valid case id (one optional group folder, for example documents/landing-page).`);
+    process.exit(1);
+}
+if (hash) {
+    const value = caseSha256(caseId);
+    if (value === null) {
+        console.error(`cases/${caseId}.md does not exist.`);
+        process.exit(1);
+    }
+    console.log(value);
+    process.exit(0);
+}
+const { notes, errors } = readExploreNotes(caseId, profile);
+const relative = (file) => path.relative(ROOT, file).split(path.sep).join('/');
+if (json) {
+    console.log(JSON.stringify({
+        caseId,
+        notes: notes.map((note) => ({ ...note, freshness: noteFreshness(note) })),
+        errors: errors.map((error) => ({ file: relative(error.file), problems: error.problems })),
+    }, null, 2));
+}
+else {
+    console.log(`Exploration notes for cases/${caseId}.md (${relative(exploreNotesDir(caseId))})`);
+    if (notes.length === 0)
+        console.log('  none yet: /gen-spec replays the case.');
+    for (const note of notes) {
+        console.log(`  ${note.profile.padEnd(24)} ${describe(note)}`);
+    }
+}
+let failed = false;
+for (const error of errors) {
+    failed = true;
+    for (const problem of error.problems)
+        console.log(`FAIL ${relative(error.file)}: ${problem}`);
+}
+if (check) {
+    const value = caseSha256(caseId);
+    if (value === null) {
+        failed = true;
+        console.log(`FAIL cases/${caseId}.md does not exist`);
+    }
+    const known = new Set(knownProfiles());
+    for (const note of notes) {
+        const freshness = noteFreshness(note);
+        if (freshness.state !== 'fresh') {
+            console.log(`WARN ${relative(exploreNotesFile(caseId, note.profile))}: ${freshness.reason}; current caseSha256 is ${value ?? '(unknown)'}`);
+        }
+        if (!known.has(note.profile)) {
+            console.log(`WARN ${relative(exploreNotesFile(caseId, note.profile))}: "${note.profile}" is not in profiles.yaml`);
+        }
+    }
+    if (failed) {
+        console.error('\nA note that fails here is ignored by /gen-spec. Fix it, or delete it and let the case be replayed.');
+        process.exit(1);
+    }
+}
+function describe(note) {
+    const freshness = noteFreshness(note);
+    const steps = Object.keys(note.steps)
+        .sort((a, b) => Number(a) - Number(b))
+        .join(',');
+    const targets = Object.keys(note.targets ?? {}).sort().join(',');
+    const state = freshness.state === 'fresh' ? 'fresh' : `stale (${freshness.reason})`;
+    return `${state}  steps ${steps || '-'}  targets ${targets || '-'}  captured ${note.capturedAt}${note.updatedAt ? `  refreshed ${note.updatedAt}` : ''}`;
+}
+function knownProfiles() {
+    try {
+        return loadProfiles().map((entry) => entry.id);
+    }
+    catch {
+        return [];
+    }
+}

package/dist/scripts/markers-confirm.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/scripts/markers-confirm.js

@@ -0,0 +1,139 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { CASES_DIR, REPORTS_DIR, ROOT, TESTS_DIR } from '../lib/paths.js';
+import { markerSentence, parseCaseDocument, removeMarkers } from '../lib/case-doc.js';
+/*
+ * `pnpm markers:confirm --case <id> --env <env> [--report <run-id|file>] [--dry-run]`
+ *
+ * Settles the `🔍 to be confirmed` lines of one case from the last run instead of from a browser: a marker goes when
+ * every test of the case passed in the newest report AND the spec carries the line as
+ * `// unconfirmed: <the sentence>`. It answers "does the page still say this?", never "is this the right
+ * expectation": that stays a person's decision, which the case panel shows as "why it needs you".
+ *
+ * A report is evidence for the wording it asserted, not for a line no assertion reads, so a marker without an
+ * `// unconfirmed:` line in the spec is left alone and named in the output.
+ */
+const RUN_ID = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(-[a-z0-9]+(-[a-z0-9]+)*)?$/;
+const UNCONFIRMED = /\/\/[ \t]*unconfirmed:[ \t]*(.+)$/gm;
+/** Shorter matches are noise: a two-word sentence would match half a spec. */
+const MIN_MATCH = 16;
+const args = process.argv.slice(2);
+const option = (name) => {
+    const index = args.indexOf(name);
+    return index >= 0 ? args[index + 1] : undefined;
+};
+const caseId = option('--case');
+const env = option('--env');
+const reportArg = option('--report');
+const dryRun = args.includes('--dry-run');
+if (args.includes('--help') || !caseId || !env) {
+    console.log('Usage: pnpm markers:confirm --case <case-id> --env <env> [--report <run-id|file>] [--dry-run]');
+    console.log('Removes the "to be confirmed" markers of one case when that run passed and the spec checks them.');
+    process.exit(args.includes('--help') ? 0 : 1);
+}
+/** The report to trust: the one named by `--report` (a run id or a summary file), else the newest run. */
+function chosenSummary() {
+    if (reportArg === undefined)
+        return newestSummary();
+    const file = reportArg.endsWith('.json')
+        ? path.resolve(ROOT, reportArg)
+        : path.join(REPORTS_DIR, reportArg, 'summary.json');
+    if (!fs.existsSync(file))
+        return null;
+    return { where: path.relative(ROOT, file).replace(/\\/g, '/'), summary: JSON.parse(fs.readFileSync(file, 'utf8')) };
+}
+/** The newest report: `latest/` when it exists, otherwise the newest run folder that has a summary. */
+function newestSummary() {
+    const latest = path.join(REPORTS_DIR, 'latest', 'summary.json');
+    if (fs.existsSync(latest))
+        return { where: 'reports/latest/summary.json', summary: JSON.parse(fs.readFileSync(latest, 'utf8')) };
+    if (!fs.existsSync(REPORTS_DIR))
+        return null;
+    const runs = fs
+        .readdirSync(REPORTS_DIR)
+        .filter((name) => RUN_ID.test(name) && fs.existsSync(path.join(REPORTS_DIR, name, 'summary.json')))
+        .sort();
+    const last = runs.at(-1);
+    if (!last)
+        return null;
+    return {
+        where: `reports/${last}/summary.json`,
+        summary: JSON.parse(fs.readFileSync(path.join(REPORTS_DIR, last, 'summary.json'), 'utf8')),
+    };
+}
+const normalize = (value) => value
+    .replace(/`/g, '')
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/[.;:,]+$/, '')
+    .trim()
+    .toLowerCase();
+const casePath = path.join(CASES_DIR, `${caseId}.md`);
+const specPath = path.join(TESTS_DIR, `${caseId}.spec.ts`);
+if (!fs.existsSync(casePath)) {
+    console.error(`FAIL ${caseId}: no case document at cases/${caseId}.md`);
+    process.exit(1);
+}
+const report = chosenSummary();
+if (report === null) {
+    console.error('FAIL: no run report found. Run the case first: pnpm asrun --env ' + env + ' --grep ' + caseId);
+    process.exit(1);
+}
+const results = report.summary.results.filter((result) => result.caseId === caseId);
+if (results.length === 0) {
+    console.error(`FAIL ${caseId}: ${report.where} holds no result for this case. Run it first.`);
+    process.exit(1);
+}
+const runEnv = report.summary.env ?? results.find((result) => result.env !== undefined)?.env;
+if (runEnv !== undefined && runEnv !== env) {
+    console.error(`FAIL ${caseId}: the newest report is a ${runEnv} run, not ${env}. Run the case in ${env} first.`);
+    process.exit(1);
+}
+const unsettled = results.filter((result) => result.status !== 'passed');
+if (unsettled.length > 0) {
+    console.error(`FAIL ${caseId}: ${unsettled.length} of ${results.length} test(s) did not pass in ${report.where} (${unsettled
+        .map((result) => `${result.profile}: ${result.status}`)
+        .join(', ')}); nothing is confirmed from a run that did not pass.`);
+    process.exit(1);
+}
+const text = fs.readFileSync(casePath, 'utf8');
+const doc = parseCaseDocument(text);
+if (doc.markers.length === 0) {
+    console.log(`${caseId}: no marker to settle.`);
+    process.exit(0);
+}
+if (!fs.existsSync(specPath)) {
+    console.log(`${caseId}: no spec yet (tests/${caseId}.spec.ts), so no run can confirm a marker.`);
+    process.exit(0);
+}
+const spec = fs.readFileSync(specPath, 'utf8');
+const quoted = [...spec.matchAll(UNCONFIRMED)].map((match) => normalize(match[1]));
+const confirmed = [];
+const left = [];
+for (const marker of doc.markers) {
+    const sentence = normalize(markerSentence(text, marker));
+    const match = sentence.length >= MIN_MATCH
+        ? quoted.find((quote) => quote.includes(sentence) || sentence.includes(quote))
+        : undefined;
+    if (match === undefined) {
+        left.push({ marker, reason: 'the spec has no "// unconfirmed:" line for it, so no assertion reads it' });
+        continue;
+    }
+    confirmed.push(marker);
+}
+for (const { marker, reason } of left) {
+    console.log(`keep ${caseId}:${marker.line}  ${markerSentence(text, marker).slice(0, 100)}`);
+    console.log(`     ${reason}`);
+}
+if (confirmed.length === 0) {
+    console.log(`${caseId}: nothing to settle from ${report.where}.`);
+    process.exit(0);
+}
+for (const marker of confirmed) {
+    console.log(`pass ${caseId}:${marker.line}  ${markerSentence(text, marker).slice(0, 100)}`);
+}
+if (dryRun) {
+    console.log(`dry run: ${confirmed.length} marker(s) would go, ${left.length} would stay.`);
+    process.exit(0);
+}
+fs.writeFileSync(casePath, removeMarkers(text, confirmed));
+console.log(`wrote cases/${caseId}.md: ${confirmed.length} marker(s) settled by ${report.where}, ${left.length} left.`);

package/dist/ui/server/actions.js

@@ -16,6 +16,21 @@ import { AI_COMMANDS, AI_COMMAND_WRITES, AI_TOOLS, continueCommandLine, continue
 const REPORTS_DIR = path.join(ROOT, 'reports');
 const REPORT_ID = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(-[a-z0-9]+(-[a-z0-9]+)*)?$/;
 const newSessionId = (tool) => tool === 'claude' ? crypto.randomUUID() : `ses_${crypto.randomBytes(12).toString('hex')}`;
+/**
+ * The request as the person made it, for "Run again". A confirmation they gave is left out on purpose: write cases and
+ * an overwritten spec are asked about again on the next run.
+ */
+function replayRequest(request) {
+    if (request.action === 'run') {
+        const { confirmWritesIn: _confirmed, ...rest } = request;
+        return rest;
+    }
+    if (request.action === 'ai') {
+        const { force: _force, ...rest } = request;
+        return rest;
+    }
+    return request;
+}
 export const startJobSchema = z.discriminatedUnion('action', [
     z.object({ action: z.literal('signin'), profile: z.string().min(1), env: z.string().min(1) }),
     z.object({ action: z.literal('visit'), profile: z.string().min(1), env: z.string().min(1) }),
@@ -157,6 +172,7 @@ export function buildJobSpec(request) {
             args: [...scriptArgs('login.ts'), request.profile],
             locks: [authLock(request.profile)],
             acceptsInput: true,
+            rerun: replayRequest(request),
             onFinish: () => void verifyProfiles({ ids: [request.profile] }),
         };
     }
@@ -175,16 +191,20 @@ export function buildJobSpec(request) {
             args: [...scriptArgs('explore.ts'), request.profile, ...(startPath ? [startPath] : [])],
             locks: [`visit:${request.profile}`],
             acceptsInput: false,
+            rerun: replayRequest(request),
         };
     }
     // An AI run on a case, a first turn or a continuation: the same permissions and locks either way.
     // Generating and healing write the spec and the case values and run the tests, so they must not overlap with a run.
-    // Reviewing only edits the case file and looks at pages, so it takes just the case lock.
+    // Reviewing only edits the case file, but every AI command drives the browser through the workspace's one
+    // `playwright-test` MCP server: two runs at once interleave on it and leave it wedged for every later run (one
+    // stops the other's paused test, and its call queue never drains). So the `ai` lock is workspace-wide, and only
+    // the case lock narrows it further.
     const aiSpec = (tool, command, caseId, env, sessionId, run, model, thinking) => ({
         kind: 'ai',
         env,
         ...run,
-        locks: [`ai:${caseId}`, ...(AI_COMMAND_WRITES[command] === 'files' ? ['run', ...allLocks()] : [])],
+        locks: ['ai', `ai:${caseId}`, ...(AI_COMMAND_WRITES[command] === 'files' ? ['run', ...allLocks()] : [])],
         acceptsInput: false,
         stream: tool,
         ai: { tool, command, caseId, env, sessionId, ...(model && { model }), ...(thinking && { thinking }) },
@@ -209,8 +229,8 @@ export function buildJobSpec(request) {
         if (request.attended && request.command !== 'explore-case') {
             throw new ActionError('Only explore-case can be attended.');
         }
-        if ((request.hints !== undefined || request.report) && request.command !== 'heal-spec') {
-            throw new ActionError('Only heal-spec takes hints and a report.');
+        if (request.report && request.command !== 'heal-spec') {
+            throw new ActionError('Only heal-spec reads a report.');
         }
         // The spec is generated for every profile the case names (and runs for every profile), so a selection could only
         // narrow it wrongly.
@@ -246,18 +266,26 @@ export function buildJobSpec(request) {
         };
         const session = { id: newSessionId(request.tool), resume: false };
         const { file, args } = headlessInvocation(request.tool, request.command, request.caseId, options, session);
-        return aiSpec(request.tool, request.command, request.caseId, request.env, session.id, {
-            command: headlessCommandLine(request.tool, request.command, request.caseId, options, session),
-            file,
-            args,
-        }, request.model, request.thinking);
+        return {
+            ...aiSpec(request.tool, request.command, request.caseId, request.env, session.id, {
+                command: headlessCommandLine(request.tool, request.command, request.caseId, options, session),
+                file,
+                args,
+            }, request.model, request.thinking),
+            rerun: replayRequest(request),
+        };
     }
     if (request.action === 'continue') {
         const earlier = jobs.get(request.jobId);
         const ai = earlier?.spec.ai;
         if (!earlier || !ai)
             throw new ActionError('Unknown job, or it was not an AI run on a case.');
-        if (earlier.info.status !== 'succeeded' || earlier.info.resultState !== 'needs-input') {
+        const finishedWithQuestions = earlier.info.status === 'succeeded' && earlier.info.resultState === 'needs-input';
+        // A server restart can leave a run that had already written its questions: answering them is still the next step.
+        const recoveredWithQuestions = earlier.info.status === 'interrupted' &&
+            earlier.info.resultState === 'needs-input' &&
+            earlier.info.leftover?.state !== 'alive';
+        if (!finishedWithQuestions && !recoveredWithQuestions) {
             throw new ActionError('Only a run that finished with RESULT: needs-input can be continued.');
         }
         if (findContinuation(jobs.list(), earlier.info.id)) {
@@ -272,12 +300,16 @@ export function buildJobSpec(request) {
         fs.writeFileSync(path.join(ROOT, replyPath), request.reply.trim() + '\n');
         // The next turn keeps how the first one was told to run (model and reasoning), so the log stays the same.
         const { file, args } = continueInvocation(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking);
-        return aiSpec(ai.tool, ai.command, ai.caseId, ai.env, ai.sessionId, {
-            command: continueCommandLine(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking),
-            file,
-            args,
-            continuesJobId: earlier.info.id,
-        }, ai.model, ai.thinking);
+        return {
+            ...aiSpec(ai.tool, ai.command, ai.caseId, ai.env, ai.sessionId, {
+                command: continueCommandLine(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking),
+                file,
+                args,
+                continuesJobId: earlier.info.id,
+            }, ai.model, ai.thinking),
+            // "Run again" on a continued run repeats the run it continued, not the answers that were sent to it.
+            ...(earlier.spec.rerun && { rerun: earlier.spec.rerun }),
+        };
     }
     if (request.action === 'make-cases') {
         if (request.memo.trim() === '')
@@ -295,6 +327,7 @@ export function buildJobSpec(request) {
             locks: ['ai:make-cases'],
             acceptsInput: false,
             stream: request.tool,
+            rerun: replayRequest(request),
         };
     }
     if (!knownEnvs.includes(request.env))
@@ -364,6 +397,7 @@ export function buildJobSpec(request) {
         reportId,
         locks: ['run', ...allLocks()],
         acceptsInput: false,
+        rerun: replayRequest(request),
         ...(updateBaselines.length > 0 && { baselineCases: updateBaselines }),
         onFinish: () => void verifyProfiles({ env: request.env }),
     };