ccqa 1.58.3 → 1.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/ccqa.mjs CHANGED
@@ -3143,7 +3143,7 @@ function formatDuration$1(ms) {
3143
3143
  return `${minutes} minute${minutes === 1 ? "" : "s"}`;
3144
3144
  }
3145
3145
  async function invokeClaudeStreaming(options, onEvent) {
3146
- const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
3146
+ const { prompt, systemPrompt, allowedTools, disableThinking = false, effort, bashGuard, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, beforeAbCommand, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
3147
3147
  const resolvedModel = resolveModel(model);
3148
3148
  const mergedEnv = buildInvocationEnv(env);
3149
3149
  const abortController = new AbortController();
@@ -3171,52 +3171,35 @@ async function invokeClaudeStreaming(options, onEvent) {
3171
3171
  env: mergedEnv,
3172
3172
  ...mcpServers ? { mcpServers } : {},
3173
3173
  ...disableThinking ? { thinking: { type: "disabled" } } : {},
3174
- hooks: onAbAction || onAbActionFailed ? {
3174
+ ...effort ? { effort } : {},
3175
+ hooks: onAbAction || onAbActionFailed || bashGuard || beforeAbCommand ? {
3175
3176
  PreToolUse: [{ hooks: [async (input) => {
3176
3177
  if (input.hook_event_name !== "PreToolUse") return {};
3177
3178
  if (input.tool_name !== "Bash") return {};
3178
3179
  const cmd = input.tool_input?.["command"];
3179
3180
  if (typeof cmd !== "string") return {};
3181
+ const guarded = bashGuard?.(cmd);
3182
+ if (guarded) return {
3183
+ decision: "block",
3184
+ reason: guarded
3185
+ };
3180
3186
  if (!relaxAbConstraints) {
3181
- if (isBlockedAbSubcommand(cmd)) return {
3182
- decision: "block",
3183
- reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
3184
- };
3185
- if (hasRefSelector(cmd)) return {
3186
- decision: "block",
3187
- reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
3188
- };
3189
- const bareTag = findPositionalBareTag(cmd);
3190
- if (bareTag !== null) return {
3191
- decision: "block",
3192
- reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
3193
- };
3194
- if (hasMultipleAbInvocations(cmd)) return {
3195
- decision: "block",
3196
- reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
3197
- };
3198
- if (hasErrorSuppression(cmd)) return {
3187
+ const reason = abCommandBlockReason(cmd, envScrubMap);
3188
+ if (reason !== null) return {
3199
3189
  decision: "block",
3200
- reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
3190
+ reason
3201
3191
  };
3202
- const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
3203
- if (runProducedUrl !== null) return {
3192
+ const checkpoint = beforeAbCommand?.(extractCcqaStepFromBashCommand(cmd) ?? void 0);
3193
+ if (checkpoint) return {
3204
3194
  decision: "block",
3205
- reason: `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`
3195
+ reason: checkpoint
3206
3196
  };
3207
3197
  }
3208
- const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
3209
- const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
3210
- if ((ab !== null || assertMarker !== null) && onAbAction) {
3198
+ const recorded = relaxAbConstraints ? null : abActionEventFromCommand(cmd);
3199
+ if (recorded !== null && onAbAction) {
3211
3200
  lastAbToolUseId = input.tool_use_id;
3212
- lastAbAnswerHolds = assertMarker === null ? null : markerHolds(assertMarker, ab);
3213
- const stepId = extractCcqaStepFromBashCommand(cmd);
3214
- onAbAction({
3215
- ...ab !== null ? { abAction: ab } : {},
3216
- ...stepId ? { stepId } : {},
3217
- ...assertMarker !== null ? { assertMarker } : {},
3218
- ...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
3219
- });
3201
+ lastAbAnswerHolds = recorded.holds;
3202
+ onAbAction(recorded.event);
3220
3203
  } else lastAbToolUseId = null;
3221
3204
  return {};
3222
3205
  }] }],
@@ -3804,6 +3787,45 @@ async function fireMockPreToolUseHooks(msg, options) {
3804
3787
  for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
3805
3788
  }
3806
3789
  }
3790
+ /**
3791
+ * Why the trace refuses this agent-browser command, or null when it may run.
3792
+ * Shared by the Bash hook and the trace's own batch tool, so both enforce the
3793
+ * same replayability contract.
3794
+ */
3795
+ function abCommandBlockReason(cmd, envScrubMap = []) {
3796
+ if (isBlockedAbSubcommand(cmd)) return "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector.";
3797
+ if (hasRefSelector(cmd)) return "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either.";
3798
+ const bareTag = findPositionalBareTag(cmd);
3799
+ if (bareTag !== null) return `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`;
3800
+ if (hasMultipleAbInvocations(cmd)) return "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command.";
3801
+ if (hasErrorSuppression(cmd)) return "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code.";
3802
+ const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
3803
+ if (runProducedUrl !== null) return `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`;
3804
+ return null;
3805
+ }
3806
+ /**
3807
+ * What recording this command produces, and — for a marked probe — what its
3808
+ * output must say for the check to hold. Null when it records nothing.
3809
+ */
3810
+ function abActionEventFromCommand(cmd) {
3811
+ const assertMarker = extractCcqaAssertFromBashCommand(cmd);
3812
+ const ab = extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
3813
+ if (ab === null && assertMarker === null) return null;
3814
+ const stepId = extractCcqaStepFromBashCommand(cmd);
3815
+ return {
3816
+ event: {
3817
+ ...ab !== null ? { abAction: ab } : {},
3818
+ ...stepId ? { stepId } : {},
3819
+ ...assertMarker !== null ? { assertMarker } : {},
3820
+ ...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
3821
+ },
3822
+ holds: assertMarker === null ? null : markerHolds(assertMarker, ab)
3823
+ };
3824
+ }
3825
+ /** Whether a shell line runs agent-browser as a command (env prefixes allowed), not just mentions it. */
3826
+ function isAgentBrowserCommand(cmd) {
3827
+ return splitShellStatements(cmd).some((st) => isAgentBrowserHead(splitLeadingEnvAssignments(st).command));
3828
+ }
3807
3829
  //#endregion
3808
3830
  //#region src/prompts/format.ts
3809
3831
  /**
@@ -3950,7 +3972,7 @@ function toAgentBrowserArgs(action) {
3950
3972
  }
3951
3973
  case "wait": {
3952
3974
  const loc = action.locator;
3953
- if (!loc) return null;
3975
+ if (!loc || loc.value.startsWith("--")) return null;
3954
3976
  if (loc.by === "text") return [
3955
3977
  lit("wait"),
3956
3978
  lit("--text"),
@@ -4005,6 +4027,61 @@ function interactionToArgs(action) {
4005
4027
  if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
4006
4028
  return out;
4007
4029
  }
4030
+ /**
4031
+ * The marked probe that records an assert during a trace — the inverse of
4032
+ * `promoteMarkedAssert`, as the trace prompt's Assertion Protocol spells it.
4033
+ */
4034
+ function markedAssertArgs(action) {
4035
+ const loc = action.locator;
4036
+ const sel = loc ? [val(locatorToSelector(loc))] : null;
4037
+ switch (action.assert) {
4038
+ case "text_visible": return action.value ? {
4039
+ marker: "1",
4040
+ tokens: [
4041
+ lit("wait"),
4042
+ lit("--text"),
4043
+ val(action.value)
4044
+ ]
4045
+ } : null;
4046
+ case "url_contains": return action.value ? {
4047
+ marker: `url_contains:${action.value}`,
4048
+ tokens: [lit("get"), lit("url")]
4049
+ } : null;
4050
+ case "element_visible":
4051
+ case "element_not_visible":
4052
+ if (loc?.by === "role") return loc.name ? {
4053
+ marker: action.assert,
4054
+ tokens: roleProbeTokens(loc.value, loc.name, loc.exact === true)
4055
+ } : null;
4056
+ return sel ? {
4057
+ marker: action.assert,
4058
+ tokens: [
4059
+ lit("get"),
4060
+ lit("count"),
4061
+ ...sel
4062
+ ]
4063
+ } : null;
4064
+ case "element_enabled":
4065
+ case "element_disabled": return sel ? {
4066
+ marker: action.assert,
4067
+ tokens: [
4068
+ lit("is"),
4069
+ lit("enabled"),
4070
+ ...sel
4071
+ ]
4072
+ } : null;
4073
+ case "element_checked":
4074
+ case "element_unchecked": return sel ? {
4075
+ marker: action.assert,
4076
+ tokens: [
4077
+ lit("is"),
4078
+ lit("checked"),
4079
+ ...sel
4080
+ ]
4081
+ } : null;
4082
+ default: return null;
4083
+ }
4084
+ }
4008
4085
  //#endregion
4009
4086
  //#region src/diagnose/prompt.ts
4010
4087
  function buildDiagnosePrompt(input) {
@@ -14759,6 +14836,12 @@ function tail(output) {
14759
14836
  const trimmed = output.trim();
14760
14837
  return trimmed.length <= OUTPUT_TAIL_CHARS ? trimmed : trimmed.slice(-OUTPUT_TAIL_CHARS);
14761
14838
  }
14839
+ const LOGGED_TAIL_LINES = 40;
14840
+ /** The end of an output tail that is already scrubbed, one log line per line. */
14841
+ function logOutputTail(scrubbed, write) {
14842
+ if (scrubbed === "") return;
14843
+ for (const line of scrubbed.split("\n").slice(-LOGGED_TAIL_LINES)) write(` | ${line}`);
14844
+ }
14762
14845
  /**
14763
14846
  * Where new support files may be created.
14764
14847
  *
@@ -14950,16 +15033,19 @@ async function runVerificationLoop(p, state) {
14950
15033
  failing = run;
14951
15034
  }
14952
15035
  }
15036
+ const outputTail = scrubEnvValues(tail(failing.output), outputScrub);
14953
15037
  if (attempt >= maxRetries) {
14954
15038
  warn(`verification still failing after ${maxRetries} fix attempt(s) — generated files kept`);
15039
+ logOutputTail(outputTail, warn);
14955
15040
  await clearCaseRun(p.ctx.ref);
14956
15041
  return finish(false);
14957
15042
  }
14958
15043
  fix(`verification failed (exit ${failing.exitCode}) — requesting a fix (${attempt + 1}/${maxRetries})`);
15044
+ logOutputTail(outputTail, fix);
14959
15045
  const fixPrompt = buildLlmFixPrompt({
14960
15046
  targetId: p.target,
14961
15047
  command: failing.command,
14962
- outputTail: scrubEnvValues(tail(failing.output), outputScrub),
15048
+ outputTail,
14963
15049
  files: [...state.entries()].map(([path, f]) => ({
14964
15050
  path,
14965
15051
  contents: f.contents,
@@ -24558,1116 +24644,1364 @@ async function readLockBody(lockPath) {
24558
24644
  }
24559
24645
  }
24560
24646
  //#endregion
24561
- //#region src/prompts/trace.ts
24562
- function generateSessionName() {
24563
- return `ccqa-trace-${buildRunId()}`;
24647
+ //#region src/ir/accessible-name.ts
24648
+ /**
24649
+ * A recorder reads an accessible name off a snapshot and writes it as the
24650
+ * attribute it guessed the name came from. The name can come from a `<label>`
24651
+ * or `aria-labelledby` instead, and the attribute's absence is not evidence
24652
+ * about the element. See docs/targets.md for the measurement.
24653
+ */
24654
+ const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
24655
+ /**
24656
+ * The name this selector may be after, or null.
24657
+ *
24658
+ * `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
24659
+ * mean what they say, and a zero from one of those is evidence.
24660
+ */
24661
+ function nameFromAttributeSelector(selector) {
24662
+ const m = NAMING_ATTR.exec(selector.trim());
24663
+ const name = m?.[1] ?? m?.[2];
24664
+ return name === void 0 || name === "" ? null : name;
24564
24665
  }
24565
24666
  /**
24566
- * Build the trace system prompt. `input.steps` is a flat list with includes
24567
- * already expanded (each step carries id / source / instruction / expected).
24568
- * The spec opens URLs via explicit step instructions (e.g.
24569
- * `instruction: "${APP_URL}/articles を開く"`).
24667
+ * The role of the first node in an accessibility snapshot whose name is
24668
+ * exactly `name`, or null.
24570
24669
  *
24571
- * In v0.4 every spec is traced from scratch — block contents are inlined
24572
- * into the spec's own step list at expand time, so the prompt has no
24573
- * special "this is a block" mode. The `source` tag on each step still
24574
- * distinguishes spec-native steps from inlined block steps for the
24575
- * `// step:` comments in the eventual codegen output.
24670
+ * agent-browser prints one node per line as `<role> "<name>"`. Matched on the
24671
+ * whole name, so a tree holding both "Priority" and "Priority *" answers for
24672
+ * the one that was asked, and refused when two roles carry it.
24576
24673
  */
24577
- function buildTraceSystemPrompt(input) {
24578
- const sessionName = input.sessionName ?? generateSessionName();
24579
- const firstStepId = input.steps[0]?.id ?? "step-01";
24580
- const callerGuidance = input.instruction ? `## Caller Guidance
24581
-
24582
- The caller provided extra guidance for this recording — for example, a drift
24583
- audit's finding about what a previous recording of this spec got wrong. Treat
24584
- it as advice on what to avoid or verify while recording; the spec's steps
24585
- above remain the contract for what to do.
24586
-
24587
- ${input.instruction}
24588
-
24589
- ` : "";
24590
- const stepsText = input.steps.map((step) => [
24591
- `### ${step.id} [${step.source}]`,
24592
- `- **Instruction**: ${step.instruction}`,
24593
- ...step.expected ? [`- **Expected**: ${step.expected}`] : []
24594
- ].join("\n")).join("\n\n");
24595
- const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
24596
- const contextText = renderContext(input.context ?? []);
24597
- const conventionsText = renderConventions(input.conventions ?? []);
24598
- return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
24599
-
24600
- ## Session
24601
-
24602
- SESSION NAME: \`${sessionName}\`
24603
-
24604
- Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
24605
-
24606
- **Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
24607
- env assignment naming the spec step it executes**, e.g.
24608
-
24609
- \`\`\`bash
24610
- CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
24611
- \`\`\`
24612
-
24613
- This prefix is what ties each recorded action to its step in the generated
24614
- test — a command without it loses its step attribution.
24615
-
24616
- **Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
24617
- in, opening the application, putting the account where the precondition says it
24618
- starts — those are not step-01, and marking them as step-01 makes the first
24619
- step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
24620
- first step the case actually lists.
24621
-
24622
- Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
24623
- itself ignores the variable.
24624
-
24625
- Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
24626
- assignment (alongside \`CCQA_STEP\`) that records the check as a test
24627
- assertion — see "Assertion Protocol".
24628
-
24629
- **A command that types into a password field carries \`CCQA_SECRET=1\`** as
24630
- well, e.g.
24631
-
24632
- \`\`\`bash
24633
- CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
24634
- \`\`\`
24635
-
24636
- A password field is an \`<input type="password">\` — the snapshot shows its
24637
- value as dots, and nothing else about the page is a password. The prefix is
24638
- what stops the typed value from being written to the recording: a value that
24639
- came from a \`\${VAR}\` is kept as that reference, and one typed literally is
24640
- dropped along with its command.
24641
-
24642
- ## Browser Commands
24643
-
24644
- \`\`\`
24645
- CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
24646
- CCQA_STEP=<step-id> agent-browser --session SESSION snapshot
24647
- CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
24648
- CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
24649
- CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
24650
- CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
24651
- CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
24652
- CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
24653
- CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
24654
- CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
24655
- CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
24656
- CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
24657
- agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
24658
- CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
24659
- CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
24660
- # See "Selector Rules" for the full \`find\` subset.
24661
- # IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
24662
- # CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
24663
- # Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
24664
- # (which waits internally). To check an element exists, use \`get count "<selector>"\`.
24665
- \`\`\`
24666
-
24667
- ## Selector Rules
24668
-
24669
- **ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
24670
-
24671
- | Format | Use when |
24672
- |--------|----------|
24673
- | \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
24674
- | \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
24675
- | \`[placeholder='text']\` | Input identified by placeholder |
24676
- | \`[type='password']\` | Password inputs only |
24677
- | \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
24678
- | \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
24679
-
24680
- **FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
24681
-
24682
- - \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
24683
- - **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
24684
- - \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
24685
- - **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
24686
- - JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
24687
-
24688
- ### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
24689
-
24690
- When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
24691
-
24692
- \`\`\`
24693
- find role <role> <action> [--name "<n>"] [--exact]
24694
- find text|label|placeholder|alt|title "<text>" <action> [--exact]
24695
- find testid "<id>" <action>
24696
- find first|last "<ALLOWED-css>" <action>
24697
- find nth <index> "<ALLOWED-css>" <action>
24698
- \`\`\`
24699
-
24700
- \`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
24701
-
24702
- **Rules for \`find\`:**
24703
-
24704
- 1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
24705
- 2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
24706
- 3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
24707
- 4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
24708
- 5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
24709
- 6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
24710
- 7. \`find\` includes its own wait; do not chain a \`wait\` before it.
24711
- 8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
24712
- 9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
24713
-
24714
- **Examples:**
24715
-
24716
- - ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
24717
- - ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
24718
- - ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
24719
- - ✗ \`find role button --name "Submit" click\` — wrong order
24720
- - ✗ \`find last "button" click\` — bare tag
24721
-
24722
- ### Selector workflow
24723
-
24724
- 1. Run \`snapshot\` and read the ARIA tree.
24725
- 2. Identify the element; note its exact \`aria-label\` if present.
24726
- 3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
24727
- For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
24728
- 4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
24729
- 5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
24730
- 6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
24731
- 7. Never guess. If a selector fails, take a fresh snapshot before retrying.
24732
- 8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
24733
-
24734
- ### Special input types
24735
-
24736
- **contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
24737
-
24738
- **combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
24739
-
24740
- **Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
24741
-
24742
- **Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
24743
-
24744
- **File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
24745
-
24746
- ## Test Specification
24747
-
24748
- Title: ${input.title}
24749
-
24750
- Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
24751
-
24752
- ## Steps
24753
-
24754
- ${stepsText}
24755
- ${expectationsText}${contextText}${conventionsText}
24756
- ## Execution Workflow
24757
-
24758
- For each step:
24759
- 1. Emit \`STEP_START|<step-id>|<short description>\`.
24760
- 2. Run \`snapshot\` and identify selectors from the ARIA tree.
24761
- 3. Execute the action using an ALLOWED selector (see Selector Rules), prefixing the command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step.
24762
- 4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
24763
- 5. Run \`snapshot\` again to verify the outcome.
24764
- 6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
24765
- 7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
24766
- 8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
24767
-
24768
- **Protocol lines are recorded ONLY from your plain assistant text.** Every
24769
- \`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
24770
- message text between tool calls — never through Bash (\`echo\` etc.) and never
24771
- omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
24772
- prefix on each command (that prefix is authoritative, not the \`STEP_START\`
24773
- line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
24774
- verification commands** — a command that ran, whose failure rolls the record
24775
- back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
24776
- proves it; it is reported and not recorded.
24777
-
24778
- **After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
24779
-
24780
- ## Guardrails
24781
-
24782
- - **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
24783
- - **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
24784
- - Do NOT retry a selector without taking a fresh snapshot first.
24785
- - Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
24786
- - **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
24787
- - **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
24788
-
24789
- ## Source Code Reference
24790
-
24791
- You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
24792
-
24793
- **When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
24794
-
24795
- **How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
24796
-
24797
- **Rules**: only READ source files, never modify them. Keep searches focused.
24798
-
24799
- ## Waiting for Async Operations
24800
-
24801
- Prefer \`wait\` over polling:
24802
-
24803
- \`\`\`bash
24804
- CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
24805
- \`\`\`
24806
-
24807
- If polling is required (e.g. waiting for a spinner to disappear):
24808
-
24809
- \`\`\`bash
24810
- for i in $(seq 1 18); do
24811
- sleep 10
24812
- result=$(agent-browser --session ${sessionName} snapshot 2>&1)
24813
- echo "$result" | grep -q "<done indicator>" && break
24814
- done
24815
- CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
24816
- \`\`\`
24817
-
24818
- After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
24819
-
24820
- ## AB_ACTION Protocol
24821
-
24822
- After **every** browser action, emit one line (outside any code block):
24823
-
24824
- \`\`\`
24825
- AB_ACTION|cookies_clear
24826
- AB_ACTION|open|<url>
24827
- AB_ACTION|click|<selector>|<visible label>
24828
- AB_ACTION|dblclick|<selector>|<visible label>
24829
- AB_ACTION|fill|<selector>|<value>|<aria label>
24830
- AB_ACTION|check|<selector>|<visible label>
24831
- AB_ACTION|uncheck|<selector>|<visible label>
24832
- AB_ACTION|press|<Key>
24833
- AB_ACTION|select|<selector>|<value>|<aria label>
24834
- AB_ACTION|hover|<selector>|<visible label>
24835
- AB_ACTION|scroll|<direction>|<pixels>
24836
- AB_ACTION|drag|<source selector>|<target selector>|<source label>
24837
- AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
24838
- AB_ACTION|wait|<selector or text>|<label>
24839
- AB_ACTION|snapshot|<key observation, max 100 chars>
24840
-
24841
- # find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
24842
- # <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
24843
- AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
24844
- AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
24845
- AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
24846
- AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
24847
- AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
24848
- AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
24849
- AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
24850
- AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
24851
- \`\`\`
24852
-
24853
- Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
24854
-
24855
- **CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
24856
-
24857
- - A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
24858
- - If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
24859
- - An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
24860
- - **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
24861
-
24862
- - **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
24863
- - If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
24864
-
24865
- ## Assertion Protocol
24866
-
24867
- Record each signal you verified as an assertion. The **primary mechanism** is
24868
- a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
24869
- marker rides the same channel as the command, so the assertion is captured
24870
- even if you print no protocol text. The check you were going to run anyway
24871
- becomes the recorded assertion:
24872
-
24873
- \`\`\`bash
24874
- # text_visible — the marked wait IS the assertion (no separate assert line)
24875
- CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
24876
- # element_visible / element_not_visible — mark the get-count probe
24877
- CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
24878
- CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
24879
- # url_contains — the substring rides in the marker; put it on \`get url\`
24880
- CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
24881
- # element state — mark the \`is\` probe that asks for it
24882
- CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
24883
- CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
24884
- \`\`\`
24885
-
24886
- **Marker semantics:**
24887
-
24888
- | Marker | On command | Records |
24889
- |--------|------------|---------|
24890
- | \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
24891
- | \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
24892
- | \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
24893
- | \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
24894
- | \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
24895
- | \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
24896
-
24897
- - A marked command that exits non-zero records nothing — fix the check and re-run it.
24898
- - A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
24899
- - \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
24900
- - **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
24901
-
24902
- - **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
24903
-
24904
- - **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
24905
-
24906
- - **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
24907
- - **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
24908
- - **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
24909
-
24910
- **Every assert type has a marker command.** There is no text form: a printed
24911
- \`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
24912
- reported rather than recorded.
24913
-
24914
- | assertType | The command that records it | Use when |
24915
- |------------|------------------------------|----------|
24916
- | \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
24917
- | \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
24918
- | \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
24919
- | \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
24920
- | \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
24921
- | \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
24922
-
24923
- **Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
24924
-
24925
- - Timestamps, session IDs, exact numeric counts that vary between runs.
24926
- - **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
24927
- - **Relative-time labels** — true only in the moment of the trace, stale by replay:
24928
- - English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
24929
- - Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
24930
- - Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
24931
- - **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
24932
-
24933
- **No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
24934
-
24935
- The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
24936
-
24937
- - ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
24938
- - ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
24939
- - ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
24940
- - If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
24941
-
24942
- **Page-context and selector rules:**
24943
-
24944
- - After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
24945
- - Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
24946
- - When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
24947
-
24948
- **MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
24949
-
24950
- The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
24951
-
24952
- 1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
24953
- 2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
24954
- 3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
24955
-
24956
- **Verify the assertion form actually resolves on the live page.** The
24957
- verification and the recorded assertion are the same marked command:
24958
-
24959
- \`\`\`bash
24960
- # text_visible
24961
- CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
24962
- # element_visible — >=1 printed means present
24963
- CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
24964
- # element_not_visible — 0 printed means absent (works for text too: "text=<text>")
24965
- CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
24966
- # element state — \`is\` prints true/false and exits 0 either way; read it
24967
- CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
24968
- CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
24969
- \`\`\`
24970
-
24971
- When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
24972
-
24973
- ## Status Protocol
24974
-
24975
- Emit exactly one status line per step (outside any code block):
24976
-
24977
- \`\`\`
24978
- STEP_START|<step-id>|<short description>
24979
- STEP_DONE|<step-id>|<what was verified>
24980
- ASSERTION_FAILED|<step-id>|<category: app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
24981
- STEP_SKIPPED|<step-id>|<reason>
24982
- RUN_COMPLETED|passed|<summary>
24983
- RUN_COMPLETED|failed|<summary>
24984
- \`\`\`
24985
-
24986
- **\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
24987
- recorded as FAILED, however many steps went well: nothing else can tell a
24988
- finished route from one you abandoned halfway.
24989
-
24990
- ${callerGuidance}## Start
24991
-
24992
- Begin by clearing cookies, then proceed straight to the first step's instruction.
24993
-
24994
- \`\`\`bash
24995
- agent-browser --session ${sessionName} cookies clear
24996
- \`\`\`
24997
-
24998
- (The cookies-clear preamble belongs to no step, so it is the one command
24999
- without a \`CCQA_STEP\` prefix.)
25000
-
25001
- Emit:
25002
- \`\`\`
25003
- AB_ACTION|cookies_clear
25004
- \`\`\`
25005
-
25006
- Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
25007
- every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
25008
- step is responsible for opening the initial URL.
25009
- `;
24674
+ function roleOfAccessibleName(snapshot, name) {
24675
+ const roles = /* @__PURE__ */ new Set();
24676
+ for (const line of snapshot.split("\n")) {
24677
+ const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
24678
+ if (m && m[2] === name) roles.add(m[1]);
24679
+ }
24680
+ return roles.size === 1 ? [...roles][0] : null;
25010
24681
  }
25011
- function buildTracePrompt(title) {
25012
- return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
24682
+ //#endregion
24683
+ //#region src/runtime/replay-validate.ts
24684
+ function isProbe(x) {
24685
+ return x !== null && !Array.isArray(x);
24686
+ }
24687
+ function runProbe(probe, sessionName) {
24688
+ return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
24689
+ }
24690
+ function runStateCheck(check, sessionName) {
24691
+ let answer = "";
24692
+ const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
24693
+ for (let round = 0; round < rounds; round++) {
24694
+ const r = spawnAB([
24695
+ "--session",
24696
+ sessionName,
24697
+ "is",
24698
+ check.state,
24699
+ check.selector
24700
+ ]);
24701
+ answer = r.status === 0 ? r.stdout.trim() : "";
24702
+ if (answer === String(check.expected)) return {
24703
+ ok: true,
24704
+ reason: ""
24705
+ };
24706
+ if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
24707
+ }
24708
+ return answer === "true" || answer === "false" ? {
24709
+ ok: false,
24710
+ reason: `expected ${check.state}=${check.expected}, page says ${answer}`
24711
+ } : {
24712
+ ok: false,
24713
+ reason: `is ${check.state} answered ${answer || "nothing"}`
24714
+ };
24715
+ }
24716
+ const SELECTOR_POLL_INTERVAL_MS = 500;
24717
+ /** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
24718
+ function runPollCheck(check, sessionName) {
24719
+ const present = (selector) => {
24720
+ const r = spawnAB([
24721
+ "--session",
24722
+ sessionName,
24723
+ "get",
24724
+ "count",
24725
+ selector
24726
+ ]);
24727
+ return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
24728
+ };
24729
+ const deadline = Date.now() + check.timeoutMs;
24730
+ const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
24731
+ let count = NaN;
24732
+ for (let round = 0;; round++) {
24733
+ count = present(check.selector);
24734
+ if (!Number.isNaN(count) && count > 0) return {
24735
+ ok: true,
24736
+ reason: ""
24737
+ };
24738
+ if (round + 1 >= rounds || Date.now() >= deadline) break;
24739
+ sleepSync(SELECTOR_POLL_INTERVAL_MS);
24740
+ }
24741
+ return {
24742
+ ok: false,
24743
+ reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
24744
+ };
24745
+ }
24746
+ /** What the validator records against an action it never replayed. */
24747
+ const CASCADE_REASON = "skipped after a preceding action failed";
24748
+ /** Whether a drop reason means "not replayed", rather than "replayed and failed". */
24749
+ function isCascadeReason(reason) {
24750
+ return reason === CASCADE_REASON;
24751
+ }
24752
+ const SHORT_TIMEOUT_MS = 5e3;
24753
+ const ASSERT_TIMEOUT_MS = 1e4;
24754
+ const INTERACTION_POLL_INTERVAL_MS = 250;
24755
+ const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
24756
+ /** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
24757
+ function attemptsFor(timeoutMs, intervalMs) {
24758
+ return Math.max(1, Math.ceil(timeoutMs / intervalMs));
25013
24759
  }
25014
24760
  /**
25015
- * The case's expectations, plus the one rule that decides where each is
25016
- * recorded. Without the rule a recorder either asserts everything at the end —
25017
- * on a page the flow has already left — or asserts each item at every step it
25018
- * might apply to, which is the over-assertion the selector rules exist to
25019
- * prevent.
24761
+ * Convert one recorded action into the `agent-browser` arg list that would
24762
+ * exercise it. Interaction commands come from the shared
24763
+ * `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
24764
+ * get validation-specific handling (poll checks, timeouts, unverifiable
24765
+ * skips). Returns `null` for actions that should not be validated (snapshot
24766
+ * has no side effect; assert types whose codegen forms aren't directly
24767
+ * verifiable here fall through to the caller's `unverifiable` fallback).
25020
24768
  */
25021
- function renderExpectations(expectations) {
25022
- if (expectations.length === 0) return "";
25023
- return `
25024
- ## Expected results
25025
-
25026
- The case states these without saying which step delivers them:
25027
-
25028
- ${expectations.map((text) => `- ${text}`).join("\n")}
25029
-
25030
- **Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
25031
- `;
24769
+ function actionToAbArgs(action, sessionName, envOverrides = {}) {
24770
+ const base = ["--session", sessionName];
24771
+ const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
24772
+ switch (action.action) {
24773
+ case "snapshot": return null;
24774
+ case "assert": return assertToAbArgs(action, sub, sessionName);
24775
+ case "wait": {
24776
+ const loc = action.locator;
24777
+ if (!loc) return null;
24778
+ if (loc.by === "text") return [
24779
+ ...base,
24780
+ "wait",
24781
+ "--text",
24782
+ sub(loc.value),
24783
+ "--timeout",
24784
+ String(SHORT_TIMEOUT_MS)
24785
+ ];
24786
+ const raw = sub(locatorToSelector(loc));
24787
+ if (!raw) return null;
24788
+ if (/^\d+$/.test(raw)) return null;
24789
+ if (raw.startsWith("--")) return null;
24790
+ if (raw.startsWith("text=")) return [
24791
+ ...base,
24792
+ "wait",
24793
+ "--text",
24794
+ raw.slice(5),
24795
+ "--timeout",
24796
+ String(SHORT_TIMEOUT_MS)
24797
+ ];
24798
+ return {
24799
+ kind: "poll-present",
24800
+ selector: raw,
24801
+ timeoutMs: SHORT_TIMEOUT_MS
24802
+ };
24803
+ }
24804
+ default: {
24805
+ const tokens = toAgentBrowserArgs(action);
24806
+ if (tokens === null) return null;
24807
+ const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
24808
+ return [...base, ...tokens.map((t) => resolve(t.text))];
24809
+ }
24810
+ }
24811
+ }
24812
+ function roleProbe(base, role, name) {
24813
+ return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
24814
+ }
24815
+ /**
24816
+ * Presence asked the way the locator addresses the element. `null` is
24817
+ * unverifiable rather than absent: a count from a selector the engine never
24818
+ * parsed is not evidence.
24819
+ */
24820
+ function presenceCheck(locator, selector, sub, base) {
24821
+ if (locator?.by === "text") return [
24822
+ ...base,
24823
+ "wait",
24824
+ "--text",
24825
+ sub(locator.value),
24826
+ "--timeout",
24827
+ String(ASSERT_TIMEOUT_MS)
24828
+ ];
24829
+ if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
24830
+ if (locator !== void 0 && locator.by !== "css") return null;
24831
+ const named = parseNotation(selector);
24832
+ if (named === "opaque") return null;
24833
+ if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
24834
+ if (named !== null) return null;
24835
+ return {
24836
+ kind: "poll-present",
24837
+ selector,
24838
+ timeoutMs: ASSERT_TIMEOUT_MS
24839
+ };
24840
+ }
24841
+ function assertToAbArgs(action, sub, sessionName) {
24842
+ const base = ["--session", sessionName];
24843
+ const val = sub(action.value ?? action.observation);
24844
+ const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
24845
+ switch (action.assert) {
24846
+ case "text_visible":
24847
+ if (!val) return null;
24848
+ return [
24849
+ ...base,
24850
+ "wait",
24851
+ "--text",
24852
+ val,
24853
+ "--timeout",
24854
+ String(ASSERT_TIMEOUT_MS)
24855
+ ];
24856
+ case "text_not_visible": return null;
24857
+ case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
24858
+ case "element_not_visible": return null;
24859
+ case "url_contains": return null;
24860
+ case "element_enabled":
24861
+ case "element_disabled":
24862
+ case "element_checked":
24863
+ case "element_unchecked":
24864
+ if (!sel || action.locator?.by !== "css") return null;
24865
+ if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
24866
+ return {
24867
+ kind: "state",
24868
+ state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
24869
+ selector: sel,
24870
+ expected: action.assert === "element_enabled" || action.assert === "element_checked"
24871
+ };
24872
+ default: return null;
24873
+ }
25032
24874
  }
24875
+ const NO_STEP_ID = "__no_step__";
25033
24876
  /**
25034
- * What the cleanup must make true. Separate from the case's expectations
25035
- * because its steps are: a deletion is confirmed by the thing being gone,
25036
- * which is only true after the undo has run, and asserting it among the
25037
- * case's own steps would check it while the thing still exists.
24877
+ * The role a labelled element has when addressing it by label finds nothing.
24878
+ *
24879
+ * `getByLabel` needs a real association — a `<label for>`, a wrapping label,
24880
+ * an `aria-labelledby`. A form that has none still shows the same string as
24881
+ * the field's accessible name, which is what the recorder read off the
24882
+ * snapshot, so the label it recorded matches nothing while the same text as a
24883
+ * role's name matches exactly. Only the actions whose element has one
24884
+ * unambiguous role are listed: a labelled `click` may be a button, a link or
24885
+ * a checkbox, and guessing there would replace a locator that failed with one
24886
+ * that finds the wrong element.
25038
24887
  */
25039
- function renderCleanupExpectations(expectations) {
25040
- if (expectations.length === 0) return "";
25041
- return `
25042
- ## Expected results of the cleanup
25043
-
25044
- The case states these about its cleanup steps:
25045
-
25046
- ${expectations.map((text) => `- ${text}`).join("\n")}
25047
-
25048
- **Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
25049
- `;
24888
+ const LABEL_FALLBACK_ROLE = {
24889
+ fill: "textbox",
24890
+ type: "textbox",
24891
+ check: "checkbox",
24892
+ uncheck: "checkbox"
24893
+ };
24894
+ /** Whether agent-browser's failure was "no element", the one the fallback answers. */
24895
+ function notFound(result) {
24896
+ return /not\s+found|no\s+element|no\s+such\s+element|none\s+match\s+name/i.test(`${result.stderr} ${result.stdout}`);
25050
24897
  }
25051
- /**
25052
- * Sections of the case ccqa does not interpret — the precondition, notes the
25053
- * author left. Passed through verbatim: they are written for whoever runs the
25054
- * case, and the recorder is now that reader.
25055
- */
25056
- function renderContext(context) {
25057
- if (context.length === 0) return "";
25058
- return `
25059
- ## About this case
25060
-
25061
- ${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
25062
- `;
24898
+ /** Whether this argv navigates — the only action safe to repeat wholesale. */
24899
+ function isOpen(argv) {
24900
+ return argv[2] === "open";
25063
24901
  }
25064
- /**
25065
- * The project's own recording guidance. Where the case says what to do, this
25066
- * says how this project is driven — which account signs in, what has to be
25067
- * true before the first step. Prose, verbatim: it is written for whoever runs
25068
- * the case, and this is now that reader.
25069
- */
25070
- function renderConventions(conventions) {
25071
- if (conventions.length === 0) return "";
25072
- return `
25073
- ## How this project is driven
25074
-
25075
- ${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
25076
- `;
24902
+ /** A navigation another navigation took away, rather than one that failed. */
24903
+ function superseded(result) {
24904
+ return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
24905
+ }
24906
+ /** The one line both callers log when the fallback rewrote a locator. */
24907
+ function formatPromotion(promotion) {
24908
+ return `locator rewritten to the form that replays: ${promotion}`;
24909
+ }
24910
+ /** The same action addressed by role + accessible name, or null when there is no such form. */
24911
+ function labelFallback(action) {
24912
+ const loc = action.locator;
24913
+ if (!loc || loc.by !== "label") return null;
24914
+ const role = LABEL_FALLBACK_ROLE[action.action];
24915
+ if (role === void 0) return null;
24916
+ return {
24917
+ ...action,
24918
+ locator: {
24919
+ by: "role",
24920
+ value: role,
24921
+ name: loc.value,
24922
+ exact: true
24923
+ }
24924
+ };
25077
24925
  }
25078
- //#endregion
25079
- //#region src/ir/accessible-name.ts
25080
24926
  /**
25081
- * A recorder reads an accessible name off a snapshot and writes it as the
25082
- * attribute it guessed the name came from. The name can come from a `<label>`
25083
- * or `aria-labelledby` instead, and the attribute's absence is not evidence
25084
- * about the element. See docs/targets.md for the measurement.
24927
+ * Spawn one interaction, waiting for its element the way an assertion waits
24928
+ * for its text: a recording pauses for a snapshot between opening a page and
24929
+ * typing into it, the replay does not, so the replay reaches a field sooner
24930
+ * than the recording ever did.
24931
+ *
24932
+ * Only a "no element" failure waits — anything else repeats identically. When
24933
+ * there is an `alternate` form it is tried on every attempt rather than after
24934
+ * the wait: a label the page never associates with its input will not start
24935
+ * matching, so waiting first spends the budget for nothing.
25085
24936
  */
25086
- const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
24937
+ function runInteraction(argv, alternate, patience, sessionName) {
24938
+ const started = Date.now();
24939
+ const deadline = started + ASSERT_TIMEOUT_MS;
24940
+ let settled = false;
24941
+ for (let attempt = 0;; attempt++) {
24942
+ let result = spawnAB(argv);
24943
+ if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
24944
+ const waitedMs = Date.now() - started;
24945
+ if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
24946
+ settled = true;
24947
+ spawnAB([
24948
+ "--session",
24949
+ sessionName,
24950
+ "wait",
24951
+ "--load",
24952
+ "networkidle"
24953
+ ]);
24954
+ continue;
24955
+ }
24956
+ if (result.status === 0 || !notFound(result)) return {
24957
+ result,
24958
+ viaAlternate: false,
24959
+ waitedMs
24960
+ };
24961
+ if (alternate !== null) {
24962
+ const alt = spawnAB(alternate);
24963
+ if (alt.status === 0) return {
24964
+ result: alt,
24965
+ viaAlternate: true,
24966
+ waitedMs
24967
+ };
24968
+ }
24969
+ if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
24970
+ patience.spent = true;
24971
+ return {
24972
+ result,
24973
+ viaAlternate: false,
24974
+ waitedMs: Date.now() - started
24975
+ };
24976
+ }
24977
+ sleepSync(INTERACTION_POLL_INTERVAL_MS);
24978
+ }
24979
+ }
25087
24980
  /**
25088
- * The name this selector may be after, or null.
24981
+ * Say what a `by: "css"` locator that is not CSS actually addresses.
25089
24982
  *
25090
- * `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
25091
- * mean what they say, and a zero from one of those is evidence.
24983
+ * `get count` answers 0 for Playwright's notation instead of failing, so a
24984
+ * route holding one reads as dead. Rewritten in place, and reported, so the
24985
+ * saved route carries the form that replays rather than the one that counts
24986
+ * nothing. Notation this cannot convert leaves the locator alone; the assert
24987
+ * path then treats it as unverifiable rather than absent.
25092
24988
  */
25093
- function nameFromAttributeSelector(selector) {
25094
- const m = NAMING_ATTR.exec(selector.trim());
25095
- const name = m?.[1] ?? m?.[2];
25096
- return name === void 0 || name === "" ? null : name;
24989
+ function nameNotation(action) {
24990
+ const loc = action.locator;
24991
+ if (action.action !== "assert" || loc?.by !== "css") return null;
24992
+ const named = parseNotation(loc.value);
24993
+ if (named === null || named === "opaque" || named.by !== "text") return null;
24994
+ action.locator = named;
24995
+ return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
25097
24996
  }
25098
24997
  /**
25099
- * The role of the first node in an accessibility snapshot whose name is
25100
- * exactly `name`, or null.
24998
+ * An attribute selector that counted nothing, asked of the accessibility tree
24999
+ * the name was read from. See docs/targets.md for why the two disagree.
25101
25000
  *
25102
- * agent-browser prints one node per line as `<role> "<name>"`. Matched on the
25103
- * whole name, so a tree holding both "Priority" and "Priority *" answers for
25104
- * the one that was asked, and refused when two roles carry it.
25001
+ * The snapshot is read for the role, then `find role` is asked as well: the
25002
+ * tree says the name exists somewhere, the probe says the replay can reach it,
25003
+ * and only the second is what the route will depend on.
25105
25004
  */
25106
- function roleOfAccessibleName(snapshot, name) {
25107
- const roles = /* @__PURE__ */ new Set();
25108
- for (const line of snapshot.split("\n")) {
25109
- const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
25110
- if (m && m[2] === name) roles.add(m[1]);
25111
- }
25112
- return roles.size === 1 ? [...roles][0] : null;
25113
- }
25114
- //#endregion
25115
- //#region src/runtime/replay-validate.ts
25116
- function isProbe(x) {
25117
- return x !== null && !Array.isArray(x);
25118
- }
25119
- function runProbe(probe, sessionName) {
25120
- return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
25005
+ function askTheAccessibilityTree(action, sub, sessionName) {
25006
+ if (action.action !== "assert" || action.locator?.by !== "css") return null;
25007
+ const name = nameFromAttributeSelector(action.locator.value);
25008
+ if (name === null) return null;
25009
+ const snapshot = spawnAB([
25010
+ "--session",
25011
+ sessionName,
25012
+ "snapshot"
25013
+ ]);
25014
+ if (snapshot.status !== 0) return null;
25015
+ const role = roleOfAccessibleName(snapshot.stdout, sub(name));
25016
+ if (role === null) return null;
25017
+ if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
25018
+ const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
25019
+ action.locator = {
25020
+ by: "role",
25021
+ value: role,
25022
+ name,
25023
+ exact: true
25024
+ };
25025
+ return {
25026
+ skipped: false,
25027
+ ok: true,
25028
+ reason: "",
25029
+ promoted
25030
+ };
25121
25031
  }
25122
- function runStateCheck(check, sessionName) {
25123
- let answer = "";
25124
- const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
25125
- for (let round = 0; round < rounds; round++) {
25126
- const r = spawnAB([
25127
- "--session",
25128
- sessionName,
25129
- "is",
25130
- check.state,
25131
- check.selector
25132
- ]);
25133
- answer = r.status === 0 ? r.stdout.trim() : "";
25134
- if (answer === String(check.expected)) return {
25032
+ /**
25033
+ * Replay one recorded action against the validation session. Element-presence
25034
+ * checks go through `runPollCheck` (which uses `get count`, never the blocking
25035
+ * `wait <selector>`); everything else spawns the agent-browser argv.
25036
+ */
25037
+ function runValidationAction(action, sessionName, envOverrides = {}, patience) {
25038
+ const named = nameNotation(action);
25039
+ const withName = (outcome) => named === null || !outcome.ok ? outcome : {
25040
+ ...outcome,
25041
+ promoted: outcome.promoted ?? named
25042
+ };
25043
+ const built = actionToAbArgs(action, sessionName, envOverrides);
25044
+ if (built === null) return {
25045
+ skipped: true,
25046
+ ok: false,
25047
+ reason: ""
25048
+ };
25049
+ if (isProbe(built)) {
25050
+ const { ok, reason } = runProbe(built, sessionName);
25051
+ if (ok) return withName({
25052
+ skipped: false,
25053
+ ok,
25054
+ reason
25055
+ });
25056
+ const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
25057
+ return askTheAccessibilityTree(action, resolve, sessionName) ?? {
25058
+ skipped: false,
25059
+ ok,
25060
+ reason
25061
+ };
25062
+ }
25063
+ const fallback = labelFallback(action);
25064
+ const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
25065
+ const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
25066
+ if (viaAlternate) {
25067
+ const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
25068
+ action.locator = fallback.locator;
25069
+ return {
25070
+ skipped: false,
25135
25071
  ok: true,
25136
- reason: ""
25072
+ reason: "",
25073
+ promoted
25137
25074
  };
25138
- if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
25139
25075
  }
25140
- return answer === "true" || answer === "false" ? {
25141
- ok: false,
25142
- reason: `expected ${check.state}=${check.expected}, page says ${answer}`
25143
- } : {
25076
+ if (result.status === 0) return withName({
25077
+ skipped: false,
25078
+ ok: true,
25079
+ reason: ""
25080
+ });
25081
+ return {
25082
+ skipped: false,
25144
25083
  ok: false,
25145
- reason: `is ${check.state} answered ${answer || "nothing"}`
25084
+ reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
25146
25085
  };
25147
25086
  }
25148
- const SELECTOR_POLL_INTERVAL_MS = 500;
25149
- /** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
25150
- function runPollCheck(check, sessionName) {
25151
- const present = (selector) => {
25152
- const r = spawnAB([
25087
+ /**
25088
+ * Replay actions in order against a live session and stop at the first that
25089
+ * fails. Unlike `validateActions` there is no cascade or rescue: the caller
25090
+ * hands what is left to someone who can look at the page. A wait this cannot
25091
+ * reproduce still settles the page, since the next action may depend on it;
25092
+ * a check it cannot run is returned as `unchecked`, never as passed.
25093
+ */
25094
+ function replayUntilFailure(actions, opts) {
25095
+ const patience = { spent: false };
25096
+ const passed = [];
25097
+ const unchecked = [];
25098
+ for (const action of actions) {
25099
+ const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25100
+ if (outcome.skipped && action.action === "wait") spawnAB([
25153
25101
  "--session",
25154
- sessionName,
25155
- "get",
25156
- "count",
25157
- selector
25102
+ opts.sessionName,
25103
+ "wait",
25104
+ "--load",
25105
+ "networkidle"
25158
25106
  ]);
25159
- return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
25160
- };
25161
- const deadline = Date.now() + check.timeoutMs;
25162
- const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
25163
- let count = NaN;
25164
- for (let round = 0;; round++) {
25165
- count = present(check.selector);
25166
- if (!Number.isNaN(count) && count > 0) return {
25167
- ok: true,
25168
- reason: ""
25107
+ else if (outcome.skipped && action.action === "assert") {
25108
+ unchecked.push(action);
25109
+ continue;
25110
+ } else if (outcome.skipped) return {
25111
+ passed,
25112
+ unchecked,
25113
+ failed: {
25114
+ action,
25115
+ reason: "the recording does not say how to replay it"
25116
+ }
25169
25117
  };
25170
- if (round + 1 >= rounds || Date.now() >= deadline) break;
25171
- sleepSync(SELECTOR_POLL_INTERVAL_MS);
25118
+ else if (!outcome.ok) return {
25119
+ passed,
25120
+ unchecked,
25121
+ failed: {
25122
+ action,
25123
+ reason: outcome.reason
25124
+ }
25125
+ };
25126
+ passed.push(action);
25172
25127
  }
25173
25128
  return {
25174
- ok: false,
25175
- reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
25129
+ passed,
25130
+ unchecked
25176
25131
  };
25177
25132
  }
25178
- /** What the validator records against an action it never replayed. */
25179
- const CASCADE_REASON = "skipped after a preceding action failed";
25180
- /** Whether a drop reason means "not replayed", rather than "replayed and failed". */
25181
- function isCascadeReason(reason) {
25182
- return reason === CASCADE_REASON;
25183
- }
25184
- const SHORT_TIMEOUT_MS = 5e3;
25185
- const ASSERT_TIMEOUT_MS = 1e4;
25186
- const INTERACTION_POLL_INTERVAL_MS = 250;
25187
- const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
25188
- /** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
25189
- function attemptsFor(timeoutMs, intervalMs) {
25190
- return Math.max(1, Math.ceil(timeoutMs / intervalMs));
25133
+ function validateActions(actions, opts) {
25134
+ const kept = [];
25135
+ const dropped = [];
25136
+ const promoted = [];
25137
+ const patience = { spent: false };
25138
+ let skipFromStepId = null;
25139
+ for (let i = 0; i < actions.length; i++) {
25140
+ const action = actions[i];
25141
+ opts.onProgress?.(i, actions.length, action);
25142
+ const stepId = action.stepId ?? NO_STEP_ID;
25143
+ if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
25144
+ if (skipFromStepId !== null && isPassiveAction(action.action)) {
25145
+ dropped.push({
25146
+ index: i,
25147
+ action,
25148
+ reason: CASCADE_REASON
25149
+ });
25150
+ continue;
25151
+ }
25152
+ const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25153
+ if (outcome.promoted) promoted.push(outcome.promoted);
25154
+ if (outcome.skipped) {
25155
+ kept.push(action);
25156
+ continue;
25157
+ }
25158
+ if (outcome.ok) {
25159
+ kept.push(action);
25160
+ if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
25161
+ continue;
25162
+ }
25163
+ dropped.push({
25164
+ index: i,
25165
+ action,
25166
+ reason: outcome.reason
25167
+ });
25168
+ if (!isPassiveAction(action.action)) skipFromStepId = stepId;
25169
+ }
25170
+ const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
25171
+ promoted.push(...afterRescue.promoted ?? []);
25172
+ return {
25173
+ ...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
25174
+ promoted
25175
+ };
25191
25176
  }
25192
25177
  /**
25193
- * Convert one recorded action into the `agent-browser` arg list that would
25194
- * exercise it. Interaction commands come from the shared
25195
- * `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
25196
- * get validation-specific handling (poll checks, timeouts, unverifiable
25197
- * skips). Returns `null` for actions that should not be validated (snapshot
25198
- * has no side effect; assert types whose codegen forms aren't directly
25199
- * verifiable here fall through to the caller's `unverifiable` fallback).
25178
+ * Translate the internal `{ kept, dropped }` result of the rescue pass
25179
+ * into the public-facing shape. In strict mode the caller sees the same
25180
+ * shape as before (kept/dropped); in lenient mode the still-failed
25181
+ * actions move to `unstable` with `replayUnstable: true` tagged on, so
25182
+ * codegen can warn about them while still emitting the line.
25200
25183
  */
25201
- function actionToAbArgs(action, sessionName, envOverrides = {}) {
25202
- const base = ["--session", sessionName];
25203
- const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
25204
- switch (action.action) {
25205
- case "snapshot": return null;
25206
- case "assert": return assertToAbArgs(action, sub, sessionName);
25207
- case "wait": {
25208
- const loc = action.locator;
25209
- if (!loc) return null;
25210
- if (loc.by === "text") return [
25211
- ...base,
25212
- "wait",
25213
- "--text",
25214
- sub(loc.value),
25215
- "--timeout",
25216
- String(SHORT_TIMEOUT_MS)
25217
- ];
25218
- const raw = sub(locatorToSelector(loc));
25219
- if (!raw) return null;
25220
- if (/^\d+$/.test(raw)) return null;
25221
- if (raw.startsWith("--")) return null;
25222
- if (raw.startsWith("text=")) return [
25223
- ...base,
25224
- "wait",
25225
- "--text",
25226
- raw.slice(5),
25227
- "--timeout",
25228
- String(SHORT_TIMEOUT_MS)
25229
- ];
25230
- return {
25231
- kind: "poll-present",
25232
- selector: raw,
25233
- timeoutMs: SHORT_TIMEOUT_MS
25234
- };
25184
+ function splitByMode(originalActions, result, mode) {
25185
+ if (mode === "strict") return {
25186
+ kept: result.kept,
25187
+ unstable: [],
25188
+ dropped: result.dropped,
25189
+ rescuedSteps: result.rescuedSteps
25190
+ };
25191
+ const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
25192
+ const keptSet = new Set(result.kept);
25193
+ const finalKept = [];
25194
+ const unstable = [];
25195
+ for (let i = 0; i < originalActions.length; i++) {
25196
+ const action = originalActions[i];
25197
+ if (keptSet.has(action)) {
25198
+ finalKept.push(action);
25199
+ continue;
25235
25200
  }
25236
- default: {
25237
- const tokens = toAgentBrowserArgs(action);
25238
- if (tokens === null) return null;
25239
- const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
25240
- return [...base, ...tokens.map((t) => resolve(t.text))];
25201
+ const drop = droppedByIndex.get(i);
25202
+ if (drop) {
25203
+ action.replayUnstable = true;
25204
+ action.replayReason = drop.reason;
25205
+ unstable.push(action);
25206
+ }
25207
+ }
25208
+ return {
25209
+ kept: finalKept,
25210
+ unstable,
25211
+ dropped: [],
25212
+ rescuedSteps: result.rescuedSteps
25213
+ };
25214
+ }
25215
+ function rescueLostSteps(actions, kept, dropped, opts, patience) {
25216
+ const stepsWithSurvivors = /* @__PURE__ */ new Set();
25217
+ for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
25218
+ const lostStepDrops = /* @__PURE__ */ new Map();
25219
+ for (const d of dropped) {
25220
+ const id = d.action.stepId;
25221
+ if (!id || stepsWithSurvivors.has(id)) continue;
25222
+ const list = lostStepDrops.get(id) ?? [];
25223
+ list.push(d);
25224
+ lostStepDrops.set(id, list);
25225
+ }
25226
+ if (lostStepDrops.size === 0) return {
25227
+ kept,
25228
+ dropped
25229
+ };
25230
+ const promoted = [];
25231
+ const rescuedIndices = /* @__PURE__ */ new Set();
25232
+ const rescuedSteps = [];
25233
+ const strict = (opts.mode ?? "lenient") === "strict";
25234
+ for (const [stepId, drops] of lostStepDrops.entries()) {
25235
+ const replayed = [];
25236
+ const unverifiable = [];
25237
+ let stillFailing = false;
25238
+ for (const d of drops) {
25239
+ const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
25240
+ if (outcome.promoted) promoted.push(outcome.promoted);
25241
+ if (outcome.skipped) {
25242
+ unverifiable.push(d.index);
25243
+ continue;
25244
+ }
25245
+ if (outcome.ok) {
25246
+ replayed.push(d.index);
25247
+ continue;
25248
+ }
25249
+ stillFailing = true;
25250
+ if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
25241
25251
  }
25252
+ if (replayed.length === 0) continue;
25253
+ for (const i of replayed) rescuedIndices.add(i);
25254
+ if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
25255
+ rescuedSteps.push(stepId);
25256
+ }
25257
+ if (rescuedIndices.size === 0) return {
25258
+ kept,
25259
+ dropped,
25260
+ promoted
25261
+ };
25262
+ const keptSet = new Set(kept);
25263
+ const newKept = [];
25264
+ for (let i = 0; i < actions.length; i++) {
25265
+ const action = actions[i];
25266
+ if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
25242
25267
  }
25243
- }
25244
- function roleProbe(base, role, name) {
25245
- return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
25246
- }
25247
- /**
25248
- * Presence asked the way the locator addresses the element. `null` is
25249
- * unverifiable rather than absent: a count from a selector the engine never
25250
- * parsed is not evidence.
25251
- */
25252
- function presenceCheck(locator, selector, sub, base) {
25253
- if (locator?.by === "text") return [
25254
- ...base,
25255
- "wait",
25256
- "--text",
25257
- sub(locator.value),
25258
- "--timeout",
25259
- String(ASSERT_TIMEOUT_MS)
25260
- ];
25261
- if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
25262
- if (locator !== void 0 && locator.by !== "css") return null;
25263
- const named = parseNotation(selector);
25264
- if (named === "opaque") return null;
25265
- if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
25266
- if (named !== null) return null;
25267
25268
  return {
25268
- kind: "poll-present",
25269
- selector,
25270
- timeoutMs: ASSERT_TIMEOUT_MS
25269
+ kept: newKept,
25270
+ dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
25271
+ rescuedSteps,
25272
+ promoted
25271
25273
  };
25272
25274
  }
25273
- function assertToAbArgs(action, sub, sessionName) {
25274
- const base = ["--session", sessionName];
25275
- const val = sub(action.value ?? action.observation);
25276
- const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
25277
- switch (action.assert) {
25278
- case "text_visible":
25279
- if (!val) return null;
25280
- return [
25281
- ...base,
25282
- "wait",
25283
- "--text",
25284
- val,
25285
- "--timeout",
25286
- String(ASSERT_TIMEOUT_MS)
25287
- ];
25288
- case "text_not_visible": return null;
25289
- case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
25290
- case "element_not_visible": return null;
25291
- case "url_contains": return null;
25292
- case "element_enabled":
25293
- case "element_disabled":
25294
- case "element_checked":
25295
- case "element_unchecked":
25296
- if (!sel || action.locator?.by !== "css") return null;
25297
- if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
25298
- return {
25299
- kind: "state",
25300
- state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
25301
- selector: sel,
25302
- expected: action.assert === "element_enabled" || action.assert === "element_checked"
25303
- };
25304
- default: return null;
25305
- }
25275
+ /**
25276
+ * Passive (read-only) actions whose only effect is observation. When a
25277
+ * preceding action fails, dropping these too is the right move because
25278
+ * they were trying to observe state the failed action would have set up.
25279
+ */
25280
+ function isPassiveAction(action) {
25281
+ return action === "snapshot" || action === "wait" || action === "assert";
25306
25282
  }
25307
- const NO_STEP_ID = "__no_step__";
25283
+ //#endregion
25284
+ //#region src/prompts/previous-recording.ts
25308
25285
  /**
25309
- * The role a labelled element has when addressing it by label finds nothing.
25310
- *
25311
- * `getByLabel` needs a real association — a `<label for>`, a wrapping label,
25312
- * an `aria-labelledby`. A form that has none still shows the same string as
25313
- * the field's accessible name, which is what the recorder read off the
25314
- * snapshot, so the label it recorded matches nothing while the same text as a
25315
- * role's name matches exactly. Only the actions whose element has one
25316
- * unambiguous role are listed: a labelled `click` may be a button, a link or
25317
- * a checkbox, and guessing there would replace a locator that failed with one
25318
- * that finds the wrong element.
25286
+ * The previous recording as the commands that perform it, for the trace prompt:
25287
+ * a re-record is usually one moved screen in an otherwise intact route.
25319
25288
  */
25320
- const LABEL_FALLBACK_ROLE = {
25321
- fill: "textbox",
25322
- type: "textbox",
25323
- check: "checkbox",
25324
- uncheck: "checkbox"
25325
- };
25326
- /** Whether agent-browser's failure was "no element", the one the fallback answers. */
25327
- function notFound(result) {
25328
- return /not\s+found|no\s+element|no\s+such\s+element/i.test(`${result.stderr} ${result.stdout}`);
25289
+ function renderPreviousRecording(actions, sessionName, currentStepIds) {
25290
+ const current = new Set([SETUP_STEP_ID, ...currentStepIds]);
25291
+ const brokenStep = actions.find(failedToReplay)?.stepId;
25292
+ const byStep = /* @__PURE__ */ new Map();
25293
+ for (const action of actions) {
25294
+ if (action.stepId === void 0 || !current.has(action.stepId)) continue;
25295
+ const command = previousCommand(action, sessionName, action.stepId === brokenStep);
25296
+ if (command === null) continue;
25297
+ const lines = byStep.get(action.stepId) ?? [];
25298
+ lines.push(command);
25299
+ byStep.set(action.stepId, lines);
25300
+ }
25301
+ if (byStep.size === 0) return "";
25302
+ return `
25303
+ ## Previous recording
25304
+
25305
+ This case was recorded before. Below is what that recording did, step by step,
25306
+ as the commands that perform it. The application may have moved since, and the
25307
+ steps above may have been edited: **the steps above are the contract, this is a
25308
+ map.**
25309
+
25310
+ For a step listed here, this replaces the snapshot-first routine of the
25311
+ Execution Workflow:
25312
+
25313
+ 1. While a step's previous commands still do what its instruction and
25314
+ \`Expected\` say, call the \`replay_step\` tool with the step id instead of
25315
+ running them yourself. It runs and records them in this session, checks
25316
+ included, and stops at the first that fails. Do not snapshot first.
25317
+ 2. When it reports they all ran, emit \`STEP_DONE\` and go to the next step
25318
+ without a confirming snapshot.
25319
+ 3. When it reports a failure, the commands before it are recorded and the
25320
+ page is where they left it: finish that step with the Execution Workflow —
25321
+ snapshot, find what works, record it — without re-running what was
25322
+ recorded. A step whose instruction or \`Expected\` asks for something these
25323
+ commands do not do skips \`replay_step\` and uses the Execution Workflow
25324
+ from the start. The next step goes back to \`replay_step\`.
25325
+ 4. A step you record yourself takes its assertions from its current
25326
+ \`Expected\`: keep a previous check only while it verifies what
25327
+ \`Expected\` says now. A step whose \`Expected\` has changed is one to
25328
+ record yourself, since \`replay_step\` records the previous checks as
25329
+ they are.
25330
+ 5. A command marked \`# did not replay last time\` is where the previous
25331
+ recording's own validation found the route broken: expect that step to
25332
+ need the Execution Workflow, and do not keep a form that only half works.
25333
+
25334
+ A step with no section here is recorded the usual way.
25335
+
25336
+ ${[...byStep].map(([stepId, lines]) => `### ${stepId}\n\n\`\`\`bash\n${lines.join("\n")}\n\`\`\``).join("\n\n")}
25337
+ `;
25329
25338
  }
25330
- /** Whether this argv navigates — the only action safe to repeat wholesale. */
25331
- function isOpen(argv) {
25332
- return argv[2] === "open";
25339
+ function previousCommand(action, sessionName, inBrokenStep) {
25340
+ if (action.action === "wait" && action.locator?.by !== "text") return null;
25341
+ const marked = action.action === "assert" ? markedAssertArgs(action) : null;
25342
+ const tokens = marked?.tokens ?? (action.action === "assert" ? null : toAgentBrowserArgs(action));
25343
+ if (tokens === null) return null;
25344
+ const line = [
25345
+ `CCQA_STEP=${action.stepId}`,
25346
+ ...marked ? [`CCQA_ASSERT=${shellWord(marked.marker)}`] : [],
25347
+ ...action.secret ? ["CCQA_SECRET=1"] : [],
25348
+ "agent-browser",
25349
+ "--session",
25350
+ sessionName,
25351
+ renderTokens(tokens)
25352
+ ].join(" ");
25353
+ return inBrokenStep && failedToReplay(action) ? `${line} # did not replay last time` : line;
25333
25354
  }
25334
- /** A navigation another navigation took away, rather than one that failed. */
25335
- function superseded(result) {
25336
- return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
25355
+ /** Failed on its own, rather than skipped behind an earlier failure. */
25356
+ function failedToReplay(action) {
25357
+ return action.replayUnstable === true && !isCascadeReason(action.replayReason);
25337
25358
  }
25338
- /** The one line both callers log when the fallback rewrote a locator. */
25339
- function formatPromotion(promotion) {
25340
- return `locator rewritten to the form that replays: ${promotion}`;
25359
+ function renderTokens(tokens) {
25360
+ return tokens.map((t) => t.expandsEnv ? quote(t.text) : t.text).join(" ");
25341
25361
  }
25342
- /** The same action addressed by role + accessible name, or null when there is no such form. */
25343
- function labelFallback(action) {
25344
- const loc = action.locator;
25345
- if (!loc || loc.by !== "label") return null;
25346
- const role = LABEL_FALLBACK_ROLE[action.action];
25347
- if (role === void 0) return null;
25348
- return {
25349
- ...action,
25350
- locator: {
25351
- by: "role",
25352
- value: role,
25353
- name: loc.value,
25354
- exact: true
25355
- }
25356
- };
25362
+ /** Double-quoted with `${VAR}` refs left live, as the recording keeps them; any other `$` is literal text. */
25363
+ function quote(s) {
25364
+ return `"${s.replace(/(["\\`])|\$(?!\{[A-Za-z_]\w*\})/g, (m, c) => `\\${c ?? m}`)}"`;
25357
25365
  }
25358
- /**
25359
- * Spawn one interaction, waiting for its element the way an assertion waits
25360
- * for its text: a recording pauses for a snapshot between opening a page and
25361
- * typing into it, the replay does not, so the replay reaches a field sooner
25362
- * than the recording ever did.
25363
- *
25364
- * Only a "no element" failure waits — anything else repeats identically. When
25365
- * there is an `alternate` form it is tried on every attempt rather than after
25366
- * the wait: a label the page never associates with its input will not start
25367
- * matching, so waiting first spends the budget for nothing.
25368
- */
25369
- function runInteraction(argv, alternate, patience, sessionName) {
25370
- const started = Date.now();
25371
- const deadline = started + ASSERT_TIMEOUT_MS;
25372
- let settled = false;
25373
- for (let attempt = 0;; attempt++) {
25374
- let result = spawnAB(argv);
25375
- if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
25376
- const waitedMs = Date.now() - started;
25377
- if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
25378
- settled = true;
25379
- spawnAB([
25380
- "--session",
25381
- sessionName,
25382
- "wait",
25383
- "--load",
25384
- "networkidle"
25385
- ]);
25386
- continue;
25387
- }
25388
- if (result.status === 0 || !notFound(result)) return {
25389
- result,
25390
- viaAlternate: false,
25391
- waitedMs
25392
- };
25393
- if (alternate !== null) {
25394
- const alt = spawnAB(alternate);
25395
- if (alt.status === 0) return {
25396
- result: alt,
25397
- viaAlternate: true,
25398
- waitedMs
25399
- };
25400
- }
25401
- if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
25402
- patience.spent = true;
25403
- return {
25404
- result,
25405
- viaAlternate: false,
25406
- waitedMs: Date.now() - started
25407
- };
25408
- }
25409
- sleepSync(INTERACTION_POLL_INTERVAL_MS);
25410
- }
25366
+ function shellWord(s) {
25367
+ return /^[\w:/.,=-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
25368
+ }
25369
+ //#endregion
25370
+ //#region src/prompts/trace.ts
25371
+ function generateSessionName() {
25372
+ return `ccqa-trace-${buildRunId()}`;
25411
25373
  }
25412
25374
  /**
25413
- * Say what a `by: "css"` locator that is not CSS actually addresses.
25375
+ * Build the trace system prompt. `input.steps` is a flat list with includes
25376
+ * already expanded (each step carries id / source / instruction / expected).
25377
+ * The spec opens URLs via explicit step instructions (e.g.
25378
+ * `instruction: "${APP_URL}/articles を開く"`).
25414
25379
  *
25415
- * `get count` answers 0 for Playwright's notation instead of failing, so a
25416
- * route holding one reads as dead. Rewritten in place, and reported, so the
25417
- * saved route carries the form that replays rather than the one that counts
25418
- * nothing. Notation this cannot convert leaves the locator alone; the assert
25419
- * path then treats it as unverifiable rather than absent.
25380
+ * In v0.4 every spec is traced from scratch — block contents are inlined
25381
+ * into the spec's own step list at expand time, so the prompt has no
25382
+ * special "this is a block" mode. The `source` tag on each step still
25383
+ * distinguishes spec-native steps from inlined block steps for the
25384
+ * `// step:` comments in the eventual codegen output.
25420
25385
  */
25421
- function nameNotation(action) {
25422
- const loc = action.locator;
25423
- if (action.action !== "assert" || loc?.by !== "css") return null;
25424
- const named = parseNotation(loc.value);
25425
- if (named === null || named === "opaque" || named.by !== "text") return null;
25426
- action.locator = named;
25427
- return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
25386
+ function buildTraceSystemPrompt(input) {
25387
+ const sessionName = input.sessionName ?? generateSessionName();
25388
+ const firstStepId = input.steps[0]?.id ?? "step-01";
25389
+ const callerGuidance = input.instruction ? `## Caller Guidance
25390
+
25391
+ The caller provided extra guidance for this recording — for example, a drift
25392
+ audit's finding about what a previous recording of this spec got wrong. Treat
25393
+ it as advice on what to avoid or verify while recording; the spec's steps
25394
+ above remain the contract for what to do.
25395
+
25396
+ ${input.instruction}
25397
+
25398
+ ` : "";
25399
+ const stepsText = input.steps.map((step) => [
25400
+ `### ${step.id} [${step.source}]`,
25401
+ `- **Instruction**: ${step.instruction}`,
25402
+ ...step.expected ? [`- **Expected**: ${step.expected}`] : []
25403
+ ].join("\n")).join("\n\n");
25404
+ const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
25405
+ const contextText = renderContext(input.context ?? []);
25406
+ const conventionsText = renderConventions(input.conventions ?? []);
25407
+ const previousText = renderPreviousRecording(input.previousRecording ?? [], sessionName, input.steps.map((s) => s.id));
25408
+ return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
25409
+
25410
+ ## Session
25411
+
25412
+ SESSION NAME: \`${sessionName}\`
25413
+
25414
+ Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
25415
+
25416
+ **Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
25417
+ env assignment naming the spec step it executes**, e.g.
25418
+
25419
+ \`\`\`bash
25420
+ CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
25421
+ \`\`\`
25422
+
25423
+ This prefix is what ties each recorded action to its step in the generated
25424
+ test — a command without it loses its step attribution.
25425
+
25426
+ **Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
25427
+ in, opening the application, putting the account where the precondition says it
25428
+ starts — those are not step-01, and marking them as step-01 makes the first
25429
+ step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
25430
+ first step the case actually lists.
25431
+
25432
+ Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
25433
+ itself ignores the variable.
25434
+
25435
+ Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
25436
+ assignment (alongside \`CCQA_STEP\`) that records the check as a test
25437
+ assertion — see "Assertion Protocol".
25438
+
25439
+ **A command that types into a password field carries \`CCQA_SECRET=1\`** as
25440
+ well, e.g.
25441
+
25442
+ \`\`\`bash
25443
+ CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
25444
+ \`\`\`
25445
+
25446
+ A password field is an \`<input type="password">\` — the snapshot shows its
25447
+ value as dots, and nothing else about the page is a password. The prefix is
25448
+ what stops the typed value from being written to the recording: a value that
25449
+ came from a \`\${VAR}\` is kept as that reference, and one typed literally is
25450
+ dropped along with its command.
25451
+
25452
+ ## Browser Commands
25453
+
25454
+ \`\`\`
25455
+ CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
25456
+ CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -i -c # interactive elements only — the default for finding controls
25457
+ CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -c -s "<css>" # the page's text, scoped to one region
25458
+ CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
25459
+ CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
25460
+ CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
25461
+ CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
25462
+ CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
25463
+ CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
25464
+ CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
25465
+ CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
25466
+ CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
25467
+ CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
25468
+ agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
25469
+ CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
25470
+ CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
25471
+ # See "Selector Rules" for the full \`find\` subset.
25472
+ # IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
25473
+ # CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
25474
+ # Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
25475
+ # (which waits internally). To check an element exists, use \`get count "<selector>"\`.
25476
+ \`\`\`
25477
+
25478
+ ## Selector Rules
25479
+
25480
+ **ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
25481
+
25482
+ | Format | Use when |
25483
+ |--------|----------|
25484
+ | \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
25485
+ | \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
25486
+ | \`[placeholder='text']\` | Input identified by placeholder |
25487
+ | \`[type='password']\` | Password inputs only |
25488
+ | \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
25489
+ | \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
25490
+
25491
+ **FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
25492
+
25493
+ - \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
25494
+ - **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
25495
+ - \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
25496
+ - **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
25497
+ - JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
25498
+
25499
+ ### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
25500
+
25501
+ When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
25502
+
25503
+ \`\`\`
25504
+ find role <role> <action> [--name "<n>"] [--exact]
25505
+ find text|label|placeholder|alt|title "<text>" <action> [--exact]
25506
+ find testid "<id>" <action>
25507
+ find first|last "<ALLOWED-css>" <action>
25508
+ find nth <index> "<ALLOWED-css>" <action>
25509
+ \`\`\`
25510
+
25511
+ \`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
25512
+
25513
+ **Rules for \`find\`:**
25514
+
25515
+ 1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
25516
+ 2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
25517
+ 3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
25518
+ 4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
25519
+ 5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
25520
+ 6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
25521
+ 7. \`find\` includes its own wait; do not chain a \`wait\` before it.
25522
+ 8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
25523
+ 9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
25524
+
25525
+ **Examples:**
25526
+
25527
+ - ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
25528
+ - ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
25529
+ - ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
25530
+ - ✗ \`find role button --name "Submit" click\` — wrong order
25531
+ - ✗ \`find last "button" click\` — bare tag
25532
+
25533
+ ### Selector workflow
25534
+
25535
+ 1. Run \`snapshot\` and read the ARIA tree.
25536
+ 2. Identify the element; note its exact \`aria-label\` if present.
25537
+ 3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
25538
+ For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
25539
+ 4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
25540
+ 5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
25541
+ 6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
25542
+ 7. Never guess. If a selector fails, take a fresh snapshot before retrying.
25543
+ 8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
25544
+
25545
+ ### Special input types
25546
+
25547
+ **contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
25548
+
25549
+ **combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
25550
+
25551
+ **Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
25552
+
25553
+ **Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
25554
+
25555
+ **File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
25556
+
25557
+ ## Test Specification
25558
+
25559
+ Title: ${input.title}
25560
+
25561
+ Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
25562
+
25563
+ ## Steps
25564
+
25565
+ ${stepsText}
25566
+ ${expectationsText}${contextText}${conventionsText}${previousText}
25567
+ ## Execution Workflow
25568
+
25569
+ For each step:
25570
+ 1. Emit \`STEP_START|<step-id>|<short description>\`.
25571
+ 2. Run \`snapshot -i -c\` and identify selectors from the ARIA tree. A full \`snapshot\` is large; when you need the page's text, scope it with \`snapshot -c -s "<css>"\`.
25572
+ 3. Execute the step's actions using ALLOWED selectors (see Selector Rules), prefixing each command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step. **Batch them:** put every command you can already write — the actions, their waits, the \`CCQA_ASSERT\` checks, and a closing \`snapshot -i -c\` when you need to see the result — into one \`run_commands\` call. Each turn costs far more than a command; use a single Bash call only when you must see one answer before you can write the next command.
25573
+ 4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
25574
+ 5. Verify the outcome from the batch's output. When a batch stops at a failure, everything before it is recorded: look at the page and continue from there.
25575
+ 6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
25576
+ 7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
25577
+ 8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
25578
+
25579
+ **Protocol lines are recorded ONLY from your plain assistant text.** Every
25580
+ \`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
25581
+ message text between tool calls — never through Bash (\`echo\` etc.) and never
25582
+ omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
25583
+ prefix on each command (that prefix is authoritative, not the \`STEP_START\`
25584
+ line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
25585
+ verification commands** — a command that ran, whose failure rolls the record
25586
+ back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
25587
+ proves it; it is reported and not recorded.
25588
+
25589
+ **After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
25590
+
25591
+ ## Guardrails
25592
+
25593
+ - **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
25594
+ - **Stay on a flow you have started.** Once a step is inside something whose state lives on the page — a conversation, a multi-step dialog, a form not yet saved — do not navigate away to investigate; that state does not come back. Look things up in the source with Read, Grep and Glob instead.
25595
+ - **Clean up before you stop.** When you end the run with \`ASSERTION_FAILED\` after the case has created something, still perform its cleanup steps, then emit \`RUN_COMPLETED|failed\`. What a failed recording leaves behind breaks the next run of every case that lists the same things.
25596
+ - **The step must match the application.** When what a step asks for is not there, or the application does it in another place or order, do not bend the recording to fit: emit \`ASSERTION_FAILED|<step-id>|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>\`. Read the source to ground the suggestion. A recording that works around the step hides the mismatch; the report gets the spec fixed. Redoing an earlier step's work to make this one possible — going back to refill or resubmit what an earlier step already did — is such a workaround.
25597
+ - **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
25598
+ - Do NOT retry a selector without taking a fresh snapshot first.
25599
+ - Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
25600
+ - **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
25601
+ - **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
25602
+
25603
+ ## Source Code Reference
25604
+
25605
+ You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
25606
+
25607
+ **When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
25608
+
25609
+ **How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
25610
+
25611
+ **Rules**: only READ source files, never modify them. Keep searches focused.
25612
+
25613
+ ## Waiting for Async Operations
25614
+
25615
+ Prefer \`wait\` over polling:
25616
+
25617
+ \`\`\`bash
25618
+ CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
25619
+ \`\`\`
25620
+
25621
+ If polling is required (e.g. waiting for a spinner to disappear):
25622
+
25623
+ \`\`\`bash
25624
+ for i in $(seq 1 18); do
25625
+ sleep 10
25626
+ result=$(agent-browser --session ${sessionName} snapshot 2>&1)
25627
+ echo "$result" | grep -q "<done indicator>" && break
25628
+ done
25629
+ CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
25630
+ \`\`\`
25631
+
25632
+ After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
25633
+
25634
+ ## AB_ACTION Protocol
25635
+
25636
+ After **every** browser action, emit one line (outside any code block):
25637
+
25638
+ \`\`\`
25639
+ AB_ACTION|cookies_clear
25640
+ AB_ACTION|open|<url>
25641
+ AB_ACTION|click|<selector>|<visible label>
25642
+ AB_ACTION|dblclick|<selector>|<visible label>
25643
+ AB_ACTION|fill|<selector>|<value>|<aria label>
25644
+ AB_ACTION|check|<selector>|<visible label>
25645
+ AB_ACTION|uncheck|<selector>|<visible label>
25646
+ AB_ACTION|press|<Key>
25647
+ AB_ACTION|select|<selector>|<value>|<aria label>
25648
+ AB_ACTION|hover|<selector>|<visible label>
25649
+ AB_ACTION|scroll|<direction>|<pixels>
25650
+ AB_ACTION|drag|<source selector>|<target selector>|<source label>
25651
+ AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
25652
+ AB_ACTION|wait|<selector or text>|<label>
25653
+ AB_ACTION|snapshot|<key observation, max 100 chars>
25654
+
25655
+ # find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
25656
+ # <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
25657
+ AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
25658
+ AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
25659
+ AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
25660
+ AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
25661
+ AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
25662
+ AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
25663
+ AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
25664
+ AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
25665
+ \`\`\`
25666
+
25667
+ Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
25668
+
25669
+ **CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
25670
+
25671
+ - A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
25672
+ - If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
25673
+ - An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
25674
+ - **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
25675
+
25676
+ - **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
25677
+ - If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
25678
+
25679
+ ## Assertion Protocol
25680
+
25681
+ Record each signal you verified as an assertion. The **primary mechanism** is
25682
+ a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
25683
+ marker rides the same channel as the command, so the assertion is captured
25684
+ even if you print no protocol text. The check you were going to run anyway
25685
+ becomes the recorded assertion:
25686
+
25687
+ \`\`\`bash
25688
+ # text_visible — the marked wait IS the assertion (no separate assert line)
25689
+ CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
25690
+ # element_visible / element_not_visible — mark the get-count probe
25691
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
25692
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
25693
+ # url_contains — the substring rides in the marker; put it on \`get url\`
25694
+ CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
25695
+ # element state — mark the \`is\` probe that asks for it
25696
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
25697
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
25698
+ \`\`\`
25699
+
25700
+ **Marker semantics:**
25701
+
25702
+ | Marker | On command | Records |
25703
+ |--------|------------|---------|
25704
+ | \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
25705
+ | \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
25706
+ | \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
25707
+ | \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
25708
+ | \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
25709
+ | \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
25710
+
25711
+ - A marked command that exits non-zero records nothing — fix the check and re-run it.
25712
+ - A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
25713
+ - \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
25714
+ - **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
25715
+
25716
+ - **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
25717
+
25718
+ - **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
25719
+
25720
+ - **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
25721
+ - **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
25722
+ - **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
25723
+
25724
+ **Every assert type has a marker command.** There is no text form: a printed
25725
+ \`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
25726
+ reported rather than recorded.
25727
+
25728
+ | assertType | The command that records it | Use when |
25729
+ |------------|------------------------------|----------|
25730
+ | \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
25731
+ | \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
25732
+ | \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
25733
+ | \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
25734
+ | \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
25735
+ | \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
25736
+
25737
+ **Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
25738
+
25739
+ - Timestamps, session IDs, exact numeric counts that vary between runs.
25740
+ - **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
25741
+ - **Relative-time labels** — true only in the moment of the trace, stale by replay:
25742
+ - English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
25743
+ - Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
25744
+ - Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
25745
+ - **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
25746
+
25747
+ **No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
25748
+
25749
+ The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
25750
+
25751
+ - ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
25752
+ - ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
25753
+ - ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
25754
+ - If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
25755
+
25756
+ **Page-context and selector rules:**
25757
+
25758
+ - After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
25759
+ - Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
25760
+ - When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
25761
+
25762
+ **MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
25763
+
25764
+ The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
25765
+
25766
+ 1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
25767
+ 2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
25768
+ 3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
25769
+
25770
+ **Verify the assertion form actually resolves on the live page.** The
25771
+ verification and the recorded assertion are the same marked command:
25772
+
25773
+ \`\`\`bash
25774
+ # text_visible
25775
+ CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
25776
+ # element_visible — >=1 printed means present
25777
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
25778
+ # element_not_visible — 0 printed means absent (works for text too: "text=<text>")
25779
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
25780
+ # element state — \`is\` prints true/false and exits 0 either way; read it
25781
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
25782
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
25783
+ \`\`\`
25784
+
25785
+ When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
25786
+
25787
+ ## Status Protocol
25788
+
25789
+ Emit exactly one status line per step (outside any code block):
25790
+
25791
+ \`\`\`
25792
+ STEP_START|<step-id>|<short description>
25793
+ STEP_DONE|<step-id>|<what was verified>
25794
+ ASSERTION_FAILED|<step-id>|<category: spec-mismatch|app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
25795
+ STEP_SKIPPED|<step-id>|<reason>
25796
+ RUN_COMPLETED|passed|<summary>
25797
+ RUN_COMPLETED|failed|<summary>
25798
+ \`\`\`
25799
+
25800
+ **\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
25801
+ recorded as FAILED, however many steps went well: nothing else can tell a
25802
+ finished route from one you abandoned halfway.
25803
+
25804
+ ${callerGuidance}## Start
25805
+
25806
+ Begin by clearing cookies, then proceed straight to the first step's instruction.
25807
+
25808
+ \`\`\`bash
25809
+ agent-browser --session ${sessionName} cookies clear
25810
+ \`\`\`
25811
+
25812
+ (The cookies-clear preamble belongs to no step, so it is the one command
25813
+ without a \`CCQA_STEP\` prefix.)
25814
+
25815
+ Emit:
25816
+ \`\`\`
25817
+ AB_ACTION|cookies_clear
25818
+ \`\`\`
25819
+
25820
+ Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
25821
+ every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
25822
+ step is responsible for opening the initial URL.
25823
+ `;
25428
25824
  }
25429
- /**
25430
- * An attribute selector that counted nothing, asked of the accessibility tree
25431
- * the name was read from. See docs/targets.md for why the two disagree.
25432
- *
25433
- * The snapshot is read for the role, then `find role` is asked as well: the
25434
- * tree says the name exists somewhere, the probe says the replay can reach it,
25435
- * and only the second is what the route will depend on.
25436
- */
25437
- function askTheAccessibilityTree(action, sub, sessionName) {
25438
- if (action.action !== "assert" || action.locator?.by !== "css") return null;
25439
- const name = nameFromAttributeSelector(action.locator.value);
25440
- if (name === null) return null;
25441
- const snapshot = spawnAB([
25442
- "--session",
25443
- sessionName,
25444
- "snapshot"
25445
- ]);
25446
- if (snapshot.status !== 0) return null;
25447
- const role = roleOfAccessibleName(snapshot.stdout, sub(name));
25448
- if (role === null) return null;
25449
- if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
25450
- const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
25451
- action.locator = {
25452
- by: "role",
25453
- value: role,
25454
- name,
25455
- exact: true
25456
- };
25457
- return {
25458
- skipped: false,
25459
- ok: true,
25460
- reason: "",
25461
- promoted
25462
- };
25825
+ function buildTracePrompt(title) {
25826
+ return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
25463
25827
  }
25464
25828
  /**
25465
- * Replay one recorded action against the validation session. Element-presence
25466
- * checks go through `runPollCheck` (which uses `get count`, never the blocking
25467
- * `wait <selector>`); everything else spawns the agent-browser argv.
25829
+ * The case's expectations, plus the one rule that decides where each is
25830
+ * recorded. Without the rule a recorder either asserts everything at the end —
25831
+ * on a page the flow has already left — or asserts each item at every step it
25832
+ * might apply to, which is the over-assertion the selector rules exist to
25833
+ * prevent.
25468
25834
  */
25469
- function runValidationAction(action, sessionName, envOverrides = {}, patience) {
25470
- const named = nameNotation(action);
25471
- const withName = (outcome) => named === null || !outcome.ok ? outcome : {
25472
- ...outcome,
25473
- promoted: outcome.promoted ?? named
25474
- };
25475
- const built = actionToAbArgs(action, sessionName, envOverrides);
25476
- if (built === null) return {
25477
- skipped: true,
25478
- ok: false,
25479
- reason: ""
25480
- };
25481
- if (isProbe(built)) {
25482
- const { ok, reason } = runProbe(built, sessionName);
25483
- if (ok) return withName({
25484
- skipped: false,
25485
- ok,
25486
- reason
25487
- });
25488
- const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
25489
- return askTheAccessibilityTree(action, resolve, sessionName) ?? {
25490
- skipped: false,
25491
- ok,
25492
- reason
25493
- };
25494
- }
25495
- const fallback = labelFallback(action);
25496
- const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
25497
- const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
25498
- if (viaAlternate) {
25499
- const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
25500
- action.locator = fallback.locator;
25501
- return {
25502
- skipped: false,
25503
- ok: true,
25504
- reason: "",
25505
- promoted
25506
- };
25507
- }
25508
- if (result.status === 0) return withName({
25509
- skipped: false,
25510
- ok: true,
25511
- reason: ""
25512
- });
25513
- return {
25514
- skipped: false,
25515
- ok: false,
25516
- reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
25517
- };
25518
- }
25519
- function validateActions(actions, opts) {
25520
- const kept = [];
25521
- const dropped = [];
25522
- const promoted = [];
25523
- const patience = { spent: false };
25524
- let skipFromStepId = null;
25525
- for (let i = 0; i < actions.length; i++) {
25526
- const action = actions[i];
25527
- opts.onProgress?.(i, actions.length, action);
25528
- const stepId = action.stepId ?? NO_STEP_ID;
25529
- if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
25530
- if (skipFromStepId !== null && isPassiveAction(action.action)) {
25531
- dropped.push({
25532
- index: i,
25533
- action,
25534
- reason: CASCADE_REASON
25535
- });
25536
- continue;
25537
- }
25538
- const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25539
- if (outcome.promoted) promoted.push(outcome.promoted);
25540
- if (outcome.skipped) {
25541
- kept.push(action);
25542
- continue;
25543
- }
25544
- if (outcome.ok) {
25545
- kept.push(action);
25546
- if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
25547
- continue;
25548
- }
25549
- dropped.push({
25550
- index: i,
25551
- action,
25552
- reason: outcome.reason
25553
- });
25554
- if (!isPassiveAction(action.action)) skipFromStepId = stepId;
25555
- }
25556
- const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
25557
- promoted.push(...afterRescue.promoted ?? []);
25558
- return {
25559
- ...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
25560
- promoted
25561
- };
25835
+ function renderExpectations(expectations) {
25836
+ if (expectations.length === 0) return "";
25837
+ return `
25838
+ ## Expected results
25839
+
25840
+ The case states these without saying which step delivers them:
25841
+
25842
+ ${expectations.map((text) => `- ${text}`).join("\n")}
25843
+
25844
+ **Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
25845
+ `;
25562
25846
  }
25563
25847
  /**
25564
- * Translate the internal `{ kept, dropped }` result of the rescue pass
25565
- * into the public-facing shape. In strict mode the caller sees the same
25566
- * shape as before (kept/dropped); in lenient mode the still-failed
25567
- * actions move to `unstable` with `replayUnstable: true` tagged on, so
25568
- * codegen can warn about them while still emitting the line.
25848
+ * What the cleanup must make true. Separate from the case's expectations
25849
+ * because its steps are: a deletion is confirmed by the thing being gone,
25850
+ * which is only true after the undo has run, and asserting it among the
25851
+ * case's own steps would check it while the thing still exists.
25569
25852
  */
25570
- function splitByMode(originalActions, result, mode) {
25571
- if (mode === "strict") return {
25572
- kept: result.kept,
25573
- unstable: [],
25574
- dropped: result.dropped,
25575
- rescuedSteps: result.rescuedSteps
25576
- };
25577
- const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
25578
- const keptSet = new Set(result.kept);
25579
- const finalKept = [];
25580
- const unstable = [];
25581
- for (let i = 0; i < originalActions.length; i++) {
25582
- const action = originalActions[i];
25583
- if (keptSet.has(action)) {
25584
- finalKept.push(action);
25585
- continue;
25586
- }
25587
- const drop = droppedByIndex.get(i);
25588
- if (drop) {
25589
- action.replayUnstable = true;
25590
- action.replayReason = drop.reason;
25591
- unstable.push(action);
25592
- }
25593
- }
25853
+ function renderCleanupExpectations(expectations) {
25854
+ if (expectations.length === 0) return "";
25855
+ return `
25856
+ ## Expected results of the cleanup
25857
+
25858
+ The case states these about its cleanup steps:
25859
+
25860
+ ${expectations.map((text) => `- ${text}`).join("\n")}
25861
+
25862
+ **Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
25863
+ `;
25864
+ }
25865
+ /**
25866
+ * Sections of the case ccqa does not interpret — the precondition, notes the
25867
+ * author left. Passed through verbatim: they are written for whoever runs the
25868
+ * case, and the recorder is now that reader.
25869
+ */
25870
+ function renderContext(context) {
25871
+ if (context.length === 0) return "";
25872
+ return `
25873
+ ## About this case
25874
+
25875
+ ${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
25876
+ `;
25877
+ }
25878
+ /**
25879
+ * The project's own recording guidance. Where the case says what to do, this
25880
+ * says how this project is driven — which account signs in, what has to be
25881
+ * true before the first step. Prose, verbatim: it is written for whoever runs
25882
+ * the case, and this is now that reader.
25883
+ */
25884
+ function renderConventions(conventions) {
25885
+ if (conventions.length === 0) return "";
25886
+ return `
25887
+ ## How this project is driven
25888
+
25889
+ ${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
25890
+ `;
25891
+ }
25892
+ //#endregion
25893
+ //#region src/cli/trace-tools.ts
25894
+ const TRACE_TOOLS_SERVER = "ccqa";
25895
+ /** A wedged command must not hold the whole trace; agent-browser's own waits are far shorter. */
25896
+ const COMMAND_TIMEOUT_MS = 12e4;
25897
+ /** Enough of a snapshot or a `get` answer to act on, without flooding the turn. */
25898
+ const OUTPUT_CAP = 6e3;
25899
+ /**
25900
+ * The trace's own tools. Both exist to cut round trips: one turn per browser
25901
+ * command is what made a re-record cost hundreds of turns.
25902
+ *
25903
+ * - `run_commands` runs a batch the recorder wrote, recording each that passes.
25904
+ * - `replay_step` runs one step's previous commands.
25905
+ */
25906
+ function buildTraceTools(input) {
25907
+ const offersReplay = input.previous.length > 0;
25594
25908
  return {
25595
- kept: finalKept,
25596
- unstable,
25597
- dropped: [],
25598
- rescuedSteps: result.rescuedSteps
25909
+ server: createSdkMcpServer({
25910
+ name: TRACE_TOOLS_SERVER,
25911
+ version: "1.0.0",
25912
+ tools: [tool("run_commands", "Run several agent-browser commands in order, each written exactly as you would run it in Bash (with its CCQA_STEP / CCQA_ASSERT / CCQA_SECRET prefixes). Each that succeeds is recorded; the batch stops at the first that fails. Returns each command's outcome and the output of the last one.", { commands: z.array(z.string()).min(1).describe("agent-browser command lines, one per entry") }, async ({ commands }) => ({ content: [{
25913
+ type: "text",
25914
+ text: await runCommands(commands, input)
25915
+ }] })), ...offersReplay ? [tool("replay_step", "Run one step's previous-recording commands in this browser session and record those that pass. Stops at the first that fails.", { step: z.string().describe("The step id, e.g. step-03") }, async ({ step }) => ({ content: [{
25916
+ type: "text",
25917
+ text: replayStep(step, input)
25918
+ }] }))] : []]
25919
+ }),
25920
+ allowedTools: ["run_commands", ...offersReplay ? ["replay_step"] : []].map((t) => `mcp__${TRACE_TOOLS_SERVER}__${t}`)
25599
25921
  };
25600
25922
  }
25601
- function rescueLostSteps(actions, kept, dropped, opts, patience) {
25602
- const stepsWithSurvivors = /* @__PURE__ */ new Set();
25603
- for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
25604
- const lostStepDrops = /* @__PURE__ */ new Map();
25605
- for (const d of dropped) {
25606
- const id = d.action.stepId;
25607
- if (!id || stepsWithSurvivors.has(id)) continue;
25608
- const list = lostStepDrops.get(id) ?? [];
25609
- list.push(d);
25610
- lostStepDrops.set(id, list);
25611
- }
25612
- if (lostStepDrops.size === 0) return {
25613
- kept,
25614
- dropped
25615
- };
25616
- const promoted = [];
25617
- const rescuedIndices = /* @__PURE__ */ new Set();
25618
- const rescuedSteps = [];
25619
- const strict = (opts.mode ?? "lenient") === "strict";
25620
- for (const [stepId, drops] of lostStepDrops.entries()) {
25621
- const replayed = [];
25622
- const unverifiable = [];
25623
- let stillFailing = false;
25624
- for (const d of drops) {
25625
- const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
25626
- if (outcome.promoted) promoted.push(outcome.promoted);
25627
- if (outcome.skipped) {
25628
- unverifiable.push(d.index);
25629
- continue;
25630
- }
25631
- if (outcome.ok) {
25632
- replayed.push(d.index);
25633
- continue;
25634
- }
25635
- stillFailing = true;
25636
- if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
25923
+ async function runCommands(commands, input) {
25924
+ const report = [];
25925
+ for (const [i, cmd] of commands.entries()) {
25926
+ const stop = (why) => [
25927
+ ...report,
25928
+ `✗ ${cmd}\n${why}`,
25929
+ ...i + 1 < commands.length ? [`The ${commands.length - i - 1} command(s) after it were not run.`] : []
25930
+ ].join("\n");
25931
+ if (!isAgentBrowserCommand(cmd)) return stop("Only agent-browser commands can be batched; run anything else with Bash.");
25932
+ const blocked = abCommandBlockReason(cmd, input.envScrubMap);
25933
+ if (blocked !== null) return stop(blocked);
25934
+ const recorded = abActionEventFromCommand(cmd);
25935
+ const checkpoint = input.beforeAbCommand(extractCcqaStepFromBashCommand(cmd) ?? void 0);
25936
+ if (checkpoint) return stop(checkpoint);
25937
+ info(`$ ${scrubEnvValues(cmd, input.envScrubMap)}`);
25938
+ const result = await runShell(cmd, {
25939
+ ...process.env,
25940
+ ...input.env
25941
+ });
25942
+ const out = result.stdout.trim();
25943
+ if (!(result.status === 0 && (recorded?.holds ? recorded.holds(out) : true))) {
25944
+ const detail = (result.stderr || out || `exit ${result.status ?? "timeout"}`).trim().slice(0, 600);
25945
+ return stop(result.status === 0 ? `The check did not hold; the page answered: ${out.slice(0, 300)}` : detail);
25637
25946
  }
25638
- if (replayed.length === 0) continue;
25639
- for (const i of replayed) rescuedIndices.add(i);
25640
- if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
25641
- rescuedSteps.push(stepId);
25642
- }
25643
- if (rescuedIndices.size === 0) return {
25644
- kept,
25645
- dropped,
25646
- promoted
25647
- };
25648
- const keptSet = new Set(kept);
25649
- const newKept = [];
25650
- for (let i = 0; i < actions.length; i++) {
25651
- const action = actions[i];
25652
- if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
25947
+ if (recorded) input.onAbAction(recorded.event);
25948
+ const last = i === commands.length - 1;
25949
+ report.push(`✓ ${cmd}${out && (last || out.length < 200) ? `\n${cap(out)}` : ""}`);
25653
25950
  }
25654
- return {
25655
- kept: newKept,
25656
- dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
25657
- rescuedSteps,
25658
- promoted
25659
- };
25951
+ return report.join("\n");
25660
25952
  }
25661
25953
  /**
25662
- * Passive (read-only) actions whose only effect is observation. When a
25663
- * preceding action fails, dropping these too is the right move because
25664
- * they were trying to observe state the failed action would have set up.
25954
+ * Async, so the trace's deadline and signal teardown still run while a batch
25955
+ * does. Settles on `exit` too: a daemon the command started may hold the pipes open.
25665
25956
  */
25666
- function isPassiveAction(action) {
25667
- return action === "snapshot" || action === "wait" || action === "assert";
25957
+ function runShell(cmd, env) {
25958
+ return new Promise((resolve) => {
25959
+ const child = spawn("bash", ["-c", cmd], {
25960
+ env,
25961
+ timeout: COMMAND_TIMEOUT_MS
25962
+ });
25963
+ let stdout = "";
25964
+ let stderr = "";
25965
+ child.stdout.setEncoding("utf8").on("data", (d) => stdout += d);
25966
+ child.stderr.setEncoding("utf8").on("data", (d) => stderr += d);
25967
+ const done = (status, err) => resolve({
25968
+ status,
25969
+ stdout,
25970
+ stderr: stderr || (err ?? "")
25971
+ });
25972
+ child.on("error", (e) => done(null, e.message));
25973
+ child.on("close", (status) => done(status));
25974
+ child.on("exit", (status) => setTimeout(() => done(status), 1e3).unref());
25975
+ });
25976
+ }
25977
+ function cap(out) {
25978
+ return out.length <= OUTPUT_CAP ? out : `${out.slice(0, OUTPUT_CAP)}\n… (${out.length - OUTPUT_CAP} more characters; scope it with snapshot -s <selector>)`;
25979
+ }
25980
+ function replayStep(step, input) {
25981
+ const actions = input.previous.filter((a) => a.stepId === step && a.action !== "snapshot").map(({ replayUnstable: _u, replayReason: _r, ...a }) => a);
25982
+ if (actions.length === 0) return `No previous commands for ${step}. Record it with the Execution Workflow.`;
25983
+ const { passed, unchecked, failed } = replayUntilFailure(actions, {
25984
+ sessionName: input.sessionName,
25985
+ envOverrides: { CCQA_RUN_ID: input.env["CCQA_RUN_ID"] ?? input.sessionName }
25986
+ });
25987
+ input.onReplayed(step, passed);
25988
+ info(`replay_step ${step}: ${passed.length}/${actions.length} replayed${failed ? "" : " — all passed"}`);
25989
+ const notRun = unchecked.length === 0 ? "" : `\nThese checks could not be run here and were not recorded — perform the ones Expected still asks for yourself:\n` + unchecked.map((a) => `- ${describeStepAction(a)}`).join("\n");
25990
+ if (!failed) {
25991
+ const next = unchecked.length === 0 ? " Emit STEP_DONE and go on." : "";
25992
+ return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded, its checks included.${next}${notRun}`;
25993
+ }
25994
+ return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded. The next one failed: ${describeStepAction(failed.action)} — ${failed.reason}\nThe page is where the recorded commands left it. Finish ${step} with the Execution Workflow (snapshot first); do not re-run the recorded commands.${notRun}`;
25668
25995
  }
25669
25996
  //#endregion
25670
25997
  //#region src/cli/trace.ts
25998
+ /** The recorder's Bash runs agent-browser only: source is read with Read/Grep/Glob. */
25999
+ function traceBashGuard(cmd) {
26000
+ if (isAgentBrowserCommand(cmd)) return null;
26001
+ return "Bash here runs agent-browser commands only. Read, search and list the source with the Read, Grep and Glob tools; wait for the page with agent-browser wait, not sleep.";
26002
+ }
26003
+ /** Browser commands in one step between spec-mismatch checkpoints (see `beforeAbCommand`). */
26004
+ const STEP_CHECKPOINT = 20;
25671
26005
  /**
25672
26006
  * Step ids (in spec order) whose kept actions include no assertion. A step
25673
26007
  * with none produced a test that performs the step but verifies nothing
@@ -25705,6 +26039,11 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25705
26039
  }
25706
26040
  const conventions = await loadConventions(opts.cwd ?? process.cwd(), { guides: opts.conventions ?? [] });
25707
26041
  for (const w of conventions.warnings) warn(w);
26042
+ const caseRef = testCase.ref;
26043
+ const previous = await tryGetRecording(caseRef);
26044
+ const previousActions = previous ? [...previous.actions, ...previous.cleanup ?? []] : [];
26045
+ if (previousActions.length > 0) meta("previous", opts.freshIr ? "ignored (--fresh-ir)" : `${previousActions.length} action(s) offered as a map`);
26046
+ const offersMap = !opts.freshIr && previousActions.length > 0;
25708
26047
  const baseSystemPrompt = buildTraceSystemPrompt({
25709
26048
  title: testCase.title,
25710
26049
  steps,
@@ -25716,7 +26055,8 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25716
26055
  ...testCase.expectations.length > 0 ? { expectations: testCase.expectations } : {},
25717
26056
  ...testCase.cleanupExpectations.length > 0 ? { cleanupExpectations: testCase.cleanupExpectations } : {},
25718
26057
  ...testCase.context.length > 0 ? { context: testCase.context } : {},
25719
- ...opts.instruction ? { instruction: opts.instruction } : {}
26058
+ ...opts.instruction ? { instruction: opts.instruction } : {},
26059
+ ...offersMap ? { previousRecording: previousActions } : {}
25720
26060
  });
25721
26061
  const promptBundle = await loadPromptBundle(opts.hubContext ?? null, "record", opts.cwd ?? process.cwd());
25722
26062
  if (promptBundle !== null) meta("prompt", promptBundle.loaded.join(" + "));
@@ -25737,40 +26077,71 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25737
26077
  } : action;
25738
26078
  };
25739
26079
  let lastCommandPushCount = 0;
26080
+ const onAbAction = ({ abAction, stepId, assertMarker, secret }) => {
26081
+ const stepForCommand = stepTracker.fromCommand(stepId);
26082
+ const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
26083
+ let recorded = null;
26084
+ if (assertMarker !== void 0) {
26085
+ recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
26086
+ if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
26087
+ }
26088
+ if (recorded === null) {
26089
+ const parsed = line === null ? null : parseAbActionLine(line);
26090
+ recorded = parsed === null ? [] : [parsed];
26091
+ }
26092
+ let pushed = 0;
26093
+ for (const action of recorded) {
26094
+ const stamped = withStepId(action, stepForCommand);
26095
+ if (stamped) {
26096
+ traceActions.push(secret ? {
26097
+ ...stamped,
26098
+ secret: true
26099
+ } : stamped);
26100
+ pushed += 1;
26101
+ }
26102
+ }
26103
+ lastCommandPushCount = pushed;
26104
+ };
26105
+ const commandsByStep = /* @__PURE__ */ new Map();
26106
+ const beforeAbCommand = (stepId) => {
26107
+ if (stepId === void 0) return null;
26108
+ const n = (commandsByStep.get(stepId) ?? 0) + 1;
26109
+ commandsByStep.set(stepId, n);
26110
+ if (n % STEP_CHECKPOINT !== 0) return null;
26111
+ return `${stepId} has taken ${n - 1} browser commands. If the application does not do what this step says — the control is somewhere else, things happen in a different order, the result never appears — stop working around it and report it: ASSERTION_FAILED|${stepId}|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>. If the step does match and you are close, run this command again to continue.`;
26112
+ };
26113
+ const invokeBase = agentBrowserInvokeBase({
26114
+ sessionName,
26115
+ runId: sessionName
26116
+ });
26117
+ const traceTools = buildTraceTools({
26118
+ previous: offersMap ? previousActions : [],
26119
+ sessionName,
26120
+ env: invokeBase.env ?? {},
26121
+ envScrubMap,
26122
+ onReplayed: (stepId, passed) => {
26123
+ stepTracker.fromCommand(stepId);
26124
+ for (const action of passed) traceActions.push({
26125
+ ...action,
26126
+ stepId
26127
+ });
26128
+ lastCommandPushCount = 0;
26129
+ },
26130
+ onAbAction,
26131
+ beforeAbCommand
26132
+ });
25740
26133
  const { isError, errorDetail } = await invokeClaudeStreaming({
25741
26134
  prompt,
25742
26135
  systemPrompt,
25743
- ...agentBrowserInvokeBase({
25744
- sessionName,
25745
- runId: sessionName
25746
- }),
26136
+ ...invokeBase,
26137
+ allowedTools: [...invokeBase.allowedTools, ...traceTools.allowedTools],
26138
+ mcpServers: { [TRACE_TOOLS_SERVER]: traceTools.server },
25747
26139
  model,
25748
26140
  envScrubMap,
25749
- onAbAction: ({ abAction, stepId, assertMarker, secret }) => {
25750
- const stepForCommand = stepTracker.fromCommand(stepId);
25751
- const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
25752
- let recorded = null;
25753
- if (assertMarker !== void 0) {
25754
- recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
25755
- if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
25756
- }
25757
- if (recorded === null) {
25758
- const parsed = line === null ? null : parseAbActionLine(line);
25759
- recorded = parsed === null ? [] : [parsed];
25760
- }
25761
- let pushed = 0;
25762
- for (const action of recorded) {
25763
- const stamped = withStepId(action, stepForCommand);
25764
- if (stamped) {
25765
- traceActions.push(secret ? {
25766
- ...stamped,
25767
- secret: true
25768
- } : stamped);
25769
- pushed += 1;
25770
- }
25771
- }
25772
- lastCommandPushCount = pushed;
25773
- },
26141
+ onAbAction,
26142
+ beforeAbCommand,
26143
+ effort: "low",
26144
+ bashGuard: traceBashGuard,
25774
26145
  onAbActionFailed: () => {
25775
26146
  if (lastCommandPushCount > 0) traceActions.splice(-lastCommandPushCount);
25776
26147
  lastCommandPushCount = 0;
@@ -25805,10 +26176,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25805
26176
  });
25806
26177
  const overallStatus = failureReason === null ? "passed" : "failed";
25807
26178
  const dedupedActions = dedupAndReport(scrubAndReport(traceActions));
26179
+ const validationStart = Date.now();
25808
26180
  const validatedActions = overallStatus === "passed" || opts.validateFailedTrace === true ? validateAndReport(dedupedActions, validationMode, { CCQA_RUN_ID: sessionName }, opts.teardown) : dedupedActions;
26181
+ if (validatedActions !== dedupedActions) meta("validation time", `${((Date.now() - validationStart) / 1e3).toFixed(1)}s`);
25809
26182
  opts.teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
25810
- const caseRef = testCase.ref;
25811
- const previous = overallStatus === "passed" ? await tryGetRecording(caseRef) : null;
25812
26183
  let recordingPath;
25813
26184
  const cleanupIds = new Set(testCase.cleanup.map((s) => s.id));
25814
26185
  const routeActions = validatedActions.filter((a) => !cleanupIds.has(a.stepId ?? ""));
@@ -25847,7 +26218,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25847
26218
  */
25848
26219
  function traceFailureReason(lines, session) {
25849
26220
  const assertionFailed = lines.find((l) => l.type === "ASSERTION_FAILED");
25850
- if (assertionFailed) return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED`;
26221
+ if (assertionFailed) {
26222
+ const detail = assertionFailed.detail.trim();
26223
+ return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED${detail ? `: ${detail}` : ""}`;
26224
+ }
25851
26225
  if (session.isError) return `the Claude session ended in an error: ${session.errorDetail ?? "no detail reported"}`;
25852
26226
  const completed = lines.filter((l) => l.type === "RUN_COMPLETED").at(-1);
25853
26227
  if (completed === void 0) return "the session ended without a RUN_COMPLETED line — the model stopped before it reported an outcome";
@@ -26675,7 +27049,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
26675
27049
  const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
26676
27050
  if (VALIDATION_MODES.includes(raw)) return raw;
26677
27051
  throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
26678
- }, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
27052
+ }, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--fresh-ir", "Record from scratch. By default an existing ir.json is shown to the recorder as a map of the route, so steps that still work are not rediscovered.").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
26679
27053
  await withCostReporting("record", () => runRecord(specPath, opts));
26680
27054
  }));
26681
27055
  async function runRecord(caseArgument, opts) {
@@ -26787,6 +27161,7 @@ async function runRecord(caseArgument, opts) {
26787
27161
  ...config.sessionState ? { sessionState: config.sessionState } : {},
26788
27162
  ...resolved.targetConfig.conventions.operate.length > 0 ? { conventions: resolved.targetConfig.conventions.operate } : {},
26789
27163
  ...opts.instruction ? { instruction: opts.instruction } : {},
27164
+ ...opts.freshIr ? { freshIr: true } : {},
26790
27165
  onStep: (stepId) => {
26791
27166
  tracingStep = stepId;
26792
27167
  }
@@ -28672,12 +29047,12 @@ function releaseAll(locks, holder) {
28672
29047
  }
28673
29048
  //#endregion
28674
29049
  //#region src/hub/core/audit-need.ts
28675
- function auditNeed(drift, key, range) {
28676
- const entry = drift.specs[key];
29050
+ function auditNeed(drift, spec, range, deployTimes) {
29051
+ const entry = drift.specs[spec.key];
28677
29052
  if (!entry) return { because: "neverAudited" };
28678
- const since = freshness(entry.gitHead, key, range);
29053
+ const since = freshness(entry.gitHead, spec.key, range);
28679
29054
  switch (since.kind) {
28680
- case "current": return { because: "current" };
29055
+ case "current": return specMovedSince(spec.changedAt, entry.gitHead, entry.at, deployTimes) ? { because: "deployReached" } : { because: "current" };
28681
29056
  case "touched": return { because: "deployReached" };
28682
29057
  case "unanswerable": return {
28683
29058
  because: "cannotTell",
@@ -28686,13 +29061,52 @@ function auditNeed(drift, key, range) {
28686
29061
  default: throw new Error(`unhandled freshness: ${String(since)}`);
28687
29062
  }
28688
29063
  }
29064
+ /**
29065
+ * When each deployed commit reached the environment. A baseline read at that
29066
+ * commit cannot have seen anything committed after it was deployed.
29067
+ */
29068
+ function deployedAt(log) {
29069
+ const out = /* @__PURE__ */ new Map();
29070
+ for (const entry of log.entries) out.set(entry.sha, entry.at);
29071
+ return out;
29072
+ }
29073
+ /**
29074
+ * Has the spec moved since the baseline was taken?
29075
+ *
29076
+ * A verdict is a claim about a (spec, product) pair, so either side moving
29077
+ * invalidates it. The deploy log covers the product side; this covers the
29078
+ * other one. Without it a spec repaired and merged stays `needsRepair` until
29079
+ * a deploy happens to reach it, and a run that passed against the previous
29080
+ * spec keeps answering `verified` for the new one.
29081
+ *
29082
+ * Compared against when the baseline commit was *deployed*, not when the audit
29083
+ * or run happened: the tree read at that commit predates its deployment, so an
29084
+ * edit after it is definitely not in it. Falls back to the baseline's own
29085
+ * timestamp when the log cannot place the commit.
29086
+ *
29087
+ * One-directional. A later edit time proves the baseline is stale; an earlier
29088
+ * one proves nothing, and this answers false rather than guessing.
29089
+ */
29090
+ function specMovedSince(changedAt, baselineSha, baselineAt, deployTimes) {
29091
+ if (!changedAt) return null;
29092
+ return isLater(changedAt, baselineSha && deployTimes.get(baselineSha) || baselineAt) ? changedAt : null;
29093
+ }
29094
+ /**
29095
+ * Compared as instants, never as strings: a spec's edit time carries its
29096
+ * committer's offset, and `+09:00` sorts after `Z` for an earlier moment.
29097
+ * An unparseable side answers false, the same as a missing one.
29098
+ */
29099
+ function isLater(a, b) {
29100
+ return Date.parse(a) > Date.parse(b);
29101
+ }
28689
29102
  /** True for every answer but `current`. */
28690
29103
  function needsAudit(need) {
28691
29104
  return need.because !== "current";
28692
29105
  }
28693
29106
  function computeAuditNeed(input) {
28694
29107
  const range = buildRange(input.log, input.touchIndex);
28695
- return Object.fromEntries(input.specs.map((spec) => [spec.key, heldBy(input.locks, spec.key, input.now) ? { because: "held" } : auditNeed(input.drift, spec.key, range)]));
29108
+ const deployTimes = deployedAt(input.log);
29109
+ return Object.fromEntries(input.specs.map((spec) => [spec.key, heldBy(input.locks, spec.key, input.now) ? { because: "held" } : auditNeed(input.drift, spec, range, deployTimes)]));
28696
29110
  }
28697
29111
  //#endregion
28698
29112
  //#region src/drift/audit-selection.ts
@@ -29254,7 +29668,8 @@ async function readChunk(cwd, paths, byRelPath, out) {
29254
29668
  let when = "";
29255
29669
  for (const line of stdout.split("\n")) {
29256
29670
  if (line.startsWith("\0")) {
29257
- when = line.slice(1).trim();
29671
+ const ms = Date.parse(line.slice(1).trim());
29672
+ when = Number.isNaN(ms) ? "" : new Date(ms).toISOString();
29258
29673
  continue;
29259
29674
  }
29260
29675
  const abs = byRelPath.get(line.trim());
@@ -31878,36 +32293,6 @@ function createGetCoverageEdgesHandler(config) {
31878
32293
  }
31879
32294
  //#endregion
31880
32295
  //#region src/hub/core/rerun.ts
31881
- /**
31882
- * When each deployed commit reached the environment. A baseline read at that
31883
- * commit cannot have seen anything committed after it was deployed.
31884
- */
31885
- function deployedAt(log) {
31886
- const out = /* @__PURE__ */ new Map();
31887
- for (const entry of log.entries) out.set(entry.sha, entry.at);
31888
- return out;
31889
- }
31890
- /**
31891
- * Has the spec moved since the baseline was taken?
31892
- *
31893
- * A verdict is a claim about a (spec, product) pair, so either side moving
31894
- * invalidates it. The deploy log covers the product side; this covers the
31895
- * other one. Without it a spec repaired and merged stays `needsRepair` until
31896
- * a deploy happens to reach it, and a run that passed against the previous
31897
- * spec keeps answering `verified` for the new one.
31898
- *
31899
- * Compared against when the baseline commit was *deployed*, not when the audit
31900
- * or run happened: the tree read at that commit predates its deployment, so an
31901
- * edit after it is definitely not in it. Falls back to the baseline's own
31902
- * timestamp when the log cannot place the commit.
31903
- *
31904
- * One-directional. A later edit time proves the baseline is stale; an earlier
31905
- * one proves nothing, and this answers false rather than guessing.
31906
- */
31907
- function specMovedSince(changedAt, baselineSha, baselineAt, deployTimes) {
31908
- if (!changedAt) return null;
31909
- return changedAt > (baselineSha && deployTimes.get(baselineSha) || baselineAt) ? changedAt : null;
31910
- }
31911
32296
  function computeRerun(input) {
31912
32297
  const { specs, ledger, log, touchIndex, drift, locks, attestations, dismissals, now } = input;
31913
32298
  const range = buildRange(log, touchIndex);
@@ -31919,7 +32304,7 @@ function computeRerun(input) {
31919
32304
  lastGreen: ledger.green[spec.key] ?? null,
31920
32305
  lastRed: ledger.red[spec.key] ?? null
31921
32306
  };
31922
- let audit = auditState(drift, spec.key, range);
32307
+ let audit = auditState(drift, spec, range, deployTimes);
31923
32308
  let execution = executionState(coords, (sha) => freshness(sha, spec.key, range));
31924
32309
  const driftEntry = drift.specs[spec.key];
31925
32310
  const dismissal = dismissals.specs[spec.key];
@@ -31927,7 +32312,6 @@ function computeRerun(input) {
31927
32312
  if (dismissed) audit = { audit: "clean" };
31928
32313
  const auditMoved = specMovedSince(spec.changedAt, driftEntry?.gitHead ?? null, driftEntry?.at ?? "", deployTimes);
31929
32314
  const runMoved = specMovedSince(spec.changedAt, coords.lastRun?.deployedSha ?? null, coords.lastRun?.at ?? "", deployTimes);
31930
- if (auditMoved && audit.audit !== "due") audit = { audit: "due" };
31931
32315
  if (runMoved && execution.execution === "passed") execution = { execution: "stale" };
31932
32316
  const manualState = readAttestation(attestations.specs[spec.key], spec, coords, range, log, deployTimes);
31933
32317
  const lapsedOverRed = manualState?.kind === "lapsed" && manualState.because !== "newerRed";
@@ -31972,7 +32356,7 @@ function computeRerun(input) {
31972
32356
  */
31973
32357
  function readAttestation(attest, spec, { lastRun, lastRed }, range, log, deployTimes) {
31974
32358
  if (!attest) return null;
31975
- const newerRed = lastRed !== null && lastRed.at > attest.at;
32359
+ const newerRed = lastRed !== null && isLater(lastRed.at, attest.at);
31976
32360
  if (newerRed && lastRed.runId === lastRun?.runId) return {
31977
32361
  kind: "lapsed",
31978
32362
  attest,
@@ -32018,8 +32402,8 @@ function readAttestation(attest, spec, { lastRun, lastRed }, range, log, deployT
32018
32402
  * rule out is treated as having landed (ADR-0014). The hole is kept as an
32019
32403
  * annotation so the answer stays explicable.
32020
32404
  */
32021
- function auditState(drift, key, range) {
32022
- const need = auditNeed(drift, key, range);
32405
+ function auditState(drift, spec, range, deployTimes) {
32406
+ const need = auditNeed(drift, spec, range, deployTimes);
32023
32407
  switch (need.because) {
32024
32408
  case "neverAudited":
32025
32409
  case "deployReached": return { audit: "due" };
@@ -32028,7 +32412,7 @@ function auditState(drift, key, range) {
32028
32412
  ...need.reason ? { auditAssumedReached: need.reason } : {}
32029
32413
  };
32030
32414
  case "current": {
32031
- const label = drift.specs[key].label;
32415
+ const label = drift.specs[spec.key].label;
32032
32416
  if (label === null) return { audit: "clean" };
32033
32417
  if (label === "UNKNOWN") return { audit: "undecided" };
32034
32418
  if (label === "PRODUCT_BUG" || label === "ENVIRONMENT") return { audit: "clean" };