ccqa 1.58.2 → 1.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/ccqa.mjs CHANGED
@@ -3143,7 +3143,7 @@ function formatDuration$1(ms) {
3143
3143
  return `${minutes} minute${minutes === 1 ? "" : "s"}`;
3144
3144
  }
3145
3145
  async function invokeClaudeStreaming(options, onEvent) {
3146
- const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
3146
+ const { prompt, systemPrompt, allowedTools, disableThinking = false, effort, bashGuard, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, beforeAbCommand, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
3147
3147
  const resolvedModel = resolveModel(model);
3148
3148
  const mergedEnv = buildInvocationEnv(env);
3149
3149
  const abortController = new AbortController();
@@ -3171,52 +3171,35 @@ async function invokeClaudeStreaming(options, onEvent) {
3171
3171
  env: mergedEnv,
3172
3172
  ...mcpServers ? { mcpServers } : {},
3173
3173
  ...disableThinking ? { thinking: { type: "disabled" } } : {},
3174
- hooks: onAbAction || onAbActionFailed ? {
3174
+ ...effort ? { effort } : {},
3175
+ hooks: onAbAction || onAbActionFailed || bashGuard || beforeAbCommand ? {
3175
3176
  PreToolUse: [{ hooks: [async (input) => {
3176
3177
  if (input.hook_event_name !== "PreToolUse") return {};
3177
3178
  if (input.tool_name !== "Bash") return {};
3178
3179
  const cmd = input.tool_input?.["command"];
3179
3180
  if (typeof cmd !== "string") return {};
3181
+ const guarded = bashGuard?.(cmd);
3182
+ if (guarded) return {
3183
+ decision: "block",
3184
+ reason: guarded
3185
+ };
3180
3186
  if (!relaxAbConstraints) {
3181
- if (isBlockedAbSubcommand(cmd)) return {
3182
- decision: "block",
3183
- reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
3184
- };
3185
- if (hasRefSelector(cmd)) return {
3186
- decision: "block",
3187
- reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
3188
- };
3189
- const bareTag = findPositionalBareTag(cmd);
3190
- if (bareTag !== null) return {
3191
- decision: "block",
3192
- reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
3193
- };
3194
- if (hasMultipleAbInvocations(cmd)) return {
3195
- decision: "block",
3196
- reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
3197
- };
3198
- if (hasErrorSuppression(cmd)) return {
3187
+ const reason = abCommandBlockReason(cmd, envScrubMap);
3188
+ if (reason !== null) return {
3199
3189
  decision: "block",
3200
- reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
3190
+ reason
3201
3191
  };
3202
- const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
3203
- if (runProducedUrl !== null) return {
3192
+ const checkpoint = beforeAbCommand?.(extractCcqaStepFromBashCommand(cmd) ?? void 0);
3193
+ if (checkpoint) return {
3204
3194
  decision: "block",
3205
- reason: `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`
3195
+ reason: checkpoint
3206
3196
  };
3207
3197
  }
3208
- const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
3209
- const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
3210
- if ((ab !== null || assertMarker !== null) && onAbAction) {
3198
+ const recorded = relaxAbConstraints ? null : abActionEventFromCommand(cmd);
3199
+ if (recorded !== null && onAbAction) {
3211
3200
  lastAbToolUseId = input.tool_use_id;
3212
- lastAbAnswerHolds = assertMarker === null ? null : markerHolds(assertMarker, ab);
3213
- const stepId = extractCcqaStepFromBashCommand(cmd);
3214
- onAbAction({
3215
- ...ab !== null ? { abAction: ab } : {},
3216
- ...stepId ? { stepId } : {},
3217
- ...assertMarker !== null ? { assertMarker } : {},
3218
- ...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
3219
- });
3201
+ lastAbAnswerHolds = recorded.holds;
3202
+ onAbAction(recorded.event);
3220
3203
  } else lastAbToolUseId = null;
3221
3204
  return {};
3222
3205
  }] }],
@@ -3804,6 +3787,45 @@ async function fireMockPreToolUseHooks(msg, options) {
3804
3787
  for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
3805
3788
  }
3806
3789
  }
3790
+ /**
3791
+ * Why the trace refuses this agent-browser command, or null when it may run.
3792
+ * Shared by the Bash hook and the trace's own batch tool, so both enforce the
3793
+ * same replayability contract.
3794
+ */
3795
+ function abCommandBlockReason(cmd, envScrubMap = []) {
3796
+ if (isBlockedAbSubcommand(cmd)) return "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector.";
3797
+ if (hasRefSelector(cmd)) return "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either.";
3798
+ const bareTag = findPositionalBareTag(cmd);
3799
+ if (bareTag !== null) return `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`;
3800
+ if (hasMultipleAbInvocations(cmd)) return "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command.";
3801
+ if (hasErrorSuppression(cmd)) return "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code.";
3802
+ const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
3803
+ if (runProducedUrl !== null) return `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`;
3804
+ return null;
3805
+ }
3806
+ /**
3807
+ * What recording this command produces, and — for a marked probe — what its
3808
+ * output must say for the check to hold. Null when it records nothing.
3809
+ */
3810
+ function abActionEventFromCommand(cmd) {
3811
+ const assertMarker = extractCcqaAssertFromBashCommand(cmd);
3812
+ const ab = extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
3813
+ if (ab === null && assertMarker === null) return null;
3814
+ const stepId = extractCcqaStepFromBashCommand(cmd);
3815
+ return {
3816
+ event: {
3817
+ ...ab !== null ? { abAction: ab } : {},
3818
+ ...stepId ? { stepId } : {},
3819
+ ...assertMarker !== null ? { assertMarker } : {},
3820
+ ...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
3821
+ },
3822
+ holds: assertMarker === null ? null : markerHolds(assertMarker, ab)
3823
+ };
3824
+ }
3825
+ /** Whether a shell line runs agent-browser as a command (env prefixes allowed), not just mentions it. */
3826
+ function isAgentBrowserCommand(cmd) {
3827
+ return splitShellStatements(cmd).some((st) => isAgentBrowserHead(splitLeadingEnvAssignments(st).command));
3828
+ }
3807
3829
  //#endregion
3808
3830
  //#region src/prompts/format.ts
3809
3831
  /**
@@ -3950,7 +3972,7 @@ function toAgentBrowserArgs(action) {
3950
3972
  }
3951
3973
  case "wait": {
3952
3974
  const loc = action.locator;
3953
- if (!loc) return null;
3975
+ if (!loc || loc.value.startsWith("--")) return null;
3954
3976
  if (loc.by === "text") return [
3955
3977
  lit("wait"),
3956
3978
  lit("--text"),
@@ -4005,6 +4027,61 @@ function interactionToArgs(action) {
4005
4027
  if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
4006
4028
  return out;
4007
4029
  }
4030
+ /**
4031
+ * The marked probe that records an assert during a trace — the inverse of
4032
+ * `promoteMarkedAssert`, as the trace prompt's Assertion Protocol spells it.
4033
+ */
4034
+ function markedAssertArgs(action) {
4035
+ const loc = action.locator;
4036
+ const sel = loc ? [val(locatorToSelector(loc))] : null;
4037
+ switch (action.assert) {
4038
+ case "text_visible": return action.value ? {
4039
+ marker: "1",
4040
+ tokens: [
4041
+ lit("wait"),
4042
+ lit("--text"),
4043
+ val(action.value)
4044
+ ]
4045
+ } : null;
4046
+ case "url_contains": return action.value ? {
4047
+ marker: `url_contains:${action.value}`,
4048
+ tokens: [lit("get"), lit("url")]
4049
+ } : null;
4050
+ case "element_visible":
4051
+ case "element_not_visible":
4052
+ if (loc?.by === "role") return loc.name ? {
4053
+ marker: action.assert,
4054
+ tokens: roleProbeTokens(loc.value, loc.name, loc.exact === true)
4055
+ } : null;
4056
+ return sel ? {
4057
+ marker: action.assert,
4058
+ tokens: [
4059
+ lit("get"),
4060
+ lit("count"),
4061
+ ...sel
4062
+ ]
4063
+ } : null;
4064
+ case "element_enabled":
4065
+ case "element_disabled": return sel ? {
4066
+ marker: action.assert,
4067
+ tokens: [
4068
+ lit("is"),
4069
+ lit("enabled"),
4070
+ ...sel
4071
+ ]
4072
+ } : null;
4073
+ case "element_checked":
4074
+ case "element_unchecked": return sel ? {
4075
+ marker: action.assert,
4076
+ tokens: [
4077
+ lit("is"),
4078
+ lit("checked"),
4079
+ ...sel
4080
+ ]
4081
+ } : null;
4082
+ default: return null;
4083
+ }
4084
+ }
4008
4085
  //#endregion
4009
4086
  //#region src/diagnose/prompt.ts
4010
4087
  function buildDiagnosePrompt(input) {
@@ -7868,7 +7945,7 @@ function reuseFirstContract(hasDraft, draftInvariant) {
7868
7945
  "3. **Match the examples.** Write in the same style as the conventions examples: naming, structure, assertion phrasing, import layout. Where you follow an existing file rather than the plain reading of a guideline — a waiting pattern, a locator shape, a teardown structure — say which file, by path, in the comment. \"Following the existing implementation\" without naming it cannot be checked by the next reader, and it is how a pattern that was right once outlives the reason for it."
7869
7946
  ];
7870
7947
  if (hasDraft) {
7871
- rules.push("4. **The draft is ground truth.** The mechanical draft below encodes the recorded route. Do not change the meaning of its operation sequence or its assertions — no reordering, dropping, or weakening. You may rewrite locators when the conventions or the sources you read justify a better one.");
7948
+ rules.push("4. **The draft is ground truth for the route, not for the form of its checks.** Keep its operation sequence: no reordering or dropping of the actions the case needs. Its `expect` lines are what the recorder observed on the way, written in one fixed form (`await expect(...)`, one per observation). Which of them the test keeps, and in which form — hard or soft, where the check lives, how the locator is scoped — is decided by the case's expected results and the project's conventions, never by the draft. Where the conventions and the draft disagree, follow the conventions. What an expected result claims must still be checked: never loosen the value it states.");
7872
7949
  if (draftInvariant) rules.push(`5. ${draftInvariant}`);
7873
7950
  }
7874
7951
  return `## Reuse contract\n\n${rules.join("\n")}`;
@@ -7916,6 +7993,8 @@ function buildLlmGenPrompt(input) {
7916
7993
  sections.push(input.taskInstructions);
7917
7994
  const steps = input.steps.map(formatStep).join("\n");
7918
7995
  sections.push(`## Test spec\n\nTitle: ${input.specTitle}\n\nSteps:\n${steps}`);
7996
+ const facts = caseFacts(input);
7997
+ if (facts) sections.push(facts);
7919
7998
  if (input.draft) sections.push(`## Mechanical draft (recorded ground truth)\n\nPath: ${input.draft.path}\n\n\`\`\`
7920
7999
  ` + input.draft.contents + "\n```");
7921
8000
  if (input.resources.length > 0) {
@@ -7936,6 +8015,19 @@ These are the project's rules for the code you write, not background reading. Co
7936
8015
  return sections.join("\n\n") + languageDirective(input.language);
7937
8016
  }
7938
8017
  /**
8018
+ * What the case states beyond its steps. Project conventions key rules on
8019
+ * these (which checks are the case's, where it may run), and the review that
8020
+ * follows already reads them — a rewrite without them can only guess.
8021
+ */
8022
+ function caseFacts(input) {
8023
+ const parts = [];
8024
+ if (input.expectations && input.expectations.length > 0) parts.push(`### Expected results\n\nThe case states these for the flow as a whole. They are the checks the test exists for:\n\n${input.expectations.map((e) => `- ${body(e)}`).join("\n")}`);
8025
+ if (input.cleanup && input.cleanup.length > 0) parts.push(`### Cleanup steps\n\n${input.cleanup.map(formatStep).join("\n")}`);
8026
+ if (input.cleanupExpectations && input.cleanupExpectations.length > 0) parts.push(`### Expected results of the cleanup\n\n${input.cleanupExpectations.map((e) => `- ${body(e)}`).join("\n")}`);
8027
+ for (const c of input.context ?? []) parts.push(`### ${c.heading}\n\n${c.body.trim()}`);
8028
+ return parts.length === 0 ? "" : `## About this case\n\n${parts.join("\n\n")}`;
8029
+ }
8030
+ /**
7939
8031
  * Prompt for one iteration of the verification loop: the generated files
7940
8032
  * failed their run command; ask for corrected files under the same contract.
7941
8033
  */
@@ -8304,7 +8396,7 @@ function weakerTwin(file, source) {
8304
8396
  function decidesNothing(input) {
8305
8397
  const source = input.files.get(input.testPath);
8306
8398
  if (source === void 0 || input.caseText.join("").trim().length === 0) return [];
8307
- if (/\bexpect\(|\bjudgeByLlm\b/.test(source)) return [];
8399
+ if (/\bexpect(?:\.soft)?\(|\bjudgeByLlm\b/.test(source)) return [];
8308
8400
  return [{
8309
8401
  file: input.testPath,
8310
8402
  line: 1,
@@ -8373,7 +8465,7 @@ function containerOfPageText(file, source) {
8373
8465
  */
8374
8466
  function explained(all, i) {
8375
8467
  let above = i - 1;
8376
- while (above >= 0 && /\bexpect\(|^\s*$|^\s*\)/.test(all[above] ?? "")) above -= 1;
8468
+ while (above >= 0 && /\bexpect(?:\.soft)?\(|^\s*$|^\s*\)/.test(all[above] ?? "")) above -= 1;
8377
8469
  return /^\s*\/\//.test(all[above] ?? "");
8378
8470
  }
8379
8471
  /**
@@ -8388,7 +8480,7 @@ function explained(all, i) {
8388
8480
  function unjustifiedFirst(file, source) {
8389
8481
  const all = lines(source);
8390
8482
  return all.flatMap((text, i) => {
8391
- if (!/\bexpect\(/.test(text) || !/\.first\(\)/.test(text)) return [];
8483
+ if (!/\bexpect(?:\.soft)?\(/.test(text) || !/\.first\(\)/.test(text)) return [];
8392
8484
  if (explained(all, i)) return [];
8393
8485
  return [{
8394
8486
  file,
@@ -8463,7 +8555,7 @@ function assertedStrings(text, locators) {
8463
8555
  function unaskedAssertions(file, source, locators, said) {
8464
8556
  const all = lines(source);
8465
8557
  return all.flatMap((text, i) => {
8466
- if (!/\bexpect\(/.test(text)) return [];
8558
+ if (!/\bexpect(?:\.soft)?\(/.test(text)) return [];
8467
8559
  const strings = assertedStrings(text, locators);
8468
8560
  if (strings.length === 0 || strings.some((s) => said.includes(s))) return [];
8469
8561
  if (explained(all, i)) return [];
@@ -14783,6 +14875,10 @@ async function generateWithLlmEngine(req) {
14783
14875
  specTitle: ctx.spec.title,
14784
14876
  steps: req.steps,
14785
14877
  draft: req.draft,
14878
+ expectations: ctx.expectations,
14879
+ cleanup: ctx.cleanup,
14880
+ cleanupExpectations: ctx.cleanupExpectations,
14881
+ context: ctx.context ?? [],
14786
14882
  ...req.draftInvariant ? { draftInvariant: req.draftInvariant } : {},
14787
14883
  resources: resources.map(toPromptResource),
14788
14884
  conventionSections: conventions.sections,
@@ -15467,7 +15563,7 @@ function headerPreserveRule(header, titleSuffix) {
15467
15563
  * rejection, which costs a whole round.
15468
15564
  */
15469
15565
  function stepCommentPreserveRule() {
15470
- return "**Keep every `test.step(...)` block, with its title exactly as the draft wrote it.** Those titles are how the evidence table, the run's per-step screenshots and the review of this test say which assertions belong to which step. A reshaped one parses as no step at all, and every step then reads as deciding nothing. Keep the wording, the numbering and the punctuation, and keep each step's actions inside its own block — a step's call to a page-object method belongs in the block, not outside it. Do not merge, split or reorder the blocks. A test already at this path may have been written by an older ccqa in a different shape: the draft is what this file must look like, not that one.";
15566
+ return "**Keep every `test.step(...)` block, with its title exactly as the draft wrote it.** Those titles are how the evidence table, the run's per-step screenshots and the review of this test say which assertions belong to which step. A reshaped one parses as no step at all, and every step then reads as deciding nothing. Keep the wording, the numbering and the punctuation, and keep each step's actions inside its own block — a step's call to a page-object method belongs in the block, not outside it. Do not merge, split or reorder the blocks. A test already at this path may have been written by an older ccqa in a different shape: the draft's step blocks are what this file must keep, not that one's.";
15471
15567
  }
15472
15568
  /**
15473
15569
  * Told to the rewrite, because the alternative failure is silent: a claim
@@ -24539,1116 +24635,1364 @@ async function readLockBody(lockPath) {
24539
24635
  }
24540
24636
  }
24541
24637
  //#endregion
24542
- //#region src/prompts/trace.ts
24543
- function generateSessionName() {
24544
- return `ccqa-trace-${buildRunId()}`;
24638
+ //#region src/ir/accessible-name.ts
24639
+ /**
24640
+ * A recorder reads an accessible name off a snapshot and writes it as the
24641
+ * attribute it guessed the name came from. The name can come from a `<label>`
24642
+ * or `aria-labelledby` instead, and the attribute's absence is not evidence
24643
+ * about the element. See docs/targets.md for the measurement.
24644
+ */
24645
+ const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
24646
+ /**
24647
+ * The name this selector may be after, or null.
24648
+ *
24649
+ * `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
24650
+ * mean what they say, and a zero from one of those is evidence.
24651
+ */
24652
+ function nameFromAttributeSelector(selector) {
24653
+ const m = NAMING_ATTR.exec(selector.trim());
24654
+ const name = m?.[1] ?? m?.[2];
24655
+ return name === void 0 || name === "" ? null : name;
24545
24656
  }
24546
24657
  /**
24547
- * Build the trace system prompt. `input.steps` is a flat list with includes
24548
- * already expanded (each step carries id / source / instruction / expected).
24549
- * The spec opens URLs via explicit step instructions (e.g.
24550
- * `instruction: "${APP_URL}/articles を開く"`).
24658
+ * The role of the first node in an accessibility snapshot whose name is
24659
+ * exactly `name`, or null.
24551
24660
  *
24552
- * In v0.4 every spec is traced from scratch — block contents are inlined
24553
- * into the spec's own step list at expand time, so the prompt has no
24554
- * special "this is a block" mode. The `source` tag on each step still
24555
- * distinguishes spec-native steps from inlined block steps for the
24556
- * `// step:` comments in the eventual codegen output.
24661
+ * agent-browser prints one node per line as `<role> "<name>"`. Matched on the
24662
+ * whole name, so a tree holding both "Priority" and "Priority *" answers for
24663
+ * the one that was asked, and refused when two roles carry it.
24557
24664
  */
24558
- function buildTraceSystemPrompt(input) {
24559
- const sessionName = input.sessionName ?? generateSessionName();
24560
- const firstStepId = input.steps[0]?.id ?? "step-01";
24561
- const callerGuidance = input.instruction ? `## Caller Guidance
24562
-
24563
- The caller provided extra guidance for this recording — for example, a drift
24564
- audit's finding about what a previous recording of this spec got wrong. Treat
24565
- it as advice on what to avoid or verify while recording; the spec's steps
24566
- above remain the contract for what to do.
24567
-
24568
- ${input.instruction}
24569
-
24570
- ` : "";
24571
- const stepsText = input.steps.map((step) => [
24572
- `### ${step.id} [${step.source}]`,
24573
- `- **Instruction**: ${step.instruction}`,
24574
- ...step.expected ? [`- **Expected**: ${step.expected}`] : []
24575
- ].join("\n")).join("\n\n");
24576
- const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
24577
- const contextText = renderContext(input.context ?? []);
24578
- const conventionsText = renderConventions(input.conventions ?? []);
24579
- return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
24580
-
24581
- ## Session
24582
-
24583
- SESSION NAME: \`${sessionName}\`
24584
-
24585
- Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
24586
-
24587
- **Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
24588
- env assignment naming the spec step it executes**, e.g.
24589
-
24590
- \`\`\`bash
24591
- CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
24592
- \`\`\`
24593
-
24594
- This prefix is what ties each recorded action to its step in the generated
24595
- test — a command without it loses its step attribution.
24596
-
24597
- **Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
24598
- in, opening the application, putting the account where the precondition says it
24599
- starts — those are not step-01, and marking them as step-01 makes the first
24600
- step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
24601
- first step the case actually lists.
24602
-
24603
- Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
24604
- itself ignores the variable.
24605
-
24606
- Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
24607
- assignment (alongside \`CCQA_STEP\`) that records the check as a test
24608
- assertion — see "Assertion Protocol".
24609
-
24610
- **A command that types into a password field carries \`CCQA_SECRET=1\`** as
24611
- well, e.g.
24612
-
24613
- \`\`\`bash
24614
- CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
24615
- \`\`\`
24616
-
24617
- A password field is an \`<input type="password">\` — the snapshot shows its
24618
- value as dots, and nothing else about the page is a password. The prefix is
24619
- what stops the typed value from being written to the recording: a value that
24620
- came from a \`\${VAR}\` is kept as that reference, and one typed literally is
24621
- dropped along with its command.
24622
-
24623
- ## Browser Commands
24624
-
24625
- \`\`\`
24626
- CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
24627
- CCQA_STEP=<step-id> agent-browser --session SESSION snapshot
24628
- CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
24629
- CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
24630
- CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
24631
- CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
24632
- CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
24633
- CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
24634
- CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
24635
- CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
24636
- CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
24637
- CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
24638
- agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
24639
- CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
24640
- CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
24641
- # See "Selector Rules" for the full \`find\` subset.
24642
- # IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
24643
- # CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
24644
- # Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
24645
- # (which waits internally). To check an element exists, use \`get count "<selector>"\`.
24646
- \`\`\`
24647
-
24648
- ## Selector Rules
24649
-
24650
- **ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
24651
-
24652
- | Format | Use when |
24653
- |--------|----------|
24654
- | \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
24655
- | \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
24656
- | \`[placeholder='text']\` | Input identified by placeholder |
24657
- | \`[type='password']\` | Password inputs only |
24658
- | \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
24659
- | \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
24660
-
24661
- **FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
24662
-
24663
- - \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
24664
- - **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
24665
- - \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
24666
- - **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
24667
- - JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
24668
-
24669
- ### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
24670
-
24671
- When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
24672
-
24673
- \`\`\`
24674
- find role <role> <action> [--name "<n>"] [--exact]
24675
- find text|label|placeholder|alt|title "<text>" <action> [--exact]
24676
- find testid "<id>" <action>
24677
- find first|last "<ALLOWED-css>" <action>
24678
- find nth <index> "<ALLOWED-css>" <action>
24679
- \`\`\`
24680
-
24681
- \`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
24682
-
24683
- **Rules for \`find\`:**
24684
-
24685
- 1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
24686
- 2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
24687
- 3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
24688
- 4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
24689
- 5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
24690
- 6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
24691
- 7. \`find\` includes its own wait; do not chain a \`wait\` before it.
24692
- 8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
24693
- 9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
24694
-
24695
- **Examples:**
24696
-
24697
- - ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
24698
- - ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
24699
- - ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
24700
- - ✗ \`find role button --name "Submit" click\` — wrong order
24701
- - ✗ \`find last "button" click\` — bare tag
24702
-
24703
- ### Selector workflow
24704
-
24705
- 1. Run \`snapshot\` and read the ARIA tree.
24706
- 2. Identify the element; note its exact \`aria-label\` if present.
24707
- 3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
24708
- For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
24709
- 4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
24710
- 5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
24711
- 6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
24712
- 7. Never guess. If a selector fails, take a fresh snapshot before retrying.
24713
- 8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
24714
-
24715
- ### Special input types
24716
-
24717
- **contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
24718
-
24719
- **combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
24720
-
24721
- **Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
24722
-
24723
- **Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
24724
-
24725
- **File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
24726
-
24727
- ## Test Specification
24728
-
24729
- Title: ${input.title}
24730
-
24731
- Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
24732
-
24733
- ## Steps
24734
-
24735
- ${stepsText}
24736
- ${expectationsText}${contextText}${conventionsText}
24737
- ## Execution Workflow
24738
-
24739
- For each step:
24740
- 1. Emit \`STEP_START|<step-id>|<short description>\`.
24741
- 2. Run \`snapshot\` and identify selectors from the ARIA tree.
24742
- 3. Execute the action using an ALLOWED selector (see Selector Rules), prefixing the command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step.
24743
- 4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
24744
- 5. Run \`snapshot\` again to verify the outcome.
24745
- 6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
24746
- 7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
24747
- 8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
24748
-
24749
- **Protocol lines are recorded ONLY from your plain assistant text.** Every
24750
- \`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
24751
- message text between tool calls — never through Bash (\`echo\` etc.) and never
24752
- omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
24753
- prefix on each command (that prefix is authoritative, not the \`STEP_START\`
24754
- line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
24755
- verification commands** — a command that ran, whose failure rolls the record
24756
- back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
24757
- proves it; it is reported and not recorded.
24758
-
24759
- **After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
24760
-
24761
- ## Guardrails
24762
-
24763
- - **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
24764
- - **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
24765
- - Do NOT retry a selector without taking a fresh snapshot first.
24766
- - Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
24767
- - **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
24768
- - **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
24769
-
24770
- ## Source Code Reference
24771
-
24772
- You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
24773
-
24774
- **When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
24775
-
24776
- **How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
24777
-
24778
- **Rules**: only READ source files, never modify them. Keep searches focused.
24779
-
24780
- ## Waiting for Async Operations
24781
-
24782
- Prefer \`wait\` over polling:
24783
-
24784
- \`\`\`bash
24785
- CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
24786
- \`\`\`
24787
-
24788
- If polling is required (e.g. waiting for a spinner to disappear):
24789
-
24790
- \`\`\`bash
24791
- for i in $(seq 1 18); do
24792
- sleep 10
24793
- result=$(agent-browser --session ${sessionName} snapshot 2>&1)
24794
- echo "$result" | grep -q "<done indicator>" && break
24795
- done
24796
- CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
24797
- \`\`\`
24798
-
24799
- After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
24800
-
24801
- ## AB_ACTION Protocol
24802
-
24803
- After **every** browser action, emit one line (outside any code block):
24804
-
24805
- \`\`\`
24806
- AB_ACTION|cookies_clear
24807
- AB_ACTION|open|<url>
24808
- AB_ACTION|click|<selector>|<visible label>
24809
- AB_ACTION|dblclick|<selector>|<visible label>
24810
- AB_ACTION|fill|<selector>|<value>|<aria label>
24811
- AB_ACTION|check|<selector>|<visible label>
24812
- AB_ACTION|uncheck|<selector>|<visible label>
24813
- AB_ACTION|press|<Key>
24814
- AB_ACTION|select|<selector>|<value>|<aria label>
24815
- AB_ACTION|hover|<selector>|<visible label>
24816
- AB_ACTION|scroll|<direction>|<pixels>
24817
- AB_ACTION|drag|<source selector>|<target selector>|<source label>
24818
- AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
24819
- AB_ACTION|wait|<selector or text>|<label>
24820
- AB_ACTION|snapshot|<key observation, max 100 chars>
24821
-
24822
- # find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
24823
- # <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
24824
- AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
24825
- AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
24826
- AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
24827
- AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
24828
- AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
24829
- AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
24830
- AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
24831
- AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
24832
- \`\`\`
24833
-
24834
- Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
24835
-
24836
- **CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
24837
-
24838
- - A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
24839
- - If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
24840
- - An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
24841
- - **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
24842
-
24843
- - **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
24844
- - If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
24845
-
24846
- ## Assertion Protocol
24847
-
24848
- Record each signal you verified as an assertion. The **primary mechanism** is
24849
- a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
24850
- marker rides the same channel as the command, so the assertion is captured
24851
- even if you print no protocol text. The check you were going to run anyway
24852
- becomes the recorded assertion:
24853
-
24854
- \`\`\`bash
24855
- # text_visible — the marked wait IS the assertion (no separate assert line)
24856
- CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
24857
- # element_visible / element_not_visible — mark the get-count probe
24858
- CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
24859
- CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
24860
- # url_contains — the substring rides in the marker; put it on \`get url\`
24861
- CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
24862
- # element state — mark the \`is\` probe that asks for it
24863
- CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
24864
- CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
24865
- \`\`\`
24866
-
24867
- **Marker semantics:**
24868
-
24869
- | Marker | On command | Records |
24870
- |--------|------------|---------|
24871
- | \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
24872
- | \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
24873
- | \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
24874
- | \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
24875
- | \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
24876
- | \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
24877
-
24878
- - A marked command that exits non-zero records nothing — fix the check and re-run it.
24879
- - A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
24880
- - \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
24881
- - **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
24882
-
24883
- - **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
24884
-
24885
- - **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
24886
-
24887
- - **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
24888
- - **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
24889
- - **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
24890
-
24891
- **Every assert type has a marker command.** There is no text form: a printed
24892
- \`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
24893
- reported rather than recorded.
24894
-
24895
- | assertType | The command that records it | Use when |
24896
- |------------|------------------------------|----------|
24897
- | \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
24898
- | \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
24899
- | \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
24900
- | \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
24901
- | \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
24902
- | \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
24903
-
24904
- **Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
24905
-
24906
- - Timestamps, session IDs, exact numeric counts that vary between runs.
24907
- - **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
24908
- - **Relative-time labels** — true only in the moment of the trace, stale by replay:
24909
- - English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
24910
- - Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
24911
- - Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
24912
- - **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
24913
-
24914
- **No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
24915
-
24916
- The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
24917
-
24918
- - ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
24919
- - ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
24920
- - ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
24921
- - If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
24922
-
24923
- **Page-context and selector rules:**
24924
-
24925
- - After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
24926
- - Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
24927
- - When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
24928
-
24929
- **MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
24930
-
24931
- The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
24932
-
24933
- 1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
24934
- 2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
24935
- 3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
24936
-
24937
- **Verify the assertion form actually resolves on the live page.** The
24938
- verification and the recorded assertion are the same marked command:
24939
-
24940
- \`\`\`bash
24941
- # text_visible
24942
- CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
24943
- # element_visible — >=1 printed means present
24944
- CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
24945
- # element_not_visible — 0 printed means absent (works for text too: "text=<text>")
24946
- CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
24947
- # element state — \`is\` prints true/false and exits 0 either way; read it
24948
- CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
24949
- CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
24950
- \`\`\`
24951
-
24952
- When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
24953
-
24954
- ## Status Protocol
24955
-
24956
- Emit exactly one status line per step (outside any code block):
24957
-
24958
- \`\`\`
24959
- STEP_START|<step-id>|<short description>
24960
- STEP_DONE|<step-id>|<what was verified>
24961
- ASSERTION_FAILED|<step-id>|<category: app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
24962
- STEP_SKIPPED|<step-id>|<reason>
24963
- RUN_COMPLETED|passed|<summary>
24964
- RUN_COMPLETED|failed|<summary>
24965
- \`\`\`
24966
-
24967
- **\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
24968
- recorded as FAILED, however many steps went well: nothing else can tell a
24969
- finished route from one you abandoned halfway.
24970
-
24971
- ${callerGuidance}## Start
24972
-
24973
- Begin by clearing cookies, then proceed straight to the first step's instruction.
24974
-
24975
- \`\`\`bash
24976
- agent-browser --session ${sessionName} cookies clear
24977
- \`\`\`
24978
-
24979
- (The cookies-clear preamble belongs to no step, so it is the one command
24980
- without a \`CCQA_STEP\` prefix.)
24981
-
24982
- Emit:
24983
- \`\`\`
24984
- AB_ACTION|cookies_clear
24985
- \`\`\`
24986
-
24987
- Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
24988
- every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
24989
- step is responsible for opening the initial URL.
24990
- `;
24665
+ function roleOfAccessibleName(snapshot, name) {
24666
+ const roles = /* @__PURE__ */ new Set();
24667
+ for (const line of snapshot.split("\n")) {
24668
+ const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
24669
+ if (m && m[2] === name) roles.add(m[1]);
24670
+ }
24671
+ return roles.size === 1 ? [...roles][0] : null;
24672
+ }
24673
+ //#endregion
24674
+ //#region src/runtime/replay-validate.ts
24675
+ function isProbe(x) {
24676
+ return x !== null && !Array.isArray(x);
24677
+ }
24678
+ function runProbe(probe, sessionName) {
24679
+ return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
24680
+ }
24681
+ function runStateCheck(check, sessionName) {
24682
+ let answer = "";
24683
+ const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
24684
+ for (let round = 0; round < rounds; round++) {
24685
+ const r = spawnAB([
24686
+ "--session",
24687
+ sessionName,
24688
+ "is",
24689
+ check.state,
24690
+ check.selector
24691
+ ]);
24692
+ answer = r.status === 0 ? r.stdout.trim() : "";
24693
+ if (answer === String(check.expected)) return {
24694
+ ok: true,
24695
+ reason: ""
24696
+ };
24697
+ if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
24698
+ }
24699
+ return answer === "true" || answer === "false" ? {
24700
+ ok: false,
24701
+ reason: `expected ${check.state}=${check.expected}, page says ${answer}`
24702
+ } : {
24703
+ ok: false,
24704
+ reason: `is ${check.state} answered ${answer || "nothing"}`
24705
+ };
24706
+ }
24707
+ const SELECTOR_POLL_INTERVAL_MS = 500;
24708
+ /** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
24709
+ function runPollCheck(check, sessionName) {
24710
+ const present = (selector) => {
24711
+ const r = spawnAB([
24712
+ "--session",
24713
+ sessionName,
24714
+ "get",
24715
+ "count",
24716
+ selector
24717
+ ]);
24718
+ return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
24719
+ };
24720
+ const deadline = Date.now() + check.timeoutMs;
24721
+ const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
24722
+ let count = NaN;
24723
+ for (let round = 0;; round++) {
24724
+ count = present(check.selector);
24725
+ if (!Number.isNaN(count) && count > 0) return {
24726
+ ok: true,
24727
+ reason: ""
24728
+ };
24729
+ if (round + 1 >= rounds || Date.now() >= deadline) break;
24730
+ sleepSync(SELECTOR_POLL_INTERVAL_MS);
24731
+ }
24732
+ return {
24733
+ ok: false,
24734
+ reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
24735
+ };
24736
+ }
24737
+ /** What the validator records against an action it never replayed. */
24738
+ const CASCADE_REASON = "skipped after a preceding action failed";
24739
+ /** Whether a drop reason means "not replayed", rather than "replayed and failed". */
24740
+ function isCascadeReason(reason) {
24741
+ return reason === CASCADE_REASON;
24742
+ }
24743
+ const SHORT_TIMEOUT_MS = 5e3;
24744
+ const ASSERT_TIMEOUT_MS = 1e4;
24745
+ const INTERACTION_POLL_INTERVAL_MS = 250;
24746
+ const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
24747
+ /** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
24748
+ function attemptsFor(timeoutMs, intervalMs) {
24749
+ return Math.max(1, Math.ceil(timeoutMs / intervalMs));
24750
+ }
24751
+ /**
24752
+ * Convert one recorded action into the `agent-browser` arg list that would
24753
+ * exercise it. Interaction commands come from the shared
24754
+ * `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
24755
+ * get validation-specific handling (poll checks, timeouts, unverifiable
24756
+ * skips). Returns `null` for actions that should not be validated (snapshot
24757
+ * has no side effect; assert types whose codegen forms aren't directly
24758
+ * verifiable here fall through to the caller's `unverifiable` fallback).
24759
+ */
24760
+ function actionToAbArgs(action, sessionName, envOverrides = {}) {
24761
+ const base = ["--session", sessionName];
24762
+ const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
24763
+ switch (action.action) {
24764
+ case "snapshot": return null;
24765
+ case "assert": return assertToAbArgs(action, sub, sessionName);
24766
+ case "wait": {
24767
+ const loc = action.locator;
24768
+ if (!loc) return null;
24769
+ if (loc.by === "text") return [
24770
+ ...base,
24771
+ "wait",
24772
+ "--text",
24773
+ sub(loc.value),
24774
+ "--timeout",
24775
+ String(SHORT_TIMEOUT_MS)
24776
+ ];
24777
+ const raw = sub(locatorToSelector(loc));
24778
+ if (!raw) return null;
24779
+ if (/^\d+$/.test(raw)) return null;
24780
+ if (raw.startsWith("--")) return null;
24781
+ if (raw.startsWith("text=")) return [
24782
+ ...base,
24783
+ "wait",
24784
+ "--text",
24785
+ raw.slice(5),
24786
+ "--timeout",
24787
+ String(SHORT_TIMEOUT_MS)
24788
+ ];
24789
+ return {
24790
+ kind: "poll-present",
24791
+ selector: raw,
24792
+ timeoutMs: SHORT_TIMEOUT_MS
24793
+ };
24794
+ }
24795
+ default: {
24796
+ const tokens = toAgentBrowserArgs(action);
24797
+ if (tokens === null) return null;
24798
+ const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
24799
+ return [...base, ...tokens.map((t) => resolve(t.text))];
24800
+ }
24801
+ }
24802
+ }
24803
+ function roleProbe(base, role, name) {
24804
+ return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
24805
+ }
24806
+ /**
24807
+ * Presence asked the way the locator addresses the element. `null` is
24808
+ * unverifiable rather than absent: a count from a selector the engine never
24809
+ * parsed is not evidence.
24810
+ */
24811
+ function presenceCheck(locator, selector, sub, base) {
24812
+ if (locator?.by === "text") return [
24813
+ ...base,
24814
+ "wait",
24815
+ "--text",
24816
+ sub(locator.value),
24817
+ "--timeout",
24818
+ String(ASSERT_TIMEOUT_MS)
24819
+ ];
24820
+ if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
24821
+ if (locator !== void 0 && locator.by !== "css") return null;
24822
+ const named = parseNotation(selector);
24823
+ if (named === "opaque") return null;
24824
+ if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
24825
+ if (named !== null) return null;
24826
+ return {
24827
+ kind: "poll-present",
24828
+ selector,
24829
+ timeoutMs: ASSERT_TIMEOUT_MS
24830
+ };
24831
+ }
24832
+ function assertToAbArgs(action, sub, sessionName) {
24833
+ const base = ["--session", sessionName];
24834
+ const val = sub(action.value ?? action.observation);
24835
+ const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
24836
+ switch (action.assert) {
24837
+ case "text_visible":
24838
+ if (!val) return null;
24839
+ return [
24840
+ ...base,
24841
+ "wait",
24842
+ "--text",
24843
+ val,
24844
+ "--timeout",
24845
+ String(ASSERT_TIMEOUT_MS)
24846
+ ];
24847
+ case "text_not_visible": return null;
24848
+ case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
24849
+ case "element_not_visible": return null;
24850
+ case "url_contains": return null;
24851
+ case "element_enabled":
24852
+ case "element_disabled":
24853
+ case "element_checked":
24854
+ case "element_unchecked":
24855
+ if (!sel || action.locator?.by !== "css") return null;
24856
+ if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
24857
+ return {
24858
+ kind: "state",
24859
+ state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
24860
+ selector: sel,
24861
+ expected: action.assert === "element_enabled" || action.assert === "element_checked"
24862
+ };
24863
+ default: return null;
24864
+ }
24991
24865
  }
24992
- function buildTracePrompt(title) {
24993
- return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
24866
+ const NO_STEP_ID = "__no_step__";
24867
+ /**
24868
+ * The role a labelled element has when addressing it by label finds nothing.
24869
+ *
24870
+ * `getByLabel` needs a real association — a `<label for>`, a wrapping label,
24871
+ * an `aria-labelledby`. A form that has none still shows the same string as
24872
+ * the field's accessible name, which is what the recorder read off the
24873
+ * snapshot, so the label it recorded matches nothing while the same text as a
24874
+ * role's name matches exactly. Only the actions whose element has one
24875
+ * unambiguous role are listed: a labelled `click` may be a button, a link or
24876
+ * a checkbox, and guessing there would replace a locator that failed with one
24877
+ * that finds the wrong element.
24878
+ */
24879
+ const LABEL_FALLBACK_ROLE = {
24880
+ fill: "textbox",
24881
+ type: "textbox",
24882
+ check: "checkbox",
24883
+ uncheck: "checkbox"
24884
+ };
24885
+ /** Whether agent-browser's failure was "no element", the one the fallback answers. */
24886
+ function notFound(result) {
24887
+ return /not\s+found|no\s+element|no\s+such\s+element|none\s+match\s+name/i.test(`${result.stderr} ${result.stdout}`);
24888
+ }
24889
+ /** Whether this argv navigates — the only action safe to repeat wholesale. */
24890
+ function isOpen(argv) {
24891
+ return argv[2] === "open";
24892
+ }
24893
+ /** A navigation another navigation took away, rather than one that failed. */
24894
+ function superseded(result) {
24895
+ return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
24896
+ }
24897
+ /** The one line both callers log when the fallback rewrote a locator. */
24898
+ function formatPromotion(promotion) {
24899
+ return `locator rewritten to the form that replays: ${promotion}`;
24900
+ }
24901
+ /** The same action addressed by role + accessible name, or null when there is no such form. */
24902
+ function labelFallback(action) {
24903
+ const loc = action.locator;
24904
+ if (!loc || loc.by !== "label") return null;
24905
+ const role = LABEL_FALLBACK_ROLE[action.action];
24906
+ if (role === void 0) return null;
24907
+ return {
24908
+ ...action,
24909
+ locator: {
24910
+ by: "role",
24911
+ value: role,
24912
+ name: loc.value,
24913
+ exact: true
24914
+ }
24915
+ };
24994
24916
  }
24995
24917
  /**
24996
- * The case's expectations, plus the one rule that decides where each is
24997
- * recorded. Without the rule a recorder either asserts everything at the end —
24998
- * on a page the flow has already left — or asserts each item at every step it
24999
- * might apply to, which is the over-assertion the selector rules exist to
25000
- * prevent.
24918
+ * Spawn one interaction, waiting for its element the way an assertion waits
24919
+ * for its text: a recording pauses for a snapshot between opening a page and
24920
+ * typing into it, the replay does not, so the replay reaches a field sooner
24921
+ * than the recording ever did.
24922
+ *
24923
+ * Only a "no element" failure waits — anything else repeats identically. When
24924
+ * there is an `alternate` form it is tried on every attempt rather than after
24925
+ * the wait: a label the page never associates with its input will not start
24926
+ * matching, so waiting first spends the budget for nothing.
25001
24927
  */
25002
- function renderExpectations(expectations) {
25003
- if (expectations.length === 0) return "";
25004
- return `
25005
- ## Expected results
25006
-
25007
- The case states these without saying which step delivers them:
25008
-
25009
- ${expectations.map((text) => `- ${text}`).join("\n")}
25010
-
25011
- **Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
25012
- `;
24928
+ function runInteraction(argv, alternate, patience, sessionName) {
24929
+ const started = Date.now();
24930
+ const deadline = started + ASSERT_TIMEOUT_MS;
24931
+ let settled = false;
24932
+ for (let attempt = 0;; attempt++) {
24933
+ let result = spawnAB(argv);
24934
+ if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
24935
+ const waitedMs = Date.now() - started;
24936
+ if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
24937
+ settled = true;
24938
+ spawnAB([
24939
+ "--session",
24940
+ sessionName,
24941
+ "wait",
24942
+ "--load",
24943
+ "networkidle"
24944
+ ]);
24945
+ continue;
24946
+ }
24947
+ if (result.status === 0 || !notFound(result)) return {
24948
+ result,
24949
+ viaAlternate: false,
24950
+ waitedMs
24951
+ };
24952
+ if (alternate !== null) {
24953
+ const alt = spawnAB(alternate);
24954
+ if (alt.status === 0) return {
24955
+ result: alt,
24956
+ viaAlternate: true,
24957
+ waitedMs
24958
+ };
24959
+ }
24960
+ if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
24961
+ patience.spent = true;
24962
+ return {
24963
+ result,
24964
+ viaAlternate: false,
24965
+ waitedMs: Date.now() - started
24966
+ };
24967
+ }
24968
+ sleepSync(INTERACTION_POLL_INTERVAL_MS);
24969
+ }
25013
24970
  }
25014
24971
  /**
25015
- * What the cleanup must make true. Separate from the case's expectations
25016
- * because its steps are: a deletion is confirmed by the thing being gone,
25017
- * which is only true after the undo has run, and asserting it among the
25018
- * case's own steps would check it while the thing still exists.
24972
+ * Say what a `by: "css"` locator that is not CSS actually addresses.
24973
+ *
24974
+ * `get count` answers 0 for Playwright's notation instead of failing, so a
24975
+ * route holding one reads as dead. Rewritten in place, and reported, so the
24976
+ * saved route carries the form that replays rather than the one that counts
24977
+ * nothing. Notation this cannot convert leaves the locator alone; the assert
24978
+ * path then treats it as unverifiable rather than absent.
25019
24979
  */
25020
- function renderCleanupExpectations(expectations) {
25021
- if (expectations.length === 0) return "";
25022
- return `
25023
- ## Expected results of the cleanup
25024
-
25025
- The case states these about its cleanup steps:
25026
-
25027
- ${expectations.map((text) => `- ${text}`).join("\n")}
25028
-
25029
- **Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
25030
- `;
24980
+ function nameNotation(action) {
24981
+ const loc = action.locator;
24982
+ if (action.action !== "assert" || loc?.by !== "css") return null;
24983
+ const named = parseNotation(loc.value);
24984
+ if (named === null || named === "opaque" || named.by !== "text") return null;
24985
+ action.locator = named;
24986
+ return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
25031
24987
  }
25032
24988
  /**
25033
- * Sections of the case ccqa does not interpret — the precondition, notes the
25034
- * author left. Passed through verbatim: they are written for whoever runs the
25035
- * case, and the recorder is now that reader.
24989
+ * An attribute selector that counted nothing, asked of the accessibility tree
24990
+ * the name was read from. See docs/targets.md for why the two disagree.
24991
+ *
24992
+ * The snapshot is read for the role, then `find role` is asked as well: the
24993
+ * tree says the name exists somewhere, the probe says the replay can reach it,
24994
+ * and only the second is what the route will depend on.
25036
24995
  */
25037
- function renderContext(context) {
25038
- if (context.length === 0) return "";
25039
- return `
25040
- ## About this case
25041
-
25042
- ${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
25043
- `;
24996
+ function askTheAccessibilityTree(action, sub, sessionName) {
24997
+ if (action.action !== "assert" || action.locator?.by !== "css") return null;
24998
+ const name = nameFromAttributeSelector(action.locator.value);
24999
+ if (name === null) return null;
25000
+ const snapshot = spawnAB([
25001
+ "--session",
25002
+ sessionName,
25003
+ "snapshot"
25004
+ ]);
25005
+ if (snapshot.status !== 0) return null;
25006
+ const role = roleOfAccessibleName(snapshot.stdout, sub(name));
25007
+ if (role === null) return null;
25008
+ if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
25009
+ const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
25010
+ action.locator = {
25011
+ by: "role",
25012
+ value: role,
25013
+ name,
25014
+ exact: true
25015
+ };
25016
+ return {
25017
+ skipped: false,
25018
+ ok: true,
25019
+ reason: "",
25020
+ promoted
25021
+ };
25022
+ }
25023
+ /**
25024
+ * Replay one recorded action against the validation session. Element-presence
25025
+ * checks go through `runPollCheck` (which uses `get count`, never the blocking
25026
+ * `wait <selector>`); everything else spawns the agent-browser argv.
25027
+ */
25028
+ function runValidationAction(action, sessionName, envOverrides = {}, patience) {
25029
+ const named = nameNotation(action);
25030
+ const withName = (outcome) => named === null || !outcome.ok ? outcome : {
25031
+ ...outcome,
25032
+ promoted: outcome.promoted ?? named
25033
+ };
25034
+ const built = actionToAbArgs(action, sessionName, envOverrides);
25035
+ if (built === null) return {
25036
+ skipped: true,
25037
+ ok: false,
25038
+ reason: ""
25039
+ };
25040
+ if (isProbe(built)) {
25041
+ const { ok, reason } = runProbe(built, sessionName);
25042
+ if (ok) return withName({
25043
+ skipped: false,
25044
+ ok,
25045
+ reason
25046
+ });
25047
+ const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
25048
+ return askTheAccessibilityTree(action, resolve, sessionName) ?? {
25049
+ skipped: false,
25050
+ ok,
25051
+ reason
25052
+ };
25053
+ }
25054
+ const fallback = labelFallback(action);
25055
+ const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
25056
+ const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
25057
+ if (viaAlternate) {
25058
+ const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
25059
+ action.locator = fallback.locator;
25060
+ return {
25061
+ skipped: false,
25062
+ ok: true,
25063
+ reason: "",
25064
+ promoted
25065
+ };
25066
+ }
25067
+ if (result.status === 0) return withName({
25068
+ skipped: false,
25069
+ ok: true,
25070
+ reason: ""
25071
+ });
25072
+ return {
25073
+ skipped: false,
25074
+ ok: false,
25075
+ reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
25076
+ };
25044
25077
  }
25045
25078
  /**
25046
- * The project's own recording guidance. Where the case says what to do, this
25047
- * says how this project is driven — which account signs in, what has to be
25048
- * true before the first step. Prose, verbatim: it is written for whoever runs
25049
- * the case, and this is now that reader.
25079
+ * Replay actions in order against a live session and stop at the first that
25080
+ * fails. Unlike `validateActions` there is no cascade or rescue: the caller
25081
+ * hands what is left to someone who can look at the page. A wait this cannot
25082
+ * reproduce still settles the page, since the next action may depend on it;
25083
+ * a check it cannot run is returned as `unchecked`, never as passed.
25050
25084
  */
25051
- function renderConventions(conventions) {
25052
- if (conventions.length === 0) return "";
25053
- return `
25054
- ## How this project is driven
25055
-
25056
- ${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
25057
- `;
25085
+ function replayUntilFailure(actions, opts) {
25086
+ const patience = { spent: false };
25087
+ const passed = [];
25088
+ const unchecked = [];
25089
+ for (const action of actions) {
25090
+ const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25091
+ if (outcome.skipped && action.action === "wait") spawnAB([
25092
+ "--session",
25093
+ opts.sessionName,
25094
+ "wait",
25095
+ "--load",
25096
+ "networkidle"
25097
+ ]);
25098
+ else if (outcome.skipped && action.action === "assert") {
25099
+ unchecked.push(action);
25100
+ continue;
25101
+ } else if (outcome.skipped) return {
25102
+ passed,
25103
+ unchecked,
25104
+ failed: {
25105
+ action,
25106
+ reason: "the recording does not say how to replay it"
25107
+ }
25108
+ };
25109
+ else if (!outcome.ok) return {
25110
+ passed,
25111
+ unchecked,
25112
+ failed: {
25113
+ action,
25114
+ reason: outcome.reason
25115
+ }
25116
+ };
25117
+ passed.push(action);
25118
+ }
25119
+ return {
25120
+ passed,
25121
+ unchecked
25122
+ };
25058
25123
  }
25059
- //#endregion
25060
- //#region src/ir/accessible-name.ts
25061
- /**
25062
- * A recorder reads an accessible name off a snapshot and writes it as the
25063
- * attribute it guessed the name came from. The name can come from a `<label>`
25064
- * or `aria-labelledby` instead, and the attribute's absence is not evidence
25065
- * about the element. See docs/targets.md for the measurement.
25066
- */
25067
- const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
25068
- /**
25069
- * The name this selector may be after, or null.
25070
- *
25071
- * `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
25072
- * mean what they say, and a zero from one of those is evidence.
25073
- */
25074
- function nameFromAttributeSelector(selector) {
25075
- const m = NAMING_ATTR.exec(selector.trim());
25076
- const name = m?.[1] ?? m?.[2];
25077
- return name === void 0 || name === "" ? null : name;
25124
+ function validateActions(actions, opts) {
25125
+ const kept = [];
25126
+ const dropped = [];
25127
+ const promoted = [];
25128
+ const patience = { spent: false };
25129
+ let skipFromStepId = null;
25130
+ for (let i = 0; i < actions.length; i++) {
25131
+ const action = actions[i];
25132
+ opts.onProgress?.(i, actions.length, action);
25133
+ const stepId = action.stepId ?? NO_STEP_ID;
25134
+ if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
25135
+ if (skipFromStepId !== null && isPassiveAction(action.action)) {
25136
+ dropped.push({
25137
+ index: i,
25138
+ action,
25139
+ reason: CASCADE_REASON
25140
+ });
25141
+ continue;
25142
+ }
25143
+ const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25144
+ if (outcome.promoted) promoted.push(outcome.promoted);
25145
+ if (outcome.skipped) {
25146
+ kept.push(action);
25147
+ continue;
25148
+ }
25149
+ if (outcome.ok) {
25150
+ kept.push(action);
25151
+ if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
25152
+ continue;
25153
+ }
25154
+ dropped.push({
25155
+ index: i,
25156
+ action,
25157
+ reason: outcome.reason
25158
+ });
25159
+ if (!isPassiveAction(action.action)) skipFromStepId = stepId;
25160
+ }
25161
+ const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
25162
+ promoted.push(...afterRescue.promoted ?? []);
25163
+ return {
25164
+ ...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
25165
+ promoted
25166
+ };
25078
25167
  }
25079
25168
  /**
25080
- * The role of the first node in an accessibility snapshot whose name is
25081
- * exactly `name`, or null.
25082
- *
25083
- * agent-browser prints one node per line as `<role> "<name>"`. Matched on the
25084
- * whole name, so a tree holding both "Priority" and "Priority *" answers for
25085
- * the one that was asked, and refused when two roles carry it.
25169
+ * Translate the internal `{ kept, dropped }` result of the rescue pass
25170
+ * into the public-facing shape. In strict mode the caller sees the same
25171
+ * shape as before (kept/dropped); in lenient mode the still-failed
25172
+ * actions move to `unstable` with `replayUnstable: true` tagged on, so
25173
+ * codegen can warn about them while still emitting the line.
25086
25174
  */
25087
- function roleOfAccessibleName(snapshot, name) {
25088
- const roles = /* @__PURE__ */ new Set();
25089
- for (const line of snapshot.split("\n")) {
25090
- const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
25091
- if (m && m[2] === name) roles.add(m[1]);
25175
+ function splitByMode(originalActions, result, mode) {
25176
+ if (mode === "strict") return {
25177
+ kept: result.kept,
25178
+ unstable: [],
25179
+ dropped: result.dropped,
25180
+ rescuedSteps: result.rescuedSteps
25181
+ };
25182
+ const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
25183
+ const keptSet = new Set(result.kept);
25184
+ const finalKept = [];
25185
+ const unstable = [];
25186
+ for (let i = 0; i < originalActions.length; i++) {
25187
+ const action = originalActions[i];
25188
+ if (keptSet.has(action)) {
25189
+ finalKept.push(action);
25190
+ continue;
25191
+ }
25192
+ const drop = droppedByIndex.get(i);
25193
+ if (drop) {
25194
+ action.replayUnstable = true;
25195
+ action.replayReason = drop.reason;
25196
+ unstable.push(action);
25197
+ }
25092
25198
  }
25093
- return roles.size === 1 ? [...roles][0] : null;
25094
- }
25095
- //#endregion
25096
- //#region src/runtime/replay-validate.ts
25097
- function isProbe(x) {
25098
- return x !== null && !Array.isArray(x);
25099
- }
25100
- function runProbe(probe, sessionName) {
25101
- return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
25199
+ return {
25200
+ kept: finalKept,
25201
+ unstable,
25202
+ dropped: [],
25203
+ rescuedSteps: result.rescuedSteps
25204
+ };
25102
25205
  }
25103
- function runStateCheck(check, sessionName) {
25104
- let answer = "";
25105
- const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
25106
- for (let round = 0; round < rounds; round++) {
25107
- const r = spawnAB([
25108
- "--session",
25109
- sessionName,
25110
- "is",
25111
- check.state,
25112
- check.selector
25113
- ]);
25114
- answer = r.status === 0 ? r.stdout.trim() : "";
25115
- if (answer === String(check.expected)) return {
25116
- ok: true,
25117
- reason: ""
25118
- };
25119
- if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
25206
+ function rescueLostSteps(actions, kept, dropped, opts, patience) {
25207
+ const stepsWithSurvivors = /* @__PURE__ */ new Set();
25208
+ for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
25209
+ const lostStepDrops = /* @__PURE__ */ new Map();
25210
+ for (const d of dropped) {
25211
+ const id = d.action.stepId;
25212
+ if (!id || stepsWithSurvivors.has(id)) continue;
25213
+ const list = lostStepDrops.get(id) ?? [];
25214
+ list.push(d);
25215
+ lostStepDrops.set(id, list);
25120
25216
  }
25121
- return answer === "true" || answer === "false" ? {
25122
- ok: false,
25123
- reason: `expected ${check.state}=${check.expected}, page says ${answer}`
25124
- } : {
25125
- ok: false,
25126
- reason: `is ${check.state} answered ${answer || "nothing"}`
25217
+ if (lostStepDrops.size === 0) return {
25218
+ kept,
25219
+ dropped
25127
25220
  };
25128
- }
25129
- const SELECTOR_POLL_INTERVAL_MS = 500;
25130
- /** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
25131
- function runPollCheck(check, sessionName) {
25132
- const present = (selector) => {
25133
- const r = spawnAB([
25134
- "--session",
25135
- sessionName,
25136
- "get",
25137
- "count",
25138
- selector
25139
- ]);
25140
- return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
25221
+ const promoted = [];
25222
+ const rescuedIndices = /* @__PURE__ */ new Set();
25223
+ const rescuedSteps = [];
25224
+ const strict = (opts.mode ?? "lenient") === "strict";
25225
+ for (const [stepId, drops] of lostStepDrops.entries()) {
25226
+ const replayed = [];
25227
+ const unverifiable = [];
25228
+ let stillFailing = false;
25229
+ for (const d of drops) {
25230
+ const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
25231
+ if (outcome.promoted) promoted.push(outcome.promoted);
25232
+ if (outcome.skipped) {
25233
+ unverifiable.push(d.index);
25234
+ continue;
25235
+ }
25236
+ if (outcome.ok) {
25237
+ replayed.push(d.index);
25238
+ continue;
25239
+ }
25240
+ stillFailing = true;
25241
+ if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
25242
+ }
25243
+ if (replayed.length === 0) continue;
25244
+ for (const i of replayed) rescuedIndices.add(i);
25245
+ if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
25246
+ rescuedSteps.push(stepId);
25247
+ }
25248
+ if (rescuedIndices.size === 0) return {
25249
+ kept,
25250
+ dropped,
25251
+ promoted
25141
25252
  };
25142
- const deadline = Date.now() + check.timeoutMs;
25143
- const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
25144
- let count = NaN;
25145
- for (let round = 0;; round++) {
25146
- count = present(check.selector);
25147
- if (!Number.isNaN(count) && count > 0) return {
25148
- ok: true,
25149
- reason: ""
25150
- };
25151
- if (round + 1 >= rounds || Date.now() >= deadline) break;
25152
- sleepSync(SELECTOR_POLL_INTERVAL_MS);
25253
+ const keptSet = new Set(kept);
25254
+ const newKept = [];
25255
+ for (let i = 0; i < actions.length; i++) {
25256
+ const action = actions[i];
25257
+ if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
25153
25258
  }
25154
25259
  return {
25155
- ok: false,
25156
- reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
25260
+ kept: newKept,
25261
+ dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
25262
+ rescuedSteps,
25263
+ promoted
25157
25264
  };
25158
25265
  }
25159
- /** What the validator records against an action it never replayed. */
25160
- const CASCADE_REASON = "skipped after a preceding action failed";
25161
- /** Whether a drop reason means "not replayed", rather than "replayed and failed". */
25162
- function isCascadeReason(reason) {
25163
- return reason === CASCADE_REASON;
25164
- }
25165
- const SHORT_TIMEOUT_MS = 5e3;
25166
- const ASSERT_TIMEOUT_MS = 1e4;
25167
- const INTERACTION_POLL_INTERVAL_MS = 250;
25168
- const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
25169
- /** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
25170
- function attemptsFor(timeoutMs, intervalMs) {
25171
- return Math.max(1, Math.ceil(timeoutMs / intervalMs));
25172
- }
25173
25266
  /**
25174
- * Convert one recorded action into the `agent-browser` arg list that would
25175
- * exercise it. Interaction commands come from the shared
25176
- * `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
25177
- * get validation-specific handling (poll checks, timeouts, unverifiable
25178
- * skips). Returns `null` for actions that should not be validated (snapshot
25179
- * has no side effect; assert types whose codegen forms aren't directly
25180
- * verifiable here fall through to the caller's `unverifiable` fallback).
25267
+ * Passive (read-only) actions whose only effect is observation. When a
25268
+ * preceding action fails, dropping these too is the right move because
25269
+ * they were trying to observe state the failed action would have set up.
25181
25270
  */
25182
- function actionToAbArgs(action, sessionName, envOverrides = {}) {
25183
- const base = ["--session", sessionName];
25184
- const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
25185
- switch (action.action) {
25186
- case "snapshot": return null;
25187
- case "assert": return assertToAbArgs(action, sub, sessionName);
25188
- case "wait": {
25189
- const loc = action.locator;
25190
- if (!loc) return null;
25191
- if (loc.by === "text") return [
25192
- ...base,
25193
- "wait",
25194
- "--text",
25195
- sub(loc.value),
25196
- "--timeout",
25197
- String(SHORT_TIMEOUT_MS)
25198
- ];
25199
- const raw = sub(locatorToSelector(loc));
25200
- if (!raw) return null;
25201
- if (/^\d+$/.test(raw)) return null;
25202
- if (raw.startsWith("--")) return null;
25203
- if (raw.startsWith("text=")) return [
25204
- ...base,
25205
- "wait",
25206
- "--text",
25207
- raw.slice(5),
25208
- "--timeout",
25209
- String(SHORT_TIMEOUT_MS)
25210
- ];
25211
- return {
25212
- kind: "poll-present",
25213
- selector: raw,
25214
- timeoutMs: SHORT_TIMEOUT_MS
25215
- };
25216
- }
25217
- default: {
25218
- const tokens = toAgentBrowserArgs(action);
25219
- if (tokens === null) return null;
25220
- const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
25221
- return [...base, ...tokens.map((t) => resolve(t.text))];
25222
- }
25223
- }
25224
- }
25225
- function roleProbe(base, role, name) {
25226
- return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
25271
+ function isPassiveAction(action) {
25272
+ return action === "snapshot" || action === "wait" || action === "assert";
25227
25273
  }
25274
+ //#endregion
25275
+ //#region src/prompts/previous-recording.ts
25228
25276
  /**
25229
- * Presence asked the way the locator addresses the element. `null` is
25230
- * unverifiable rather than absent: a count from a selector the engine never
25231
- * parsed is not evidence.
25277
+ * The previous recording as the commands that perform it, for the trace prompt:
25278
+ * a re-record is usually one moved screen in an otherwise intact route.
25232
25279
  */
25233
- function presenceCheck(locator, selector, sub, base) {
25234
- if (locator?.by === "text") return [
25235
- ...base,
25236
- "wait",
25237
- "--text",
25238
- sub(locator.value),
25239
- "--timeout",
25240
- String(ASSERT_TIMEOUT_MS)
25241
- ];
25242
- if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
25243
- if (locator !== void 0 && locator.by !== "css") return null;
25244
- const named = parseNotation(selector);
25245
- if (named === "opaque") return null;
25246
- if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
25247
- if (named !== null) return null;
25248
- return {
25249
- kind: "poll-present",
25250
- selector,
25251
- timeoutMs: ASSERT_TIMEOUT_MS
25252
- };
25280
+ function renderPreviousRecording(actions, sessionName, currentStepIds) {
25281
+ const current = new Set([SETUP_STEP_ID, ...currentStepIds]);
25282
+ const brokenStep = actions.find(failedToReplay)?.stepId;
25283
+ const byStep = /* @__PURE__ */ new Map();
25284
+ for (const action of actions) {
25285
+ if (action.stepId === void 0 || !current.has(action.stepId)) continue;
25286
+ const command = previousCommand(action, sessionName, action.stepId === brokenStep);
25287
+ if (command === null) continue;
25288
+ const lines = byStep.get(action.stepId) ?? [];
25289
+ lines.push(command);
25290
+ byStep.set(action.stepId, lines);
25291
+ }
25292
+ if (byStep.size === 0) return "";
25293
+ return `
25294
+ ## Previous recording
25295
+
25296
+ This case was recorded before. Below is what that recording did, step by step,
25297
+ as the commands that perform it. The application may have moved since, and the
25298
+ steps above may have been edited: **the steps above are the contract, this is a
25299
+ map.**
25300
+
25301
+ For a step listed here, this replaces the snapshot-first routine of the
25302
+ Execution Workflow:
25303
+
25304
+ 1. While a step's previous commands still do what its instruction and
25305
+ \`Expected\` say, call the \`replay_step\` tool with the step id instead of
25306
+ running them yourself. It runs and records them in this session, checks
25307
+ included, and stops at the first that fails. Do not snapshot first.
25308
+ 2. When it reports they all ran, emit \`STEP_DONE\` and go to the next step
25309
+ without a confirming snapshot.
25310
+ 3. When it reports a failure, the commands before it are recorded and the
25311
+ page is where they left it: finish that step with the Execution Workflow —
25312
+ snapshot, find what works, record it — without re-running what was
25313
+ recorded. A step whose instruction or \`Expected\` asks for something these
25314
+ commands do not do skips \`replay_step\` and uses the Execution Workflow
25315
+ from the start. The next step goes back to \`replay_step\`.
25316
+ 4. A step you record yourself takes its assertions from its current
25317
+ \`Expected\`: keep a previous check only while it verifies what
25318
+ \`Expected\` says now. A step whose \`Expected\` has changed is one to
25319
+ record yourself, since \`replay_step\` records the previous checks as
25320
+ they are.
25321
+ 5. A command marked \`# did not replay last time\` is where the previous
25322
+ recording's own validation found the route broken: expect that step to
25323
+ need the Execution Workflow, and do not keep a form that only half works.
25324
+
25325
+ A step with no section here is recorded the usual way.
25326
+
25327
+ ${[...byStep].map(([stepId, lines]) => `### ${stepId}\n\n\`\`\`bash\n${lines.join("\n")}\n\`\`\``).join("\n\n")}
25328
+ `;
25253
25329
  }
25254
- function assertToAbArgs(action, sub, sessionName) {
25255
- const base = ["--session", sessionName];
25256
- const val = sub(action.value ?? action.observation);
25257
- const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
25258
- switch (action.assert) {
25259
- case "text_visible":
25260
- if (!val) return null;
25261
- return [
25262
- ...base,
25263
- "wait",
25264
- "--text",
25265
- val,
25266
- "--timeout",
25267
- String(ASSERT_TIMEOUT_MS)
25268
- ];
25269
- case "text_not_visible": return null;
25270
- case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
25271
- case "element_not_visible": return null;
25272
- case "url_contains": return null;
25273
- case "element_enabled":
25274
- case "element_disabled":
25275
- case "element_checked":
25276
- case "element_unchecked":
25277
- if (!sel || action.locator?.by !== "css") return null;
25278
- if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
25279
- return {
25280
- kind: "state",
25281
- state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
25282
- selector: sel,
25283
- expected: action.assert === "element_enabled" || action.assert === "element_checked"
25284
- };
25285
- default: return null;
25286
- }
25330
+ function previousCommand(action, sessionName, inBrokenStep) {
25331
+ if (action.action === "wait" && action.locator?.by !== "text") return null;
25332
+ const marked = action.action === "assert" ? markedAssertArgs(action) : null;
25333
+ const tokens = marked?.tokens ?? (action.action === "assert" ? null : toAgentBrowserArgs(action));
25334
+ if (tokens === null) return null;
25335
+ const line = [
25336
+ `CCQA_STEP=${action.stepId}`,
25337
+ ...marked ? [`CCQA_ASSERT=${shellWord(marked.marker)}`] : [],
25338
+ ...action.secret ? ["CCQA_SECRET=1"] : [],
25339
+ "agent-browser",
25340
+ "--session",
25341
+ sessionName,
25342
+ renderTokens(tokens)
25343
+ ].join(" ");
25344
+ return inBrokenStep && failedToReplay(action) ? `${line} # did not replay last time` : line;
25287
25345
  }
25288
- const NO_STEP_ID = "__no_step__";
25289
- /**
25290
- * The role a labelled element has when addressing it by label finds nothing.
25291
- *
25292
- * `getByLabel` needs a real association — a `<label for>`, a wrapping label,
25293
- * an `aria-labelledby`. A form that has none still shows the same string as
25294
- * the field's accessible name, which is what the recorder read off the
25295
- * snapshot, so the label it recorded matches nothing while the same text as a
25296
- * role's name matches exactly. Only the actions whose element has one
25297
- * unambiguous role are listed: a labelled `click` may be a button, a link or
25298
- * a checkbox, and guessing there would replace a locator that failed with one
25299
- * that finds the wrong element.
25300
- */
25301
- const LABEL_FALLBACK_ROLE = {
25302
- fill: "textbox",
25303
- type: "textbox",
25304
- check: "checkbox",
25305
- uncheck: "checkbox"
25306
- };
25307
- /** Whether agent-browser's failure was "no element", the one the fallback answers. */
25308
- function notFound(result) {
25309
- return /not\s+found|no\s+element|no\s+such\s+element/i.test(`${result.stderr} ${result.stdout}`);
25346
+ /** Failed on its own, rather than skipped behind an earlier failure. */
25347
+ function failedToReplay(action) {
25348
+ return action.replayUnstable === true && !isCascadeReason(action.replayReason);
25310
25349
  }
25311
- /** Whether this argv navigates — the only action safe to repeat wholesale. */
25312
- function isOpen(argv) {
25313
- return argv[2] === "open";
25350
+ function renderTokens(tokens) {
25351
+ return tokens.map((t) => t.expandsEnv ? quote(t.text) : t.text).join(" ");
25314
25352
  }
25315
- /** A navigation another navigation took away, rather than one that failed. */
25316
- function superseded(result) {
25317
- return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
25353
+ /** Double-quoted with `${VAR}` refs left live, as the recording keeps them; any other `$` is literal text. */
25354
+ function quote(s) {
25355
+ return `"${s.replace(/(["\\`])|\$(?!\{[A-Za-z_]\w*\})/g, (m, c) => `\\${c ?? m}`)}"`;
25318
25356
  }
25319
- /** The one line both callers log when the fallback rewrote a locator. */
25320
- function formatPromotion(promotion) {
25321
- return `locator rewritten to the form that replays: ${promotion}`;
25357
+ function shellWord(s) {
25358
+ return /^[\w:/.,=-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
25359
+ }
25360
+ //#endregion
25361
+ //#region src/prompts/trace.ts
25362
+ function generateSessionName() {
25363
+ return `ccqa-trace-${buildRunId()}`;
25364
+ }
25365
+ /**
25366
+ * Build the trace system prompt. `input.steps` is a flat list with includes
25367
+ * already expanded (each step carries id / source / instruction / expected).
25368
+ * The spec opens URLs via explicit step instructions (e.g.
25369
+ * `instruction: "${APP_URL}/articles を開く"`).
25370
+ *
25371
+ * In v0.4 every spec is traced from scratch — block contents are inlined
25372
+ * into the spec's own step list at expand time, so the prompt has no
25373
+ * special "this is a block" mode. The `source` tag on each step still
25374
+ * distinguishes spec-native steps from inlined block steps for the
25375
+ * `// step:` comments in the eventual codegen output.
25376
+ */
25377
+ function buildTraceSystemPrompt(input) {
25378
+ const sessionName = input.sessionName ?? generateSessionName();
25379
+ const firstStepId = input.steps[0]?.id ?? "step-01";
25380
+ const callerGuidance = input.instruction ? `## Caller Guidance
25381
+
25382
+ The caller provided extra guidance for this recording — for example, a drift
25383
+ audit's finding about what a previous recording of this spec got wrong. Treat
25384
+ it as advice on what to avoid or verify while recording; the spec's steps
25385
+ above remain the contract for what to do.
25386
+
25387
+ ${input.instruction}
25388
+
25389
+ ` : "";
25390
+ const stepsText = input.steps.map((step) => [
25391
+ `### ${step.id} [${step.source}]`,
25392
+ `- **Instruction**: ${step.instruction}`,
25393
+ ...step.expected ? [`- **Expected**: ${step.expected}`] : []
25394
+ ].join("\n")).join("\n\n");
25395
+ const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
25396
+ const contextText = renderContext(input.context ?? []);
25397
+ const conventionsText = renderConventions(input.conventions ?? []);
25398
+ const previousText = renderPreviousRecording(input.previousRecording ?? [], sessionName, input.steps.map((s) => s.id));
25399
+ return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
25400
+
25401
+ ## Session
25402
+
25403
+ SESSION NAME: \`${sessionName}\`
25404
+
25405
+ Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
25406
+
25407
+ **Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
25408
+ env assignment naming the spec step it executes**, e.g.
25409
+
25410
+ \`\`\`bash
25411
+ CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
25412
+ \`\`\`
25413
+
25414
+ This prefix is what ties each recorded action to its step in the generated
25415
+ test — a command without it loses its step attribution.
25416
+
25417
+ **Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
25418
+ in, opening the application, putting the account where the precondition says it
25419
+ starts — those are not step-01, and marking them as step-01 makes the first
25420
+ step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
25421
+ first step the case actually lists.
25422
+
25423
+ Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
25424
+ itself ignores the variable.
25425
+
25426
+ Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
25427
+ assignment (alongside \`CCQA_STEP\`) that records the check as a test
25428
+ assertion — see "Assertion Protocol".
25429
+
25430
+ **A command that types into a password field carries \`CCQA_SECRET=1\`** as
25431
+ well, e.g.
25432
+
25433
+ \`\`\`bash
25434
+ CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
25435
+ \`\`\`
25436
+
25437
+ A password field is an \`<input type="password">\` — the snapshot shows its
25438
+ value as dots, and nothing else about the page is a password. The prefix is
25439
+ what stops the typed value from being written to the recording: a value that
25440
+ came from a \`\${VAR}\` is kept as that reference, and one typed literally is
25441
+ dropped along with its command.
25442
+
25443
+ ## Browser Commands
25444
+
25445
+ \`\`\`
25446
+ CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
25447
+ CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -i -c # interactive elements only — the default for finding controls
25448
+ CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -c -s "<css>" # the page's text, scoped to one region
25449
+ CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
25450
+ CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
25451
+ CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
25452
+ CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
25453
+ CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
25454
+ CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
25455
+ CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
25456
+ CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
25457
+ CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
25458
+ CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
25459
+ agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
25460
+ CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
25461
+ CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
25462
+ # See "Selector Rules" for the full \`find\` subset.
25463
+ # IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
25464
+ # CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
25465
+ # Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
25466
+ # (which waits internally). To check an element exists, use \`get count "<selector>"\`.
25467
+ \`\`\`
25468
+
25469
+ ## Selector Rules
25470
+
25471
+ **ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
25472
+
25473
+ | Format | Use when |
25474
+ |--------|----------|
25475
+ | \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
25476
+ | \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
25477
+ | \`[placeholder='text']\` | Input identified by placeholder |
25478
+ | \`[type='password']\` | Password inputs only |
25479
+ | \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
25480
+ | \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
25481
+
25482
+ **FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
25483
+
25484
+ - \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
25485
+ - **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
25486
+ - \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
25487
+ - **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
25488
+ - JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
25489
+
25490
+ ### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
25491
+
25492
+ When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
25493
+
25494
+ \`\`\`
25495
+ find role <role> <action> [--name "<n>"] [--exact]
25496
+ find text|label|placeholder|alt|title "<text>" <action> [--exact]
25497
+ find testid "<id>" <action>
25498
+ find first|last "<ALLOWED-css>" <action>
25499
+ find nth <index> "<ALLOWED-css>" <action>
25500
+ \`\`\`
25501
+
25502
+ \`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
25503
+
25504
+ **Rules for \`find\`:**
25505
+
25506
+ 1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
25507
+ 2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
25508
+ 3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
25509
+ 4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
25510
+ 5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
25511
+ 6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
25512
+ 7. \`find\` includes its own wait; do not chain a \`wait\` before it.
25513
+ 8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
25514
+ 9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
25515
+
25516
+ **Examples:**
25517
+
25518
+ - ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
25519
+ - ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
25520
+ - ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
25521
+ - ✗ \`find role button --name "Submit" click\` — wrong order
25522
+ - ✗ \`find last "button" click\` — bare tag
25523
+
25524
+ ### Selector workflow
25525
+
25526
+ 1. Run \`snapshot\` and read the ARIA tree.
25527
+ 2. Identify the element; note its exact \`aria-label\` if present.
25528
+ 3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
25529
+ For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
25530
+ 4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
25531
+ 5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
25532
+ 6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
25533
+ 7. Never guess. If a selector fails, take a fresh snapshot before retrying.
25534
+ 8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
25535
+
25536
+ ### Special input types
25537
+
25538
+ **contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
25539
+
25540
+ **combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
25541
+
25542
+ **Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
25543
+
25544
+ **Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
25545
+
25546
+ **File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
25547
+
25548
+ ## Test Specification
25549
+
25550
+ Title: ${input.title}
25551
+
25552
+ Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
25553
+
25554
+ ## Steps
25555
+
25556
+ ${stepsText}
25557
+ ${expectationsText}${contextText}${conventionsText}${previousText}
25558
+ ## Execution Workflow
25559
+
25560
+ For each step:
25561
+ 1. Emit \`STEP_START|<step-id>|<short description>\`.
25562
+ 2. Run \`snapshot -i -c\` and identify selectors from the ARIA tree. A full \`snapshot\` is large; when you need the page's text, scope it with \`snapshot -c -s "<css>"\`.
25563
+ 3. Execute the step's actions using ALLOWED selectors (see Selector Rules), prefixing each command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step. **Batch them:** put every command you can already write — the actions, their waits, the \`CCQA_ASSERT\` checks, and a closing \`snapshot -i -c\` when you need to see the result — into one \`run_commands\` call. Each turn costs far more than a command; use a single Bash call only when you must see one answer before you can write the next command.
25564
+ 4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
25565
+ 5. Verify the outcome from the batch's output. When a batch stops at a failure, everything before it is recorded: look at the page and continue from there.
25566
+ 6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
25567
+ 7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
25568
+ 8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
25569
+
25570
+ **Protocol lines are recorded ONLY from your plain assistant text.** Every
25571
+ \`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
25572
+ message text between tool calls — never through Bash (\`echo\` etc.) and never
25573
+ omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
25574
+ prefix on each command (that prefix is authoritative, not the \`STEP_START\`
25575
+ line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
25576
+ verification commands** — a command that ran, whose failure rolls the record
25577
+ back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
25578
+ proves it; it is reported and not recorded.
25579
+
25580
+ **After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
25581
+
25582
+ ## Guardrails
25583
+
25584
+ - **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
25585
+ - **Stay on a flow you have started.** Once a step is inside something whose state lives on the page — a conversation, a multi-step dialog, a form not yet saved — do not navigate away to investigate; that state does not come back. Look things up in the source with Read, Grep and Glob instead.
25586
+ - **Clean up before you stop.** When you end the run with \`ASSERTION_FAILED\` after the case has created something, still perform its cleanup steps, then emit \`RUN_COMPLETED|failed\`. What a failed recording leaves behind breaks the next run of every case that lists the same things.
25587
+ - **The step must match the application.** When what a step asks for is not there, or the application does it in another place or order, do not bend the recording to fit: emit \`ASSERTION_FAILED|<step-id>|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>\`. Read the source to ground the suggestion. A recording that works around the step hides the mismatch; the report gets the spec fixed. Redoing an earlier step's work to make this one possible — going back to refill or resubmit what an earlier step already did — is such a workaround.
25588
+ - **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
25589
+ - Do NOT retry a selector without taking a fresh snapshot first.
25590
+ - Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
25591
+ - **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
25592
+ - **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
25593
+
25594
+ ## Source Code Reference
25595
+
25596
+ You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
25597
+
25598
+ **When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
25599
+
25600
+ **How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
25601
+
25602
+ **Rules**: only READ source files, never modify them. Keep searches focused.
25603
+
25604
+ ## Waiting for Async Operations
25605
+
25606
+ Prefer \`wait\` over polling:
25607
+
25608
+ \`\`\`bash
25609
+ CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
25610
+ \`\`\`
25611
+
25612
+ If polling is required (e.g. waiting for a spinner to disappear):
25613
+
25614
+ \`\`\`bash
25615
+ for i in $(seq 1 18); do
25616
+ sleep 10
25617
+ result=$(agent-browser --session ${sessionName} snapshot 2>&1)
25618
+ echo "$result" | grep -q "<done indicator>" && break
25619
+ done
25620
+ CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
25621
+ \`\`\`
25622
+
25623
+ After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
25624
+
25625
+ ## AB_ACTION Protocol
25626
+
25627
+ After **every** browser action, emit one line (outside any code block):
25628
+
25629
+ \`\`\`
25630
+ AB_ACTION|cookies_clear
25631
+ AB_ACTION|open|<url>
25632
+ AB_ACTION|click|<selector>|<visible label>
25633
+ AB_ACTION|dblclick|<selector>|<visible label>
25634
+ AB_ACTION|fill|<selector>|<value>|<aria label>
25635
+ AB_ACTION|check|<selector>|<visible label>
25636
+ AB_ACTION|uncheck|<selector>|<visible label>
25637
+ AB_ACTION|press|<Key>
25638
+ AB_ACTION|select|<selector>|<value>|<aria label>
25639
+ AB_ACTION|hover|<selector>|<visible label>
25640
+ AB_ACTION|scroll|<direction>|<pixels>
25641
+ AB_ACTION|drag|<source selector>|<target selector>|<source label>
25642
+ AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
25643
+ AB_ACTION|wait|<selector or text>|<label>
25644
+ AB_ACTION|snapshot|<key observation, max 100 chars>
25645
+
25646
+ # find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
25647
+ # <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
25648
+ AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
25649
+ AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
25650
+ AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
25651
+ AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
25652
+ AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
25653
+ AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
25654
+ AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
25655
+ AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
25656
+ \`\`\`
25657
+
25658
+ Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
25659
+
25660
+ **CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
25661
+
25662
+ - A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
25663
+ - If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
25664
+ - An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
25665
+ - **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
25666
+
25667
+ - **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
25668
+ - If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
25669
+
25670
+ ## Assertion Protocol
25671
+
25672
+ Record each signal you verified as an assertion. The **primary mechanism** is
25673
+ a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
25674
+ marker rides the same channel as the command, so the assertion is captured
25675
+ even if you print no protocol text. The check you were going to run anyway
25676
+ becomes the recorded assertion:
25677
+
25678
+ \`\`\`bash
25679
+ # text_visible — the marked wait IS the assertion (no separate assert line)
25680
+ CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
25681
+ # element_visible / element_not_visible — mark the get-count probe
25682
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
25683
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
25684
+ # url_contains — the substring rides in the marker; put it on \`get url\`
25685
+ CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
25686
+ # element state — mark the \`is\` probe that asks for it
25687
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
25688
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
25689
+ \`\`\`
25690
+
25691
+ **Marker semantics:**
25692
+
25693
+ | Marker | On command | Records |
25694
+ |--------|------------|---------|
25695
+ | \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
25696
+ | \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
25697
+ | \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
25698
+ | \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
25699
+ | \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
25700
+ | \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
25701
+
25702
+ - A marked command that exits non-zero records nothing — fix the check and re-run it.
25703
+ - A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
25704
+ - \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
25705
+ - **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
25706
+
25707
+ - **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
25708
+
25709
+ - **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
25710
+
25711
+ - **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
25712
+ - **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
25713
+ - **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
25714
+
25715
+ **Every assert type has a marker command.** There is no text form: a printed
25716
+ \`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
25717
+ reported rather than recorded.
25718
+
25719
+ | assertType | The command that records it | Use when |
25720
+ |------------|------------------------------|----------|
25721
+ | \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
25722
+ | \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
25723
+ | \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
25724
+ | \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
25725
+ | \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
25726
+ | \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
25727
+
25728
+ **Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
25729
+
25730
+ - Timestamps, session IDs, exact numeric counts that vary between runs.
25731
+ - **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
25732
+ - **Relative-time labels** — true only in the moment of the trace, stale by replay:
25733
+ - English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
25734
+ - Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
25735
+ - Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
25736
+ - **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
25737
+
25738
+ **No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
25739
+
25740
+ The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
25741
+
25742
+ - ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
25743
+ - ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
25744
+ - ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
25745
+ - If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
25746
+
25747
+ **Page-context and selector rules:**
25748
+
25749
+ - After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
25750
+ - Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
25751
+ - When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
25752
+
25753
+ **MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
25754
+
25755
+ The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
25756
+
25757
+ 1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
25758
+ 2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
25759
+ 3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
25760
+
25761
+ **Verify the assertion form actually resolves on the live page.** The
25762
+ verification and the recorded assertion are the same marked command:
25763
+
25764
+ \`\`\`bash
25765
+ # text_visible
25766
+ CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
25767
+ # element_visible — >=1 printed means present
25768
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
25769
+ # element_not_visible — 0 printed means absent (works for text too: "text=<text>")
25770
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
25771
+ # element state — \`is\` prints true/false and exits 0 either way; read it
25772
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
25773
+ CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
25774
+ \`\`\`
25775
+
25776
+ When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
25777
+
25778
+ ## Status Protocol
25779
+
25780
+ Emit exactly one status line per step (outside any code block):
25781
+
25782
+ \`\`\`
25783
+ STEP_START|<step-id>|<short description>
25784
+ STEP_DONE|<step-id>|<what was verified>
25785
+ ASSERTION_FAILED|<step-id>|<category: spec-mismatch|app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
25786
+ STEP_SKIPPED|<step-id>|<reason>
25787
+ RUN_COMPLETED|passed|<summary>
25788
+ RUN_COMPLETED|failed|<summary>
25789
+ \`\`\`
25790
+
25791
+ **\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
25792
+ recorded as FAILED, however many steps went well: nothing else can tell a
25793
+ finished route from one you abandoned halfway.
25794
+
25795
+ ${callerGuidance}## Start
25796
+
25797
+ Begin by clearing cookies, then proceed straight to the first step's instruction.
25798
+
25799
+ \`\`\`bash
25800
+ agent-browser --session ${sessionName} cookies clear
25801
+ \`\`\`
25802
+
25803
+ (The cookies-clear preamble belongs to no step, so it is the one command
25804
+ without a \`CCQA_STEP\` prefix.)
25805
+
25806
+ Emit:
25807
+ \`\`\`
25808
+ AB_ACTION|cookies_clear
25809
+ \`\`\`
25810
+
25811
+ Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
25812
+ every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
25813
+ step is responsible for opening the initial URL.
25814
+ `;
25322
25815
  }
25323
- /** The same action addressed by role + accessible name, or null when there is no such form. */
25324
- function labelFallback(action) {
25325
- const loc = action.locator;
25326
- if (!loc || loc.by !== "label") return null;
25327
- const role = LABEL_FALLBACK_ROLE[action.action];
25328
- if (role === void 0) return null;
25329
- return {
25330
- ...action,
25331
- locator: {
25332
- by: "role",
25333
- value: role,
25334
- name: loc.value,
25335
- exact: true
25336
- }
25337
- };
25816
+ function buildTracePrompt(title) {
25817
+ return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
25338
25818
  }
25339
25819
  /**
25340
- * Spawn one interaction, waiting for its element the way an assertion waits
25341
- * for its text: a recording pauses for a snapshot between opening a page and
25342
- * typing into it, the replay does not, so the replay reaches a field sooner
25343
- * than the recording ever did.
25344
- *
25345
- * Only a "no element" failure waits — anything else repeats identically. When
25346
- * there is an `alternate` form it is tried on every attempt rather than after
25347
- * the wait: a label the page never associates with its input will not start
25348
- * matching, so waiting first spends the budget for nothing.
25820
+ * The case's expectations, plus the one rule that decides where each is
25821
+ * recorded. Without the rule a recorder either asserts everything at the end —
25822
+ * on a page the flow has already left — or asserts each item at every step it
25823
+ * might apply to, which is the over-assertion the selector rules exist to
25824
+ * prevent.
25349
25825
  */
25350
- function runInteraction(argv, alternate, patience, sessionName) {
25351
- const started = Date.now();
25352
- const deadline = started + ASSERT_TIMEOUT_MS;
25353
- let settled = false;
25354
- for (let attempt = 0;; attempt++) {
25355
- let result = spawnAB(argv);
25356
- if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
25357
- const waitedMs = Date.now() - started;
25358
- if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
25359
- settled = true;
25360
- spawnAB([
25361
- "--session",
25362
- sessionName,
25363
- "wait",
25364
- "--load",
25365
- "networkidle"
25366
- ]);
25367
- continue;
25368
- }
25369
- if (result.status === 0 || !notFound(result)) return {
25370
- result,
25371
- viaAlternate: false,
25372
- waitedMs
25373
- };
25374
- if (alternate !== null) {
25375
- const alt = spawnAB(alternate);
25376
- if (alt.status === 0) return {
25377
- result: alt,
25378
- viaAlternate: true,
25379
- waitedMs
25380
- };
25381
- }
25382
- if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
25383
- patience.spent = true;
25384
- return {
25385
- result,
25386
- viaAlternate: false,
25387
- waitedMs: Date.now() - started
25388
- };
25389
- }
25390
- sleepSync(INTERACTION_POLL_INTERVAL_MS);
25391
- }
25826
+ function renderExpectations(expectations) {
25827
+ if (expectations.length === 0) return "";
25828
+ return `
25829
+ ## Expected results
25830
+
25831
+ The case states these without saying which step delivers them:
25832
+
25833
+ ${expectations.map((text) => `- ${text}`).join("\n")}
25834
+
25835
+ **Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
25836
+ `;
25392
25837
  }
25393
25838
  /**
25394
- * Say what a `by: "css"` locator that is not CSS actually addresses.
25395
- *
25396
- * `get count` answers 0 for Playwright's notation instead of failing, so a
25397
- * route holding one reads as dead. Rewritten in place, and reported, so the
25398
- * saved route carries the form that replays rather than the one that counts
25399
- * nothing. Notation this cannot convert leaves the locator alone; the assert
25400
- * path then treats it as unverifiable rather than absent.
25839
+ * What the cleanup must make true. Separate from the case's expectations
25840
+ * because its steps are: a deletion is confirmed by the thing being gone,
25841
+ * which is only true after the undo has run, and asserting it among the
25842
+ * case's own steps would check it while the thing still exists.
25401
25843
  */
25402
- function nameNotation(action) {
25403
- const loc = action.locator;
25404
- if (action.action !== "assert" || loc?.by !== "css") return null;
25405
- const named = parseNotation(loc.value);
25406
- if (named === null || named === "opaque" || named.by !== "text") return null;
25407
- action.locator = named;
25408
- return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
25844
+ function renderCleanupExpectations(expectations) {
25845
+ if (expectations.length === 0) return "";
25846
+ return `
25847
+ ## Expected results of the cleanup
25848
+
25849
+ The case states these about its cleanup steps:
25850
+
25851
+ ${expectations.map((text) => `- ${text}`).join("\n")}
25852
+
25853
+ **Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
25854
+ `;
25409
25855
  }
25410
25856
  /**
25411
- * An attribute selector that counted nothing, asked of the accessibility tree
25412
- * the name was read from. See docs/targets.md for why the two disagree.
25413
- *
25414
- * The snapshot is read for the role, then `find role` is asked as well: the
25415
- * tree says the name exists somewhere, the probe says the replay can reach it,
25416
- * and only the second is what the route will depend on.
25417
- */
25418
- function askTheAccessibilityTree(action, sub, sessionName) {
25419
- if (action.action !== "assert" || action.locator?.by !== "css") return null;
25420
- const name = nameFromAttributeSelector(action.locator.value);
25421
- if (name === null) return null;
25422
- const snapshot = spawnAB([
25423
- "--session",
25424
- sessionName,
25425
- "snapshot"
25426
- ]);
25427
- if (snapshot.status !== 0) return null;
25428
- const role = roleOfAccessibleName(snapshot.stdout, sub(name));
25429
- if (role === null) return null;
25430
- if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
25431
- const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
25432
- action.locator = {
25433
- by: "role",
25434
- value: role,
25435
- name,
25436
- exact: true
25437
- };
25438
- return {
25439
- skipped: false,
25440
- ok: true,
25441
- reason: "",
25442
- promoted
25443
- };
25857
+ * Sections of the case ccqa does not interpret — the precondition, notes the
25858
+ * author left. Passed through verbatim: they are written for whoever runs the
25859
+ * case, and the recorder is now that reader.
25860
+ */
25861
+ function renderContext(context) {
25862
+ if (context.length === 0) return "";
25863
+ return `
25864
+ ## About this case
25865
+
25866
+ ${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
25867
+ `;
25444
25868
  }
25445
25869
  /**
25446
- * Replay one recorded action against the validation session. Element-presence
25447
- * checks go through `runPollCheck` (which uses `get count`, never the blocking
25448
- * `wait <selector>`); everything else spawns the agent-browser argv.
25870
+ * The project's own recording guidance. Where the case says what to do, this
25871
+ * says how this project is driven — which account signs in, what has to be
25872
+ * true before the first step. Prose, verbatim: it is written for whoever runs
25873
+ * the case, and this is now that reader.
25449
25874
  */
25450
- function runValidationAction(action, sessionName, envOverrides = {}, patience) {
25451
- const named = nameNotation(action);
25452
- const withName = (outcome) => named === null || !outcome.ok ? outcome : {
25453
- ...outcome,
25454
- promoted: outcome.promoted ?? named
25455
- };
25456
- const built = actionToAbArgs(action, sessionName, envOverrides);
25457
- if (built === null) return {
25458
- skipped: true,
25459
- ok: false,
25460
- reason: ""
25461
- };
25462
- if (isProbe(built)) {
25463
- const { ok, reason } = runProbe(built, sessionName);
25464
- if (ok) return withName({
25465
- skipped: false,
25466
- ok,
25467
- reason
25468
- });
25469
- const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
25470
- return askTheAccessibilityTree(action, resolve, sessionName) ?? {
25471
- skipped: false,
25472
- ok,
25473
- reason
25474
- };
25475
- }
25476
- const fallback = labelFallback(action);
25477
- const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
25478
- const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
25479
- if (viaAlternate) {
25480
- const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
25481
- action.locator = fallback.locator;
25482
- return {
25483
- skipped: false,
25484
- ok: true,
25485
- reason: "",
25486
- promoted
25487
- };
25488
- }
25489
- if (result.status === 0) return withName({
25490
- skipped: false,
25491
- ok: true,
25492
- reason: ""
25493
- });
25494
- return {
25495
- skipped: false,
25496
- ok: false,
25497
- reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
25498
- };
25499
- }
25500
- function validateActions(actions, opts) {
25501
- const kept = [];
25502
- const dropped = [];
25503
- const promoted = [];
25504
- const patience = { spent: false };
25505
- let skipFromStepId = null;
25506
- for (let i = 0; i < actions.length; i++) {
25507
- const action = actions[i];
25508
- opts.onProgress?.(i, actions.length, action);
25509
- const stepId = action.stepId ?? NO_STEP_ID;
25510
- if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
25511
- if (skipFromStepId !== null && isPassiveAction(action.action)) {
25512
- dropped.push({
25513
- index: i,
25514
- action,
25515
- reason: CASCADE_REASON
25516
- });
25517
- continue;
25518
- }
25519
- const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
25520
- if (outcome.promoted) promoted.push(outcome.promoted);
25521
- if (outcome.skipped) {
25522
- kept.push(action);
25523
- continue;
25524
- }
25525
- if (outcome.ok) {
25526
- kept.push(action);
25527
- if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
25528
- continue;
25529
- }
25530
- dropped.push({
25531
- index: i,
25532
- action,
25533
- reason: outcome.reason
25534
- });
25535
- if (!isPassiveAction(action.action)) skipFromStepId = stepId;
25536
- }
25537
- const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
25538
- promoted.push(...afterRescue.promoted ?? []);
25539
- return {
25540
- ...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
25541
- promoted
25542
- };
25875
+ function renderConventions(conventions) {
25876
+ if (conventions.length === 0) return "";
25877
+ return `
25878
+ ## How this project is driven
25879
+
25880
+ ${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
25881
+ `;
25543
25882
  }
25883
+ //#endregion
25884
+ //#region src/cli/trace-tools.ts
25885
+ const TRACE_TOOLS_SERVER = "ccqa";
25886
+ /** A wedged command must not hold the whole trace; agent-browser's own waits are far shorter. */
25887
+ const COMMAND_TIMEOUT_MS = 12e4;
25888
+ /** Enough of a snapshot or a `get` answer to act on, without flooding the turn. */
25889
+ const OUTPUT_CAP = 6e3;
25544
25890
  /**
25545
- * Translate the internal `{ kept, dropped }` result of the rescue pass
25546
- * into the public-facing shape. In strict mode the caller sees the same
25547
- * shape as before (kept/dropped); in lenient mode the still-failed
25548
- * actions move to `unstable` with `replayUnstable: true` tagged on, so
25549
- * codegen can warn about them while still emitting the line.
25891
+ * The trace's own tools. Both exist to cut round trips: one turn per browser
25892
+ * command is what made a re-record cost hundreds of turns.
25893
+ *
25894
+ * - `run_commands` runs a batch the recorder wrote, recording each that passes.
25895
+ * - `replay_step` runs one step's previous commands.
25550
25896
  */
25551
- function splitByMode(originalActions, result, mode) {
25552
- if (mode === "strict") return {
25553
- kept: result.kept,
25554
- unstable: [],
25555
- dropped: result.dropped,
25556
- rescuedSteps: result.rescuedSteps
25557
- };
25558
- const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
25559
- const keptSet = new Set(result.kept);
25560
- const finalKept = [];
25561
- const unstable = [];
25562
- for (let i = 0; i < originalActions.length; i++) {
25563
- const action = originalActions[i];
25564
- if (keptSet.has(action)) {
25565
- finalKept.push(action);
25566
- continue;
25567
- }
25568
- const drop = droppedByIndex.get(i);
25569
- if (drop) {
25570
- action.replayUnstable = true;
25571
- action.replayReason = drop.reason;
25572
- unstable.push(action);
25573
- }
25574
- }
25897
+ function buildTraceTools(input) {
25898
+ const offersReplay = input.previous.length > 0;
25575
25899
  return {
25576
- kept: finalKept,
25577
- unstable,
25578
- dropped: [],
25579
- rescuedSteps: result.rescuedSteps
25900
+ server: createSdkMcpServer({
25901
+ name: TRACE_TOOLS_SERVER,
25902
+ version: "1.0.0",
25903
+ tools: [tool("run_commands", "Run several agent-browser commands in order, each written exactly as you would run it in Bash (with its CCQA_STEP / CCQA_ASSERT / CCQA_SECRET prefixes). Each that succeeds is recorded; the batch stops at the first that fails. Returns each command's outcome and the output of the last one.", { commands: z.array(z.string()).min(1).describe("agent-browser command lines, one per entry") }, async ({ commands }) => ({ content: [{
25904
+ type: "text",
25905
+ text: await runCommands(commands, input)
25906
+ }] })), ...offersReplay ? [tool("replay_step", "Run one step's previous-recording commands in this browser session and record those that pass. Stops at the first that fails.", { step: z.string().describe("The step id, e.g. step-03") }, async ({ step }) => ({ content: [{
25907
+ type: "text",
25908
+ text: replayStep(step, input)
25909
+ }] }))] : []]
25910
+ }),
25911
+ allowedTools: ["run_commands", ...offersReplay ? ["replay_step"] : []].map((t) => `mcp__${TRACE_TOOLS_SERVER}__${t}`)
25580
25912
  };
25581
25913
  }
25582
- function rescueLostSteps(actions, kept, dropped, opts, patience) {
25583
- const stepsWithSurvivors = /* @__PURE__ */ new Set();
25584
- for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
25585
- const lostStepDrops = /* @__PURE__ */ new Map();
25586
- for (const d of dropped) {
25587
- const id = d.action.stepId;
25588
- if (!id || stepsWithSurvivors.has(id)) continue;
25589
- const list = lostStepDrops.get(id) ?? [];
25590
- list.push(d);
25591
- lostStepDrops.set(id, list);
25592
- }
25593
- if (lostStepDrops.size === 0) return {
25594
- kept,
25595
- dropped
25596
- };
25597
- const promoted = [];
25598
- const rescuedIndices = /* @__PURE__ */ new Set();
25599
- const rescuedSteps = [];
25600
- const strict = (opts.mode ?? "lenient") === "strict";
25601
- for (const [stepId, drops] of lostStepDrops.entries()) {
25602
- const replayed = [];
25603
- const unverifiable = [];
25604
- let stillFailing = false;
25605
- for (const d of drops) {
25606
- const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
25607
- if (outcome.promoted) promoted.push(outcome.promoted);
25608
- if (outcome.skipped) {
25609
- unverifiable.push(d.index);
25610
- continue;
25611
- }
25612
- if (outcome.ok) {
25613
- replayed.push(d.index);
25614
- continue;
25615
- }
25616
- stillFailing = true;
25617
- if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
25914
+ async function runCommands(commands, input) {
25915
+ const report = [];
25916
+ for (const [i, cmd] of commands.entries()) {
25917
+ const stop = (why) => [
25918
+ ...report,
25919
+ `✗ ${cmd}\n${why}`,
25920
+ ...i + 1 < commands.length ? [`The ${commands.length - i - 1} command(s) after it were not run.`] : []
25921
+ ].join("\n");
25922
+ if (!isAgentBrowserCommand(cmd)) return stop("Only agent-browser commands can be batched; run anything else with Bash.");
25923
+ const blocked = abCommandBlockReason(cmd, input.envScrubMap);
25924
+ if (blocked !== null) return stop(blocked);
25925
+ const recorded = abActionEventFromCommand(cmd);
25926
+ const checkpoint = input.beforeAbCommand(extractCcqaStepFromBashCommand(cmd) ?? void 0);
25927
+ if (checkpoint) return stop(checkpoint);
25928
+ info(`$ ${scrubEnvValues(cmd, input.envScrubMap)}`);
25929
+ const result = await runShell(cmd, {
25930
+ ...process.env,
25931
+ ...input.env
25932
+ });
25933
+ const out = result.stdout.trim();
25934
+ if (!(result.status === 0 && (recorded?.holds ? recorded.holds(out) : true))) {
25935
+ const detail = (result.stderr || out || `exit ${result.status ?? "timeout"}`).trim().slice(0, 600);
25936
+ return stop(result.status === 0 ? `The check did not hold; the page answered: ${out.slice(0, 300)}` : detail);
25618
25937
  }
25619
- if (replayed.length === 0) continue;
25620
- for (const i of replayed) rescuedIndices.add(i);
25621
- if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
25622
- rescuedSteps.push(stepId);
25623
- }
25624
- if (rescuedIndices.size === 0) return {
25625
- kept,
25626
- dropped,
25627
- promoted
25628
- };
25629
- const keptSet = new Set(kept);
25630
- const newKept = [];
25631
- for (let i = 0; i < actions.length; i++) {
25632
- const action = actions[i];
25633
- if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
25938
+ if (recorded) input.onAbAction(recorded.event);
25939
+ const last = i === commands.length - 1;
25940
+ report.push(`✓ ${cmd}${out && (last || out.length < 200) ? `\n${cap(out)}` : ""}`);
25634
25941
  }
25635
- return {
25636
- kept: newKept,
25637
- dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
25638
- rescuedSteps,
25639
- promoted
25640
- };
25942
+ return report.join("\n");
25641
25943
  }
25642
25944
  /**
25643
- * Passive (read-only) actions whose only effect is observation. When a
25644
- * preceding action fails, dropping these too is the right move because
25645
- * they were trying to observe state the failed action would have set up.
25945
+ * Async, so the trace's deadline and signal teardown still run while a batch
25946
+ * does. Settles on `exit` too: a daemon the command started may hold the pipes open.
25646
25947
  */
25647
- function isPassiveAction(action) {
25648
- return action === "snapshot" || action === "wait" || action === "assert";
25948
+ function runShell(cmd, env) {
25949
+ return new Promise((resolve) => {
25950
+ const child = spawn("bash", ["-c", cmd], {
25951
+ env,
25952
+ timeout: COMMAND_TIMEOUT_MS
25953
+ });
25954
+ let stdout = "";
25955
+ let stderr = "";
25956
+ child.stdout.setEncoding("utf8").on("data", (d) => stdout += d);
25957
+ child.stderr.setEncoding("utf8").on("data", (d) => stderr += d);
25958
+ const done = (status, err) => resolve({
25959
+ status,
25960
+ stdout,
25961
+ stderr: stderr || (err ?? "")
25962
+ });
25963
+ child.on("error", (e) => done(null, e.message));
25964
+ child.on("close", (status) => done(status));
25965
+ child.on("exit", (status) => setTimeout(() => done(status), 1e3).unref());
25966
+ });
25967
+ }
25968
+ function cap(out) {
25969
+ return out.length <= OUTPUT_CAP ? out : `${out.slice(0, OUTPUT_CAP)}\n… (${out.length - OUTPUT_CAP} more characters; scope it with snapshot -s <selector>)`;
25970
+ }
25971
+ function replayStep(step, input) {
25972
+ const actions = input.previous.filter((a) => a.stepId === step && a.action !== "snapshot").map(({ replayUnstable: _u, replayReason: _r, ...a }) => a);
25973
+ if (actions.length === 0) return `No previous commands for ${step}. Record it with the Execution Workflow.`;
25974
+ const { passed, unchecked, failed } = replayUntilFailure(actions, {
25975
+ sessionName: input.sessionName,
25976
+ envOverrides: { CCQA_RUN_ID: input.env["CCQA_RUN_ID"] ?? input.sessionName }
25977
+ });
25978
+ input.onReplayed(step, passed);
25979
+ info(`replay_step ${step}: ${passed.length}/${actions.length} replayed${failed ? "" : " — all passed"}`);
25980
+ const notRun = unchecked.length === 0 ? "" : `\nThese checks could not be run here and were not recorded — perform the ones Expected still asks for yourself:\n` + unchecked.map((a) => `- ${describeStepAction(a)}`).join("\n");
25981
+ if (!failed) {
25982
+ const next = unchecked.length === 0 ? " Emit STEP_DONE and go on." : "";
25983
+ return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded, its checks included.${next}${notRun}`;
25984
+ }
25985
+ return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded. The next one failed: ${describeStepAction(failed.action)} — ${failed.reason}\nThe page is where the recorded commands left it. Finish ${step} with the Execution Workflow (snapshot first); do not re-run the recorded commands.${notRun}`;
25649
25986
  }
25650
25987
  //#endregion
25651
25988
  //#region src/cli/trace.ts
25989
+ /** The recorder's Bash runs agent-browser only: source is read with Read/Grep/Glob. */
25990
+ function traceBashGuard(cmd) {
25991
+ if (isAgentBrowserCommand(cmd)) return null;
25992
+ return "Bash here runs agent-browser commands only. Read, search and list the source with the Read, Grep and Glob tools; wait for the page with agent-browser wait, not sleep.";
25993
+ }
25994
+ /** Browser commands in one step between spec-mismatch checkpoints (see `beforeAbCommand`). */
25995
+ const STEP_CHECKPOINT = 20;
25652
25996
  /**
25653
25997
  * Step ids (in spec order) whose kept actions include no assertion. A step
25654
25998
  * with none produced a test that performs the step but verifies nothing
@@ -25686,6 +26030,11 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25686
26030
  }
25687
26031
  const conventions = await loadConventions(opts.cwd ?? process.cwd(), { guides: opts.conventions ?? [] });
25688
26032
  for (const w of conventions.warnings) warn(w);
26033
+ const caseRef = testCase.ref;
26034
+ const previous = await tryGetRecording(caseRef);
26035
+ const previousActions = previous ? [...previous.actions, ...previous.cleanup ?? []] : [];
26036
+ if (previousActions.length > 0) meta("previous", opts.freshIr ? "ignored (--fresh-ir)" : `${previousActions.length} action(s) offered as a map`);
26037
+ const offersMap = !opts.freshIr && previousActions.length > 0;
25689
26038
  const baseSystemPrompt = buildTraceSystemPrompt({
25690
26039
  title: testCase.title,
25691
26040
  steps,
@@ -25697,7 +26046,8 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25697
26046
  ...testCase.expectations.length > 0 ? { expectations: testCase.expectations } : {},
25698
26047
  ...testCase.cleanupExpectations.length > 0 ? { cleanupExpectations: testCase.cleanupExpectations } : {},
25699
26048
  ...testCase.context.length > 0 ? { context: testCase.context } : {},
25700
- ...opts.instruction ? { instruction: opts.instruction } : {}
26049
+ ...opts.instruction ? { instruction: opts.instruction } : {},
26050
+ ...offersMap ? { previousRecording: previousActions } : {}
25701
26051
  });
25702
26052
  const promptBundle = await loadPromptBundle(opts.hubContext ?? null, "record", opts.cwd ?? process.cwd());
25703
26053
  if (promptBundle !== null) meta("prompt", promptBundle.loaded.join(" + "));
@@ -25718,40 +26068,71 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25718
26068
  } : action;
25719
26069
  };
25720
26070
  let lastCommandPushCount = 0;
26071
+ const onAbAction = ({ abAction, stepId, assertMarker, secret }) => {
26072
+ const stepForCommand = stepTracker.fromCommand(stepId);
26073
+ const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
26074
+ let recorded = null;
26075
+ if (assertMarker !== void 0) {
26076
+ recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
26077
+ if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
26078
+ }
26079
+ if (recorded === null) {
26080
+ const parsed = line === null ? null : parseAbActionLine(line);
26081
+ recorded = parsed === null ? [] : [parsed];
26082
+ }
26083
+ let pushed = 0;
26084
+ for (const action of recorded) {
26085
+ const stamped = withStepId(action, stepForCommand);
26086
+ if (stamped) {
26087
+ traceActions.push(secret ? {
26088
+ ...stamped,
26089
+ secret: true
26090
+ } : stamped);
26091
+ pushed += 1;
26092
+ }
26093
+ }
26094
+ lastCommandPushCount = pushed;
26095
+ };
26096
+ const commandsByStep = /* @__PURE__ */ new Map();
26097
+ const beforeAbCommand = (stepId) => {
26098
+ if (stepId === void 0) return null;
26099
+ const n = (commandsByStep.get(stepId) ?? 0) + 1;
26100
+ commandsByStep.set(stepId, n);
26101
+ if (n % STEP_CHECKPOINT !== 0) return null;
26102
+ return `${stepId} has taken ${n - 1} browser commands. If the application does not do what this step says — the control is somewhere else, things happen in a different order, the result never appears — stop working around it and report it: ASSERTION_FAILED|${stepId}|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>. If the step does match and you are close, run this command again to continue.`;
26103
+ };
26104
+ const invokeBase = agentBrowserInvokeBase({
26105
+ sessionName,
26106
+ runId: sessionName
26107
+ });
26108
+ const traceTools = buildTraceTools({
26109
+ previous: offersMap ? previousActions : [],
26110
+ sessionName,
26111
+ env: invokeBase.env ?? {},
26112
+ envScrubMap,
26113
+ onReplayed: (stepId, passed) => {
26114
+ stepTracker.fromCommand(stepId);
26115
+ for (const action of passed) traceActions.push({
26116
+ ...action,
26117
+ stepId
26118
+ });
26119
+ lastCommandPushCount = 0;
26120
+ },
26121
+ onAbAction,
26122
+ beforeAbCommand
26123
+ });
25721
26124
  const { isError, errorDetail } = await invokeClaudeStreaming({
25722
26125
  prompt,
25723
26126
  systemPrompt,
25724
- ...agentBrowserInvokeBase({
25725
- sessionName,
25726
- runId: sessionName
25727
- }),
26127
+ ...invokeBase,
26128
+ allowedTools: [...invokeBase.allowedTools, ...traceTools.allowedTools],
26129
+ mcpServers: { [TRACE_TOOLS_SERVER]: traceTools.server },
25728
26130
  model,
25729
26131
  envScrubMap,
25730
- onAbAction: ({ abAction, stepId, assertMarker, secret }) => {
25731
- const stepForCommand = stepTracker.fromCommand(stepId);
25732
- const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
25733
- let recorded = null;
25734
- if (assertMarker !== void 0) {
25735
- recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
25736
- if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
25737
- }
25738
- if (recorded === null) {
25739
- const parsed = line === null ? null : parseAbActionLine(line);
25740
- recorded = parsed === null ? [] : [parsed];
25741
- }
25742
- let pushed = 0;
25743
- for (const action of recorded) {
25744
- const stamped = withStepId(action, stepForCommand);
25745
- if (stamped) {
25746
- traceActions.push(secret ? {
25747
- ...stamped,
25748
- secret: true
25749
- } : stamped);
25750
- pushed += 1;
25751
- }
25752
- }
25753
- lastCommandPushCount = pushed;
25754
- },
26132
+ onAbAction,
26133
+ beforeAbCommand,
26134
+ effort: "low",
26135
+ bashGuard: traceBashGuard,
25755
26136
  onAbActionFailed: () => {
25756
26137
  if (lastCommandPushCount > 0) traceActions.splice(-lastCommandPushCount);
25757
26138
  lastCommandPushCount = 0;
@@ -25786,10 +26167,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25786
26167
  });
25787
26168
  const overallStatus = failureReason === null ? "passed" : "failed";
25788
26169
  const dedupedActions = dedupAndReport(scrubAndReport(traceActions));
26170
+ const validationStart = Date.now();
25789
26171
  const validatedActions = overallStatus === "passed" || opts.validateFailedTrace === true ? validateAndReport(dedupedActions, validationMode, { CCQA_RUN_ID: sessionName }, opts.teardown) : dedupedActions;
26172
+ if (validatedActions !== dedupedActions) meta("validation time", `${((Date.now() - validationStart) / 1e3).toFixed(1)}s`);
25790
26173
  opts.teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
25791
- const caseRef = testCase.ref;
25792
- const previous = overallStatus === "passed" ? await tryGetRecording(caseRef) : null;
25793
26174
  let recordingPath;
25794
26175
  const cleanupIds = new Set(testCase.cleanup.map((s) => s.id));
25795
26176
  const routeActions = validatedActions.filter((a) => !cleanupIds.has(a.stepId ?? ""));
@@ -25828,7 +26209,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
25828
26209
  */
25829
26210
  function traceFailureReason(lines, session) {
25830
26211
  const assertionFailed = lines.find((l) => l.type === "ASSERTION_FAILED");
25831
- if (assertionFailed) return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED`;
26212
+ if (assertionFailed) {
26213
+ const detail = assertionFailed.detail.trim();
26214
+ return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED${detail ? `: ${detail}` : ""}`;
26215
+ }
25832
26216
  if (session.isError) return `the Claude session ended in an error: ${session.errorDetail ?? "no detail reported"}`;
25833
26217
  const completed = lines.filter((l) => l.type === "RUN_COMPLETED").at(-1);
25834
26218
  if (completed === void 0) return "the session ended without a RUN_COMPLETED line — the model stopped before it reported an outcome";
@@ -26481,6 +26865,7 @@ async function runGenerateLocked(resolved, opts, cwd) {
26481
26865
  cleanup: testCase.cleanup,
26482
26866
  expectations: testCase.expectations,
26483
26867
  cleanupExpectations: testCase.cleanupExpectations,
26868
+ ...testCase.context.length > 0 ? { context: testCase.context } : {},
26484
26869
  fields: testCase.fields,
26485
26870
  cwd,
26486
26871
  testPath,
@@ -26655,7 +27040,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
26655
27040
  const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
26656
27041
  if (VALIDATION_MODES.includes(raw)) return raw;
26657
27042
  throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
26658
- }, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
27043
+ }, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--fresh-ir", "Record from scratch. By default an existing ir.json is shown to the recorder as a map of the route, so steps that still work are not rediscovered.").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
26659
27044
  await withCostReporting("record", () => runRecord(specPath, opts));
26660
27045
  }));
26661
27046
  async function runRecord(caseArgument, opts) {
@@ -26767,6 +27152,7 @@ async function runRecord(caseArgument, opts) {
26767
27152
  ...config.sessionState ? { sessionState: config.sessionState } : {},
26768
27153
  ...resolved.targetConfig.conventions.operate.length > 0 ? { conventions: resolved.targetConfig.conventions.operate } : {},
26769
27154
  ...opts.instruction ? { instruction: opts.instruction } : {},
27155
+ ...opts.freshIr ? { freshIr: true } : {},
26770
27156
  onStep: (stepId) => {
26771
27157
  tracingStep = stepId;
26772
27158
  }