ccqa 1.58.3 → 1.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +1531 -1147
- package/dist/package.json +1 -1
- package/dist/runtime/judge.cjs +53 -35
- package/dist/runtime/judge.mjs +53 -35
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -3143,7 +3143,7 @@ function formatDuration$1(ms) {
|
|
|
3143
3143
|
return `${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
3144
3144
|
}
|
|
3145
3145
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
3146
|
-
const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
|
|
3146
|
+
const { prompt, systemPrompt, allowedTools, disableThinking = false, effort, bashGuard, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, beforeAbCommand, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
|
|
3147
3147
|
const resolvedModel = resolveModel(model);
|
|
3148
3148
|
const mergedEnv = buildInvocationEnv(env);
|
|
3149
3149
|
const abortController = new AbortController();
|
|
@@ -3171,52 +3171,35 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
3171
3171
|
env: mergedEnv,
|
|
3172
3172
|
...mcpServers ? { mcpServers } : {},
|
|
3173
3173
|
...disableThinking ? { thinking: { type: "disabled" } } : {},
|
|
3174
|
-
|
|
3174
|
+
...effort ? { effort } : {},
|
|
3175
|
+
hooks: onAbAction || onAbActionFailed || bashGuard || beforeAbCommand ? {
|
|
3175
3176
|
PreToolUse: [{ hooks: [async (input) => {
|
|
3176
3177
|
if (input.hook_event_name !== "PreToolUse") return {};
|
|
3177
3178
|
if (input.tool_name !== "Bash") return {};
|
|
3178
3179
|
const cmd = input.tool_input?.["command"];
|
|
3179
3180
|
if (typeof cmd !== "string") return {};
|
|
3181
|
+
const guarded = bashGuard?.(cmd);
|
|
3182
|
+
if (guarded) return {
|
|
3183
|
+
decision: "block",
|
|
3184
|
+
reason: guarded
|
|
3185
|
+
};
|
|
3180
3186
|
if (!relaxAbConstraints) {
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
|
|
3184
|
-
};
|
|
3185
|
-
if (hasRefSelector(cmd)) return {
|
|
3186
|
-
decision: "block",
|
|
3187
|
-
reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
|
|
3188
|
-
};
|
|
3189
|
-
const bareTag = findPositionalBareTag(cmd);
|
|
3190
|
-
if (bareTag !== null) return {
|
|
3191
|
-
decision: "block",
|
|
3192
|
-
reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
|
|
3193
|
-
};
|
|
3194
|
-
if (hasMultipleAbInvocations(cmd)) return {
|
|
3195
|
-
decision: "block",
|
|
3196
|
-
reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
|
|
3197
|
-
};
|
|
3198
|
-
if (hasErrorSuppression(cmd)) return {
|
|
3187
|
+
const reason = abCommandBlockReason(cmd, envScrubMap);
|
|
3188
|
+
if (reason !== null) return {
|
|
3199
3189
|
decision: "block",
|
|
3200
|
-
reason
|
|
3190
|
+
reason
|
|
3201
3191
|
};
|
|
3202
|
-
const
|
|
3203
|
-
if (
|
|
3192
|
+
const checkpoint = beforeAbCommand?.(extractCcqaStepFromBashCommand(cmd) ?? void 0);
|
|
3193
|
+
if (checkpoint) return {
|
|
3204
3194
|
decision: "block",
|
|
3205
|
-
reason:
|
|
3195
|
+
reason: checkpoint
|
|
3206
3196
|
};
|
|
3207
3197
|
}
|
|
3208
|
-
const
|
|
3209
|
-
|
|
3210
|
-
if ((ab !== null || assertMarker !== null) && onAbAction) {
|
|
3198
|
+
const recorded = relaxAbConstraints ? null : abActionEventFromCommand(cmd);
|
|
3199
|
+
if (recorded !== null && onAbAction) {
|
|
3211
3200
|
lastAbToolUseId = input.tool_use_id;
|
|
3212
|
-
lastAbAnswerHolds =
|
|
3213
|
-
|
|
3214
|
-
onAbAction({
|
|
3215
|
-
...ab !== null ? { abAction: ab } : {},
|
|
3216
|
-
...stepId ? { stepId } : {},
|
|
3217
|
-
...assertMarker !== null ? { assertMarker } : {},
|
|
3218
|
-
...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
|
|
3219
|
-
});
|
|
3201
|
+
lastAbAnswerHolds = recorded.holds;
|
|
3202
|
+
onAbAction(recorded.event);
|
|
3220
3203
|
} else lastAbToolUseId = null;
|
|
3221
3204
|
return {};
|
|
3222
3205
|
}] }],
|
|
@@ -3804,6 +3787,45 @@ async function fireMockPreToolUseHooks(msg, options) {
|
|
|
3804
3787
|
for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
|
|
3805
3788
|
}
|
|
3806
3789
|
}
|
|
3790
|
+
/**
|
|
3791
|
+
* Why the trace refuses this agent-browser command, or null when it may run.
|
|
3792
|
+
* Shared by the Bash hook and the trace's own batch tool, so both enforce the
|
|
3793
|
+
* same replayability contract.
|
|
3794
|
+
*/
|
|
3795
|
+
function abCommandBlockReason(cmd, envScrubMap = []) {
|
|
3796
|
+
if (isBlockedAbSubcommand(cmd)) return "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector.";
|
|
3797
|
+
if (hasRefSelector(cmd)) return "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either.";
|
|
3798
|
+
const bareTag = findPositionalBareTag(cmd);
|
|
3799
|
+
if (bareTag !== null) return `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`;
|
|
3800
|
+
if (hasMultipleAbInvocations(cmd)) return "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command.";
|
|
3801
|
+
if (hasErrorSuppression(cmd)) return "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code.";
|
|
3802
|
+
const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
|
|
3803
|
+
if (runProducedUrl !== null) return `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`;
|
|
3804
|
+
return null;
|
|
3805
|
+
}
|
|
3806
|
+
/**
|
|
3807
|
+
* What recording this command produces, and — for a marked probe — what its
|
|
3808
|
+
* output must say for the check to hold. Null when it records nothing.
|
|
3809
|
+
*/
|
|
3810
|
+
function abActionEventFromCommand(cmd) {
|
|
3811
|
+
const assertMarker = extractCcqaAssertFromBashCommand(cmd);
|
|
3812
|
+
const ab = extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
|
|
3813
|
+
if (ab === null && assertMarker === null) return null;
|
|
3814
|
+
const stepId = extractCcqaStepFromBashCommand(cmd);
|
|
3815
|
+
return {
|
|
3816
|
+
event: {
|
|
3817
|
+
...ab !== null ? { abAction: ab } : {},
|
|
3818
|
+
...stepId ? { stepId } : {},
|
|
3819
|
+
...assertMarker !== null ? { assertMarker } : {},
|
|
3820
|
+
...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
|
|
3821
|
+
},
|
|
3822
|
+
holds: assertMarker === null ? null : markerHolds(assertMarker, ab)
|
|
3823
|
+
};
|
|
3824
|
+
}
|
|
3825
|
+
/** Whether a shell line runs agent-browser as a command (env prefixes allowed), not just mentions it. */
|
|
3826
|
+
function isAgentBrowserCommand(cmd) {
|
|
3827
|
+
return splitShellStatements(cmd).some((st) => isAgentBrowserHead(splitLeadingEnvAssignments(st).command));
|
|
3828
|
+
}
|
|
3807
3829
|
//#endregion
|
|
3808
3830
|
//#region src/prompts/format.ts
|
|
3809
3831
|
/**
|
|
@@ -3950,7 +3972,7 @@ function toAgentBrowserArgs(action) {
|
|
|
3950
3972
|
}
|
|
3951
3973
|
case "wait": {
|
|
3952
3974
|
const loc = action.locator;
|
|
3953
|
-
if (!loc) return null;
|
|
3975
|
+
if (!loc || loc.value.startsWith("--")) return null;
|
|
3954
3976
|
if (loc.by === "text") return [
|
|
3955
3977
|
lit("wait"),
|
|
3956
3978
|
lit("--text"),
|
|
@@ -4005,6 +4027,61 @@ function interactionToArgs(action) {
|
|
|
4005
4027
|
if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
|
|
4006
4028
|
return out;
|
|
4007
4029
|
}
|
|
4030
|
+
/**
|
|
4031
|
+
* The marked probe that records an assert during a trace — the inverse of
|
|
4032
|
+
* `promoteMarkedAssert`, as the trace prompt's Assertion Protocol spells it.
|
|
4033
|
+
*/
|
|
4034
|
+
function markedAssertArgs(action) {
|
|
4035
|
+
const loc = action.locator;
|
|
4036
|
+
const sel = loc ? [val(locatorToSelector(loc))] : null;
|
|
4037
|
+
switch (action.assert) {
|
|
4038
|
+
case "text_visible": return action.value ? {
|
|
4039
|
+
marker: "1",
|
|
4040
|
+
tokens: [
|
|
4041
|
+
lit("wait"),
|
|
4042
|
+
lit("--text"),
|
|
4043
|
+
val(action.value)
|
|
4044
|
+
]
|
|
4045
|
+
} : null;
|
|
4046
|
+
case "url_contains": return action.value ? {
|
|
4047
|
+
marker: `url_contains:${action.value}`,
|
|
4048
|
+
tokens: [lit("get"), lit("url")]
|
|
4049
|
+
} : null;
|
|
4050
|
+
case "element_visible":
|
|
4051
|
+
case "element_not_visible":
|
|
4052
|
+
if (loc?.by === "role") return loc.name ? {
|
|
4053
|
+
marker: action.assert,
|
|
4054
|
+
tokens: roleProbeTokens(loc.value, loc.name, loc.exact === true)
|
|
4055
|
+
} : null;
|
|
4056
|
+
return sel ? {
|
|
4057
|
+
marker: action.assert,
|
|
4058
|
+
tokens: [
|
|
4059
|
+
lit("get"),
|
|
4060
|
+
lit("count"),
|
|
4061
|
+
...sel
|
|
4062
|
+
]
|
|
4063
|
+
} : null;
|
|
4064
|
+
case "element_enabled":
|
|
4065
|
+
case "element_disabled": return sel ? {
|
|
4066
|
+
marker: action.assert,
|
|
4067
|
+
tokens: [
|
|
4068
|
+
lit("is"),
|
|
4069
|
+
lit("enabled"),
|
|
4070
|
+
...sel
|
|
4071
|
+
]
|
|
4072
|
+
} : null;
|
|
4073
|
+
case "element_checked":
|
|
4074
|
+
case "element_unchecked": return sel ? {
|
|
4075
|
+
marker: action.assert,
|
|
4076
|
+
tokens: [
|
|
4077
|
+
lit("is"),
|
|
4078
|
+
lit("checked"),
|
|
4079
|
+
...sel
|
|
4080
|
+
]
|
|
4081
|
+
} : null;
|
|
4082
|
+
default: return null;
|
|
4083
|
+
}
|
|
4084
|
+
}
|
|
4008
4085
|
//#endregion
|
|
4009
4086
|
//#region src/diagnose/prompt.ts
|
|
4010
4087
|
function buildDiagnosePrompt(input) {
|
|
@@ -14759,6 +14836,12 @@ function tail(output) {
|
|
|
14759
14836
|
const trimmed = output.trim();
|
|
14760
14837
|
return trimmed.length <= OUTPUT_TAIL_CHARS ? trimmed : trimmed.slice(-OUTPUT_TAIL_CHARS);
|
|
14761
14838
|
}
|
|
14839
|
+
const LOGGED_TAIL_LINES = 40;
|
|
14840
|
+
/** The end of an output tail that is already scrubbed, one log line per line. */
|
|
14841
|
+
function logOutputTail(scrubbed, write) {
|
|
14842
|
+
if (scrubbed === "") return;
|
|
14843
|
+
for (const line of scrubbed.split("\n").slice(-LOGGED_TAIL_LINES)) write(` | ${line}`);
|
|
14844
|
+
}
|
|
14762
14845
|
/**
|
|
14763
14846
|
* Where new support files may be created.
|
|
14764
14847
|
*
|
|
@@ -14950,16 +15033,19 @@ async function runVerificationLoop(p, state) {
|
|
|
14950
15033
|
failing = run;
|
|
14951
15034
|
}
|
|
14952
15035
|
}
|
|
15036
|
+
const outputTail = scrubEnvValues(tail(failing.output), outputScrub);
|
|
14953
15037
|
if (attempt >= maxRetries) {
|
|
14954
15038
|
warn(`verification still failing after ${maxRetries} fix attempt(s) — generated files kept`);
|
|
15039
|
+
logOutputTail(outputTail, warn);
|
|
14955
15040
|
await clearCaseRun(p.ctx.ref);
|
|
14956
15041
|
return finish(false);
|
|
14957
15042
|
}
|
|
14958
15043
|
fix(`verification failed (exit ${failing.exitCode}) — requesting a fix (${attempt + 1}/${maxRetries})`);
|
|
15044
|
+
logOutputTail(outputTail, fix);
|
|
14959
15045
|
const fixPrompt = buildLlmFixPrompt({
|
|
14960
15046
|
targetId: p.target,
|
|
14961
15047
|
command: failing.command,
|
|
14962
|
-
outputTail
|
|
15048
|
+
outputTail,
|
|
14963
15049
|
files: [...state.entries()].map(([path, f]) => ({
|
|
14964
15050
|
path,
|
|
14965
15051
|
contents: f.contents,
|
|
@@ -24558,1116 +24644,1364 @@ async function readLockBody(lockPath) {
|
|
|
24558
24644
|
}
|
|
24559
24645
|
}
|
|
24560
24646
|
//#endregion
|
|
24561
|
-
//#region src/
|
|
24562
|
-
|
|
24563
|
-
|
|
24647
|
+
//#region src/ir/accessible-name.ts
|
|
24648
|
+
/**
|
|
24649
|
+
* A recorder reads an accessible name off a snapshot and writes it as the
|
|
24650
|
+
* attribute it guessed the name came from. The name can come from a `<label>`
|
|
24651
|
+
* or `aria-labelledby` instead, and the attribute's absence is not evidence
|
|
24652
|
+
* about the element. See docs/targets.md for the measurement.
|
|
24653
|
+
*/
|
|
24654
|
+
const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
|
|
24655
|
+
/**
|
|
24656
|
+
* The name this selector may be after, or null.
|
|
24657
|
+
*
|
|
24658
|
+
* `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
|
|
24659
|
+
* mean what they say, and a zero from one of those is evidence.
|
|
24660
|
+
*/
|
|
24661
|
+
function nameFromAttributeSelector(selector) {
|
|
24662
|
+
const m = NAMING_ATTR.exec(selector.trim());
|
|
24663
|
+
const name = m?.[1] ?? m?.[2];
|
|
24664
|
+
return name === void 0 || name === "" ? null : name;
|
|
24564
24665
|
}
|
|
24565
24666
|
/**
|
|
24566
|
-
*
|
|
24567
|
-
*
|
|
24568
|
-
* The spec opens URLs via explicit step instructions (e.g.
|
|
24569
|
-
* `instruction: "${APP_URL}/articles を開く"`).
|
|
24667
|
+
* The role of the first node in an accessibility snapshot whose name is
|
|
24668
|
+
* exactly `name`, or null.
|
|
24570
24669
|
*
|
|
24571
|
-
*
|
|
24572
|
-
*
|
|
24573
|
-
*
|
|
24574
|
-
* distinguishes spec-native steps from inlined block steps for the
|
|
24575
|
-
* `// step:` comments in the eventual codegen output.
|
|
24670
|
+
* agent-browser prints one node per line as `<role> "<name>"`. Matched on the
|
|
24671
|
+
* whole name, so a tree holding both "Priority" and "Priority *" answers for
|
|
24672
|
+
* the one that was asked, and refused when two roles carry it.
|
|
24576
24673
|
*/
|
|
24577
|
-
function
|
|
24578
|
-
const
|
|
24579
|
-
const
|
|
24580
|
-
|
|
24581
|
-
|
|
24582
|
-
|
|
24583
|
-
|
|
24584
|
-
it as advice on what to avoid or verify while recording; the spec's steps
|
|
24585
|
-
above remain the contract for what to do.
|
|
24586
|
-
|
|
24587
|
-
${input.instruction}
|
|
24588
|
-
|
|
24589
|
-
` : "";
|
|
24590
|
-
const stepsText = input.steps.map((step) => [
|
|
24591
|
-
`### ${step.id} [${step.source}]`,
|
|
24592
|
-
`- **Instruction**: ${step.instruction}`,
|
|
24593
|
-
...step.expected ? [`- **Expected**: ${step.expected}`] : []
|
|
24594
|
-
].join("\n")).join("\n\n");
|
|
24595
|
-
const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
|
|
24596
|
-
const contextText = renderContext(input.context ?? []);
|
|
24597
|
-
const conventionsText = renderConventions(input.conventions ?? []);
|
|
24598
|
-
return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
|
|
24599
|
-
|
|
24600
|
-
## Session
|
|
24601
|
-
|
|
24602
|
-
SESSION NAME: \`${sessionName}\`
|
|
24603
|
-
|
|
24604
|
-
Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
|
|
24605
|
-
|
|
24606
|
-
**Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
|
|
24607
|
-
env assignment naming the spec step it executes**, e.g.
|
|
24608
|
-
|
|
24609
|
-
\`\`\`bash
|
|
24610
|
-
CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
|
|
24611
|
-
\`\`\`
|
|
24612
|
-
|
|
24613
|
-
This prefix is what ties each recorded action to its step in the generated
|
|
24614
|
-
test — a command without it loses its step attribution.
|
|
24615
|
-
|
|
24616
|
-
**Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
|
|
24617
|
-
in, opening the application, putting the account where the precondition says it
|
|
24618
|
-
starts — those are not step-01, and marking them as step-01 makes the first
|
|
24619
|
-
step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
|
|
24620
|
-
first step the case actually lists.
|
|
24621
|
-
|
|
24622
|
-
Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
|
|
24623
|
-
itself ignores the variable.
|
|
24624
|
-
|
|
24625
|
-
Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
|
|
24626
|
-
assignment (alongside \`CCQA_STEP\`) that records the check as a test
|
|
24627
|
-
assertion — see "Assertion Protocol".
|
|
24628
|
-
|
|
24629
|
-
**A command that types into a password field carries \`CCQA_SECRET=1\`** as
|
|
24630
|
-
well, e.g.
|
|
24631
|
-
|
|
24632
|
-
\`\`\`bash
|
|
24633
|
-
CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
|
|
24634
|
-
\`\`\`
|
|
24635
|
-
|
|
24636
|
-
A password field is an \`<input type="password">\` — the snapshot shows its
|
|
24637
|
-
value as dots, and nothing else about the page is a password. The prefix is
|
|
24638
|
-
what stops the typed value from being written to the recording: a value that
|
|
24639
|
-
came from a \`\${VAR}\` is kept as that reference, and one typed literally is
|
|
24640
|
-
dropped along with its command.
|
|
24641
|
-
|
|
24642
|
-
## Browser Commands
|
|
24643
|
-
|
|
24644
|
-
\`\`\`
|
|
24645
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
|
|
24646
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION snapshot
|
|
24647
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
|
|
24648
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
|
|
24649
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
|
|
24650
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
|
|
24651
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
|
|
24652
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
|
|
24653
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
|
|
24654
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
|
|
24655
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
|
|
24656
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
|
|
24657
|
-
agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
|
|
24658
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
|
|
24659
|
-
CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
|
|
24660
|
-
# See "Selector Rules" for the full \`find\` subset.
|
|
24661
|
-
# IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
|
|
24662
|
-
# CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
|
|
24663
|
-
# Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
|
|
24664
|
-
# (which waits internally). To check an element exists, use \`get count "<selector>"\`.
|
|
24665
|
-
\`\`\`
|
|
24666
|
-
|
|
24667
|
-
## Selector Rules
|
|
24668
|
-
|
|
24669
|
-
**ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
|
|
24670
|
-
|
|
24671
|
-
| Format | Use when |
|
|
24672
|
-
|--------|----------|
|
|
24673
|
-
| \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
|
|
24674
|
-
| \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
|
|
24675
|
-
| \`[placeholder='text']\` | Input identified by placeholder |
|
|
24676
|
-
| \`[type='password']\` | Password inputs only |
|
|
24677
|
-
| \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
|
|
24678
|
-
| \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
|
|
24679
|
-
|
|
24680
|
-
**FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
|
|
24681
|
-
|
|
24682
|
-
- \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
|
|
24683
|
-
- **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
|
|
24684
|
-
- \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
|
|
24685
|
-
- **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
|
|
24686
|
-
- JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
|
|
24687
|
-
|
|
24688
|
-
### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
|
|
24689
|
-
|
|
24690
|
-
When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
|
|
24691
|
-
|
|
24692
|
-
\`\`\`
|
|
24693
|
-
find role <role> <action> [--name "<n>"] [--exact]
|
|
24694
|
-
find text|label|placeholder|alt|title "<text>" <action> [--exact]
|
|
24695
|
-
find testid "<id>" <action>
|
|
24696
|
-
find first|last "<ALLOWED-css>" <action>
|
|
24697
|
-
find nth <index> "<ALLOWED-css>" <action>
|
|
24698
|
-
\`\`\`
|
|
24699
|
-
|
|
24700
|
-
\`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
|
|
24701
|
-
|
|
24702
|
-
**Rules for \`find\`:**
|
|
24703
|
-
|
|
24704
|
-
1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
|
|
24705
|
-
2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
|
|
24706
|
-
3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
|
|
24707
|
-
4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
|
|
24708
|
-
5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
|
|
24709
|
-
6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
|
|
24710
|
-
7. \`find\` includes its own wait; do not chain a \`wait\` before it.
|
|
24711
|
-
8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
|
|
24712
|
-
9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
|
|
24713
|
-
|
|
24714
|
-
**Examples:**
|
|
24715
|
-
|
|
24716
|
-
- ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
|
|
24717
|
-
- ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
|
|
24718
|
-
- ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
|
|
24719
|
-
- ✗ \`find role button --name "Submit" click\` — wrong order
|
|
24720
|
-
- ✗ \`find last "button" click\` — bare tag
|
|
24721
|
-
|
|
24722
|
-
### Selector workflow
|
|
24723
|
-
|
|
24724
|
-
1. Run \`snapshot\` and read the ARIA tree.
|
|
24725
|
-
2. Identify the element; note its exact \`aria-label\` if present.
|
|
24726
|
-
3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
|
|
24727
|
-
For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
|
|
24728
|
-
4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
|
|
24729
|
-
5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
|
|
24730
|
-
6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
|
|
24731
|
-
7. Never guess. If a selector fails, take a fresh snapshot before retrying.
|
|
24732
|
-
8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
|
|
24733
|
-
|
|
24734
|
-
### Special input types
|
|
24735
|
-
|
|
24736
|
-
**contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
|
|
24737
|
-
|
|
24738
|
-
**combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
|
|
24739
|
-
|
|
24740
|
-
**Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
|
|
24741
|
-
|
|
24742
|
-
**Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
|
|
24743
|
-
|
|
24744
|
-
**File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
|
|
24745
|
-
|
|
24746
|
-
## Test Specification
|
|
24747
|
-
|
|
24748
|
-
Title: ${input.title}
|
|
24749
|
-
|
|
24750
|
-
Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
|
|
24751
|
-
|
|
24752
|
-
## Steps
|
|
24753
|
-
|
|
24754
|
-
${stepsText}
|
|
24755
|
-
${expectationsText}${contextText}${conventionsText}
|
|
24756
|
-
## Execution Workflow
|
|
24757
|
-
|
|
24758
|
-
For each step:
|
|
24759
|
-
1. Emit \`STEP_START|<step-id>|<short description>\`.
|
|
24760
|
-
2. Run \`snapshot\` and identify selectors from the ARIA tree.
|
|
24761
|
-
3. Execute the action using an ALLOWED selector (see Selector Rules), prefixing the command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step.
|
|
24762
|
-
4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
|
|
24763
|
-
5. Run \`snapshot\` again to verify the outcome.
|
|
24764
|
-
6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
|
|
24765
|
-
7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
|
|
24766
|
-
8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
|
|
24767
|
-
|
|
24768
|
-
**Protocol lines are recorded ONLY from your plain assistant text.** Every
|
|
24769
|
-
\`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
|
|
24770
|
-
message text between tool calls — never through Bash (\`echo\` etc.) and never
|
|
24771
|
-
omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
|
|
24772
|
-
prefix on each command (that prefix is authoritative, not the \`STEP_START\`
|
|
24773
|
-
line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
|
|
24774
|
-
verification commands** — a command that ran, whose failure rolls the record
|
|
24775
|
-
back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
|
|
24776
|
-
proves it; it is reported and not recorded.
|
|
24777
|
-
|
|
24778
|
-
**After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
|
|
24779
|
-
|
|
24780
|
-
## Guardrails
|
|
24781
|
-
|
|
24782
|
-
- **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
|
|
24783
|
-
- **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
|
|
24784
|
-
- Do NOT retry a selector without taking a fresh snapshot first.
|
|
24785
|
-
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
|
|
24786
|
-
- **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
|
|
24787
|
-
- **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
|
|
24788
|
-
|
|
24789
|
-
## Source Code Reference
|
|
24790
|
-
|
|
24791
|
-
You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
|
|
24792
|
-
|
|
24793
|
-
**When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
|
|
24794
|
-
|
|
24795
|
-
**How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
|
|
24796
|
-
|
|
24797
|
-
**Rules**: only READ source files, never modify them. Keep searches focused.
|
|
24798
|
-
|
|
24799
|
-
## Waiting for Async Operations
|
|
24800
|
-
|
|
24801
|
-
Prefer \`wait\` over polling:
|
|
24802
|
-
|
|
24803
|
-
\`\`\`bash
|
|
24804
|
-
CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
|
|
24805
|
-
\`\`\`
|
|
24806
|
-
|
|
24807
|
-
If polling is required (e.g. waiting for a spinner to disappear):
|
|
24808
|
-
|
|
24809
|
-
\`\`\`bash
|
|
24810
|
-
for i in $(seq 1 18); do
|
|
24811
|
-
sleep 10
|
|
24812
|
-
result=$(agent-browser --session ${sessionName} snapshot 2>&1)
|
|
24813
|
-
echo "$result" | grep -q "<done indicator>" && break
|
|
24814
|
-
done
|
|
24815
|
-
CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
|
|
24816
|
-
\`\`\`
|
|
24817
|
-
|
|
24818
|
-
After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
|
|
24819
|
-
|
|
24820
|
-
## AB_ACTION Protocol
|
|
24821
|
-
|
|
24822
|
-
After **every** browser action, emit one line (outside any code block):
|
|
24823
|
-
|
|
24824
|
-
\`\`\`
|
|
24825
|
-
AB_ACTION|cookies_clear
|
|
24826
|
-
AB_ACTION|open|<url>
|
|
24827
|
-
AB_ACTION|click|<selector>|<visible label>
|
|
24828
|
-
AB_ACTION|dblclick|<selector>|<visible label>
|
|
24829
|
-
AB_ACTION|fill|<selector>|<value>|<aria label>
|
|
24830
|
-
AB_ACTION|check|<selector>|<visible label>
|
|
24831
|
-
AB_ACTION|uncheck|<selector>|<visible label>
|
|
24832
|
-
AB_ACTION|press|<Key>
|
|
24833
|
-
AB_ACTION|select|<selector>|<value>|<aria label>
|
|
24834
|
-
AB_ACTION|hover|<selector>|<visible label>
|
|
24835
|
-
AB_ACTION|scroll|<direction>|<pixels>
|
|
24836
|
-
AB_ACTION|drag|<source selector>|<target selector>|<source label>
|
|
24837
|
-
AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
|
|
24838
|
-
AB_ACTION|wait|<selector or text>|<label>
|
|
24839
|
-
AB_ACTION|snapshot|<key observation, max 100 chars>
|
|
24840
|
-
|
|
24841
|
-
# find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
|
|
24842
|
-
# <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
|
|
24843
|
-
AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
|
|
24844
|
-
AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
|
|
24845
|
-
AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
|
|
24846
|
-
AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
|
|
24847
|
-
AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
|
|
24848
|
-
AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
|
|
24849
|
-
AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
24850
|
-
AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
24851
|
-
\`\`\`
|
|
24852
|
-
|
|
24853
|
-
Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
|
|
24854
|
-
|
|
24855
|
-
**CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
|
|
24856
|
-
|
|
24857
|
-
- A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
|
|
24858
|
-
- If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
|
|
24859
|
-
- An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
|
|
24860
|
-
- **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
|
|
24861
|
-
|
|
24862
|
-
- **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
|
|
24863
|
-
- If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
|
|
24864
|
-
|
|
24865
|
-
## Assertion Protocol
|
|
24866
|
-
|
|
24867
|
-
Record each signal you verified as an assertion. The **primary mechanism** is
|
|
24868
|
-
a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
|
|
24869
|
-
marker rides the same channel as the command, so the assertion is captured
|
|
24870
|
-
even if you print no protocol text. The check you were going to run anyway
|
|
24871
|
-
becomes the recorded assertion:
|
|
24872
|
-
|
|
24873
|
-
\`\`\`bash
|
|
24874
|
-
# text_visible — the marked wait IS the assertion (no separate assert line)
|
|
24875
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
|
|
24876
|
-
# element_visible / element_not_visible — mark the get-count probe
|
|
24877
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
|
|
24878
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
|
|
24879
|
-
# url_contains — the substring rides in the marker; put it on \`get url\`
|
|
24880
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
|
|
24881
|
-
# element state — mark the \`is\` probe that asks for it
|
|
24882
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
|
|
24883
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
|
|
24884
|
-
\`\`\`
|
|
24885
|
-
|
|
24886
|
-
**Marker semantics:**
|
|
24887
|
-
|
|
24888
|
-
| Marker | On command | Records |
|
|
24889
|
-
|--------|------------|---------|
|
|
24890
|
-
| \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
|
|
24891
|
-
| \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
|
|
24892
|
-
| \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
|
|
24893
|
-
| \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
|
|
24894
|
-
| \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
|
|
24895
|
-
| \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
|
|
24896
|
-
|
|
24897
|
-
- A marked command that exits non-zero records nothing — fix the check and re-run it.
|
|
24898
|
-
- A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
|
|
24899
|
-
- \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
|
|
24900
|
-
- **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
|
|
24901
|
-
|
|
24902
|
-
- **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
|
|
24903
|
-
|
|
24904
|
-
- **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
|
|
24905
|
-
|
|
24906
|
-
- **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
|
|
24907
|
-
- **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
|
|
24908
|
-
- **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
|
|
24909
|
-
|
|
24910
|
-
**Every assert type has a marker command.** There is no text form: a printed
|
|
24911
|
-
\`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
|
|
24912
|
-
reported rather than recorded.
|
|
24913
|
-
|
|
24914
|
-
| assertType | The command that records it | Use when |
|
|
24915
|
-
|------------|------------------------------|----------|
|
|
24916
|
-
| \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
|
|
24917
|
-
| \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
|
|
24918
|
-
| \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
|
|
24919
|
-
| \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
|
|
24920
|
-
| \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
|
|
24921
|
-
| \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
|
|
24922
|
-
|
|
24923
|
-
**Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
|
|
24924
|
-
|
|
24925
|
-
- Timestamps, session IDs, exact numeric counts that vary between runs.
|
|
24926
|
-
- **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
|
|
24927
|
-
- **Relative-time labels** — true only in the moment of the trace, stale by replay:
|
|
24928
|
-
- English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
|
|
24929
|
-
- Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
|
|
24930
|
-
- Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
|
|
24931
|
-
- **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
|
|
24932
|
-
|
|
24933
|
-
**No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
|
|
24934
|
-
|
|
24935
|
-
The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
|
|
24936
|
-
|
|
24937
|
-
- ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
|
|
24938
|
-
- ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
|
|
24939
|
-
- ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
|
|
24940
|
-
- If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
|
|
24941
|
-
|
|
24942
|
-
**Page-context and selector rules:**
|
|
24943
|
-
|
|
24944
|
-
- After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
|
|
24945
|
-
- Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
|
|
24946
|
-
- When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
|
|
24947
|
-
|
|
24948
|
-
**MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
|
|
24949
|
-
|
|
24950
|
-
The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
|
|
24951
|
-
|
|
24952
|
-
1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
|
|
24953
|
-
2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
|
|
24954
|
-
3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
|
|
24955
|
-
|
|
24956
|
-
**Verify the assertion form actually resolves on the live page.** The
|
|
24957
|
-
verification and the recorded assertion are the same marked command:
|
|
24958
|
-
|
|
24959
|
-
\`\`\`bash
|
|
24960
|
-
# text_visible
|
|
24961
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
|
|
24962
|
-
# element_visible — >=1 printed means present
|
|
24963
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
|
|
24964
|
-
# element_not_visible — 0 printed means absent (works for text too: "text=<text>")
|
|
24965
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
|
|
24966
|
-
# element state — \`is\` prints true/false and exits 0 either way; read it
|
|
24967
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
|
|
24968
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
|
|
24969
|
-
\`\`\`
|
|
24970
|
-
|
|
24971
|
-
When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
|
|
24972
|
-
|
|
24973
|
-
## Status Protocol
|
|
24974
|
-
|
|
24975
|
-
Emit exactly one status line per step (outside any code block):
|
|
24976
|
-
|
|
24977
|
-
\`\`\`
|
|
24978
|
-
STEP_START|<step-id>|<short description>
|
|
24979
|
-
STEP_DONE|<step-id>|<what was verified>
|
|
24980
|
-
ASSERTION_FAILED|<step-id>|<category: app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
|
|
24981
|
-
STEP_SKIPPED|<step-id>|<reason>
|
|
24982
|
-
RUN_COMPLETED|passed|<summary>
|
|
24983
|
-
RUN_COMPLETED|failed|<summary>
|
|
24984
|
-
\`\`\`
|
|
24985
|
-
|
|
24986
|
-
**\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
|
|
24987
|
-
recorded as FAILED, however many steps went well: nothing else can tell a
|
|
24988
|
-
finished route from one you abandoned halfway.
|
|
24989
|
-
|
|
24990
|
-
${callerGuidance}## Start
|
|
24991
|
-
|
|
24992
|
-
Begin by clearing cookies, then proceed straight to the first step's instruction.
|
|
24993
|
-
|
|
24994
|
-
\`\`\`bash
|
|
24995
|
-
agent-browser --session ${sessionName} cookies clear
|
|
24996
|
-
\`\`\`
|
|
24997
|
-
|
|
24998
|
-
(The cookies-clear preamble belongs to no step, so it is the one command
|
|
24999
|
-
without a \`CCQA_STEP\` prefix.)
|
|
25000
|
-
|
|
25001
|
-
Emit:
|
|
25002
|
-
\`\`\`
|
|
25003
|
-
AB_ACTION|cookies_clear
|
|
25004
|
-
\`\`\`
|
|
25005
|
-
|
|
25006
|
-
Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
|
|
25007
|
-
every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
|
|
25008
|
-
step is responsible for opening the initial URL.
|
|
25009
|
-
`;
|
|
24674
|
+
function roleOfAccessibleName(snapshot, name) {
|
|
24675
|
+
const roles = /* @__PURE__ */ new Set();
|
|
24676
|
+
for (const line of snapshot.split("\n")) {
|
|
24677
|
+
const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
|
|
24678
|
+
if (m && m[2] === name) roles.add(m[1]);
|
|
24679
|
+
}
|
|
24680
|
+
return roles.size === 1 ? [...roles][0] : null;
|
|
25010
24681
|
}
|
|
25011
|
-
|
|
25012
|
-
|
|
24682
|
+
//#endregion
|
|
24683
|
+
//#region src/runtime/replay-validate.ts
|
|
24684
|
+
function isProbe(x) {
|
|
24685
|
+
return x !== null && !Array.isArray(x);
|
|
24686
|
+
}
|
|
24687
|
+
function runProbe(probe, sessionName) {
|
|
24688
|
+
return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
|
|
24689
|
+
}
|
|
24690
|
+
function runStateCheck(check, sessionName) {
|
|
24691
|
+
let answer = "";
|
|
24692
|
+
const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
|
|
24693
|
+
for (let round = 0; round < rounds; round++) {
|
|
24694
|
+
const r = spawnAB([
|
|
24695
|
+
"--session",
|
|
24696
|
+
sessionName,
|
|
24697
|
+
"is",
|
|
24698
|
+
check.state,
|
|
24699
|
+
check.selector
|
|
24700
|
+
]);
|
|
24701
|
+
answer = r.status === 0 ? r.stdout.trim() : "";
|
|
24702
|
+
if (answer === String(check.expected)) return {
|
|
24703
|
+
ok: true,
|
|
24704
|
+
reason: ""
|
|
24705
|
+
};
|
|
24706
|
+
if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
24707
|
+
}
|
|
24708
|
+
return answer === "true" || answer === "false" ? {
|
|
24709
|
+
ok: false,
|
|
24710
|
+
reason: `expected ${check.state}=${check.expected}, page says ${answer}`
|
|
24711
|
+
} : {
|
|
24712
|
+
ok: false,
|
|
24713
|
+
reason: `is ${check.state} answered ${answer || "nothing"}`
|
|
24714
|
+
};
|
|
24715
|
+
}
|
|
24716
|
+
const SELECTOR_POLL_INTERVAL_MS = 500;
|
|
24717
|
+
/** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
|
|
24718
|
+
function runPollCheck(check, sessionName) {
|
|
24719
|
+
const present = (selector) => {
|
|
24720
|
+
const r = spawnAB([
|
|
24721
|
+
"--session",
|
|
24722
|
+
sessionName,
|
|
24723
|
+
"get",
|
|
24724
|
+
"count",
|
|
24725
|
+
selector
|
|
24726
|
+
]);
|
|
24727
|
+
return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
|
|
24728
|
+
};
|
|
24729
|
+
const deadline = Date.now() + check.timeoutMs;
|
|
24730
|
+
const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
|
|
24731
|
+
let count = NaN;
|
|
24732
|
+
for (let round = 0;; round++) {
|
|
24733
|
+
count = present(check.selector);
|
|
24734
|
+
if (!Number.isNaN(count) && count > 0) return {
|
|
24735
|
+
ok: true,
|
|
24736
|
+
reason: ""
|
|
24737
|
+
};
|
|
24738
|
+
if (round + 1 >= rounds || Date.now() >= deadline) break;
|
|
24739
|
+
sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
24740
|
+
}
|
|
24741
|
+
return {
|
|
24742
|
+
ok: false,
|
|
24743
|
+
reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
|
|
24744
|
+
};
|
|
24745
|
+
}
|
|
24746
|
+
/** What the validator records against an action it never replayed. */
|
|
24747
|
+
const CASCADE_REASON = "skipped after a preceding action failed";
|
|
24748
|
+
/** Whether a drop reason means "not replayed", rather than "replayed and failed". */
|
|
24749
|
+
function isCascadeReason(reason) {
|
|
24750
|
+
return reason === CASCADE_REASON;
|
|
24751
|
+
}
|
|
24752
|
+
const SHORT_TIMEOUT_MS = 5e3;
|
|
24753
|
+
const ASSERT_TIMEOUT_MS = 1e4;
|
|
24754
|
+
const INTERACTION_POLL_INTERVAL_MS = 250;
|
|
24755
|
+
const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
|
|
24756
|
+
/** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
|
|
24757
|
+
function attemptsFor(timeoutMs, intervalMs) {
|
|
24758
|
+
return Math.max(1, Math.ceil(timeoutMs / intervalMs));
|
|
25013
24759
|
}
|
|
25014
24760
|
/**
|
|
25015
|
-
*
|
|
25016
|
-
*
|
|
25017
|
-
*
|
|
25018
|
-
*
|
|
25019
|
-
*
|
|
24761
|
+
* Convert one recorded action into the `agent-browser` arg list that would
|
|
24762
|
+
* exercise it. Interaction commands come from the shared
|
|
24763
|
+
* `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
|
|
24764
|
+
* get validation-specific handling (poll checks, timeouts, unverifiable
|
|
24765
|
+
* skips). Returns `null` for actions that should not be validated (snapshot
|
|
24766
|
+
* has no side effect; assert types whose codegen forms aren't directly
|
|
24767
|
+
* verifiable here fall through to the caller's `unverifiable` fallback).
|
|
25020
24768
|
*/
|
|
25021
|
-
function
|
|
25022
|
-
|
|
25023
|
-
|
|
25024
|
-
|
|
25025
|
-
|
|
25026
|
-
|
|
25027
|
-
|
|
25028
|
-
|
|
25029
|
-
|
|
25030
|
-
|
|
25031
|
-
|
|
24769
|
+
function actionToAbArgs(action, sessionName, envOverrides = {}) {
|
|
24770
|
+
const base = ["--session", sessionName];
|
|
24771
|
+
const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
|
|
24772
|
+
switch (action.action) {
|
|
24773
|
+
case "snapshot": return null;
|
|
24774
|
+
case "assert": return assertToAbArgs(action, sub, sessionName);
|
|
24775
|
+
case "wait": {
|
|
24776
|
+
const loc = action.locator;
|
|
24777
|
+
if (!loc) return null;
|
|
24778
|
+
if (loc.by === "text") return [
|
|
24779
|
+
...base,
|
|
24780
|
+
"wait",
|
|
24781
|
+
"--text",
|
|
24782
|
+
sub(loc.value),
|
|
24783
|
+
"--timeout",
|
|
24784
|
+
String(SHORT_TIMEOUT_MS)
|
|
24785
|
+
];
|
|
24786
|
+
const raw = sub(locatorToSelector(loc));
|
|
24787
|
+
if (!raw) return null;
|
|
24788
|
+
if (/^\d+$/.test(raw)) return null;
|
|
24789
|
+
if (raw.startsWith("--")) return null;
|
|
24790
|
+
if (raw.startsWith("text=")) return [
|
|
24791
|
+
...base,
|
|
24792
|
+
"wait",
|
|
24793
|
+
"--text",
|
|
24794
|
+
raw.slice(5),
|
|
24795
|
+
"--timeout",
|
|
24796
|
+
String(SHORT_TIMEOUT_MS)
|
|
24797
|
+
];
|
|
24798
|
+
return {
|
|
24799
|
+
kind: "poll-present",
|
|
24800
|
+
selector: raw,
|
|
24801
|
+
timeoutMs: SHORT_TIMEOUT_MS
|
|
24802
|
+
};
|
|
24803
|
+
}
|
|
24804
|
+
default: {
|
|
24805
|
+
const tokens = toAgentBrowserArgs(action);
|
|
24806
|
+
if (tokens === null) return null;
|
|
24807
|
+
const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
|
|
24808
|
+
return [...base, ...tokens.map((t) => resolve(t.text))];
|
|
24809
|
+
}
|
|
24810
|
+
}
|
|
24811
|
+
}
|
|
24812
|
+
function roleProbe(base, role, name) {
|
|
24813
|
+
return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
|
|
24814
|
+
}
|
|
24815
|
+
/**
|
|
24816
|
+
* Presence asked the way the locator addresses the element. `null` is
|
|
24817
|
+
* unverifiable rather than absent: a count from a selector the engine never
|
|
24818
|
+
* parsed is not evidence.
|
|
24819
|
+
*/
|
|
24820
|
+
function presenceCheck(locator, selector, sub, base) {
|
|
24821
|
+
if (locator?.by === "text") return [
|
|
24822
|
+
...base,
|
|
24823
|
+
"wait",
|
|
24824
|
+
"--text",
|
|
24825
|
+
sub(locator.value),
|
|
24826
|
+
"--timeout",
|
|
24827
|
+
String(ASSERT_TIMEOUT_MS)
|
|
24828
|
+
];
|
|
24829
|
+
if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
|
|
24830
|
+
if (locator !== void 0 && locator.by !== "css") return null;
|
|
24831
|
+
const named = parseNotation(selector);
|
|
24832
|
+
if (named === "opaque") return null;
|
|
24833
|
+
if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
|
|
24834
|
+
if (named !== null) return null;
|
|
24835
|
+
return {
|
|
24836
|
+
kind: "poll-present",
|
|
24837
|
+
selector,
|
|
24838
|
+
timeoutMs: ASSERT_TIMEOUT_MS
|
|
24839
|
+
};
|
|
24840
|
+
}
|
|
24841
|
+
function assertToAbArgs(action, sub, sessionName) {
|
|
24842
|
+
const base = ["--session", sessionName];
|
|
24843
|
+
const val = sub(action.value ?? action.observation);
|
|
24844
|
+
const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
|
|
24845
|
+
switch (action.assert) {
|
|
24846
|
+
case "text_visible":
|
|
24847
|
+
if (!val) return null;
|
|
24848
|
+
return [
|
|
24849
|
+
...base,
|
|
24850
|
+
"wait",
|
|
24851
|
+
"--text",
|
|
24852
|
+
val,
|
|
24853
|
+
"--timeout",
|
|
24854
|
+
String(ASSERT_TIMEOUT_MS)
|
|
24855
|
+
];
|
|
24856
|
+
case "text_not_visible": return null;
|
|
24857
|
+
case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
|
|
24858
|
+
case "element_not_visible": return null;
|
|
24859
|
+
case "url_contains": return null;
|
|
24860
|
+
case "element_enabled":
|
|
24861
|
+
case "element_disabled":
|
|
24862
|
+
case "element_checked":
|
|
24863
|
+
case "element_unchecked":
|
|
24864
|
+
if (!sel || action.locator?.by !== "css") return null;
|
|
24865
|
+
if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
|
|
24866
|
+
return {
|
|
24867
|
+
kind: "state",
|
|
24868
|
+
state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
|
|
24869
|
+
selector: sel,
|
|
24870
|
+
expected: action.assert === "element_enabled" || action.assert === "element_checked"
|
|
24871
|
+
};
|
|
24872
|
+
default: return null;
|
|
24873
|
+
}
|
|
25032
24874
|
}
|
|
24875
|
+
const NO_STEP_ID = "__no_step__";
|
|
25033
24876
|
/**
|
|
25034
|
-
*
|
|
25035
|
-
*
|
|
25036
|
-
*
|
|
25037
|
-
*
|
|
24877
|
+
* The role a labelled element has when addressing it by label finds nothing.
|
|
24878
|
+
*
|
|
24879
|
+
* `getByLabel` needs a real association — a `<label for>`, a wrapping label,
|
|
24880
|
+
* an `aria-labelledby`. A form that has none still shows the same string as
|
|
24881
|
+
* the field's accessible name, which is what the recorder read off the
|
|
24882
|
+
* snapshot, so the label it recorded matches nothing while the same text as a
|
|
24883
|
+
* role's name matches exactly. Only the actions whose element has one
|
|
24884
|
+
* unambiguous role are listed: a labelled `click` may be a button, a link or
|
|
24885
|
+
* a checkbox, and guessing there would replace a locator that failed with one
|
|
24886
|
+
* that finds the wrong element.
|
|
25038
24887
|
*/
|
|
25039
|
-
|
|
25040
|
-
|
|
25041
|
-
|
|
25042
|
-
|
|
25043
|
-
|
|
25044
|
-
|
|
25045
|
-
|
|
25046
|
-
|
|
25047
|
-
|
|
25048
|
-
**Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
|
|
25049
|
-
`;
|
|
24888
|
+
const LABEL_FALLBACK_ROLE = {
|
|
24889
|
+
fill: "textbox",
|
|
24890
|
+
type: "textbox",
|
|
24891
|
+
check: "checkbox",
|
|
24892
|
+
uncheck: "checkbox"
|
|
24893
|
+
};
|
|
24894
|
+
/** Whether agent-browser's failure was "no element", the one the fallback answers. */
|
|
24895
|
+
function notFound(result) {
|
|
24896
|
+
return /not\s+found|no\s+element|no\s+such\s+element|none\s+match\s+name/i.test(`${result.stderr} ${result.stdout}`);
|
|
25050
24897
|
}
|
|
25051
|
-
/**
|
|
25052
|
-
|
|
25053
|
-
|
|
25054
|
-
* case, and the recorder is now that reader.
|
|
25055
|
-
*/
|
|
25056
|
-
function renderContext(context) {
|
|
25057
|
-
if (context.length === 0) return "";
|
|
25058
|
-
return `
|
|
25059
|
-
## About this case
|
|
25060
|
-
|
|
25061
|
-
${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
|
|
25062
|
-
`;
|
|
24898
|
+
/** Whether this argv navigates — the only action safe to repeat wholesale. */
|
|
24899
|
+
function isOpen(argv) {
|
|
24900
|
+
return argv[2] === "open";
|
|
25063
24901
|
}
|
|
25064
|
-
/**
|
|
25065
|
-
|
|
25066
|
-
|
|
25067
|
-
|
|
25068
|
-
|
|
25069
|
-
|
|
25070
|
-
|
|
25071
|
-
|
|
25072
|
-
|
|
25073
|
-
|
|
25074
|
-
|
|
25075
|
-
|
|
25076
|
-
|
|
24902
|
+
/** A navigation another navigation took away, rather than one that failed. */
|
|
24903
|
+
function superseded(result) {
|
|
24904
|
+
return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
|
|
24905
|
+
}
|
|
24906
|
+
/** The one line both callers log when the fallback rewrote a locator. */
|
|
24907
|
+
function formatPromotion(promotion) {
|
|
24908
|
+
return `locator rewritten to the form that replays: ${promotion}`;
|
|
24909
|
+
}
|
|
24910
|
+
/** The same action addressed by role + accessible name, or null when there is no such form. */
|
|
24911
|
+
function labelFallback(action) {
|
|
24912
|
+
const loc = action.locator;
|
|
24913
|
+
if (!loc || loc.by !== "label") return null;
|
|
24914
|
+
const role = LABEL_FALLBACK_ROLE[action.action];
|
|
24915
|
+
if (role === void 0) return null;
|
|
24916
|
+
return {
|
|
24917
|
+
...action,
|
|
24918
|
+
locator: {
|
|
24919
|
+
by: "role",
|
|
24920
|
+
value: role,
|
|
24921
|
+
name: loc.value,
|
|
24922
|
+
exact: true
|
|
24923
|
+
}
|
|
24924
|
+
};
|
|
25077
24925
|
}
|
|
25078
|
-
//#endregion
|
|
25079
|
-
//#region src/ir/accessible-name.ts
|
|
25080
24926
|
/**
|
|
25081
|
-
*
|
|
25082
|
-
*
|
|
25083
|
-
*
|
|
25084
|
-
*
|
|
24927
|
+
* Spawn one interaction, waiting for its element the way an assertion waits
|
|
24928
|
+
* for its text: a recording pauses for a snapshot between opening a page and
|
|
24929
|
+
* typing into it, the replay does not, so the replay reaches a field sooner
|
|
24930
|
+
* than the recording ever did.
|
|
24931
|
+
*
|
|
24932
|
+
* Only a "no element" failure waits — anything else repeats identically. When
|
|
24933
|
+
* there is an `alternate` form it is tried on every attempt rather than after
|
|
24934
|
+
* the wait: a label the page never associates with its input will not start
|
|
24935
|
+
* matching, so waiting first spends the budget for nothing.
|
|
25085
24936
|
*/
|
|
25086
|
-
|
|
24937
|
+
function runInteraction(argv, alternate, patience, sessionName) {
|
|
24938
|
+
const started = Date.now();
|
|
24939
|
+
const deadline = started + ASSERT_TIMEOUT_MS;
|
|
24940
|
+
let settled = false;
|
|
24941
|
+
for (let attempt = 0;; attempt++) {
|
|
24942
|
+
let result = spawnAB(argv);
|
|
24943
|
+
if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
|
|
24944
|
+
const waitedMs = Date.now() - started;
|
|
24945
|
+
if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
|
|
24946
|
+
settled = true;
|
|
24947
|
+
spawnAB([
|
|
24948
|
+
"--session",
|
|
24949
|
+
sessionName,
|
|
24950
|
+
"wait",
|
|
24951
|
+
"--load",
|
|
24952
|
+
"networkidle"
|
|
24953
|
+
]);
|
|
24954
|
+
continue;
|
|
24955
|
+
}
|
|
24956
|
+
if (result.status === 0 || !notFound(result)) return {
|
|
24957
|
+
result,
|
|
24958
|
+
viaAlternate: false,
|
|
24959
|
+
waitedMs
|
|
24960
|
+
};
|
|
24961
|
+
if (alternate !== null) {
|
|
24962
|
+
const alt = spawnAB(alternate);
|
|
24963
|
+
if (alt.status === 0) return {
|
|
24964
|
+
result: alt,
|
|
24965
|
+
viaAlternate: true,
|
|
24966
|
+
waitedMs
|
|
24967
|
+
};
|
|
24968
|
+
}
|
|
24969
|
+
if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
|
|
24970
|
+
patience.spent = true;
|
|
24971
|
+
return {
|
|
24972
|
+
result,
|
|
24973
|
+
viaAlternate: false,
|
|
24974
|
+
waitedMs: Date.now() - started
|
|
24975
|
+
};
|
|
24976
|
+
}
|
|
24977
|
+
sleepSync(INTERACTION_POLL_INTERVAL_MS);
|
|
24978
|
+
}
|
|
24979
|
+
}
|
|
25087
24980
|
/**
|
|
25088
|
-
*
|
|
24981
|
+
* Say what a `by: "css"` locator that is not CSS actually addresses.
|
|
25089
24982
|
*
|
|
25090
|
-
* `
|
|
25091
|
-
*
|
|
24983
|
+
* `get count` answers 0 for Playwright's notation instead of failing, so a
|
|
24984
|
+
* route holding one reads as dead. Rewritten in place, and reported, so the
|
|
24985
|
+
* saved route carries the form that replays rather than the one that counts
|
|
24986
|
+
* nothing. Notation this cannot convert leaves the locator alone; the assert
|
|
24987
|
+
* path then treats it as unverifiable rather than absent.
|
|
25092
24988
|
*/
|
|
25093
|
-
function
|
|
25094
|
-
const
|
|
25095
|
-
|
|
25096
|
-
|
|
24989
|
+
function nameNotation(action) {
|
|
24990
|
+
const loc = action.locator;
|
|
24991
|
+
if (action.action !== "assert" || loc?.by !== "css") return null;
|
|
24992
|
+
const named = parseNotation(loc.value);
|
|
24993
|
+
if (named === null || named === "opaque" || named.by !== "text") return null;
|
|
24994
|
+
action.locator = named;
|
|
24995
|
+
return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
|
|
25097
24996
|
}
|
|
25098
24997
|
/**
|
|
25099
|
-
*
|
|
25100
|
-
*
|
|
24998
|
+
* An attribute selector that counted nothing, asked of the accessibility tree
|
|
24999
|
+
* the name was read from. See docs/targets.md for why the two disagree.
|
|
25101
25000
|
*
|
|
25102
|
-
*
|
|
25103
|
-
*
|
|
25104
|
-
*
|
|
25001
|
+
* The snapshot is read for the role, then `find role` is asked as well: the
|
|
25002
|
+
* tree says the name exists somewhere, the probe says the replay can reach it,
|
|
25003
|
+
* and only the second is what the route will depend on.
|
|
25105
25004
|
*/
|
|
25106
|
-
function
|
|
25107
|
-
|
|
25108
|
-
|
|
25109
|
-
|
|
25110
|
-
|
|
25111
|
-
|
|
25112
|
-
|
|
25113
|
-
|
|
25114
|
-
|
|
25115
|
-
|
|
25116
|
-
|
|
25117
|
-
|
|
25118
|
-
|
|
25119
|
-
|
|
25120
|
-
|
|
25005
|
+
function askTheAccessibilityTree(action, sub, sessionName) {
|
|
25006
|
+
if (action.action !== "assert" || action.locator?.by !== "css") return null;
|
|
25007
|
+
const name = nameFromAttributeSelector(action.locator.value);
|
|
25008
|
+
if (name === null) return null;
|
|
25009
|
+
const snapshot = spawnAB([
|
|
25010
|
+
"--session",
|
|
25011
|
+
sessionName,
|
|
25012
|
+
"snapshot"
|
|
25013
|
+
]);
|
|
25014
|
+
if (snapshot.status !== 0) return null;
|
|
25015
|
+
const role = roleOfAccessibleName(snapshot.stdout, sub(name));
|
|
25016
|
+
if (role === null) return null;
|
|
25017
|
+
if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
|
|
25018
|
+
const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
|
|
25019
|
+
action.locator = {
|
|
25020
|
+
by: "role",
|
|
25021
|
+
value: role,
|
|
25022
|
+
name,
|
|
25023
|
+
exact: true
|
|
25024
|
+
};
|
|
25025
|
+
return {
|
|
25026
|
+
skipped: false,
|
|
25027
|
+
ok: true,
|
|
25028
|
+
reason: "",
|
|
25029
|
+
promoted
|
|
25030
|
+
};
|
|
25121
25031
|
}
|
|
25122
|
-
|
|
25123
|
-
|
|
25124
|
-
|
|
25125
|
-
|
|
25126
|
-
|
|
25127
|
-
|
|
25128
|
-
|
|
25129
|
-
|
|
25130
|
-
|
|
25131
|
-
|
|
25132
|
-
|
|
25133
|
-
|
|
25134
|
-
|
|
25032
|
+
/**
|
|
25033
|
+
* Replay one recorded action against the validation session. Element-presence
|
|
25034
|
+
* checks go through `runPollCheck` (which uses `get count`, never the blocking
|
|
25035
|
+
* `wait <selector>`); everything else spawns the agent-browser argv.
|
|
25036
|
+
*/
|
|
25037
|
+
function runValidationAction(action, sessionName, envOverrides = {}, patience) {
|
|
25038
|
+
const named = nameNotation(action);
|
|
25039
|
+
const withName = (outcome) => named === null || !outcome.ok ? outcome : {
|
|
25040
|
+
...outcome,
|
|
25041
|
+
promoted: outcome.promoted ?? named
|
|
25042
|
+
};
|
|
25043
|
+
const built = actionToAbArgs(action, sessionName, envOverrides);
|
|
25044
|
+
if (built === null) return {
|
|
25045
|
+
skipped: true,
|
|
25046
|
+
ok: false,
|
|
25047
|
+
reason: ""
|
|
25048
|
+
};
|
|
25049
|
+
if (isProbe(built)) {
|
|
25050
|
+
const { ok, reason } = runProbe(built, sessionName);
|
|
25051
|
+
if (ok) return withName({
|
|
25052
|
+
skipped: false,
|
|
25053
|
+
ok,
|
|
25054
|
+
reason
|
|
25055
|
+
});
|
|
25056
|
+
const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
|
|
25057
|
+
return askTheAccessibilityTree(action, resolve, sessionName) ?? {
|
|
25058
|
+
skipped: false,
|
|
25059
|
+
ok,
|
|
25060
|
+
reason
|
|
25061
|
+
};
|
|
25062
|
+
}
|
|
25063
|
+
const fallback = labelFallback(action);
|
|
25064
|
+
const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
|
|
25065
|
+
const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
|
|
25066
|
+
if (viaAlternate) {
|
|
25067
|
+
const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
|
|
25068
|
+
action.locator = fallback.locator;
|
|
25069
|
+
return {
|
|
25070
|
+
skipped: false,
|
|
25135
25071
|
ok: true,
|
|
25136
|
-
reason: ""
|
|
25072
|
+
reason: "",
|
|
25073
|
+
promoted
|
|
25137
25074
|
};
|
|
25138
|
-
if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
25139
25075
|
}
|
|
25140
|
-
|
|
25141
|
-
|
|
25142
|
-
|
|
25143
|
-
|
|
25076
|
+
if (result.status === 0) return withName({
|
|
25077
|
+
skipped: false,
|
|
25078
|
+
ok: true,
|
|
25079
|
+
reason: ""
|
|
25080
|
+
});
|
|
25081
|
+
return {
|
|
25082
|
+
skipped: false,
|
|
25144
25083
|
ok: false,
|
|
25145
|
-
reason: `
|
|
25084
|
+
reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
|
|
25146
25085
|
};
|
|
25147
25086
|
}
|
|
25148
|
-
|
|
25149
|
-
|
|
25150
|
-
|
|
25151
|
-
|
|
25152
|
-
|
|
25087
|
+
/**
|
|
25088
|
+
* Replay actions in order against a live session and stop at the first that
|
|
25089
|
+
* fails. Unlike `validateActions` there is no cascade or rescue: the caller
|
|
25090
|
+
* hands what is left to someone who can look at the page. A wait this cannot
|
|
25091
|
+
* reproduce still settles the page, since the next action may depend on it;
|
|
25092
|
+
* a check it cannot run is returned as `unchecked`, never as passed.
|
|
25093
|
+
*/
|
|
25094
|
+
function replayUntilFailure(actions, opts) {
|
|
25095
|
+
const patience = { spent: false };
|
|
25096
|
+
const passed = [];
|
|
25097
|
+
const unchecked = [];
|
|
25098
|
+
for (const action of actions) {
|
|
25099
|
+
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25100
|
+
if (outcome.skipped && action.action === "wait") spawnAB([
|
|
25153
25101
|
"--session",
|
|
25154
|
-
sessionName,
|
|
25155
|
-
"
|
|
25156
|
-
"
|
|
25157
|
-
|
|
25102
|
+
opts.sessionName,
|
|
25103
|
+
"wait",
|
|
25104
|
+
"--load",
|
|
25105
|
+
"networkidle"
|
|
25158
25106
|
]);
|
|
25159
|
-
|
|
25160
|
-
|
|
25161
|
-
|
|
25162
|
-
|
|
25163
|
-
|
|
25164
|
-
|
|
25165
|
-
|
|
25166
|
-
|
|
25167
|
-
|
|
25168
|
-
|
|
25107
|
+
else if (outcome.skipped && action.action === "assert") {
|
|
25108
|
+
unchecked.push(action);
|
|
25109
|
+
continue;
|
|
25110
|
+
} else if (outcome.skipped) return {
|
|
25111
|
+
passed,
|
|
25112
|
+
unchecked,
|
|
25113
|
+
failed: {
|
|
25114
|
+
action,
|
|
25115
|
+
reason: "the recording does not say how to replay it"
|
|
25116
|
+
}
|
|
25169
25117
|
};
|
|
25170
|
-
if (
|
|
25171
|
-
|
|
25118
|
+
else if (!outcome.ok) return {
|
|
25119
|
+
passed,
|
|
25120
|
+
unchecked,
|
|
25121
|
+
failed: {
|
|
25122
|
+
action,
|
|
25123
|
+
reason: outcome.reason
|
|
25124
|
+
}
|
|
25125
|
+
};
|
|
25126
|
+
passed.push(action);
|
|
25172
25127
|
}
|
|
25173
25128
|
return {
|
|
25174
|
-
|
|
25175
|
-
|
|
25129
|
+
passed,
|
|
25130
|
+
unchecked
|
|
25176
25131
|
};
|
|
25177
25132
|
}
|
|
25178
|
-
|
|
25179
|
-
const
|
|
25180
|
-
|
|
25181
|
-
|
|
25182
|
-
|
|
25183
|
-
|
|
25184
|
-
|
|
25185
|
-
const
|
|
25186
|
-
|
|
25187
|
-
const
|
|
25188
|
-
|
|
25189
|
-
|
|
25190
|
-
|
|
25133
|
+
function validateActions(actions, opts) {
|
|
25134
|
+
const kept = [];
|
|
25135
|
+
const dropped = [];
|
|
25136
|
+
const promoted = [];
|
|
25137
|
+
const patience = { spent: false };
|
|
25138
|
+
let skipFromStepId = null;
|
|
25139
|
+
for (let i = 0; i < actions.length; i++) {
|
|
25140
|
+
const action = actions[i];
|
|
25141
|
+
opts.onProgress?.(i, actions.length, action);
|
|
25142
|
+
const stepId = action.stepId ?? NO_STEP_ID;
|
|
25143
|
+
if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
|
|
25144
|
+
if (skipFromStepId !== null && isPassiveAction(action.action)) {
|
|
25145
|
+
dropped.push({
|
|
25146
|
+
index: i,
|
|
25147
|
+
action,
|
|
25148
|
+
reason: CASCADE_REASON
|
|
25149
|
+
});
|
|
25150
|
+
continue;
|
|
25151
|
+
}
|
|
25152
|
+
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25153
|
+
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25154
|
+
if (outcome.skipped) {
|
|
25155
|
+
kept.push(action);
|
|
25156
|
+
continue;
|
|
25157
|
+
}
|
|
25158
|
+
if (outcome.ok) {
|
|
25159
|
+
kept.push(action);
|
|
25160
|
+
if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
|
|
25161
|
+
continue;
|
|
25162
|
+
}
|
|
25163
|
+
dropped.push({
|
|
25164
|
+
index: i,
|
|
25165
|
+
action,
|
|
25166
|
+
reason: outcome.reason
|
|
25167
|
+
});
|
|
25168
|
+
if (!isPassiveAction(action.action)) skipFromStepId = stepId;
|
|
25169
|
+
}
|
|
25170
|
+
const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
|
|
25171
|
+
promoted.push(...afterRescue.promoted ?? []);
|
|
25172
|
+
return {
|
|
25173
|
+
...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
|
|
25174
|
+
promoted
|
|
25175
|
+
};
|
|
25191
25176
|
}
|
|
25192
25177
|
/**
|
|
25193
|
-
*
|
|
25194
|
-
*
|
|
25195
|
-
*
|
|
25196
|
-
*
|
|
25197
|
-
*
|
|
25198
|
-
* has no side effect; assert types whose codegen forms aren't directly
|
|
25199
|
-
* verifiable here fall through to the caller's `unverifiable` fallback).
|
|
25178
|
+
* Translate the internal `{ kept, dropped }` result of the rescue pass
|
|
25179
|
+
* into the public-facing shape. In strict mode the caller sees the same
|
|
25180
|
+
* shape as before (kept/dropped); in lenient mode the still-failed
|
|
25181
|
+
* actions move to `unstable` with `replayUnstable: true` tagged on, so
|
|
25182
|
+
* codegen can warn about them while still emitting the line.
|
|
25200
25183
|
*/
|
|
25201
|
-
function
|
|
25202
|
-
|
|
25203
|
-
|
|
25204
|
-
|
|
25205
|
-
|
|
25206
|
-
|
|
25207
|
-
|
|
25208
|
-
|
|
25209
|
-
|
|
25210
|
-
|
|
25211
|
-
|
|
25212
|
-
|
|
25213
|
-
|
|
25214
|
-
|
|
25215
|
-
|
|
25216
|
-
|
|
25217
|
-
];
|
|
25218
|
-
const raw = sub(locatorToSelector(loc));
|
|
25219
|
-
if (!raw) return null;
|
|
25220
|
-
if (/^\d+$/.test(raw)) return null;
|
|
25221
|
-
if (raw.startsWith("--")) return null;
|
|
25222
|
-
if (raw.startsWith("text=")) return [
|
|
25223
|
-
...base,
|
|
25224
|
-
"wait",
|
|
25225
|
-
"--text",
|
|
25226
|
-
raw.slice(5),
|
|
25227
|
-
"--timeout",
|
|
25228
|
-
String(SHORT_TIMEOUT_MS)
|
|
25229
|
-
];
|
|
25230
|
-
return {
|
|
25231
|
-
kind: "poll-present",
|
|
25232
|
-
selector: raw,
|
|
25233
|
-
timeoutMs: SHORT_TIMEOUT_MS
|
|
25234
|
-
};
|
|
25184
|
+
function splitByMode(originalActions, result, mode) {
|
|
25185
|
+
if (mode === "strict") return {
|
|
25186
|
+
kept: result.kept,
|
|
25187
|
+
unstable: [],
|
|
25188
|
+
dropped: result.dropped,
|
|
25189
|
+
rescuedSteps: result.rescuedSteps
|
|
25190
|
+
};
|
|
25191
|
+
const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
|
|
25192
|
+
const keptSet = new Set(result.kept);
|
|
25193
|
+
const finalKept = [];
|
|
25194
|
+
const unstable = [];
|
|
25195
|
+
for (let i = 0; i < originalActions.length; i++) {
|
|
25196
|
+
const action = originalActions[i];
|
|
25197
|
+
if (keptSet.has(action)) {
|
|
25198
|
+
finalKept.push(action);
|
|
25199
|
+
continue;
|
|
25235
25200
|
}
|
|
25236
|
-
|
|
25237
|
-
|
|
25238
|
-
|
|
25239
|
-
|
|
25240
|
-
|
|
25201
|
+
const drop = droppedByIndex.get(i);
|
|
25202
|
+
if (drop) {
|
|
25203
|
+
action.replayUnstable = true;
|
|
25204
|
+
action.replayReason = drop.reason;
|
|
25205
|
+
unstable.push(action);
|
|
25206
|
+
}
|
|
25207
|
+
}
|
|
25208
|
+
return {
|
|
25209
|
+
kept: finalKept,
|
|
25210
|
+
unstable,
|
|
25211
|
+
dropped: [],
|
|
25212
|
+
rescuedSteps: result.rescuedSteps
|
|
25213
|
+
};
|
|
25214
|
+
}
|
|
25215
|
+
function rescueLostSteps(actions, kept, dropped, opts, patience) {
|
|
25216
|
+
const stepsWithSurvivors = /* @__PURE__ */ new Set();
|
|
25217
|
+
for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
|
|
25218
|
+
const lostStepDrops = /* @__PURE__ */ new Map();
|
|
25219
|
+
for (const d of dropped) {
|
|
25220
|
+
const id = d.action.stepId;
|
|
25221
|
+
if (!id || stepsWithSurvivors.has(id)) continue;
|
|
25222
|
+
const list = lostStepDrops.get(id) ?? [];
|
|
25223
|
+
list.push(d);
|
|
25224
|
+
lostStepDrops.set(id, list);
|
|
25225
|
+
}
|
|
25226
|
+
if (lostStepDrops.size === 0) return {
|
|
25227
|
+
kept,
|
|
25228
|
+
dropped
|
|
25229
|
+
};
|
|
25230
|
+
const promoted = [];
|
|
25231
|
+
const rescuedIndices = /* @__PURE__ */ new Set();
|
|
25232
|
+
const rescuedSteps = [];
|
|
25233
|
+
const strict = (opts.mode ?? "lenient") === "strict";
|
|
25234
|
+
for (const [stepId, drops] of lostStepDrops.entries()) {
|
|
25235
|
+
const replayed = [];
|
|
25236
|
+
const unverifiable = [];
|
|
25237
|
+
let stillFailing = false;
|
|
25238
|
+
for (const d of drops) {
|
|
25239
|
+
const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
|
|
25240
|
+
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25241
|
+
if (outcome.skipped) {
|
|
25242
|
+
unverifiable.push(d.index);
|
|
25243
|
+
continue;
|
|
25244
|
+
}
|
|
25245
|
+
if (outcome.ok) {
|
|
25246
|
+
replayed.push(d.index);
|
|
25247
|
+
continue;
|
|
25248
|
+
}
|
|
25249
|
+
stillFailing = true;
|
|
25250
|
+
if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
|
|
25241
25251
|
}
|
|
25252
|
+
if (replayed.length === 0) continue;
|
|
25253
|
+
for (const i of replayed) rescuedIndices.add(i);
|
|
25254
|
+
if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
|
|
25255
|
+
rescuedSteps.push(stepId);
|
|
25256
|
+
}
|
|
25257
|
+
if (rescuedIndices.size === 0) return {
|
|
25258
|
+
kept,
|
|
25259
|
+
dropped,
|
|
25260
|
+
promoted
|
|
25261
|
+
};
|
|
25262
|
+
const keptSet = new Set(kept);
|
|
25263
|
+
const newKept = [];
|
|
25264
|
+
for (let i = 0; i < actions.length; i++) {
|
|
25265
|
+
const action = actions[i];
|
|
25266
|
+
if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
|
|
25242
25267
|
}
|
|
25243
|
-
}
|
|
25244
|
-
function roleProbe(base, role, name) {
|
|
25245
|
-
return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
|
|
25246
|
-
}
|
|
25247
|
-
/**
|
|
25248
|
-
* Presence asked the way the locator addresses the element. `null` is
|
|
25249
|
-
* unverifiable rather than absent: a count from a selector the engine never
|
|
25250
|
-
* parsed is not evidence.
|
|
25251
|
-
*/
|
|
25252
|
-
function presenceCheck(locator, selector, sub, base) {
|
|
25253
|
-
if (locator?.by === "text") return [
|
|
25254
|
-
...base,
|
|
25255
|
-
"wait",
|
|
25256
|
-
"--text",
|
|
25257
|
-
sub(locator.value),
|
|
25258
|
-
"--timeout",
|
|
25259
|
-
String(ASSERT_TIMEOUT_MS)
|
|
25260
|
-
];
|
|
25261
|
-
if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
|
|
25262
|
-
if (locator !== void 0 && locator.by !== "css") return null;
|
|
25263
|
-
const named = parseNotation(selector);
|
|
25264
|
-
if (named === "opaque") return null;
|
|
25265
|
-
if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
|
|
25266
|
-
if (named !== null) return null;
|
|
25267
25268
|
return {
|
|
25268
|
-
|
|
25269
|
-
|
|
25270
|
-
|
|
25269
|
+
kept: newKept,
|
|
25270
|
+
dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
|
|
25271
|
+
rescuedSteps,
|
|
25272
|
+
promoted
|
|
25271
25273
|
};
|
|
25272
25274
|
}
|
|
25273
|
-
|
|
25274
|
-
|
|
25275
|
-
|
|
25276
|
-
|
|
25277
|
-
|
|
25278
|
-
|
|
25279
|
-
|
|
25280
|
-
return [
|
|
25281
|
-
...base,
|
|
25282
|
-
"wait",
|
|
25283
|
-
"--text",
|
|
25284
|
-
val,
|
|
25285
|
-
"--timeout",
|
|
25286
|
-
String(ASSERT_TIMEOUT_MS)
|
|
25287
|
-
];
|
|
25288
|
-
case "text_not_visible": return null;
|
|
25289
|
-
case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
|
|
25290
|
-
case "element_not_visible": return null;
|
|
25291
|
-
case "url_contains": return null;
|
|
25292
|
-
case "element_enabled":
|
|
25293
|
-
case "element_disabled":
|
|
25294
|
-
case "element_checked":
|
|
25295
|
-
case "element_unchecked":
|
|
25296
|
-
if (!sel || action.locator?.by !== "css") return null;
|
|
25297
|
-
if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
|
|
25298
|
-
return {
|
|
25299
|
-
kind: "state",
|
|
25300
|
-
state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
|
|
25301
|
-
selector: sel,
|
|
25302
|
-
expected: action.assert === "element_enabled" || action.assert === "element_checked"
|
|
25303
|
-
};
|
|
25304
|
-
default: return null;
|
|
25305
|
-
}
|
|
25275
|
+
/**
|
|
25276
|
+
* Passive (read-only) actions whose only effect is observation. When a
|
|
25277
|
+
* preceding action fails, dropping these too is the right move because
|
|
25278
|
+
* they were trying to observe state the failed action would have set up.
|
|
25279
|
+
*/
|
|
25280
|
+
function isPassiveAction(action) {
|
|
25281
|
+
return action === "snapshot" || action === "wait" || action === "assert";
|
|
25306
25282
|
}
|
|
25307
|
-
|
|
25283
|
+
//#endregion
|
|
25284
|
+
//#region src/prompts/previous-recording.ts
|
|
25308
25285
|
/**
|
|
25309
|
-
* The
|
|
25310
|
-
*
|
|
25311
|
-
* `getByLabel` needs a real association — a `<label for>`, a wrapping label,
|
|
25312
|
-
* an `aria-labelledby`. A form that has none still shows the same string as
|
|
25313
|
-
* the field's accessible name, which is what the recorder read off the
|
|
25314
|
-
* snapshot, so the label it recorded matches nothing while the same text as a
|
|
25315
|
-
* role's name matches exactly. Only the actions whose element has one
|
|
25316
|
-
* unambiguous role are listed: a labelled `click` may be a button, a link or
|
|
25317
|
-
* a checkbox, and guessing there would replace a locator that failed with one
|
|
25318
|
-
* that finds the wrong element.
|
|
25286
|
+
* The previous recording as the commands that perform it, for the trace prompt:
|
|
25287
|
+
* a re-record is usually one moved screen in an otherwise intact route.
|
|
25319
25288
|
*/
|
|
25320
|
-
|
|
25321
|
-
|
|
25322
|
-
|
|
25323
|
-
|
|
25324
|
-
|
|
25325
|
-
|
|
25326
|
-
|
|
25327
|
-
|
|
25328
|
-
|
|
25289
|
+
function renderPreviousRecording(actions, sessionName, currentStepIds) {
|
|
25290
|
+
const current = new Set([SETUP_STEP_ID, ...currentStepIds]);
|
|
25291
|
+
const brokenStep = actions.find(failedToReplay)?.stepId;
|
|
25292
|
+
const byStep = /* @__PURE__ */ new Map();
|
|
25293
|
+
for (const action of actions) {
|
|
25294
|
+
if (action.stepId === void 0 || !current.has(action.stepId)) continue;
|
|
25295
|
+
const command = previousCommand(action, sessionName, action.stepId === brokenStep);
|
|
25296
|
+
if (command === null) continue;
|
|
25297
|
+
const lines = byStep.get(action.stepId) ?? [];
|
|
25298
|
+
lines.push(command);
|
|
25299
|
+
byStep.set(action.stepId, lines);
|
|
25300
|
+
}
|
|
25301
|
+
if (byStep.size === 0) return "";
|
|
25302
|
+
return `
|
|
25303
|
+
## Previous recording
|
|
25304
|
+
|
|
25305
|
+
This case was recorded before. Below is what that recording did, step by step,
|
|
25306
|
+
as the commands that perform it. The application may have moved since, and the
|
|
25307
|
+
steps above may have been edited: **the steps above are the contract, this is a
|
|
25308
|
+
map.**
|
|
25309
|
+
|
|
25310
|
+
For a step listed here, this replaces the snapshot-first routine of the
|
|
25311
|
+
Execution Workflow:
|
|
25312
|
+
|
|
25313
|
+
1. While a step's previous commands still do what its instruction and
|
|
25314
|
+
\`Expected\` say, call the \`replay_step\` tool with the step id instead of
|
|
25315
|
+
running them yourself. It runs and records them in this session, checks
|
|
25316
|
+
included, and stops at the first that fails. Do not snapshot first.
|
|
25317
|
+
2. When it reports they all ran, emit \`STEP_DONE\` and go to the next step
|
|
25318
|
+
without a confirming snapshot.
|
|
25319
|
+
3. When it reports a failure, the commands before it are recorded and the
|
|
25320
|
+
page is where they left it: finish that step with the Execution Workflow —
|
|
25321
|
+
snapshot, find what works, record it — without re-running what was
|
|
25322
|
+
recorded. A step whose instruction or \`Expected\` asks for something these
|
|
25323
|
+
commands do not do skips \`replay_step\` and uses the Execution Workflow
|
|
25324
|
+
from the start. The next step goes back to \`replay_step\`.
|
|
25325
|
+
4. A step you record yourself takes its assertions from its current
|
|
25326
|
+
\`Expected\`: keep a previous check only while it verifies what
|
|
25327
|
+
\`Expected\` says now. A step whose \`Expected\` has changed is one to
|
|
25328
|
+
record yourself, since \`replay_step\` records the previous checks as
|
|
25329
|
+
they are.
|
|
25330
|
+
5. A command marked \`# did not replay last time\` is where the previous
|
|
25331
|
+
recording's own validation found the route broken: expect that step to
|
|
25332
|
+
need the Execution Workflow, and do not keep a form that only half works.
|
|
25333
|
+
|
|
25334
|
+
A step with no section here is recorded the usual way.
|
|
25335
|
+
|
|
25336
|
+
${[...byStep].map(([stepId, lines]) => `### ${stepId}\n\n\`\`\`bash\n${lines.join("\n")}\n\`\`\``).join("\n\n")}
|
|
25337
|
+
`;
|
|
25329
25338
|
}
|
|
25330
|
-
|
|
25331
|
-
|
|
25332
|
-
|
|
25339
|
+
function previousCommand(action, sessionName, inBrokenStep) {
|
|
25340
|
+
if (action.action === "wait" && action.locator?.by !== "text") return null;
|
|
25341
|
+
const marked = action.action === "assert" ? markedAssertArgs(action) : null;
|
|
25342
|
+
const tokens = marked?.tokens ?? (action.action === "assert" ? null : toAgentBrowserArgs(action));
|
|
25343
|
+
if (tokens === null) return null;
|
|
25344
|
+
const line = [
|
|
25345
|
+
`CCQA_STEP=${action.stepId}`,
|
|
25346
|
+
...marked ? [`CCQA_ASSERT=${shellWord(marked.marker)}`] : [],
|
|
25347
|
+
...action.secret ? ["CCQA_SECRET=1"] : [],
|
|
25348
|
+
"agent-browser",
|
|
25349
|
+
"--session",
|
|
25350
|
+
sessionName,
|
|
25351
|
+
renderTokens(tokens)
|
|
25352
|
+
].join(" ");
|
|
25353
|
+
return inBrokenStep && failedToReplay(action) ? `${line} # did not replay last time` : line;
|
|
25333
25354
|
}
|
|
25334
|
-
/**
|
|
25335
|
-
function
|
|
25336
|
-
return
|
|
25355
|
+
/** Failed on its own, rather than skipped behind an earlier failure. */
|
|
25356
|
+
function failedToReplay(action) {
|
|
25357
|
+
return action.replayUnstable === true && !isCascadeReason(action.replayReason);
|
|
25337
25358
|
}
|
|
25338
|
-
|
|
25339
|
-
|
|
25340
|
-
return `locator rewritten to the form that replays: ${promotion}`;
|
|
25359
|
+
function renderTokens(tokens) {
|
|
25360
|
+
return tokens.map((t) => t.expandsEnv ? quote(t.text) : t.text).join(" ");
|
|
25341
25361
|
}
|
|
25342
|
-
/**
|
|
25343
|
-
function
|
|
25344
|
-
|
|
25345
|
-
if (!loc || loc.by !== "label") return null;
|
|
25346
|
-
const role = LABEL_FALLBACK_ROLE[action.action];
|
|
25347
|
-
if (role === void 0) return null;
|
|
25348
|
-
return {
|
|
25349
|
-
...action,
|
|
25350
|
-
locator: {
|
|
25351
|
-
by: "role",
|
|
25352
|
-
value: role,
|
|
25353
|
-
name: loc.value,
|
|
25354
|
-
exact: true
|
|
25355
|
-
}
|
|
25356
|
-
};
|
|
25362
|
+
/** Double-quoted with `${VAR}` refs left live, as the recording keeps them; any other `$` is literal text. */
|
|
25363
|
+
function quote(s) {
|
|
25364
|
+
return `"${s.replace(/(["\\`])|\$(?!\{[A-Za-z_]\w*\})/g, (m, c) => `\\${c ?? m}`)}"`;
|
|
25357
25365
|
}
|
|
25358
|
-
|
|
25359
|
-
|
|
25360
|
-
|
|
25361
|
-
|
|
25362
|
-
|
|
25363
|
-
|
|
25364
|
-
|
|
25365
|
-
* there is an `alternate` form it is tried on every attempt rather than after
|
|
25366
|
-
* the wait: a label the page never associates with its input will not start
|
|
25367
|
-
* matching, so waiting first spends the budget for nothing.
|
|
25368
|
-
*/
|
|
25369
|
-
function runInteraction(argv, alternate, patience, sessionName) {
|
|
25370
|
-
const started = Date.now();
|
|
25371
|
-
const deadline = started + ASSERT_TIMEOUT_MS;
|
|
25372
|
-
let settled = false;
|
|
25373
|
-
for (let attempt = 0;; attempt++) {
|
|
25374
|
-
let result = spawnAB(argv);
|
|
25375
|
-
if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
|
|
25376
|
-
const waitedMs = Date.now() - started;
|
|
25377
|
-
if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
|
|
25378
|
-
settled = true;
|
|
25379
|
-
spawnAB([
|
|
25380
|
-
"--session",
|
|
25381
|
-
sessionName,
|
|
25382
|
-
"wait",
|
|
25383
|
-
"--load",
|
|
25384
|
-
"networkidle"
|
|
25385
|
-
]);
|
|
25386
|
-
continue;
|
|
25387
|
-
}
|
|
25388
|
-
if (result.status === 0 || !notFound(result)) return {
|
|
25389
|
-
result,
|
|
25390
|
-
viaAlternate: false,
|
|
25391
|
-
waitedMs
|
|
25392
|
-
};
|
|
25393
|
-
if (alternate !== null) {
|
|
25394
|
-
const alt = spawnAB(alternate);
|
|
25395
|
-
if (alt.status === 0) return {
|
|
25396
|
-
result: alt,
|
|
25397
|
-
viaAlternate: true,
|
|
25398
|
-
waitedMs
|
|
25399
|
-
};
|
|
25400
|
-
}
|
|
25401
|
-
if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
|
|
25402
|
-
patience.spent = true;
|
|
25403
|
-
return {
|
|
25404
|
-
result,
|
|
25405
|
-
viaAlternate: false,
|
|
25406
|
-
waitedMs: Date.now() - started
|
|
25407
|
-
};
|
|
25408
|
-
}
|
|
25409
|
-
sleepSync(INTERACTION_POLL_INTERVAL_MS);
|
|
25410
|
-
}
|
|
25366
|
+
function shellWord(s) {
|
|
25367
|
+
return /^[\w:/.,=-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
|
|
25368
|
+
}
|
|
25369
|
+
//#endregion
|
|
25370
|
+
//#region src/prompts/trace.ts
|
|
25371
|
+
function generateSessionName() {
|
|
25372
|
+
return `ccqa-trace-${buildRunId()}`;
|
|
25411
25373
|
}
|
|
25412
25374
|
/**
|
|
25413
|
-
*
|
|
25375
|
+
* Build the trace system prompt. `input.steps` is a flat list with includes
|
|
25376
|
+
* already expanded (each step carries id / source / instruction / expected).
|
|
25377
|
+
* The spec opens URLs via explicit step instructions (e.g.
|
|
25378
|
+
* `instruction: "${APP_URL}/articles を開く"`).
|
|
25414
25379
|
*
|
|
25415
|
-
*
|
|
25416
|
-
*
|
|
25417
|
-
*
|
|
25418
|
-
*
|
|
25419
|
-
*
|
|
25380
|
+
* In v0.4 every spec is traced from scratch — block contents are inlined
|
|
25381
|
+
* into the spec's own step list at expand time, so the prompt has no
|
|
25382
|
+
* special "this is a block" mode. The `source` tag on each step still
|
|
25383
|
+
* distinguishes spec-native steps from inlined block steps for the
|
|
25384
|
+
* `// step:` comments in the eventual codegen output.
|
|
25420
25385
|
*/
|
|
25421
|
-
function
|
|
25422
|
-
const
|
|
25423
|
-
|
|
25424
|
-
const
|
|
25425
|
-
|
|
25426
|
-
|
|
25427
|
-
|
|
25386
|
+
function buildTraceSystemPrompt(input) {
|
|
25387
|
+
const sessionName = input.sessionName ?? generateSessionName();
|
|
25388
|
+
const firstStepId = input.steps[0]?.id ?? "step-01";
|
|
25389
|
+
const callerGuidance = input.instruction ? `## Caller Guidance
|
|
25390
|
+
|
|
25391
|
+
The caller provided extra guidance for this recording — for example, a drift
|
|
25392
|
+
audit's finding about what a previous recording of this spec got wrong. Treat
|
|
25393
|
+
it as advice on what to avoid or verify while recording; the spec's steps
|
|
25394
|
+
above remain the contract for what to do.
|
|
25395
|
+
|
|
25396
|
+
${input.instruction}
|
|
25397
|
+
|
|
25398
|
+
` : "";
|
|
25399
|
+
const stepsText = input.steps.map((step) => [
|
|
25400
|
+
`### ${step.id} [${step.source}]`,
|
|
25401
|
+
`- **Instruction**: ${step.instruction}`,
|
|
25402
|
+
...step.expected ? [`- **Expected**: ${step.expected}`] : []
|
|
25403
|
+
].join("\n")).join("\n\n");
|
|
25404
|
+
const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
|
|
25405
|
+
const contextText = renderContext(input.context ?? []);
|
|
25406
|
+
const conventionsText = renderConventions(input.conventions ?? []);
|
|
25407
|
+
const previousText = renderPreviousRecording(input.previousRecording ?? [], sessionName, input.steps.map((s) => s.id));
|
|
25408
|
+
return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
|
|
25409
|
+
|
|
25410
|
+
## Session
|
|
25411
|
+
|
|
25412
|
+
SESSION NAME: \`${sessionName}\`
|
|
25413
|
+
|
|
25414
|
+
Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
|
|
25415
|
+
|
|
25416
|
+
**Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
|
|
25417
|
+
env assignment naming the spec step it executes**, e.g.
|
|
25418
|
+
|
|
25419
|
+
\`\`\`bash
|
|
25420
|
+
CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
|
|
25421
|
+
\`\`\`
|
|
25422
|
+
|
|
25423
|
+
This prefix is what ties each recorded action to its step in the generated
|
|
25424
|
+
test — a command without it loses its step attribution.
|
|
25425
|
+
|
|
25426
|
+
**Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
|
|
25427
|
+
in, opening the application, putting the account where the precondition says it
|
|
25428
|
+
starts — those are not step-01, and marking them as step-01 makes the first
|
|
25429
|
+
step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
|
|
25430
|
+
first step the case actually lists.
|
|
25431
|
+
|
|
25432
|
+
Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
|
|
25433
|
+
itself ignores the variable.
|
|
25434
|
+
|
|
25435
|
+
Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
|
|
25436
|
+
assignment (alongside \`CCQA_STEP\`) that records the check as a test
|
|
25437
|
+
assertion — see "Assertion Protocol".
|
|
25438
|
+
|
|
25439
|
+
**A command that types into a password field carries \`CCQA_SECRET=1\`** as
|
|
25440
|
+
well, e.g.
|
|
25441
|
+
|
|
25442
|
+
\`\`\`bash
|
|
25443
|
+
CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
|
|
25444
|
+
\`\`\`
|
|
25445
|
+
|
|
25446
|
+
A password field is an \`<input type="password">\` — the snapshot shows its
|
|
25447
|
+
value as dots, and nothing else about the page is a password. The prefix is
|
|
25448
|
+
what stops the typed value from being written to the recording: a value that
|
|
25449
|
+
came from a \`\${VAR}\` is kept as that reference, and one typed literally is
|
|
25450
|
+
dropped along with its command.
|
|
25451
|
+
|
|
25452
|
+
## Browser Commands
|
|
25453
|
+
|
|
25454
|
+
\`\`\`
|
|
25455
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
|
|
25456
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -i -c # interactive elements only — the default for finding controls
|
|
25457
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -c -s "<css>" # the page's text, scoped to one region
|
|
25458
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
|
|
25459
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
|
|
25460
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
|
|
25461
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
|
|
25462
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
|
|
25463
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
|
|
25464
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
|
|
25465
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
|
|
25466
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
|
|
25467
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
|
|
25468
|
+
agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
|
|
25469
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
|
|
25470
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
|
|
25471
|
+
# See "Selector Rules" for the full \`find\` subset.
|
|
25472
|
+
# IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
|
|
25473
|
+
# CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
|
|
25474
|
+
# Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
|
|
25475
|
+
# (which waits internally). To check an element exists, use \`get count "<selector>"\`.
|
|
25476
|
+
\`\`\`
|
|
25477
|
+
|
|
25478
|
+
## Selector Rules
|
|
25479
|
+
|
|
25480
|
+
**ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
|
|
25481
|
+
|
|
25482
|
+
| Format | Use when |
|
|
25483
|
+
|--------|----------|
|
|
25484
|
+
| \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
|
|
25485
|
+
| \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
|
|
25486
|
+
| \`[placeholder='text']\` | Input identified by placeholder |
|
|
25487
|
+
| \`[type='password']\` | Password inputs only |
|
|
25488
|
+
| \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
|
|
25489
|
+
| \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
|
|
25490
|
+
|
|
25491
|
+
**FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
|
|
25492
|
+
|
|
25493
|
+
- \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
|
|
25494
|
+
- **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
|
|
25495
|
+
- \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
|
|
25496
|
+
- **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
|
|
25497
|
+
- JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
|
|
25498
|
+
|
|
25499
|
+
### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
|
|
25500
|
+
|
|
25501
|
+
When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
|
|
25502
|
+
|
|
25503
|
+
\`\`\`
|
|
25504
|
+
find role <role> <action> [--name "<n>"] [--exact]
|
|
25505
|
+
find text|label|placeholder|alt|title "<text>" <action> [--exact]
|
|
25506
|
+
find testid "<id>" <action>
|
|
25507
|
+
find first|last "<ALLOWED-css>" <action>
|
|
25508
|
+
find nth <index> "<ALLOWED-css>" <action>
|
|
25509
|
+
\`\`\`
|
|
25510
|
+
|
|
25511
|
+
\`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
|
|
25512
|
+
|
|
25513
|
+
**Rules for \`find\`:**
|
|
25514
|
+
|
|
25515
|
+
1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
|
|
25516
|
+
2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
|
|
25517
|
+
3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
|
|
25518
|
+
4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
|
|
25519
|
+
5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
|
|
25520
|
+
6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
|
|
25521
|
+
7. \`find\` includes its own wait; do not chain a \`wait\` before it.
|
|
25522
|
+
8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
|
|
25523
|
+
9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
|
|
25524
|
+
|
|
25525
|
+
**Examples:**
|
|
25526
|
+
|
|
25527
|
+
- ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
|
|
25528
|
+
- ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
|
|
25529
|
+
- ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
|
|
25530
|
+
- ✗ \`find role button --name "Submit" click\` — wrong order
|
|
25531
|
+
- ✗ \`find last "button" click\` — bare tag
|
|
25532
|
+
|
|
25533
|
+
### Selector workflow
|
|
25534
|
+
|
|
25535
|
+
1. Run \`snapshot\` and read the ARIA tree.
|
|
25536
|
+
2. Identify the element; note its exact \`aria-label\` if present.
|
|
25537
|
+
3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
|
|
25538
|
+
For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
|
|
25539
|
+
4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
|
|
25540
|
+
5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
|
|
25541
|
+
6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
|
|
25542
|
+
7. Never guess. If a selector fails, take a fresh snapshot before retrying.
|
|
25543
|
+
8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
|
|
25544
|
+
|
|
25545
|
+
### Special input types
|
|
25546
|
+
|
|
25547
|
+
**contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
|
|
25548
|
+
|
|
25549
|
+
**combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
|
|
25550
|
+
|
|
25551
|
+
**Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
|
|
25552
|
+
|
|
25553
|
+
**Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
|
|
25554
|
+
|
|
25555
|
+
**File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
|
|
25556
|
+
|
|
25557
|
+
## Test Specification
|
|
25558
|
+
|
|
25559
|
+
Title: ${input.title}
|
|
25560
|
+
|
|
25561
|
+
Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
|
|
25562
|
+
|
|
25563
|
+
## Steps
|
|
25564
|
+
|
|
25565
|
+
${stepsText}
|
|
25566
|
+
${expectationsText}${contextText}${conventionsText}${previousText}
|
|
25567
|
+
## Execution Workflow
|
|
25568
|
+
|
|
25569
|
+
For each step:
|
|
25570
|
+
1. Emit \`STEP_START|<step-id>|<short description>\`.
|
|
25571
|
+
2. Run \`snapshot -i -c\` and identify selectors from the ARIA tree. A full \`snapshot\` is large; when you need the page's text, scope it with \`snapshot -c -s "<css>"\`.
|
|
25572
|
+
3. Execute the step's actions using ALLOWED selectors (see Selector Rules), prefixing each command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step. **Batch them:** put every command you can already write — the actions, their waits, the \`CCQA_ASSERT\` checks, and a closing \`snapshot -i -c\` when you need to see the result — into one \`run_commands\` call. Each turn costs far more than a command; use a single Bash call only when you must see one answer before you can write the next command.
|
|
25573
|
+
4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
|
|
25574
|
+
5. Verify the outcome from the batch's output. When a batch stops at a failure, everything before it is recorded: look at the page and continue from there.
|
|
25575
|
+
6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
|
|
25576
|
+
7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
|
|
25577
|
+
8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
|
|
25578
|
+
|
|
25579
|
+
**Protocol lines are recorded ONLY from your plain assistant text.** Every
|
|
25580
|
+
\`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
|
|
25581
|
+
message text between tool calls — never through Bash (\`echo\` etc.) and never
|
|
25582
|
+
omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
|
|
25583
|
+
prefix on each command (that prefix is authoritative, not the \`STEP_START\`
|
|
25584
|
+
line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
|
|
25585
|
+
verification commands** — a command that ran, whose failure rolls the record
|
|
25586
|
+
back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
|
|
25587
|
+
proves it; it is reported and not recorded.
|
|
25588
|
+
|
|
25589
|
+
**After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
|
|
25590
|
+
|
|
25591
|
+
## Guardrails
|
|
25592
|
+
|
|
25593
|
+
- **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
|
|
25594
|
+
- **Stay on a flow you have started.** Once a step is inside something whose state lives on the page — a conversation, a multi-step dialog, a form not yet saved — do not navigate away to investigate; that state does not come back. Look things up in the source with Read, Grep and Glob instead.
|
|
25595
|
+
- **Clean up before you stop.** When you end the run with \`ASSERTION_FAILED\` after the case has created something, still perform its cleanup steps, then emit \`RUN_COMPLETED|failed\`. What a failed recording leaves behind breaks the next run of every case that lists the same things.
|
|
25596
|
+
- **The step must match the application.** When what a step asks for is not there, or the application does it in another place or order, do not bend the recording to fit: emit \`ASSERTION_FAILED|<step-id>|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>\`. Read the source to ground the suggestion. A recording that works around the step hides the mismatch; the report gets the spec fixed. Redoing an earlier step's work to make this one possible — going back to refill or resubmit what an earlier step already did — is such a workaround.
|
|
25597
|
+
- **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
|
|
25598
|
+
- Do NOT retry a selector without taking a fresh snapshot first.
|
|
25599
|
+
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
|
|
25600
|
+
- **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
|
|
25601
|
+
- **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
|
|
25602
|
+
|
|
25603
|
+
## Source Code Reference
|
|
25604
|
+
|
|
25605
|
+
You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
|
|
25606
|
+
|
|
25607
|
+
**When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
|
|
25608
|
+
|
|
25609
|
+
**How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
|
|
25610
|
+
|
|
25611
|
+
**Rules**: only READ source files, never modify them. Keep searches focused.
|
|
25612
|
+
|
|
25613
|
+
## Waiting for Async Operations
|
|
25614
|
+
|
|
25615
|
+
Prefer \`wait\` over polling:
|
|
25616
|
+
|
|
25617
|
+
\`\`\`bash
|
|
25618
|
+
CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
|
|
25619
|
+
\`\`\`
|
|
25620
|
+
|
|
25621
|
+
If polling is required (e.g. waiting for a spinner to disappear):
|
|
25622
|
+
|
|
25623
|
+
\`\`\`bash
|
|
25624
|
+
for i in $(seq 1 18); do
|
|
25625
|
+
sleep 10
|
|
25626
|
+
result=$(agent-browser --session ${sessionName} snapshot 2>&1)
|
|
25627
|
+
echo "$result" | grep -q "<done indicator>" && break
|
|
25628
|
+
done
|
|
25629
|
+
CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
|
|
25630
|
+
\`\`\`
|
|
25631
|
+
|
|
25632
|
+
After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
|
|
25633
|
+
|
|
25634
|
+
## AB_ACTION Protocol
|
|
25635
|
+
|
|
25636
|
+
After **every** browser action, emit one line (outside any code block):
|
|
25637
|
+
|
|
25638
|
+
\`\`\`
|
|
25639
|
+
AB_ACTION|cookies_clear
|
|
25640
|
+
AB_ACTION|open|<url>
|
|
25641
|
+
AB_ACTION|click|<selector>|<visible label>
|
|
25642
|
+
AB_ACTION|dblclick|<selector>|<visible label>
|
|
25643
|
+
AB_ACTION|fill|<selector>|<value>|<aria label>
|
|
25644
|
+
AB_ACTION|check|<selector>|<visible label>
|
|
25645
|
+
AB_ACTION|uncheck|<selector>|<visible label>
|
|
25646
|
+
AB_ACTION|press|<Key>
|
|
25647
|
+
AB_ACTION|select|<selector>|<value>|<aria label>
|
|
25648
|
+
AB_ACTION|hover|<selector>|<visible label>
|
|
25649
|
+
AB_ACTION|scroll|<direction>|<pixels>
|
|
25650
|
+
AB_ACTION|drag|<source selector>|<target selector>|<source label>
|
|
25651
|
+
AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
|
|
25652
|
+
AB_ACTION|wait|<selector or text>|<label>
|
|
25653
|
+
AB_ACTION|snapshot|<key observation, max 100 chars>
|
|
25654
|
+
|
|
25655
|
+
# find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
|
|
25656
|
+
# <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
|
|
25657
|
+
AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
|
|
25658
|
+
AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
|
|
25659
|
+
AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
|
|
25660
|
+
AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
|
|
25661
|
+
AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
|
|
25662
|
+
AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
|
|
25663
|
+
AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
25664
|
+
AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
25665
|
+
\`\`\`
|
|
25666
|
+
|
|
25667
|
+
Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
|
|
25668
|
+
|
|
25669
|
+
**CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
|
|
25670
|
+
|
|
25671
|
+
- A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
|
|
25672
|
+
- If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
|
|
25673
|
+
- An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
|
|
25674
|
+
- **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
|
|
25675
|
+
|
|
25676
|
+
- **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
|
|
25677
|
+
- If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
|
|
25678
|
+
|
|
25679
|
+
## Assertion Protocol
|
|
25680
|
+
|
|
25681
|
+
Record each signal you verified as an assertion. The **primary mechanism** is
|
|
25682
|
+
a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
|
|
25683
|
+
marker rides the same channel as the command, so the assertion is captured
|
|
25684
|
+
even if you print no protocol text. The check you were going to run anyway
|
|
25685
|
+
becomes the recorded assertion:
|
|
25686
|
+
|
|
25687
|
+
\`\`\`bash
|
|
25688
|
+
# text_visible — the marked wait IS the assertion (no separate assert line)
|
|
25689
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
|
|
25690
|
+
# element_visible / element_not_visible — mark the get-count probe
|
|
25691
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
|
|
25692
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
|
|
25693
|
+
# url_contains — the substring rides in the marker; put it on \`get url\`
|
|
25694
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
|
|
25695
|
+
# element state — mark the \`is\` probe that asks for it
|
|
25696
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
|
|
25697
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
|
|
25698
|
+
\`\`\`
|
|
25699
|
+
|
|
25700
|
+
**Marker semantics:**
|
|
25701
|
+
|
|
25702
|
+
| Marker | On command | Records |
|
|
25703
|
+
|--------|------------|---------|
|
|
25704
|
+
| \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
|
|
25705
|
+
| \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
|
|
25706
|
+
| \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
|
|
25707
|
+
| \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
|
|
25708
|
+
| \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
|
|
25709
|
+
| \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
|
|
25710
|
+
|
|
25711
|
+
- A marked command that exits non-zero records nothing — fix the check and re-run it.
|
|
25712
|
+
- A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
|
|
25713
|
+
- \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
|
|
25714
|
+
- **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
|
|
25715
|
+
|
|
25716
|
+
- **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
|
|
25717
|
+
|
|
25718
|
+
- **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
|
|
25719
|
+
|
|
25720
|
+
- **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
|
|
25721
|
+
- **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
|
|
25722
|
+
- **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
|
|
25723
|
+
|
|
25724
|
+
**Every assert type has a marker command.** There is no text form: a printed
|
|
25725
|
+
\`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
|
|
25726
|
+
reported rather than recorded.
|
|
25727
|
+
|
|
25728
|
+
| assertType | The command that records it | Use when |
|
|
25729
|
+
|------------|------------------------------|----------|
|
|
25730
|
+
| \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
|
|
25731
|
+
| \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
|
|
25732
|
+
| \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
|
|
25733
|
+
| \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
|
|
25734
|
+
| \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
|
|
25735
|
+
| \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
|
|
25736
|
+
|
|
25737
|
+
**Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
|
|
25738
|
+
|
|
25739
|
+
- Timestamps, session IDs, exact numeric counts that vary between runs.
|
|
25740
|
+
- **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
|
|
25741
|
+
- **Relative-time labels** — true only in the moment of the trace, stale by replay:
|
|
25742
|
+
- English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
|
|
25743
|
+
- Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
|
|
25744
|
+
- Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
|
|
25745
|
+
- **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
|
|
25746
|
+
|
|
25747
|
+
**No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
|
|
25748
|
+
|
|
25749
|
+
The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
|
|
25750
|
+
|
|
25751
|
+
- ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
|
|
25752
|
+
- ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
|
|
25753
|
+
- ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
|
|
25754
|
+
- If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
|
|
25755
|
+
|
|
25756
|
+
**Page-context and selector rules:**
|
|
25757
|
+
|
|
25758
|
+
- After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
|
|
25759
|
+
- Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
|
|
25760
|
+
- When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
|
|
25761
|
+
|
|
25762
|
+
**MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
|
|
25763
|
+
|
|
25764
|
+
The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
|
|
25765
|
+
|
|
25766
|
+
1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
|
|
25767
|
+
2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
|
|
25768
|
+
3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
|
|
25769
|
+
|
|
25770
|
+
**Verify the assertion form actually resolves on the live page.** The
|
|
25771
|
+
verification and the recorded assertion are the same marked command:
|
|
25772
|
+
|
|
25773
|
+
\`\`\`bash
|
|
25774
|
+
# text_visible
|
|
25775
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
|
|
25776
|
+
# element_visible — >=1 printed means present
|
|
25777
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
|
|
25778
|
+
# element_not_visible — 0 printed means absent (works for text too: "text=<text>")
|
|
25779
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
|
|
25780
|
+
# element state — \`is\` prints true/false and exits 0 either way; read it
|
|
25781
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
|
|
25782
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
|
|
25783
|
+
\`\`\`
|
|
25784
|
+
|
|
25785
|
+
When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
|
|
25786
|
+
|
|
25787
|
+
## Status Protocol
|
|
25788
|
+
|
|
25789
|
+
Emit exactly one status line per step (outside any code block):
|
|
25790
|
+
|
|
25791
|
+
\`\`\`
|
|
25792
|
+
STEP_START|<step-id>|<short description>
|
|
25793
|
+
STEP_DONE|<step-id>|<what was verified>
|
|
25794
|
+
ASSERTION_FAILED|<step-id>|<category: spec-mismatch|app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
|
|
25795
|
+
STEP_SKIPPED|<step-id>|<reason>
|
|
25796
|
+
RUN_COMPLETED|passed|<summary>
|
|
25797
|
+
RUN_COMPLETED|failed|<summary>
|
|
25798
|
+
\`\`\`
|
|
25799
|
+
|
|
25800
|
+
**\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
|
|
25801
|
+
recorded as FAILED, however many steps went well: nothing else can tell a
|
|
25802
|
+
finished route from one you abandoned halfway.
|
|
25803
|
+
|
|
25804
|
+
${callerGuidance}## Start
|
|
25805
|
+
|
|
25806
|
+
Begin by clearing cookies, then proceed straight to the first step's instruction.
|
|
25807
|
+
|
|
25808
|
+
\`\`\`bash
|
|
25809
|
+
agent-browser --session ${sessionName} cookies clear
|
|
25810
|
+
\`\`\`
|
|
25811
|
+
|
|
25812
|
+
(The cookies-clear preamble belongs to no step, so it is the one command
|
|
25813
|
+
without a \`CCQA_STEP\` prefix.)
|
|
25814
|
+
|
|
25815
|
+
Emit:
|
|
25816
|
+
\`\`\`
|
|
25817
|
+
AB_ACTION|cookies_clear
|
|
25818
|
+
\`\`\`
|
|
25819
|
+
|
|
25820
|
+
Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
|
|
25821
|
+
every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
|
|
25822
|
+
step is responsible for opening the initial URL.
|
|
25823
|
+
`;
|
|
25428
25824
|
}
|
|
25429
|
-
|
|
25430
|
-
|
|
25431
|
-
* the name was read from. See docs/targets.md for why the two disagree.
|
|
25432
|
-
*
|
|
25433
|
-
* The snapshot is read for the role, then `find role` is asked as well: the
|
|
25434
|
-
* tree says the name exists somewhere, the probe says the replay can reach it,
|
|
25435
|
-
* and only the second is what the route will depend on.
|
|
25436
|
-
*/
|
|
25437
|
-
function askTheAccessibilityTree(action, sub, sessionName) {
|
|
25438
|
-
if (action.action !== "assert" || action.locator?.by !== "css") return null;
|
|
25439
|
-
const name = nameFromAttributeSelector(action.locator.value);
|
|
25440
|
-
if (name === null) return null;
|
|
25441
|
-
const snapshot = spawnAB([
|
|
25442
|
-
"--session",
|
|
25443
|
-
sessionName,
|
|
25444
|
-
"snapshot"
|
|
25445
|
-
]);
|
|
25446
|
-
if (snapshot.status !== 0) return null;
|
|
25447
|
-
const role = roleOfAccessibleName(snapshot.stdout, sub(name));
|
|
25448
|
-
if (role === null) return null;
|
|
25449
|
-
if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
|
|
25450
|
-
const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
|
|
25451
|
-
action.locator = {
|
|
25452
|
-
by: "role",
|
|
25453
|
-
value: role,
|
|
25454
|
-
name,
|
|
25455
|
-
exact: true
|
|
25456
|
-
};
|
|
25457
|
-
return {
|
|
25458
|
-
skipped: false,
|
|
25459
|
-
ok: true,
|
|
25460
|
-
reason: "",
|
|
25461
|
-
promoted
|
|
25462
|
-
};
|
|
25825
|
+
function buildTracePrompt(title) {
|
|
25826
|
+
return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
|
|
25463
25827
|
}
|
|
25464
25828
|
/**
|
|
25465
|
-
*
|
|
25466
|
-
*
|
|
25467
|
-
*
|
|
25829
|
+
* The case's expectations, plus the one rule that decides where each is
|
|
25830
|
+
* recorded. Without the rule a recorder either asserts everything at the end —
|
|
25831
|
+
* on a page the flow has already left — or asserts each item at every step it
|
|
25832
|
+
* might apply to, which is the over-assertion the selector rules exist to
|
|
25833
|
+
* prevent.
|
|
25468
25834
|
*/
|
|
25469
|
-
function
|
|
25470
|
-
|
|
25471
|
-
|
|
25472
|
-
|
|
25473
|
-
|
|
25474
|
-
|
|
25475
|
-
|
|
25476
|
-
|
|
25477
|
-
|
|
25478
|
-
|
|
25479
|
-
|
|
25480
|
-
};
|
|
25481
|
-
if (isProbe(built)) {
|
|
25482
|
-
const { ok, reason } = runProbe(built, sessionName);
|
|
25483
|
-
if (ok) return withName({
|
|
25484
|
-
skipped: false,
|
|
25485
|
-
ok,
|
|
25486
|
-
reason
|
|
25487
|
-
});
|
|
25488
|
-
const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
|
|
25489
|
-
return askTheAccessibilityTree(action, resolve, sessionName) ?? {
|
|
25490
|
-
skipped: false,
|
|
25491
|
-
ok,
|
|
25492
|
-
reason
|
|
25493
|
-
};
|
|
25494
|
-
}
|
|
25495
|
-
const fallback = labelFallback(action);
|
|
25496
|
-
const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
|
|
25497
|
-
const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
|
|
25498
|
-
if (viaAlternate) {
|
|
25499
|
-
const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
|
|
25500
|
-
action.locator = fallback.locator;
|
|
25501
|
-
return {
|
|
25502
|
-
skipped: false,
|
|
25503
|
-
ok: true,
|
|
25504
|
-
reason: "",
|
|
25505
|
-
promoted
|
|
25506
|
-
};
|
|
25507
|
-
}
|
|
25508
|
-
if (result.status === 0) return withName({
|
|
25509
|
-
skipped: false,
|
|
25510
|
-
ok: true,
|
|
25511
|
-
reason: ""
|
|
25512
|
-
});
|
|
25513
|
-
return {
|
|
25514
|
-
skipped: false,
|
|
25515
|
-
ok: false,
|
|
25516
|
-
reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
|
|
25517
|
-
};
|
|
25518
|
-
}
|
|
25519
|
-
function validateActions(actions, opts) {
|
|
25520
|
-
const kept = [];
|
|
25521
|
-
const dropped = [];
|
|
25522
|
-
const promoted = [];
|
|
25523
|
-
const patience = { spent: false };
|
|
25524
|
-
let skipFromStepId = null;
|
|
25525
|
-
for (let i = 0; i < actions.length; i++) {
|
|
25526
|
-
const action = actions[i];
|
|
25527
|
-
opts.onProgress?.(i, actions.length, action);
|
|
25528
|
-
const stepId = action.stepId ?? NO_STEP_ID;
|
|
25529
|
-
if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
|
|
25530
|
-
if (skipFromStepId !== null && isPassiveAction(action.action)) {
|
|
25531
|
-
dropped.push({
|
|
25532
|
-
index: i,
|
|
25533
|
-
action,
|
|
25534
|
-
reason: CASCADE_REASON
|
|
25535
|
-
});
|
|
25536
|
-
continue;
|
|
25537
|
-
}
|
|
25538
|
-
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25539
|
-
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25540
|
-
if (outcome.skipped) {
|
|
25541
|
-
kept.push(action);
|
|
25542
|
-
continue;
|
|
25543
|
-
}
|
|
25544
|
-
if (outcome.ok) {
|
|
25545
|
-
kept.push(action);
|
|
25546
|
-
if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
|
|
25547
|
-
continue;
|
|
25548
|
-
}
|
|
25549
|
-
dropped.push({
|
|
25550
|
-
index: i,
|
|
25551
|
-
action,
|
|
25552
|
-
reason: outcome.reason
|
|
25553
|
-
});
|
|
25554
|
-
if (!isPassiveAction(action.action)) skipFromStepId = stepId;
|
|
25555
|
-
}
|
|
25556
|
-
const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
|
|
25557
|
-
promoted.push(...afterRescue.promoted ?? []);
|
|
25558
|
-
return {
|
|
25559
|
-
...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
|
|
25560
|
-
promoted
|
|
25561
|
-
};
|
|
25835
|
+
function renderExpectations(expectations) {
|
|
25836
|
+
if (expectations.length === 0) return "";
|
|
25837
|
+
return `
|
|
25838
|
+
## Expected results
|
|
25839
|
+
|
|
25840
|
+
The case states these without saying which step delivers them:
|
|
25841
|
+
|
|
25842
|
+
${expectations.map((text) => `- ${text}`).join("\n")}
|
|
25843
|
+
|
|
25844
|
+
**Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
|
|
25845
|
+
`;
|
|
25562
25846
|
}
|
|
25563
25847
|
/**
|
|
25564
|
-
*
|
|
25565
|
-
*
|
|
25566
|
-
*
|
|
25567
|
-
*
|
|
25568
|
-
* codegen can warn about them while still emitting the line.
|
|
25848
|
+
* What the cleanup must make true. Separate from the case's expectations
|
|
25849
|
+
* because its steps are: a deletion is confirmed by the thing being gone,
|
|
25850
|
+
* which is only true after the undo has run, and asserting it among the
|
|
25851
|
+
* case's own steps would check it while the thing still exists.
|
|
25569
25852
|
*/
|
|
25570
|
-
function
|
|
25571
|
-
if (
|
|
25572
|
-
|
|
25573
|
-
|
|
25574
|
-
|
|
25575
|
-
|
|
25576
|
-
|
|
25577
|
-
|
|
25578
|
-
|
|
25579
|
-
|
|
25580
|
-
|
|
25581
|
-
|
|
25582
|
-
|
|
25583
|
-
|
|
25584
|
-
|
|
25585
|
-
|
|
25586
|
-
|
|
25587
|
-
|
|
25588
|
-
|
|
25589
|
-
|
|
25590
|
-
|
|
25591
|
-
|
|
25592
|
-
|
|
25593
|
-
|
|
25853
|
+
function renderCleanupExpectations(expectations) {
|
|
25854
|
+
if (expectations.length === 0) return "";
|
|
25855
|
+
return `
|
|
25856
|
+
## Expected results of the cleanup
|
|
25857
|
+
|
|
25858
|
+
The case states these about its cleanup steps:
|
|
25859
|
+
|
|
25860
|
+
${expectations.map((text) => `- ${text}`).join("\n")}
|
|
25861
|
+
|
|
25862
|
+
**Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
|
|
25863
|
+
`;
|
|
25864
|
+
}
|
|
25865
|
+
/**
|
|
25866
|
+
* Sections of the case ccqa does not interpret — the precondition, notes the
|
|
25867
|
+
* author left. Passed through verbatim: they are written for whoever runs the
|
|
25868
|
+
* case, and the recorder is now that reader.
|
|
25869
|
+
*/
|
|
25870
|
+
function renderContext(context) {
|
|
25871
|
+
if (context.length === 0) return "";
|
|
25872
|
+
return `
|
|
25873
|
+
## About this case
|
|
25874
|
+
|
|
25875
|
+
${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
|
|
25876
|
+
`;
|
|
25877
|
+
}
|
|
25878
|
+
/**
|
|
25879
|
+
* The project's own recording guidance. Where the case says what to do, this
|
|
25880
|
+
* says how this project is driven — which account signs in, what has to be
|
|
25881
|
+
* true before the first step. Prose, verbatim: it is written for whoever runs
|
|
25882
|
+
* the case, and this is now that reader.
|
|
25883
|
+
*/
|
|
25884
|
+
function renderConventions(conventions) {
|
|
25885
|
+
if (conventions.length === 0) return "";
|
|
25886
|
+
return `
|
|
25887
|
+
## How this project is driven
|
|
25888
|
+
|
|
25889
|
+
${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
|
|
25890
|
+
`;
|
|
25891
|
+
}
|
|
25892
|
+
//#endregion
|
|
25893
|
+
//#region src/cli/trace-tools.ts
|
|
25894
|
+
const TRACE_TOOLS_SERVER = "ccqa";
|
|
25895
|
+
/** A wedged command must not hold the whole trace; agent-browser's own waits are far shorter. */
|
|
25896
|
+
const COMMAND_TIMEOUT_MS = 12e4;
|
|
25897
|
+
/** Enough of a snapshot or a `get` answer to act on, without flooding the turn. */
|
|
25898
|
+
const OUTPUT_CAP = 6e3;
|
|
25899
|
+
/**
|
|
25900
|
+
* The trace's own tools. Both exist to cut round trips: one turn per browser
|
|
25901
|
+
* command is what made a re-record cost hundreds of turns.
|
|
25902
|
+
*
|
|
25903
|
+
* - `run_commands` runs a batch the recorder wrote, recording each that passes.
|
|
25904
|
+
* - `replay_step` runs one step's previous commands.
|
|
25905
|
+
*/
|
|
25906
|
+
function buildTraceTools(input) {
|
|
25907
|
+
const offersReplay = input.previous.length > 0;
|
|
25594
25908
|
return {
|
|
25595
|
-
|
|
25596
|
-
|
|
25597
|
-
|
|
25598
|
-
|
|
25909
|
+
server: createSdkMcpServer({
|
|
25910
|
+
name: TRACE_TOOLS_SERVER,
|
|
25911
|
+
version: "1.0.0",
|
|
25912
|
+
tools: [tool("run_commands", "Run several agent-browser commands in order, each written exactly as you would run it in Bash (with its CCQA_STEP / CCQA_ASSERT / CCQA_SECRET prefixes). Each that succeeds is recorded; the batch stops at the first that fails. Returns each command's outcome and the output of the last one.", { commands: z.array(z.string()).min(1).describe("agent-browser command lines, one per entry") }, async ({ commands }) => ({ content: [{
|
|
25913
|
+
type: "text",
|
|
25914
|
+
text: await runCommands(commands, input)
|
|
25915
|
+
}] })), ...offersReplay ? [tool("replay_step", "Run one step's previous-recording commands in this browser session and record those that pass. Stops at the first that fails.", { step: z.string().describe("The step id, e.g. step-03") }, async ({ step }) => ({ content: [{
|
|
25916
|
+
type: "text",
|
|
25917
|
+
text: replayStep(step, input)
|
|
25918
|
+
}] }))] : []]
|
|
25919
|
+
}),
|
|
25920
|
+
allowedTools: ["run_commands", ...offersReplay ? ["replay_step"] : []].map((t) => `mcp__${TRACE_TOOLS_SERVER}__${t}`)
|
|
25599
25921
|
};
|
|
25600
25922
|
}
|
|
25601
|
-
function
|
|
25602
|
-
const
|
|
25603
|
-
for (const
|
|
25604
|
-
|
|
25605
|
-
|
|
25606
|
-
|
|
25607
|
-
|
|
25608
|
-
|
|
25609
|
-
|
|
25610
|
-
|
|
25611
|
-
|
|
25612
|
-
|
|
25613
|
-
|
|
25614
|
-
|
|
25615
|
-
|
|
25616
|
-
|
|
25617
|
-
|
|
25618
|
-
|
|
25619
|
-
|
|
25620
|
-
|
|
25621
|
-
|
|
25622
|
-
|
|
25623
|
-
|
|
25624
|
-
for (const d of drops) {
|
|
25625
|
-
const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
|
|
25626
|
-
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25627
|
-
if (outcome.skipped) {
|
|
25628
|
-
unverifiable.push(d.index);
|
|
25629
|
-
continue;
|
|
25630
|
-
}
|
|
25631
|
-
if (outcome.ok) {
|
|
25632
|
-
replayed.push(d.index);
|
|
25633
|
-
continue;
|
|
25634
|
-
}
|
|
25635
|
-
stillFailing = true;
|
|
25636
|
-
if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
|
|
25923
|
+
async function runCommands(commands, input) {
|
|
25924
|
+
const report = [];
|
|
25925
|
+
for (const [i, cmd] of commands.entries()) {
|
|
25926
|
+
const stop = (why) => [
|
|
25927
|
+
...report,
|
|
25928
|
+
`✗ ${cmd}\n${why}`,
|
|
25929
|
+
...i + 1 < commands.length ? [`The ${commands.length - i - 1} command(s) after it were not run.`] : []
|
|
25930
|
+
].join("\n");
|
|
25931
|
+
if (!isAgentBrowserCommand(cmd)) return stop("Only agent-browser commands can be batched; run anything else with Bash.");
|
|
25932
|
+
const blocked = abCommandBlockReason(cmd, input.envScrubMap);
|
|
25933
|
+
if (blocked !== null) return stop(blocked);
|
|
25934
|
+
const recorded = abActionEventFromCommand(cmd);
|
|
25935
|
+
const checkpoint = input.beforeAbCommand(extractCcqaStepFromBashCommand(cmd) ?? void 0);
|
|
25936
|
+
if (checkpoint) return stop(checkpoint);
|
|
25937
|
+
info(`$ ${scrubEnvValues(cmd, input.envScrubMap)}`);
|
|
25938
|
+
const result = await runShell(cmd, {
|
|
25939
|
+
...process.env,
|
|
25940
|
+
...input.env
|
|
25941
|
+
});
|
|
25942
|
+
const out = result.stdout.trim();
|
|
25943
|
+
if (!(result.status === 0 && (recorded?.holds ? recorded.holds(out) : true))) {
|
|
25944
|
+
const detail = (result.stderr || out || `exit ${result.status ?? "timeout"}`).trim().slice(0, 600);
|
|
25945
|
+
return stop(result.status === 0 ? `The check did not hold; the page answered: ${out.slice(0, 300)}` : detail);
|
|
25637
25946
|
}
|
|
25638
|
-
if (
|
|
25639
|
-
|
|
25640
|
-
|
|
25641
|
-
rescuedSteps.push(stepId);
|
|
25642
|
-
}
|
|
25643
|
-
if (rescuedIndices.size === 0) return {
|
|
25644
|
-
kept,
|
|
25645
|
-
dropped,
|
|
25646
|
-
promoted
|
|
25647
|
-
};
|
|
25648
|
-
const keptSet = new Set(kept);
|
|
25649
|
-
const newKept = [];
|
|
25650
|
-
for (let i = 0; i < actions.length; i++) {
|
|
25651
|
-
const action = actions[i];
|
|
25652
|
-
if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
|
|
25947
|
+
if (recorded) input.onAbAction(recorded.event);
|
|
25948
|
+
const last = i === commands.length - 1;
|
|
25949
|
+
report.push(`✓ ${cmd}${out && (last || out.length < 200) ? `\n${cap(out)}` : ""}`);
|
|
25653
25950
|
}
|
|
25654
|
-
return
|
|
25655
|
-
kept: newKept,
|
|
25656
|
-
dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
|
|
25657
|
-
rescuedSteps,
|
|
25658
|
-
promoted
|
|
25659
|
-
};
|
|
25951
|
+
return report.join("\n");
|
|
25660
25952
|
}
|
|
25661
25953
|
/**
|
|
25662
|
-
*
|
|
25663
|
-
*
|
|
25664
|
-
* they were trying to observe state the failed action would have set up.
|
|
25954
|
+
* Async, so the trace's deadline and signal teardown still run while a batch
|
|
25955
|
+
* does. Settles on `exit` too: a daemon the command started may hold the pipes open.
|
|
25665
25956
|
*/
|
|
25666
|
-
function
|
|
25667
|
-
return
|
|
25957
|
+
function runShell(cmd, env) {
|
|
25958
|
+
return new Promise((resolve) => {
|
|
25959
|
+
const child = spawn("bash", ["-c", cmd], {
|
|
25960
|
+
env,
|
|
25961
|
+
timeout: COMMAND_TIMEOUT_MS
|
|
25962
|
+
});
|
|
25963
|
+
let stdout = "";
|
|
25964
|
+
let stderr = "";
|
|
25965
|
+
child.stdout.setEncoding("utf8").on("data", (d) => stdout += d);
|
|
25966
|
+
child.stderr.setEncoding("utf8").on("data", (d) => stderr += d);
|
|
25967
|
+
const done = (status, err) => resolve({
|
|
25968
|
+
status,
|
|
25969
|
+
stdout,
|
|
25970
|
+
stderr: stderr || (err ?? "")
|
|
25971
|
+
});
|
|
25972
|
+
child.on("error", (e) => done(null, e.message));
|
|
25973
|
+
child.on("close", (status) => done(status));
|
|
25974
|
+
child.on("exit", (status) => setTimeout(() => done(status), 1e3).unref());
|
|
25975
|
+
});
|
|
25976
|
+
}
|
|
25977
|
+
function cap(out) {
|
|
25978
|
+
return out.length <= OUTPUT_CAP ? out : `${out.slice(0, OUTPUT_CAP)}\n… (${out.length - OUTPUT_CAP} more characters; scope it with snapshot -s <selector>)`;
|
|
25979
|
+
}
|
|
25980
|
+
function replayStep(step, input) {
|
|
25981
|
+
const actions = input.previous.filter((a) => a.stepId === step && a.action !== "snapshot").map(({ replayUnstable: _u, replayReason: _r, ...a }) => a);
|
|
25982
|
+
if (actions.length === 0) return `No previous commands for ${step}. Record it with the Execution Workflow.`;
|
|
25983
|
+
const { passed, unchecked, failed } = replayUntilFailure(actions, {
|
|
25984
|
+
sessionName: input.sessionName,
|
|
25985
|
+
envOverrides: { CCQA_RUN_ID: input.env["CCQA_RUN_ID"] ?? input.sessionName }
|
|
25986
|
+
});
|
|
25987
|
+
input.onReplayed(step, passed);
|
|
25988
|
+
info(`replay_step ${step}: ${passed.length}/${actions.length} replayed${failed ? "" : " — all passed"}`);
|
|
25989
|
+
const notRun = unchecked.length === 0 ? "" : `\nThese checks could not be run here and were not recorded — perform the ones Expected still asks for yourself:\n` + unchecked.map((a) => `- ${describeStepAction(a)}`).join("\n");
|
|
25990
|
+
if (!failed) {
|
|
25991
|
+
const next = unchecked.length === 0 ? " Emit STEP_DONE and go on." : "";
|
|
25992
|
+
return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded, its checks included.${next}${notRun}`;
|
|
25993
|
+
}
|
|
25994
|
+
return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded. The next one failed: ${describeStepAction(failed.action)} — ${failed.reason}\nThe page is where the recorded commands left it. Finish ${step} with the Execution Workflow (snapshot first); do not re-run the recorded commands.${notRun}`;
|
|
25668
25995
|
}
|
|
25669
25996
|
//#endregion
|
|
25670
25997
|
//#region src/cli/trace.ts
|
|
25998
|
+
/** The recorder's Bash runs agent-browser only: source is read with Read/Grep/Glob. */
|
|
25999
|
+
function traceBashGuard(cmd) {
|
|
26000
|
+
if (isAgentBrowserCommand(cmd)) return null;
|
|
26001
|
+
return "Bash here runs agent-browser commands only. Read, search and list the source with the Read, Grep and Glob tools; wait for the page with agent-browser wait, not sleep.";
|
|
26002
|
+
}
|
|
26003
|
+
/** Browser commands in one step between spec-mismatch checkpoints (see `beforeAbCommand`). */
|
|
26004
|
+
const STEP_CHECKPOINT = 20;
|
|
25671
26005
|
/**
|
|
25672
26006
|
* Step ids (in spec order) whose kept actions include no assertion. A step
|
|
25673
26007
|
* with none produced a test that performs the step but verifies nothing
|
|
@@ -25705,6 +26039,11 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25705
26039
|
}
|
|
25706
26040
|
const conventions = await loadConventions(opts.cwd ?? process.cwd(), { guides: opts.conventions ?? [] });
|
|
25707
26041
|
for (const w of conventions.warnings) warn(w);
|
|
26042
|
+
const caseRef = testCase.ref;
|
|
26043
|
+
const previous = await tryGetRecording(caseRef);
|
|
26044
|
+
const previousActions = previous ? [...previous.actions, ...previous.cleanup ?? []] : [];
|
|
26045
|
+
if (previousActions.length > 0) meta("previous", opts.freshIr ? "ignored (--fresh-ir)" : `${previousActions.length} action(s) offered as a map`);
|
|
26046
|
+
const offersMap = !opts.freshIr && previousActions.length > 0;
|
|
25708
26047
|
const baseSystemPrompt = buildTraceSystemPrompt({
|
|
25709
26048
|
title: testCase.title,
|
|
25710
26049
|
steps,
|
|
@@ -25716,7 +26055,8 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25716
26055
|
...testCase.expectations.length > 0 ? { expectations: testCase.expectations } : {},
|
|
25717
26056
|
...testCase.cleanupExpectations.length > 0 ? { cleanupExpectations: testCase.cleanupExpectations } : {},
|
|
25718
26057
|
...testCase.context.length > 0 ? { context: testCase.context } : {},
|
|
25719
|
-
...opts.instruction ? { instruction: opts.instruction } : {}
|
|
26058
|
+
...opts.instruction ? { instruction: opts.instruction } : {},
|
|
26059
|
+
...offersMap ? { previousRecording: previousActions } : {}
|
|
25720
26060
|
});
|
|
25721
26061
|
const promptBundle = await loadPromptBundle(opts.hubContext ?? null, "record", opts.cwd ?? process.cwd());
|
|
25722
26062
|
if (promptBundle !== null) meta("prompt", promptBundle.loaded.join(" + "));
|
|
@@ -25737,40 +26077,71 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25737
26077
|
} : action;
|
|
25738
26078
|
};
|
|
25739
26079
|
let lastCommandPushCount = 0;
|
|
26080
|
+
const onAbAction = ({ abAction, stepId, assertMarker, secret }) => {
|
|
26081
|
+
const stepForCommand = stepTracker.fromCommand(stepId);
|
|
26082
|
+
const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
|
|
26083
|
+
let recorded = null;
|
|
26084
|
+
if (assertMarker !== void 0) {
|
|
26085
|
+
recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
|
|
26086
|
+
if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
|
|
26087
|
+
}
|
|
26088
|
+
if (recorded === null) {
|
|
26089
|
+
const parsed = line === null ? null : parseAbActionLine(line);
|
|
26090
|
+
recorded = parsed === null ? [] : [parsed];
|
|
26091
|
+
}
|
|
26092
|
+
let pushed = 0;
|
|
26093
|
+
for (const action of recorded) {
|
|
26094
|
+
const stamped = withStepId(action, stepForCommand);
|
|
26095
|
+
if (stamped) {
|
|
26096
|
+
traceActions.push(secret ? {
|
|
26097
|
+
...stamped,
|
|
26098
|
+
secret: true
|
|
26099
|
+
} : stamped);
|
|
26100
|
+
pushed += 1;
|
|
26101
|
+
}
|
|
26102
|
+
}
|
|
26103
|
+
lastCommandPushCount = pushed;
|
|
26104
|
+
};
|
|
26105
|
+
const commandsByStep = /* @__PURE__ */ new Map();
|
|
26106
|
+
const beforeAbCommand = (stepId) => {
|
|
26107
|
+
if (stepId === void 0) return null;
|
|
26108
|
+
const n = (commandsByStep.get(stepId) ?? 0) + 1;
|
|
26109
|
+
commandsByStep.set(stepId, n);
|
|
26110
|
+
if (n % STEP_CHECKPOINT !== 0) return null;
|
|
26111
|
+
return `${stepId} has taken ${n - 1} browser commands. If the application does not do what this step says — the control is somewhere else, things happen in a different order, the result never appears — stop working around it and report it: ASSERTION_FAILED|${stepId}|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>. If the step does match and you are close, run this command again to continue.`;
|
|
26112
|
+
};
|
|
26113
|
+
const invokeBase = agentBrowserInvokeBase({
|
|
26114
|
+
sessionName,
|
|
26115
|
+
runId: sessionName
|
|
26116
|
+
});
|
|
26117
|
+
const traceTools = buildTraceTools({
|
|
26118
|
+
previous: offersMap ? previousActions : [],
|
|
26119
|
+
sessionName,
|
|
26120
|
+
env: invokeBase.env ?? {},
|
|
26121
|
+
envScrubMap,
|
|
26122
|
+
onReplayed: (stepId, passed) => {
|
|
26123
|
+
stepTracker.fromCommand(stepId);
|
|
26124
|
+
for (const action of passed) traceActions.push({
|
|
26125
|
+
...action,
|
|
26126
|
+
stepId
|
|
26127
|
+
});
|
|
26128
|
+
lastCommandPushCount = 0;
|
|
26129
|
+
},
|
|
26130
|
+
onAbAction,
|
|
26131
|
+
beforeAbCommand
|
|
26132
|
+
});
|
|
25740
26133
|
const { isError, errorDetail } = await invokeClaudeStreaming({
|
|
25741
26134
|
prompt,
|
|
25742
26135
|
systemPrompt,
|
|
25743
|
-
...
|
|
25744
|
-
|
|
25745
|
-
|
|
25746
|
-
}),
|
|
26136
|
+
...invokeBase,
|
|
26137
|
+
allowedTools: [...invokeBase.allowedTools, ...traceTools.allowedTools],
|
|
26138
|
+
mcpServers: { [TRACE_TOOLS_SERVER]: traceTools.server },
|
|
25747
26139
|
model,
|
|
25748
26140
|
envScrubMap,
|
|
25749
|
-
onAbAction
|
|
25750
|
-
|
|
25751
|
-
|
|
25752
|
-
|
|
25753
|
-
if (assertMarker !== void 0) {
|
|
25754
|
-
recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
|
|
25755
|
-
if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
|
|
25756
|
-
}
|
|
25757
|
-
if (recorded === null) {
|
|
25758
|
-
const parsed = line === null ? null : parseAbActionLine(line);
|
|
25759
|
-
recorded = parsed === null ? [] : [parsed];
|
|
25760
|
-
}
|
|
25761
|
-
let pushed = 0;
|
|
25762
|
-
for (const action of recorded) {
|
|
25763
|
-
const stamped = withStepId(action, stepForCommand);
|
|
25764
|
-
if (stamped) {
|
|
25765
|
-
traceActions.push(secret ? {
|
|
25766
|
-
...stamped,
|
|
25767
|
-
secret: true
|
|
25768
|
-
} : stamped);
|
|
25769
|
-
pushed += 1;
|
|
25770
|
-
}
|
|
25771
|
-
}
|
|
25772
|
-
lastCommandPushCount = pushed;
|
|
25773
|
-
},
|
|
26141
|
+
onAbAction,
|
|
26142
|
+
beforeAbCommand,
|
|
26143
|
+
effort: "low",
|
|
26144
|
+
bashGuard: traceBashGuard,
|
|
25774
26145
|
onAbActionFailed: () => {
|
|
25775
26146
|
if (lastCommandPushCount > 0) traceActions.splice(-lastCommandPushCount);
|
|
25776
26147
|
lastCommandPushCount = 0;
|
|
@@ -25805,10 +26176,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25805
26176
|
});
|
|
25806
26177
|
const overallStatus = failureReason === null ? "passed" : "failed";
|
|
25807
26178
|
const dedupedActions = dedupAndReport(scrubAndReport(traceActions));
|
|
26179
|
+
const validationStart = Date.now();
|
|
25808
26180
|
const validatedActions = overallStatus === "passed" || opts.validateFailedTrace === true ? validateAndReport(dedupedActions, validationMode, { CCQA_RUN_ID: sessionName }, opts.teardown) : dedupedActions;
|
|
26181
|
+
if (validatedActions !== dedupedActions) meta("validation time", `${((Date.now() - validationStart) / 1e3).toFixed(1)}s`);
|
|
25809
26182
|
opts.teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
|
|
25810
|
-
const caseRef = testCase.ref;
|
|
25811
|
-
const previous = overallStatus === "passed" ? await tryGetRecording(caseRef) : null;
|
|
25812
26183
|
let recordingPath;
|
|
25813
26184
|
const cleanupIds = new Set(testCase.cleanup.map((s) => s.id));
|
|
25814
26185
|
const routeActions = validatedActions.filter((a) => !cleanupIds.has(a.stepId ?? ""));
|
|
@@ -25847,7 +26218,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25847
26218
|
*/
|
|
25848
26219
|
function traceFailureReason(lines, session) {
|
|
25849
26220
|
const assertionFailed = lines.find((l) => l.type === "ASSERTION_FAILED");
|
|
25850
|
-
if (assertionFailed)
|
|
26221
|
+
if (assertionFailed) {
|
|
26222
|
+
const detail = assertionFailed.detail.trim();
|
|
26223
|
+
return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED${detail ? `: ${detail}` : ""}`;
|
|
26224
|
+
}
|
|
25851
26225
|
if (session.isError) return `the Claude session ended in an error: ${session.errorDetail ?? "no detail reported"}`;
|
|
25852
26226
|
const completed = lines.filter((l) => l.type === "RUN_COMPLETED").at(-1);
|
|
25853
26227
|
if (completed === void 0) return "the session ended without a RUN_COMPLETED line — the model stopped before it reported an outcome";
|
|
@@ -26675,7 +27049,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
|
|
|
26675
27049
|
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
26676
27050
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
26677
27051
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
26678
|
-
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
27052
|
+
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--fresh-ir", "Record from scratch. By default an existing ir.json is shown to the recorder as a map of the route, so steps that still work are not rediscovered.").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
26679
27053
|
await withCostReporting("record", () => runRecord(specPath, opts));
|
|
26680
27054
|
}));
|
|
26681
27055
|
async function runRecord(caseArgument, opts) {
|
|
@@ -26787,6 +27161,7 @@ async function runRecord(caseArgument, opts) {
|
|
|
26787
27161
|
...config.sessionState ? { sessionState: config.sessionState } : {},
|
|
26788
27162
|
...resolved.targetConfig.conventions.operate.length > 0 ? { conventions: resolved.targetConfig.conventions.operate } : {},
|
|
26789
27163
|
...opts.instruction ? { instruction: opts.instruction } : {},
|
|
27164
|
+
...opts.freshIr ? { freshIr: true } : {},
|
|
26790
27165
|
onStep: (stepId) => {
|
|
26791
27166
|
tracingStep = stepId;
|
|
26792
27167
|
}
|
|
@@ -28672,12 +29047,12 @@ function releaseAll(locks, holder) {
|
|
|
28672
29047
|
}
|
|
28673
29048
|
//#endregion
|
|
28674
29049
|
//#region src/hub/core/audit-need.ts
|
|
28675
|
-
function auditNeed(drift,
|
|
28676
|
-
const entry = drift.specs[key];
|
|
29050
|
+
function auditNeed(drift, spec, range, deployTimes) {
|
|
29051
|
+
const entry = drift.specs[spec.key];
|
|
28677
29052
|
if (!entry) return { because: "neverAudited" };
|
|
28678
|
-
const since = freshness(entry.gitHead, key, range);
|
|
29053
|
+
const since = freshness(entry.gitHead, spec.key, range);
|
|
28679
29054
|
switch (since.kind) {
|
|
28680
|
-
case "current": return { because: "current" };
|
|
29055
|
+
case "current": return specMovedSince(spec.changedAt, entry.gitHead, entry.at, deployTimes) ? { because: "deployReached" } : { because: "current" };
|
|
28681
29056
|
case "touched": return { because: "deployReached" };
|
|
28682
29057
|
case "unanswerable": return {
|
|
28683
29058
|
because: "cannotTell",
|
|
@@ -28686,13 +29061,52 @@ function auditNeed(drift, key, range) {
|
|
|
28686
29061
|
default: throw new Error(`unhandled freshness: ${String(since)}`);
|
|
28687
29062
|
}
|
|
28688
29063
|
}
|
|
29064
|
+
/**
|
|
29065
|
+
* When each deployed commit reached the environment. A baseline read at that
|
|
29066
|
+
* commit cannot have seen anything committed after it was deployed.
|
|
29067
|
+
*/
|
|
29068
|
+
function deployedAt(log) {
|
|
29069
|
+
const out = /* @__PURE__ */ new Map();
|
|
29070
|
+
for (const entry of log.entries) out.set(entry.sha, entry.at);
|
|
29071
|
+
return out;
|
|
29072
|
+
}
|
|
29073
|
+
/**
|
|
29074
|
+
* Has the spec moved since the baseline was taken?
|
|
29075
|
+
*
|
|
29076
|
+
* A verdict is a claim about a (spec, product) pair, so either side moving
|
|
29077
|
+
* invalidates it. The deploy log covers the product side; this covers the
|
|
29078
|
+
* other one. Without it a spec repaired and merged stays `needsRepair` until
|
|
29079
|
+
* a deploy happens to reach it, and a run that passed against the previous
|
|
29080
|
+
* spec keeps answering `verified` for the new one.
|
|
29081
|
+
*
|
|
29082
|
+
* Compared against when the baseline commit was *deployed*, not when the audit
|
|
29083
|
+
* or run happened: the tree read at that commit predates its deployment, so an
|
|
29084
|
+
* edit after it is definitely not in it. Falls back to the baseline's own
|
|
29085
|
+
* timestamp when the log cannot place the commit.
|
|
29086
|
+
*
|
|
29087
|
+
* One-directional. A later edit time proves the baseline is stale; an earlier
|
|
29088
|
+
* one proves nothing, and this answers false rather than guessing.
|
|
29089
|
+
*/
|
|
29090
|
+
function specMovedSince(changedAt, baselineSha, baselineAt, deployTimes) {
|
|
29091
|
+
if (!changedAt) return null;
|
|
29092
|
+
return isLater(changedAt, baselineSha && deployTimes.get(baselineSha) || baselineAt) ? changedAt : null;
|
|
29093
|
+
}
|
|
29094
|
+
/**
|
|
29095
|
+
* Compared as instants, never as strings: a spec's edit time carries its
|
|
29096
|
+
* committer's offset, and `+09:00` sorts after `Z` for an earlier moment.
|
|
29097
|
+
* An unparseable side answers false, the same as a missing one.
|
|
29098
|
+
*/
|
|
29099
|
+
function isLater(a, b) {
|
|
29100
|
+
return Date.parse(a) > Date.parse(b);
|
|
29101
|
+
}
|
|
28689
29102
|
/** True for every answer but `current`. */
|
|
28690
29103
|
function needsAudit(need) {
|
|
28691
29104
|
return need.because !== "current";
|
|
28692
29105
|
}
|
|
28693
29106
|
function computeAuditNeed(input) {
|
|
28694
29107
|
const range = buildRange(input.log, input.touchIndex);
|
|
28695
|
-
|
|
29108
|
+
const deployTimes = deployedAt(input.log);
|
|
29109
|
+
return Object.fromEntries(input.specs.map((spec) => [spec.key, heldBy(input.locks, spec.key, input.now) ? { because: "held" } : auditNeed(input.drift, spec, range, deployTimes)]));
|
|
28696
29110
|
}
|
|
28697
29111
|
//#endregion
|
|
28698
29112
|
//#region src/drift/audit-selection.ts
|
|
@@ -29254,7 +29668,8 @@ async function readChunk(cwd, paths, byRelPath, out) {
|
|
|
29254
29668
|
let when = "";
|
|
29255
29669
|
for (const line of stdout.split("\n")) {
|
|
29256
29670
|
if (line.startsWith("\0")) {
|
|
29257
|
-
|
|
29671
|
+
const ms = Date.parse(line.slice(1).trim());
|
|
29672
|
+
when = Number.isNaN(ms) ? "" : new Date(ms).toISOString();
|
|
29258
29673
|
continue;
|
|
29259
29674
|
}
|
|
29260
29675
|
const abs = byRelPath.get(line.trim());
|
|
@@ -31878,36 +32293,6 @@ function createGetCoverageEdgesHandler(config) {
|
|
|
31878
32293
|
}
|
|
31879
32294
|
//#endregion
|
|
31880
32295
|
//#region src/hub/core/rerun.ts
|
|
31881
|
-
/**
|
|
31882
|
-
* When each deployed commit reached the environment. A baseline read at that
|
|
31883
|
-
* commit cannot have seen anything committed after it was deployed.
|
|
31884
|
-
*/
|
|
31885
|
-
function deployedAt(log) {
|
|
31886
|
-
const out = /* @__PURE__ */ new Map();
|
|
31887
|
-
for (const entry of log.entries) out.set(entry.sha, entry.at);
|
|
31888
|
-
return out;
|
|
31889
|
-
}
|
|
31890
|
-
/**
|
|
31891
|
-
* Has the spec moved since the baseline was taken?
|
|
31892
|
-
*
|
|
31893
|
-
* A verdict is a claim about a (spec, product) pair, so either side moving
|
|
31894
|
-
* invalidates it. The deploy log covers the product side; this covers the
|
|
31895
|
-
* other one. Without it a spec repaired and merged stays `needsRepair` until
|
|
31896
|
-
* a deploy happens to reach it, and a run that passed against the previous
|
|
31897
|
-
* spec keeps answering `verified` for the new one.
|
|
31898
|
-
*
|
|
31899
|
-
* Compared against when the baseline commit was *deployed*, not when the audit
|
|
31900
|
-
* or run happened: the tree read at that commit predates its deployment, so an
|
|
31901
|
-
* edit after it is definitely not in it. Falls back to the baseline's own
|
|
31902
|
-
* timestamp when the log cannot place the commit.
|
|
31903
|
-
*
|
|
31904
|
-
* One-directional. A later edit time proves the baseline is stale; an earlier
|
|
31905
|
-
* one proves nothing, and this answers false rather than guessing.
|
|
31906
|
-
*/
|
|
31907
|
-
function specMovedSince(changedAt, baselineSha, baselineAt, deployTimes) {
|
|
31908
|
-
if (!changedAt) return null;
|
|
31909
|
-
return changedAt > (baselineSha && deployTimes.get(baselineSha) || baselineAt) ? changedAt : null;
|
|
31910
|
-
}
|
|
31911
32296
|
function computeRerun(input) {
|
|
31912
32297
|
const { specs, ledger, log, touchIndex, drift, locks, attestations, dismissals, now } = input;
|
|
31913
32298
|
const range = buildRange(log, touchIndex);
|
|
@@ -31919,7 +32304,7 @@ function computeRerun(input) {
|
|
|
31919
32304
|
lastGreen: ledger.green[spec.key] ?? null,
|
|
31920
32305
|
lastRed: ledger.red[spec.key] ?? null
|
|
31921
32306
|
};
|
|
31922
|
-
let audit = auditState(drift, spec
|
|
32307
|
+
let audit = auditState(drift, spec, range, deployTimes);
|
|
31923
32308
|
let execution = executionState(coords, (sha) => freshness(sha, spec.key, range));
|
|
31924
32309
|
const driftEntry = drift.specs[spec.key];
|
|
31925
32310
|
const dismissal = dismissals.specs[spec.key];
|
|
@@ -31927,7 +32312,6 @@ function computeRerun(input) {
|
|
|
31927
32312
|
if (dismissed) audit = { audit: "clean" };
|
|
31928
32313
|
const auditMoved = specMovedSince(spec.changedAt, driftEntry?.gitHead ?? null, driftEntry?.at ?? "", deployTimes);
|
|
31929
32314
|
const runMoved = specMovedSince(spec.changedAt, coords.lastRun?.deployedSha ?? null, coords.lastRun?.at ?? "", deployTimes);
|
|
31930
|
-
if (auditMoved && audit.audit !== "due") audit = { audit: "due" };
|
|
31931
32315
|
if (runMoved && execution.execution === "passed") execution = { execution: "stale" };
|
|
31932
32316
|
const manualState = readAttestation(attestations.specs[spec.key], spec, coords, range, log, deployTimes);
|
|
31933
32317
|
const lapsedOverRed = manualState?.kind === "lapsed" && manualState.because !== "newerRed";
|
|
@@ -31972,7 +32356,7 @@ function computeRerun(input) {
|
|
|
31972
32356
|
*/
|
|
31973
32357
|
function readAttestation(attest, spec, { lastRun, lastRed }, range, log, deployTimes) {
|
|
31974
32358
|
if (!attest) return null;
|
|
31975
|
-
const newerRed = lastRed !== null && lastRed.at
|
|
32359
|
+
const newerRed = lastRed !== null && isLater(lastRed.at, attest.at);
|
|
31976
32360
|
if (newerRed && lastRed.runId === lastRun?.runId) return {
|
|
31977
32361
|
kind: "lapsed",
|
|
31978
32362
|
attest,
|
|
@@ -32018,8 +32402,8 @@ function readAttestation(attest, spec, { lastRun, lastRed }, range, log, deployT
|
|
|
32018
32402
|
* rule out is treated as having landed (ADR-0014). The hole is kept as an
|
|
32019
32403
|
* annotation so the answer stays explicable.
|
|
32020
32404
|
*/
|
|
32021
|
-
function auditState(drift,
|
|
32022
|
-
const need = auditNeed(drift,
|
|
32405
|
+
function auditState(drift, spec, range, deployTimes) {
|
|
32406
|
+
const need = auditNeed(drift, spec, range, deployTimes);
|
|
32023
32407
|
switch (need.because) {
|
|
32024
32408
|
case "neverAudited":
|
|
32025
32409
|
case "deployReached": return { audit: "due" };
|
|
@@ -32028,7 +32412,7 @@ function auditState(drift, key, range) {
|
|
|
32028
32412
|
...need.reason ? { auditAssumedReached: need.reason } : {}
|
|
32029
32413
|
};
|
|
32030
32414
|
case "current": {
|
|
32031
|
-
const label = drift.specs[key].label;
|
|
32415
|
+
const label = drift.specs[spec.key].label;
|
|
32032
32416
|
if (label === null) return { audit: "clean" };
|
|
32033
32417
|
if (label === "UNKNOWN") return { audit: "undecided" };
|
|
32034
32418
|
if (label === "PRODUCT_BUG" || label === "ENVIRONMENT") return { audit: "clean" };
|