ccqa 1.58.2 → 1.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +1506 -1120
- package/dist/hub-client/index.d.cts +2 -2
- package/dist/hub-client/index.d.mts +2 -2
- package/dist/package.json +1 -1
- package/dist/runtime/judge.cjs +53 -35
- package/dist/runtime/judge.mjs +53 -35
- package/package.json +1 -1
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -3143,7 +3143,7 @@ function formatDuration$1(ms) {
|
|
|
3143
3143
|
return `${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
3144
3144
|
}
|
|
3145
3145
|
async function invokeClaudeStreaming(options, onEvent) {
|
|
3146
|
-
const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
|
|
3146
|
+
const { prompt, systemPrompt, allowedTools, disableThinking = false, effort, bashGuard, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, beforeAbCommand, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
|
|
3147
3147
|
const resolvedModel = resolveModel(model);
|
|
3148
3148
|
const mergedEnv = buildInvocationEnv(env);
|
|
3149
3149
|
const abortController = new AbortController();
|
|
@@ -3171,52 +3171,35 @@ async function invokeClaudeStreaming(options, onEvent) {
|
|
|
3171
3171
|
env: mergedEnv,
|
|
3172
3172
|
...mcpServers ? { mcpServers } : {},
|
|
3173
3173
|
...disableThinking ? { thinking: { type: "disabled" } } : {},
|
|
3174
|
-
|
|
3174
|
+
...effort ? { effort } : {},
|
|
3175
|
+
hooks: onAbAction || onAbActionFailed || bashGuard || beforeAbCommand ? {
|
|
3175
3176
|
PreToolUse: [{ hooks: [async (input) => {
|
|
3176
3177
|
if (input.hook_event_name !== "PreToolUse") return {};
|
|
3177
3178
|
if (input.tool_name !== "Bash") return {};
|
|
3178
3179
|
const cmd = input.tool_input?.["command"];
|
|
3179
3180
|
if (typeof cmd !== "string") return {};
|
|
3181
|
+
const guarded = bashGuard?.(cmd);
|
|
3182
|
+
if (guarded) return {
|
|
3183
|
+
decision: "block",
|
|
3184
|
+
reason: guarded
|
|
3185
|
+
};
|
|
3180
3186
|
if (!relaxAbConstraints) {
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
|
|
3184
|
-
};
|
|
3185
|
-
if (hasRefSelector(cmd)) return {
|
|
3186
|
-
decision: "block",
|
|
3187
|
-
reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
|
|
3188
|
-
};
|
|
3189
|
-
const bareTag = findPositionalBareTag(cmd);
|
|
3190
|
-
if (bareTag !== null) return {
|
|
3191
|
-
decision: "block",
|
|
3192
|
-
reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
|
|
3193
|
-
};
|
|
3194
|
-
if (hasMultipleAbInvocations(cmd)) return {
|
|
3195
|
-
decision: "block",
|
|
3196
|
-
reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
|
|
3197
|
-
};
|
|
3198
|
-
if (hasErrorSuppression(cmd)) return {
|
|
3187
|
+
const reason = abCommandBlockReason(cmd, envScrubMap);
|
|
3188
|
+
if (reason !== null) return {
|
|
3199
3189
|
decision: "block",
|
|
3200
|
-
reason
|
|
3190
|
+
reason
|
|
3201
3191
|
};
|
|
3202
|
-
const
|
|
3203
|
-
if (
|
|
3192
|
+
const checkpoint = beforeAbCommand?.(extractCcqaStepFromBashCommand(cmd) ?? void 0);
|
|
3193
|
+
if (checkpoint) return {
|
|
3204
3194
|
decision: "block",
|
|
3205
|
-
reason:
|
|
3195
|
+
reason: checkpoint
|
|
3206
3196
|
};
|
|
3207
3197
|
}
|
|
3208
|
-
const
|
|
3209
|
-
|
|
3210
|
-
if ((ab !== null || assertMarker !== null) && onAbAction) {
|
|
3198
|
+
const recorded = relaxAbConstraints ? null : abActionEventFromCommand(cmd);
|
|
3199
|
+
if (recorded !== null && onAbAction) {
|
|
3211
3200
|
lastAbToolUseId = input.tool_use_id;
|
|
3212
|
-
lastAbAnswerHolds =
|
|
3213
|
-
|
|
3214
|
-
onAbAction({
|
|
3215
|
-
...ab !== null ? { abAction: ab } : {},
|
|
3216
|
-
...stepId ? { stepId } : {},
|
|
3217
|
-
...assertMarker !== null ? { assertMarker } : {},
|
|
3218
|
-
...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
|
|
3219
|
-
});
|
|
3201
|
+
lastAbAnswerHolds = recorded.holds;
|
|
3202
|
+
onAbAction(recorded.event);
|
|
3220
3203
|
} else lastAbToolUseId = null;
|
|
3221
3204
|
return {};
|
|
3222
3205
|
}] }],
|
|
@@ -3804,6 +3787,45 @@ async function fireMockPreToolUseHooks(msg, options) {
|
|
|
3804
3787
|
for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
|
|
3805
3788
|
}
|
|
3806
3789
|
}
|
|
3790
|
+
/**
|
|
3791
|
+
* Why the trace refuses this agent-browser command, or null when it may run.
|
|
3792
|
+
* Shared by the Bash hook and the trace's own batch tool, so both enforce the
|
|
3793
|
+
* same replayability contract.
|
|
3794
|
+
*/
|
|
3795
|
+
function abCommandBlockReason(cmd, envScrubMap = []) {
|
|
3796
|
+
if (isBlockedAbSubcommand(cmd)) return "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector.";
|
|
3797
|
+
if (hasRefSelector(cmd)) return "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either.";
|
|
3798
|
+
const bareTag = findPositionalBareTag(cmd);
|
|
3799
|
+
if (bareTag !== null) return `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`;
|
|
3800
|
+
if (hasMultipleAbInvocations(cmd)) return "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command.";
|
|
3801
|
+
if (hasErrorSuppression(cmd)) return "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code.";
|
|
3802
|
+
const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
|
|
3803
|
+
if (runProducedUrl !== null) return `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`;
|
|
3804
|
+
return null;
|
|
3805
|
+
}
|
|
3806
|
+
/**
|
|
3807
|
+
* What recording this command produces, and — for a marked probe — what its
|
|
3808
|
+
* output must say for the check to hold. Null when it records nothing.
|
|
3809
|
+
*/
|
|
3810
|
+
function abActionEventFromCommand(cmd) {
|
|
3811
|
+
const assertMarker = extractCcqaAssertFromBashCommand(cmd);
|
|
3812
|
+
const ab = extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
|
|
3813
|
+
if (ab === null && assertMarker === null) return null;
|
|
3814
|
+
const stepId = extractCcqaStepFromBashCommand(cmd);
|
|
3815
|
+
return {
|
|
3816
|
+
event: {
|
|
3817
|
+
...ab !== null ? { abAction: ab } : {},
|
|
3818
|
+
...stepId ? { stepId } : {},
|
|
3819
|
+
...assertMarker !== null ? { assertMarker } : {},
|
|
3820
|
+
...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
|
|
3821
|
+
},
|
|
3822
|
+
holds: assertMarker === null ? null : markerHolds(assertMarker, ab)
|
|
3823
|
+
};
|
|
3824
|
+
}
|
|
3825
|
+
/** Whether a shell line runs agent-browser as a command (env prefixes allowed), not just mentions it. */
|
|
3826
|
+
function isAgentBrowserCommand(cmd) {
|
|
3827
|
+
return splitShellStatements(cmd).some((st) => isAgentBrowserHead(splitLeadingEnvAssignments(st).command));
|
|
3828
|
+
}
|
|
3807
3829
|
//#endregion
|
|
3808
3830
|
//#region src/prompts/format.ts
|
|
3809
3831
|
/**
|
|
@@ -3950,7 +3972,7 @@ function toAgentBrowserArgs(action) {
|
|
|
3950
3972
|
}
|
|
3951
3973
|
case "wait": {
|
|
3952
3974
|
const loc = action.locator;
|
|
3953
|
-
if (!loc) return null;
|
|
3975
|
+
if (!loc || loc.value.startsWith("--")) return null;
|
|
3954
3976
|
if (loc.by === "text") return [
|
|
3955
3977
|
lit("wait"),
|
|
3956
3978
|
lit("--text"),
|
|
@@ -4005,6 +4027,61 @@ function interactionToArgs(action) {
|
|
|
4005
4027
|
if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
|
|
4006
4028
|
return out;
|
|
4007
4029
|
}
|
|
4030
|
+
/**
|
|
4031
|
+
* The marked probe that records an assert during a trace — the inverse of
|
|
4032
|
+
* `promoteMarkedAssert`, as the trace prompt's Assertion Protocol spells it.
|
|
4033
|
+
*/
|
|
4034
|
+
function markedAssertArgs(action) {
|
|
4035
|
+
const loc = action.locator;
|
|
4036
|
+
const sel = loc ? [val(locatorToSelector(loc))] : null;
|
|
4037
|
+
switch (action.assert) {
|
|
4038
|
+
case "text_visible": return action.value ? {
|
|
4039
|
+
marker: "1",
|
|
4040
|
+
tokens: [
|
|
4041
|
+
lit("wait"),
|
|
4042
|
+
lit("--text"),
|
|
4043
|
+
val(action.value)
|
|
4044
|
+
]
|
|
4045
|
+
} : null;
|
|
4046
|
+
case "url_contains": return action.value ? {
|
|
4047
|
+
marker: `url_contains:${action.value}`,
|
|
4048
|
+
tokens: [lit("get"), lit("url")]
|
|
4049
|
+
} : null;
|
|
4050
|
+
case "element_visible":
|
|
4051
|
+
case "element_not_visible":
|
|
4052
|
+
if (loc?.by === "role") return loc.name ? {
|
|
4053
|
+
marker: action.assert,
|
|
4054
|
+
tokens: roleProbeTokens(loc.value, loc.name, loc.exact === true)
|
|
4055
|
+
} : null;
|
|
4056
|
+
return sel ? {
|
|
4057
|
+
marker: action.assert,
|
|
4058
|
+
tokens: [
|
|
4059
|
+
lit("get"),
|
|
4060
|
+
lit("count"),
|
|
4061
|
+
...sel
|
|
4062
|
+
]
|
|
4063
|
+
} : null;
|
|
4064
|
+
case "element_enabled":
|
|
4065
|
+
case "element_disabled": return sel ? {
|
|
4066
|
+
marker: action.assert,
|
|
4067
|
+
tokens: [
|
|
4068
|
+
lit("is"),
|
|
4069
|
+
lit("enabled"),
|
|
4070
|
+
...sel
|
|
4071
|
+
]
|
|
4072
|
+
} : null;
|
|
4073
|
+
case "element_checked":
|
|
4074
|
+
case "element_unchecked": return sel ? {
|
|
4075
|
+
marker: action.assert,
|
|
4076
|
+
tokens: [
|
|
4077
|
+
lit("is"),
|
|
4078
|
+
lit("checked"),
|
|
4079
|
+
...sel
|
|
4080
|
+
]
|
|
4081
|
+
} : null;
|
|
4082
|
+
default: return null;
|
|
4083
|
+
}
|
|
4084
|
+
}
|
|
4008
4085
|
//#endregion
|
|
4009
4086
|
//#region src/diagnose/prompt.ts
|
|
4010
4087
|
function buildDiagnosePrompt(input) {
|
|
@@ -7868,7 +7945,7 @@ function reuseFirstContract(hasDraft, draftInvariant) {
|
|
|
7868
7945
|
"3. **Match the examples.** Write in the same style as the conventions examples: naming, structure, assertion phrasing, import layout. Where you follow an existing file rather than the plain reading of a guideline — a waiting pattern, a locator shape, a teardown structure — say which file, by path, in the comment. \"Following the existing implementation\" without naming it cannot be checked by the next reader, and it is how a pattern that was right once outlives the reason for it."
|
|
7869
7946
|
];
|
|
7870
7947
|
if (hasDraft) {
|
|
7871
|
-
rules.push("4. **The draft is ground truth.**
|
|
7948
|
+
rules.push("4. **The draft is ground truth for the route, not for the form of its checks.** Keep its operation sequence: no reordering or dropping of the actions the case needs. Its `expect` lines are what the recorder observed on the way, written in one fixed form (`await expect(...)`, one per observation). Which of them the test keeps, and in which form — hard or soft, where the check lives, how the locator is scoped — is decided by the case's expected results and the project's conventions, never by the draft. Where the conventions and the draft disagree, follow the conventions. What an expected result claims must still be checked: never loosen the value it states.");
|
|
7872
7949
|
if (draftInvariant) rules.push(`5. ${draftInvariant}`);
|
|
7873
7950
|
}
|
|
7874
7951
|
return `## Reuse contract\n\n${rules.join("\n")}`;
|
|
@@ -7916,6 +7993,8 @@ function buildLlmGenPrompt(input) {
|
|
|
7916
7993
|
sections.push(input.taskInstructions);
|
|
7917
7994
|
const steps = input.steps.map(formatStep).join("\n");
|
|
7918
7995
|
sections.push(`## Test spec\n\nTitle: ${input.specTitle}\n\nSteps:\n${steps}`);
|
|
7996
|
+
const facts = caseFacts(input);
|
|
7997
|
+
if (facts) sections.push(facts);
|
|
7919
7998
|
if (input.draft) sections.push(`## Mechanical draft (recorded ground truth)\n\nPath: ${input.draft.path}\n\n\`\`\`
|
|
7920
7999
|
` + input.draft.contents + "\n```");
|
|
7921
8000
|
if (input.resources.length > 0) {
|
|
@@ -7936,6 +8015,19 @@ These are the project's rules for the code you write, not background reading. Co
|
|
|
7936
8015
|
return sections.join("\n\n") + languageDirective(input.language);
|
|
7937
8016
|
}
|
|
7938
8017
|
/**
|
|
8018
|
+
* What the case states beyond its steps. Project conventions key rules on
|
|
8019
|
+
* these (which checks are the case's, where it may run), and the review that
|
|
8020
|
+
* follows already reads them — a rewrite without them can only guess.
|
|
8021
|
+
*/
|
|
8022
|
+
function caseFacts(input) {
|
|
8023
|
+
const parts = [];
|
|
8024
|
+
if (input.expectations && input.expectations.length > 0) parts.push(`### Expected results\n\nThe case states these for the flow as a whole. They are the checks the test exists for:\n\n${input.expectations.map((e) => `- ${body(e)}`).join("\n")}`);
|
|
8025
|
+
if (input.cleanup && input.cleanup.length > 0) parts.push(`### Cleanup steps\n\n${input.cleanup.map(formatStep).join("\n")}`);
|
|
8026
|
+
if (input.cleanupExpectations && input.cleanupExpectations.length > 0) parts.push(`### Expected results of the cleanup\n\n${input.cleanupExpectations.map((e) => `- ${body(e)}`).join("\n")}`);
|
|
8027
|
+
for (const c of input.context ?? []) parts.push(`### ${c.heading}\n\n${c.body.trim()}`);
|
|
8028
|
+
return parts.length === 0 ? "" : `## About this case\n\n${parts.join("\n\n")}`;
|
|
8029
|
+
}
|
|
8030
|
+
/**
|
|
7939
8031
|
* Prompt for one iteration of the verification loop: the generated files
|
|
7940
8032
|
* failed their run command; ask for corrected files under the same contract.
|
|
7941
8033
|
*/
|
|
@@ -8304,7 +8396,7 @@ function weakerTwin(file, source) {
|
|
|
8304
8396
|
function decidesNothing(input) {
|
|
8305
8397
|
const source = input.files.get(input.testPath);
|
|
8306
8398
|
if (source === void 0 || input.caseText.join("").trim().length === 0) return [];
|
|
8307
|
-
if (/\bexpect
|
|
8399
|
+
if (/\bexpect(?:\.soft)?\(|\bjudgeByLlm\b/.test(source)) return [];
|
|
8308
8400
|
return [{
|
|
8309
8401
|
file: input.testPath,
|
|
8310
8402
|
line: 1,
|
|
@@ -8373,7 +8465,7 @@ function containerOfPageText(file, source) {
|
|
|
8373
8465
|
*/
|
|
8374
8466
|
function explained(all, i) {
|
|
8375
8467
|
let above = i - 1;
|
|
8376
|
-
while (above >= 0 && /\bexpect
|
|
8468
|
+
while (above >= 0 && /\bexpect(?:\.soft)?\(|^\s*$|^\s*\)/.test(all[above] ?? "")) above -= 1;
|
|
8377
8469
|
return /^\s*\/\//.test(all[above] ?? "");
|
|
8378
8470
|
}
|
|
8379
8471
|
/**
|
|
@@ -8388,7 +8480,7 @@ function explained(all, i) {
|
|
|
8388
8480
|
function unjustifiedFirst(file, source) {
|
|
8389
8481
|
const all = lines(source);
|
|
8390
8482
|
return all.flatMap((text, i) => {
|
|
8391
|
-
if (!/\bexpect
|
|
8483
|
+
if (!/\bexpect(?:\.soft)?\(/.test(text) || !/\.first\(\)/.test(text)) return [];
|
|
8392
8484
|
if (explained(all, i)) return [];
|
|
8393
8485
|
return [{
|
|
8394
8486
|
file,
|
|
@@ -8463,7 +8555,7 @@ function assertedStrings(text, locators) {
|
|
|
8463
8555
|
function unaskedAssertions(file, source, locators, said) {
|
|
8464
8556
|
const all = lines(source);
|
|
8465
8557
|
return all.flatMap((text, i) => {
|
|
8466
|
-
if (!/\bexpect
|
|
8558
|
+
if (!/\bexpect(?:\.soft)?\(/.test(text)) return [];
|
|
8467
8559
|
const strings = assertedStrings(text, locators);
|
|
8468
8560
|
if (strings.length === 0 || strings.some((s) => said.includes(s))) return [];
|
|
8469
8561
|
if (explained(all, i)) return [];
|
|
@@ -14783,6 +14875,10 @@ async function generateWithLlmEngine(req) {
|
|
|
14783
14875
|
specTitle: ctx.spec.title,
|
|
14784
14876
|
steps: req.steps,
|
|
14785
14877
|
draft: req.draft,
|
|
14878
|
+
expectations: ctx.expectations,
|
|
14879
|
+
cleanup: ctx.cleanup,
|
|
14880
|
+
cleanupExpectations: ctx.cleanupExpectations,
|
|
14881
|
+
context: ctx.context ?? [],
|
|
14786
14882
|
...req.draftInvariant ? { draftInvariant: req.draftInvariant } : {},
|
|
14787
14883
|
resources: resources.map(toPromptResource),
|
|
14788
14884
|
conventionSections: conventions.sections,
|
|
@@ -15467,7 +15563,7 @@ function headerPreserveRule(header, titleSuffix) {
|
|
|
15467
15563
|
* rejection, which costs a whole round.
|
|
15468
15564
|
*/
|
|
15469
15565
|
function stepCommentPreserveRule() {
|
|
15470
|
-
return "**Keep every `test.step(...)` block, with its title exactly as the draft wrote it.** Those titles are how the evidence table, the run's per-step screenshots and the review of this test say which assertions belong to which step. A reshaped one parses as no step at all, and every step then reads as deciding nothing. Keep the wording, the numbering and the punctuation, and keep each step's actions inside its own block — a step's call to a page-object method belongs in the block, not outside it. Do not merge, split or reorder the blocks. A test already at this path may have been written by an older ccqa in a different shape: the draft
|
|
15566
|
+
return "**Keep every `test.step(...)` block, with its title exactly as the draft wrote it.** Those titles are how the evidence table, the run's per-step screenshots and the review of this test say which assertions belong to which step. A reshaped one parses as no step at all, and every step then reads as deciding nothing. Keep the wording, the numbering and the punctuation, and keep each step's actions inside its own block — a step's call to a page-object method belongs in the block, not outside it. Do not merge, split or reorder the blocks. A test already at this path may have been written by an older ccqa in a different shape: the draft's step blocks are what this file must keep, not that one's.";
|
|
15471
15567
|
}
|
|
15472
15568
|
/**
|
|
15473
15569
|
* Told to the rewrite, because the alternative failure is silent: a claim
|
|
@@ -24539,1116 +24635,1364 @@ async function readLockBody(lockPath) {
|
|
|
24539
24635
|
}
|
|
24540
24636
|
}
|
|
24541
24637
|
//#endregion
|
|
24542
|
-
//#region src/
|
|
24543
|
-
|
|
24544
|
-
|
|
24638
|
+
//#region src/ir/accessible-name.ts
|
|
24639
|
+
/**
|
|
24640
|
+
* A recorder reads an accessible name off a snapshot and writes it as the
|
|
24641
|
+
* attribute it guessed the name came from. The name can come from a `<label>`
|
|
24642
|
+
* or `aria-labelledby` instead, and the attribute's absence is not evidence
|
|
24643
|
+
* about the element. See docs/targets.md for the measurement.
|
|
24644
|
+
*/
|
|
24645
|
+
const NAMING_ATTR = /^\[\s*aria-label\s*=\s*(?:"([^"]*)"|'([^']*)')\s*\]$/;
|
|
24646
|
+
/**
|
|
24647
|
+
* The name this selector may be after, or null.
|
|
24648
|
+
*
|
|
24649
|
+
* `aria-label` only. `[name='email']` and `[title='Close']` are attributes that
|
|
24650
|
+
* mean what they say, and a zero from one of those is evidence.
|
|
24651
|
+
*/
|
|
24652
|
+
function nameFromAttributeSelector(selector) {
|
|
24653
|
+
const m = NAMING_ATTR.exec(selector.trim());
|
|
24654
|
+
const name = m?.[1] ?? m?.[2];
|
|
24655
|
+
return name === void 0 || name === "" ? null : name;
|
|
24545
24656
|
}
|
|
24546
24657
|
/**
|
|
24547
|
-
*
|
|
24548
|
-
*
|
|
24549
|
-
* The spec opens URLs via explicit step instructions (e.g.
|
|
24550
|
-
* `instruction: "${APP_URL}/articles を開く"`).
|
|
24658
|
+
* The role of the first node in an accessibility snapshot whose name is
|
|
24659
|
+
* exactly `name`, or null.
|
|
24551
24660
|
*
|
|
24552
|
-
*
|
|
24553
|
-
*
|
|
24554
|
-
*
|
|
24555
|
-
* distinguishes spec-native steps from inlined block steps for the
|
|
24556
|
-
* `// step:` comments in the eventual codegen output.
|
|
24661
|
+
* agent-browser prints one node per line as `<role> "<name>"`. Matched on the
|
|
24662
|
+
* whole name, so a tree holding both "Priority" and "Priority *" answers for
|
|
24663
|
+
* the one that was asked, and refused when two roles carry it.
|
|
24557
24664
|
*/
|
|
24558
|
-
function
|
|
24559
|
-
const
|
|
24560
|
-
const
|
|
24561
|
-
|
|
24562
|
-
|
|
24563
|
-
|
|
24564
|
-
|
|
24565
|
-
|
|
24566
|
-
|
|
24567
|
-
|
|
24568
|
-
|
|
24569
|
-
|
|
24570
|
-
|
|
24571
|
-
|
|
24572
|
-
|
|
24573
|
-
|
|
24574
|
-
|
|
24575
|
-
|
|
24576
|
-
const
|
|
24577
|
-
|
|
24578
|
-
|
|
24579
|
-
|
|
24580
|
-
|
|
24581
|
-
|
|
24582
|
-
|
|
24583
|
-
|
|
24584
|
-
|
|
24585
|
-
|
|
24586
|
-
|
|
24587
|
-
|
|
24588
|
-
|
|
24589
|
-
|
|
24590
|
-
|
|
24591
|
-
|
|
24592
|
-
|
|
24593
|
-
|
|
24594
|
-
|
|
24595
|
-
|
|
24596
|
-
|
|
24597
|
-
|
|
24598
|
-
|
|
24599
|
-
|
|
24600
|
-
|
|
24601
|
-
|
|
24602
|
-
|
|
24603
|
-
|
|
24604
|
-
|
|
24605
|
-
|
|
24606
|
-
|
|
24607
|
-
|
|
24608
|
-
|
|
24609
|
-
|
|
24610
|
-
|
|
24611
|
-
|
|
24612
|
-
|
|
24613
|
-
|
|
24614
|
-
|
|
24615
|
-
|
|
24616
|
-
|
|
24617
|
-
|
|
24618
|
-
|
|
24619
|
-
|
|
24620
|
-
|
|
24621
|
-
|
|
24622
|
-
|
|
24623
|
-
|
|
24624
|
-
|
|
24625
|
-
|
|
24626
|
-
|
|
24627
|
-
|
|
24628
|
-
|
|
24629
|
-
|
|
24630
|
-
|
|
24631
|
-
|
|
24632
|
-
|
|
24633
|
-
|
|
24634
|
-
|
|
24635
|
-
|
|
24636
|
-
|
|
24637
|
-
|
|
24638
|
-
|
|
24639
|
-
|
|
24640
|
-
|
|
24641
|
-
|
|
24642
|
-
|
|
24643
|
-
|
|
24644
|
-
|
|
24645
|
-
|
|
24646
|
-
|
|
24647
|
-
|
|
24648
|
-
|
|
24649
|
-
|
|
24650
|
-
|
|
24651
|
-
|
|
24652
|
-
|
|
24653
|
-
|
|
24654
|
-
|
|
24655
|
-
|
|
24656
|
-
|
|
24657
|
-
|
|
24658
|
-
|
|
24659
|
-
|
|
24660
|
-
|
|
24661
|
-
|
|
24662
|
-
|
|
24663
|
-
|
|
24664
|
-
|
|
24665
|
-
|
|
24666
|
-
|
|
24667
|
-
|
|
24668
|
-
|
|
24669
|
-
|
|
24670
|
-
|
|
24671
|
-
|
|
24672
|
-
|
|
24673
|
-
|
|
24674
|
-
|
|
24675
|
-
|
|
24676
|
-
|
|
24677
|
-
|
|
24678
|
-
|
|
24679
|
-
|
|
24680
|
-
|
|
24681
|
-
|
|
24682
|
-
|
|
24683
|
-
|
|
24684
|
-
|
|
24685
|
-
|
|
24686
|
-
|
|
24687
|
-
|
|
24688
|
-
|
|
24689
|
-
|
|
24690
|
-
|
|
24691
|
-
|
|
24692
|
-
|
|
24693
|
-
|
|
24694
|
-
|
|
24695
|
-
|
|
24696
|
-
|
|
24697
|
-
|
|
24698
|
-
|
|
24699
|
-
|
|
24700
|
-
|
|
24701
|
-
|
|
24702
|
-
|
|
24703
|
-
|
|
24704
|
-
|
|
24705
|
-
|
|
24706
|
-
|
|
24707
|
-
|
|
24708
|
-
|
|
24709
|
-
|
|
24710
|
-
|
|
24711
|
-
|
|
24712
|
-
|
|
24713
|
-
|
|
24714
|
-
|
|
24715
|
-
|
|
24716
|
-
|
|
24717
|
-
|
|
24718
|
-
|
|
24719
|
-
|
|
24720
|
-
|
|
24721
|
-
|
|
24722
|
-
|
|
24723
|
-
|
|
24724
|
-
|
|
24725
|
-
|
|
24726
|
-
|
|
24727
|
-
|
|
24728
|
-
|
|
24729
|
-
|
|
24730
|
-
|
|
24731
|
-
|
|
24732
|
-
|
|
24733
|
-
|
|
24734
|
-
|
|
24735
|
-
|
|
24736
|
-
|
|
24737
|
-
|
|
24738
|
-
|
|
24739
|
-
|
|
24740
|
-
|
|
24741
|
-
|
|
24742
|
-
|
|
24743
|
-
|
|
24744
|
-
|
|
24745
|
-
|
|
24746
|
-
|
|
24747
|
-
|
|
24748
|
-
|
|
24749
|
-
|
|
24750
|
-
|
|
24751
|
-
|
|
24752
|
-
|
|
24753
|
-
|
|
24754
|
-
|
|
24755
|
-
|
|
24756
|
-
|
|
24757
|
-
|
|
24758
|
-
|
|
24759
|
-
**After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
|
|
24760
|
-
|
|
24761
|
-
## Guardrails
|
|
24762
|
-
|
|
24763
|
-
- **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
|
|
24764
|
-
- **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
|
|
24765
|
-
- Do NOT retry a selector without taking a fresh snapshot first.
|
|
24766
|
-
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
|
|
24767
|
-
- **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
|
|
24768
|
-
- **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
|
|
24769
|
-
|
|
24770
|
-
## Source Code Reference
|
|
24771
|
-
|
|
24772
|
-
You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
|
|
24773
|
-
|
|
24774
|
-
**When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
|
|
24775
|
-
|
|
24776
|
-
**How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
|
|
24777
|
-
|
|
24778
|
-
**Rules**: only READ source files, never modify them. Keep searches focused.
|
|
24779
|
-
|
|
24780
|
-
## Waiting for Async Operations
|
|
24781
|
-
|
|
24782
|
-
Prefer \`wait\` over polling:
|
|
24783
|
-
|
|
24784
|
-
\`\`\`bash
|
|
24785
|
-
CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
|
|
24786
|
-
\`\`\`
|
|
24787
|
-
|
|
24788
|
-
If polling is required (e.g. waiting for a spinner to disappear):
|
|
24789
|
-
|
|
24790
|
-
\`\`\`bash
|
|
24791
|
-
for i in $(seq 1 18); do
|
|
24792
|
-
sleep 10
|
|
24793
|
-
result=$(agent-browser --session ${sessionName} snapshot 2>&1)
|
|
24794
|
-
echo "$result" | grep -q "<done indicator>" && break
|
|
24795
|
-
done
|
|
24796
|
-
CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
|
|
24797
|
-
\`\`\`
|
|
24798
|
-
|
|
24799
|
-
After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
|
|
24800
|
-
|
|
24801
|
-
## AB_ACTION Protocol
|
|
24802
|
-
|
|
24803
|
-
After **every** browser action, emit one line (outside any code block):
|
|
24804
|
-
|
|
24805
|
-
\`\`\`
|
|
24806
|
-
AB_ACTION|cookies_clear
|
|
24807
|
-
AB_ACTION|open|<url>
|
|
24808
|
-
AB_ACTION|click|<selector>|<visible label>
|
|
24809
|
-
AB_ACTION|dblclick|<selector>|<visible label>
|
|
24810
|
-
AB_ACTION|fill|<selector>|<value>|<aria label>
|
|
24811
|
-
AB_ACTION|check|<selector>|<visible label>
|
|
24812
|
-
AB_ACTION|uncheck|<selector>|<visible label>
|
|
24813
|
-
AB_ACTION|press|<Key>
|
|
24814
|
-
AB_ACTION|select|<selector>|<value>|<aria label>
|
|
24815
|
-
AB_ACTION|hover|<selector>|<visible label>
|
|
24816
|
-
AB_ACTION|scroll|<direction>|<pixels>
|
|
24817
|
-
AB_ACTION|drag|<source selector>|<target selector>|<source label>
|
|
24818
|
-
AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
|
|
24819
|
-
AB_ACTION|wait|<selector or text>|<label>
|
|
24820
|
-
AB_ACTION|snapshot|<key observation, max 100 chars>
|
|
24821
|
-
|
|
24822
|
-
# find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
|
|
24823
|
-
# <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
|
|
24824
|
-
AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
|
|
24825
|
-
AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
|
|
24826
|
-
AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
|
|
24827
|
-
AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
|
|
24828
|
-
AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
|
|
24829
|
-
AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
|
|
24830
|
-
AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
24831
|
-
AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
24832
|
-
\`\`\`
|
|
24833
|
-
|
|
24834
|
-
Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
|
|
24835
|
-
|
|
24836
|
-
**CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
|
|
24837
|
-
|
|
24838
|
-
- A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
|
|
24839
|
-
- If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
|
|
24840
|
-
- An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
|
|
24841
|
-
- **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
|
|
24842
|
-
|
|
24843
|
-
- **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
|
|
24844
|
-
- If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
|
|
24845
|
-
|
|
24846
|
-
## Assertion Protocol
|
|
24847
|
-
|
|
24848
|
-
Record each signal you verified as an assertion. The **primary mechanism** is
|
|
24849
|
-
a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
|
|
24850
|
-
marker rides the same channel as the command, so the assertion is captured
|
|
24851
|
-
even if you print no protocol text. The check you were going to run anyway
|
|
24852
|
-
becomes the recorded assertion:
|
|
24853
|
-
|
|
24854
|
-
\`\`\`bash
|
|
24855
|
-
# text_visible — the marked wait IS the assertion (no separate assert line)
|
|
24856
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
|
|
24857
|
-
# element_visible / element_not_visible — mark the get-count probe
|
|
24858
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
|
|
24859
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
|
|
24860
|
-
# url_contains — the substring rides in the marker; put it on \`get url\`
|
|
24861
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
|
|
24862
|
-
# element state — mark the \`is\` probe that asks for it
|
|
24863
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
|
|
24864
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
|
|
24865
|
-
\`\`\`
|
|
24866
|
-
|
|
24867
|
-
**Marker semantics:**
|
|
24868
|
-
|
|
24869
|
-
| Marker | On command | Records |
|
|
24870
|
-
|--------|------------|---------|
|
|
24871
|
-
| \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
|
|
24872
|
-
| \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
|
|
24873
|
-
| \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
|
|
24874
|
-
| \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
|
|
24875
|
-
| \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
|
|
24876
|
-
| \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
|
|
24877
|
-
|
|
24878
|
-
- A marked command that exits non-zero records nothing — fix the check and re-run it.
|
|
24879
|
-
- A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
|
|
24880
|
-
- \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
|
|
24881
|
-
- **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
|
|
24882
|
-
|
|
24883
|
-
- **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
|
|
24884
|
-
|
|
24885
|
-
- **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
|
|
24886
|
-
|
|
24887
|
-
- **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
|
|
24888
|
-
- **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
|
|
24889
|
-
- **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
|
|
24890
|
-
|
|
24891
|
-
**Every assert type has a marker command.** There is no text form: a printed
|
|
24892
|
-
\`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
|
|
24893
|
-
reported rather than recorded.
|
|
24894
|
-
|
|
24895
|
-
| assertType | The command that records it | Use when |
|
|
24896
|
-
|------------|------------------------------|----------|
|
|
24897
|
-
| \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
|
|
24898
|
-
| \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
|
|
24899
|
-
| \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
|
|
24900
|
-
| \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
|
|
24901
|
-
| \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
|
|
24902
|
-
| \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
|
|
24903
|
-
|
|
24904
|
-
**Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
|
|
24905
|
-
|
|
24906
|
-
- Timestamps, session IDs, exact numeric counts that vary between runs.
|
|
24907
|
-
- **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
|
|
24908
|
-
- **Relative-time labels** — true only in the moment of the trace, stale by replay:
|
|
24909
|
-
- English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
|
|
24910
|
-
- Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
|
|
24911
|
-
- Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
|
|
24912
|
-
- **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
|
|
24913
|
-
|
|
24914
|
-
**No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
|
|
24915
|
-
|
|
24916
|
-
The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
|
|
24917
|
-
|
|
24918
|
-
- ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
|
|
24919
|
-
- ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
|
|
24920
|
-
- ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
|
|
24921
|
-
- If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
|
|
24922
|
-
|
|
24923
|
-
**Page-context and selector rules:**
|
|
24924
|
-
|
|
24925
|
-
- After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
|
|
24926
|
-
- Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
|
|
24927
|
-
- When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
|
|
24928
|
-
|
|
24929
|
-
**MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
|
|
24930
|
-
|
|
24931
|
-
The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
|
|
24932
|
-
|
|
24933
|
-
1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
|
|
24934
|
-
2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
|
|
24935
|
-
3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
|
|
24936
|
-
|
|
24937
|
-
**Verify the assertion form actually resolves on the live page.** The
|
|
24938
|
-
verification and the recorded assertion are the same marked command:
|
|
24939
|
-
|
|
24940
|
-
\`\`\`bash
|
|
24941
|
-
# text_visible
|
|
24942
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
|
|
24943
|
-
# element_visible — >=1 printed means present
|
|
24944
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
|
|
24945
|
-
# element_not_visible — 0 printed means absent (works for text too: "text=<text>")
|
|
24946
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
|
|
24947
|
-
# element state — \`is\` prints true/false and exits 0 either way; read it
|
|
24948
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
|
|
24949
|
-
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
|
|
24950
|
-
\`\`\`
|
|
24951
|
-
|
|
24952
|
-
When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
|
|
24953
|
-
|
|
24954
|
-
## Status Protocol
|
|
24955
|
-
|
|
24956
|
-
Emit exactly one status line per step (outside any code block):
|
|
24957
|
-
|
|
24958
|
-
\`\`\`
|
|
24959
|
-
STEP_START|<step-id>|<short description>
|
|
24960
|
-
STEP_DONE|<step-id>|<what was verified>
|
|
24961
|
-
ASSERTION_FAILED|<step-id>|<category: app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
|
|
24962
|
-
STEP_SKIPPED|<step-id>|<reason>
|
|
24963
|
-
RUN_COMPLETED|passed|<summary>
|
|
24964
|
-
RUN_COMPLETED|failed|<summary>
|
|
24965
|
-
\`\`\`
|
|
24966
|
-
|
|
24967
|
-
**\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
|
|
24968
|
-
recorded as FAILED, however many steps went well: nothing else can tell a
|
|
24969
|
-
finished route from one you abandoned halfway.
|
|
24970
|
-
|
|
24971
|
-
${callerGuidance}## Start
|
|
24972
|
-
|
|
24973
|
-
Begin by clearing cookies, then proceed straight to the first step's instruction.
|
|
24974
|
-
|
|
24975
|
-
\`\`\`bash
|
|
24976
|
-
agent-browser --session ${sessionName} cookies clear
|
|
24977
|
-
\`\`\`
|
|
24978
|
-
|
|
24979
|
-
(The cookies-clear preamble belongs to no step, so it is the one command
|
|
24980
|
-
without a \`CCQA_STEP\` prefix.)
|
|
24981
|
-
|
|
24982
|
-
Emit:
|
|
24983
|
-
\`\`\`
|
|
24984
|
-
AB_ACTION|cookies_clear
|
|
24985
|
-
\`\`\`
|
|
24986
|
-
|
|
24987
|
-
Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
|
|
24988
|
-
every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
|
|
24989
|
-
step is responsible for opening the initial URL.
|
|
24990
|
-
`;
|
|
24665
|
+
function roleOfAccessibleName(snapshot, name) {
|
|
24666
|
+
const roles = /* @__PURE__ */ new Set();
|
|
24667
|
+
for (const line of snapshot.split("\n")) {
|
|
24668
|
+
const m = /([A-Za-z]+)\s+"([^"]*)"/.exec(line);
|
|
24669
|
+
if (m && m[2] === name) roles.add(m[1]);
|
|
24670
|
+
}
|
|
24671
|
+
return roles.size === 1 ? [...roles][0] : null;
|
|
24672
|
+
}
|
|
24673
|
+
//#endregion
|
|
24674
|
+
//#region src/runtime/replay-validate.ts
|
|
24675
|
+
function isProbe(x) {
|
|
24676
|
+
return x !== null && !Array.isArray(x);
|
|
24677
|
+
}
|
|
24678
|
+
function runProbe(probe, sessionName) {
|
|
24679
|
+
return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
|
|
24680
|
+
}
|
|
24681
|
+
function runStateCheck(check, sessionName) {
|
|
24682
|
+
let answer = "";
|
|
24683
|
+
const rounds = attemptsFor(ASSERT_TIMEOUT_MS, SELECTOR_POLL_INTERVAL_MS) + 1;
|
|
24684
|
+
for (let round = 0; round < rounds; round++) {
|
|
24685
|
+
const r = spawnAB([
|
|
24686
|
+
"--session",
|
|
24687
|
+
sessionName,
|
|
24688
|
+
"is",
|
|
24689
|
+
check.state,
|
|
24690
|
+
check.selector
|
|
24691
|
+
]);
|
|
24692
|
+
answer = r.status === 0 ? r.stdout.trim() : "";
|
|
24693
|
+
if (answer === String(check.expected)) return {
|
|
24694
|
+
ok: true,
|
|
24695
|
+
reason: ""
|
|
24696
|
+
};
|
|
24697
|
+
if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
24698
|
+
}
|
|
24699
|
+
return answer === "true" || answer === "false" ? {
|
|
24700
|
+
ok: false,
|
|
24701
|
+
reason: `expected ${check.state}=${check.expected}, page says ${answer}`
|
|
24702
|
+
} : {
|
|
24703
|
+
ok: false,
|
|
24704
|
+
reason: `is ${check.state} answered ${answer || "nothing"}`
|
|
24705
|
+
};
|
|
24706
|
+
}
|
|
24707
|
+
const SELECTOR_POLL_INTERVAL_MS = 500;
|
|
24708
|
+
/** Poll `get count <selector>` until it matches (>=1) or the timeout elapses. */
|
|
24709
|
+
function runPollCheck(check, sessionName) {
|
|
24710
|
+
const present = (selector) => {
|
|
24711
|
+
const r = spawnAB([
|
|
24712
|
+
"--session",
|
|
24713
|
+
sessionName,
|
|
24714
|
+
"get",
|
|
24715
|
+
"count",
|
|
24716
|
+
selector
|
|
24717
|
+
]);
|
|
24718
|
+
return r.status === 0 ? Number.parseInt(r.stdout.trim(), 10) : NaN;
|
|
24719
|
+
};
|
|
24720
|
+
const deadline = Date.now() + check.timeoutMs;
|
|
24721
|
+
const rounds = attemptsFor(check.timeoutMs, SELECTOR_POLL_INTERVAL_MS) + 1;
|
|
24722
|
+
let count = NaN;
|
|
24723
|
+
for (let round = 0;; round++) {
|
|
24724
|
+
count = present(check.selector);
|
|
24725
|
+
if (!Number.isNaN(count) && count > 0) return {
|
|
24726
|
+
ok: true,
|
|
24727
|
+
reason: ""
|
|
24728
|
+
};
|
|
24729
|
+
if (round + 1 >= rounds || Date.now() >= deadline) break;
|
|
24730
|
+
sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
24731
|
+
}
|
|
24732
|
+
return {
|
|
24733
|
+
ok: false,
|
|
24734
|
+
reason: `selector not present within ${check.timeoutMs}ms (get count returned ${Number.isNaN(count) ? "error" : count})`
|
|
24735
|
+
};
|
|
24736
|
+
}
|
|
24737
|
+
/** What the validator records against an action it never replayed. */
|
|
24738
|
+
const CASCADE_REASON = "skipped after a preceding action failed";
|
|
24739
|
+
/** Whether a drop reason means "not replayed", rather than "replayed and failed". */
|
|
24740
|
+
function isCascadeReason(reason) {
|
|
24741
|
+
return reason === CASCADE_REASON;
|
|
24742
|
+
}
|
|
24743
|
+
const SHORT_TIMEOUT_MS = 5e3;
|
|
24744
|
+
const ASSERT_TIMEOUT_MS = 1e4;
|
|
24745
|
+
const INTERACTION_POLL_INTERVAL_MS = 250;
|
|
24746
|
+
const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
|
|
24747
|
+
/** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
|
|
24748
|
+
function attemptsFor(timeoutMs, intervalMs) {
|
|
24749
|
+
return Math.max(1, Math.ceil(timeoutMs / intervalMs));
|
|
24750
|
+
}
|
|
24751
|
+
/**
|
|
24752
|
+
* Convert one recorded action into the `agent-browser` arg list that would
|
|
24753
|
+
* exercise it. Interaction commands come from the shared
|
|
24754
|
+
* `ir/to-agent-browser.ts` mapping with env refs resolved; `wait` / `assert`
|
|
24755
|
+
* get validation-specific handling (poll checks, timeouts, unverifiable
|
|
24756
|
+
* skips). Returns `null` for actions that should not be validated (snapshot
|
|
24757
|
+
* has no side effect; assert types whose codegen forms aren't directly
|
|
24758
|
+
* verifiable here fall through to the caller's `unverifiable` fallback).
|
|
24759
|
+
*/
|
|
24760
|
+
function actionToAbArgs(action, sessionName, envOverrides = {}) {
|
|
24761
|
+
const base = ["--session", sessionName];
|
|
24762
|
+
const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
|
|
24763
|
+
switch (action.action) {
|
|
24764
|
+
case "snapshot": return null;
|
|
24765
|
+
case "assert": return assertToAbArgs(action, sub, sessionName);
|
|
24766
|
+
case "wait": {
|
|
24767
|
+
const loc = action.locator;
|
|
24768
|
+
if (!loc) return null;
|
|
24769
|
+
if (loc.by === "text") return [
|
|
24770
|
+
...base,
|
|
24771
|
+
"wait",
|
|
24772
|
+
"--text",
|
|
24773
|
+
sub(loc.value),
|
|
24774
|
+
"--timeout",
|
|
24775
|
+
String(SHORT_TIMEOUT_MS)
|
|
24776
|
+
];
|
|
24777
|
+
const raw = sub(locatorToSelector(loc));
|
|
24778
|
+
if (!raw) return null;
|
|
24779
|
+
if (/^\d+$/.test(raw)) return null;
|
|
24780
|
+
if (raw.startsWith("--")) return null;
|
|
24781
|
+
if (raw.startsWith("text=")) return [
|
|
24782
|
+
...base,
|
|
24783
|
+
"wait",
|
|
24784
|
+
"--text",
|
|
24785
|
+
raw.slice(5),
|
|
24786
|
+
"--timeout",
|
|
24787
|
+
String(SHORT_TIMEOUT_MS)
|
|
24788
|
+
];
|
|
24789
|
+
return {
|
|
24790
|
+
kind: "poll-present",
|
|
24791
|
+
selector: raw,
|
|
24792
|
+
timeoutMs: SHORT_TIMEOUT_MS
|
|
24793
|
+
};
|
|
24794
|
+
}
|
|
24795
|
+
default: {
|
|
24796
|
+
const tokens = toAgentBrowserArgs(action);
|
|
24797
|
+
if (tokens === null) return null;
|
|
24798
|
+
const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
|
|
24799
|
+
return [...base, ...tokens.map((t) => resolve(t.text))];
|
|
24800
|
+
}
|
|
24801
|
+
}
|
|
24802
|
+
}
|
|
24803
|
+
function roleProbe(base, role, name) {
|
|
24804
|
+
return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
|
|
24805
|
+
}
|
|
24806
|
+
/**
|
|
24807
|
+
* Presence asked the way the locator addresses the element. `null` is
|
|
24808
|
+
* unverifiable rather than absent: a count from a selector the engine never
|
|
24809
|
+
* parsed is not evidence.
|
|
24810
|
+
*/
|
|
24811
|
+
function presenceCheck(locator, selector, sub, base) {
|
|
24812
|
+
if (locator?.by === "text") return [
|
|
24813
|
+
...base,
|
|
24814
|
+
"wait",
|
|
24815
|
+
"--text",
|
|
24816
|
+
sub(locator.value),
|
|
24817
|
+
"--timeout",
|
|
24818
|
+
String(ASSERT_TIMEOUT_MS)
|
|
24819
|
+
];
|
|
24820
|
+
if (locator?.by === "role" && locator.name) return roleProbe(base, sub(locator.value), sub(locator.name));
|
|
24821
|
+
if (locator !== void 0 && locator.by !== "css") return null;
|
|
24822
|
+
const named = parseNotation(selector);
|
|
24823
|
+
if (named === "opaque") return null;
|
|
24824
|
+
if (named !== null && named.by === "role" && named.name) return roleProbe(base, sub(named.value), sub(named.name));
|
|
24825
|
+
if (named !== null) return null;
|
|
24826
|
+
return {
|
|
24827
|
+
kind: "poll-present",
|
|
24828
|
+
selector,
|
|
24829
|
+
timeoutMs: ASSERT_TIMEOUT_MS
|
|
24830
|
+
};
|
|
24831
|
+
}
|
|
24832
|
+
function assertToAbArgs(action, sub, sessionName) {
|
|
24833
|
+
const base = ["--session", sessionName];
|
|
24834
|
+
const val = sub(action.value ?? action.observation);
|
|
24835
|
+
const sel = sub(action.locator ? locatorToSelector(action.locator) : action.observation);
|
|
24836
|
+
switch (action.assert) {
|
|
24837
|
+
case "text_visible":
|
|
24838
|
+
if (!val) return null;
|
|
24839
|
+
return [
|
|
24840
|
+
...base,
|
|
24841
|
+
"wait",
|
|
24842
|
+
"--text",
|
|
24843
|
+
val,
|
|
24844
|
+
"--timeout",
|
|
24845
|
+
String(ASSERT_TIMEOUT_MS)
|
|
24846
|
+
];
|
|
24847
|
+
case "text_not_visible": return null;
|
|
24848
|
+
case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
|
|
24849
|
+
case "element_not_visible": return null;
|
|
24850
|
+
case "url_contains": return null;
|
|
24851
|
+
case "element_enabled":
|
|
24852
|
+
case "element_disabled":
|
|
24853
|
+
case "element_checked":
|
|
24854
|
+
case "element_unchecked":
|
|
24855
|
+
if (!sel || action.locator?.by !== "css") return null;
|
|
24856
|
+
if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
|
|
24857
|
+
return {
|
|
24858
|
+
kind: "state",
|
|
24859
|
+
state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
|
|
24860
|
+
selector: sel,
|
|
24861
|
+
expected: action.assert === "element_enabled" || action.assert === "element_checked"
|
|
24862
|
+
};
|
|
24863
|
+
default: return null;
|
|
24864
|
+
}
|
|
24991
24865
|
}
|
|
24992
|
-
|
|
24993
|
-
|
|
24866
|
+
const NO_STEP_ID = "__no_step__";
|
|
24867
|
+
/**
|
|
24868
|
+
* The role a labelled element has when addressing it by label finds nothing.
|
|
24869
|
+
*
|
|
24870
|
+
* `getByLabel` needs a real association — a `<label for>`, a wrapping label,
|
|
24871
|
+
* an `aria-labelledby`. A form that has none still shows the same string as
|
|
24872
|
+
* the field's accessible name, which is what the recorder read off the
|
|
24873
|
+
* snapshot, so the label it recorded matches nothing while the same text as a
|
|
24874
|
+
* role's name matches exactly. Only the actions whose element has one
|
|
24875
|
+
* unambiguous role are listed: a labelled `click` may be a button, a link or
|
|
24876
|
+
* a checkbox, and guessing there would replace a locator that failed with one
|
|
24877
|
+
* that finds the wrong element.
|
|
24878
|
+
*/
|
|
24879
|
+
const LABEL_FALLBACK_ROLE = {
|
|
24880
|
+
fill: "textbox",
|
|
24881
|
+
type: "textbox",
|
|
24882
|
+
check: "checkbox",
|
|
24883
|
+
uncheck: "checkbox"
|
|
24884
|
+
};
|
|
24885
|
+
/** Whether agent-browser's failure was "no element", the one the fallback answers. */
|
|
24886
|
+
function notFound(result) {
|
|
24887
|
+
return /not\s+found|no\s+element|no\s+such\s+element|none\s+match\s+name/i.test(`${result.stderr} ${result.stdout}`);
|
|
24888
|
+
}
|
|
24889
|
+
/** Whether this argv navigates — the only action safe to repeat wholesale. */
|
|
24890
|
+
function isOpen(argv) {
|
|
24891
|
+
return argv[2] === "open";
|
|
24892
|
+
}
|
|
24893
|
+
/** A navigation another navigation took away, rather than one that failed. */
|
|
24894
|
+
function superseded(result) {
|
|
24895
|
+
return /ERR_ABORTED|navigation (?:was )?(?:interrupted|superseded|cancell?ed)/i.test(`${result.stderr} ${result.stdout}`);
|
|
24896
|
+
}
|
|
24897
|
+
/** The one line both callers log when the fallback rewrote a locator. */
|
|
24898
|
+
function formatPromotion(promotion) {
|
|
24899
|
+
return `locator rewritten to the form that replays: ${promotion}`;
|
|
24900
|
+
}
|
|
24901
|
+
/** The same action addressed by role + accessible name, or null when there is no such form. */
|
|
24902
|
+
function labelFallback(action) {
|
|
24903
|
+
const loc = action.locator;
|
|
24904
|
+
if (!loc || loc.by !== "label") return null;
|
|
24905
|
+
const role = LABEL_FALLBACK_ROLE[action.action];
|
|
24906
|
+
if (role === void 0) return null;
|
|
24907
|
+
return {
|
|
24908
|
+
...action,
|
|
24909
|
+
locator: {
|
|
24910
|
+
by: "role",
|
|
24911
|
+
value: role,
|
|
24912
|
+
name: loc.value,
|
|
24913
|
+
exact: true
|
|
24914
|
+
}
|
|
24915
|
+
};
|
|
24994
24916
|
}
|
|
24995
24917
|
/**
|
|
24996
|
-
*
|
|
24997
|
-
*
|
|
24998
|
-
*
|
|
24999
|
-
*
|
|
25000
|
-
*
|
|
24918
|
+
* Spawn one interaction, waiting for its element the way an assertion waits
|
|
24919
|
+
* for its text: a recording pauses for a snapshot between opening a page and
|
|
24920
|
+
* typing into it, the replay does not, so the replay reaches a field sooner
|
|
24921
|
+
* than the recording ever did.
|
|
24922
|
+
*
|
|
24923
|
+
* Only a "no element" failure waits — anything else repeats identically. When
|
|
24924
|
+
* there is an `alternate` form it is tried on every attempt rather than after
|
|
24925
|
+
* the wait: a label the page never associates with its input will not start
|
|
24926
|
+
* matching, so waiting first spends the budget for nothing.
|
|
25001
24927
|
*/
|
|
25002
|
-
function
|
|
25003
|
-
|
|
25004
|
-
|
|
25005
|
-
|
|
25006
|
-
|
|
25007
|
-
|
|
25008
|
-
|
|
25009
|
-
|
|
25010
|
-
|
|
25011
|
-
|
|
25012
|
-
|
|
24928
|
+
function runInteraction(argv, alternate, patience, sessionName) {
|
|
24929
|
+
const started = Date.now();
|
|
24930
|
+
const deadline = started + ASSERT_TIMEOUT_MS;
|
|
24931
|
+
let settled = false;
|
|
24932
|
+
for (let attempt = 0;; attempt++) {
|
|
24933
|
+
let result = spawnAB(argv);
|
|
24934
|
+
if (result.status !== 0 && result.wedged === true) result = spawnAB(argv);
|
|
24935
|
+
const waitedMs = Date.now() - started;
|
|
24936
|
+
if (result.status !== 0 && superseded(result) && !settled && isOpen(argv)) {
|
|
24937
|
+
settled = true;
|
|
24938
|
+
spawnAB([
|
|
24939
|
+
"--session",
|
|
24940
|
+
sessionName,
|
|
24941
|
+
"wait",
|
|
24942
|
+
"--load",
|
|
24943
|
+
"networkidle"
|
|
24944
|
+
]);
|
|
24945
|
+
continue;
|
|
24946
|
+
}
|
|
24947
|
+
if (result.status === 0 || !notFound(result)) return {
|
|
24948
|
+
result,
|
|
24949
|
+
viaAlternate: false,
|
|
24950
|
+
waitedMs
|
|
24951
|
+
};
|
|
24952
|
+
if (alternate !== null) {
|
|
24953
|
+
const alt = spawnAB(alternate);
|
|
24954
|
+
if (alt.status === 0) return {
|
|
24955
|
+
result: alt,
|
|
24956
|
+
viaAlternate: true,
|
|
24957
|
+
waitedMs
|
|
24958
|
+
};
|
|
24959
|
+
}
|
|
24960
|
+
if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
|
|
24961
|
+
patience.spent = true;
|
|
24962
|
+
return {
|
|
24963
|
+
result,
|
|
24964
|
+
viaAlternate: false,
|
|
24965
|
+
waitedMs: Date.now() - started
|
|
24966
|
+
};
|
|
24967
|
+
}
|
|
24968
|
+
sleepSync(INTERACTION_POLL_INTERVAL_MS);
|
|
24969
|
+
}
|
|
25013
24970
|
}
|
|
25014
24971
|
/**
|
|
25015
|
-
*
|
|
25016
|
-
*
|
|
25017
|
-
*
|
|
25018
|
-
*
|
|
24972
|
+
* Say what a `by: "css"` locator that is not CSS actually addresses.
|
|
24973
|
+
*
|
|
24974
|
+
* `get count` answers 0 for Playwright's notation instead of failing, so a
|
|
24975
|
+
* route holding one reads as dead. Rewritten in place, and reported, so the
|
|
24976
|
+
* saved route carries the form that replays rather than the one that counts
|
|
24977
|
+
* nothing. Notation this cannot convert leaves the locator alone; the assert
|
|
24978
|
+
* path then treats it as unverifiable rather than absent.
|
|
25019
24979
|
*/
|
|
25020
|
-
function
|
|
25021
|
-
|
|
25022
|
-
return
|
|
25023
|
-
|
|
25024
|
-
|
|
25025
|
-
|
|
25026
|
-
|
|
25027
|
-
${expectations.map((text) => `- ${text}`).join("\n")}
|
|
25028
|
-
|
|
25029
|
-
**Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
|
|
25030
|
-
`;
|
|
24980
|
+
function nameNotation(action) {
|
|
24981
|
+
const loc = action.locator;
|
|
24982
|
+
if (action.action !== "assert" || loc?.by !== "css") return null;
|
|
24983
|
+
const named = parseNotation(loc.value);
|
|
24984
|
+
if (named === null || named === "opaque" || named.by !== "text") return null;
|
|
24985
|
+
action.locator = named;
|
|
24986
|
+
return `${loc.value} is not a CSS selector — asked as ${describeLocator(named)}`;
|
|
25031
24987
|
}
|
|
25032
24988
|
/**
|
|
25033
|
-
*
|
|
25034
|
-
*
|
|
25035
|
-
*
|
|
24989
|
+
* An attribute selector that counted nothing, asked of the accessibility tree
|
|
24990
|
+
* the name was read from. See docs/targets.md for why the two disagree.
|
|
24991
|
+
*
|
|
24992
|
+
* The snapshot is read for the role, then `find role` is asked as well: the
|
|
24993
|
+
* tree says the name exists somewhere, the probe says the replay can reach it,
|
|
24994
|
+
* and only the second is what the route will depend on.
|
|
25036
24995
|
*/
|
|
25037
|
-
function
|
|
25038
|
-
if (
|
|
25039
|
-
|
|
25040
|
-
|
|
25041
|
-
|
|
25042
|
-
|
|
25043
|
-
|
|
24996
|
+
function askTheAccessibilityTree(action, sub, sessionName) {
|
|
24997
|
+
if (action.action !== "assert" || action.locator?.by !== "css") return null;
|
|
24998
|
+
const name = nameFromAttributeSelector(action.locator.value);
|
|
24999
|
+
if (name === null) return null;
|
|
25000
|
+
const snapshot = spawnAB([
|
|
25001
|
+
"--session",
|
|
25002
|
+
sessionName,
|
|
25003
|
+
"snapshot"
|
|
25004
|
+
]);
|
|
25005
|
+
if (snapshot.status !== 0) return null;
|
|
25006
|
+
const role = roleOfAccessibleName(snapshot.stdout, sub(name));
|
|
25007
|
+
if (role === null) return null;
|
|
25008
|
+
if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
|
|
25009
|
+
const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
|
|
25010
|
+
action.locator = {
|
|
25011
|
+
by: "role",
|
|
25012
|
+
value: role,
|
|
25013
|
+
name,
|
|
25014
|
+
exact: true
|
|
25015
|
+
};
|
|
25016
|
+
return {
|
|
25017
|
+
skipped: false,
|
|
25018
|
+
ok: true,
|
|
25019
|
+
reason: "",
|
|
25020
|
+
promoted
|
|
25021
|
+
};
|
|
25022
|
+
}
|
|
25023
|
+
/**
|
|
25024
|
+
* Replay one recorded action against the validation session. Element-presence
|
|
25025
|
+
* checks go through `runPollCheck` (which uses `get count`, never the blocking
|
|
25026
|
+
* `wait <selector>`); everything else spawns the agent-browser argv.
|
|
25027
|
+
*/
|
|
25028
|
+
function runValidationAction(action, sessionName, envOverrides = {}, patience) {
|
|
25029
|
+
const named = nameNotation(action);
|
|
25030
|
+
const withName = (outcome) => named === null || !outcome.ok ? outcome : {
|
|
25031
|
+
...outcome,
|
|
25032
|
+
promoted: outcome.promoted ?? named
|
|
25033
|
+
};
|
|
25034
|
+
const built = actionToAbArgs(action, sessionName, envOverrides);
|
|
25035
|
+
if (built === null) return {
|
|
25036
|
+
skipped: true,
|
|
25037
|
+
ok: false,
|
|
25038
|
+
reason: ""
|
|
25039
|
+
};
|
|
25040
|
+
if (isProbe(built)) {
|
|
25041
|
+
const { ok, reason } = runProbe(built, sessionName);
|
|
25042
|
+
if (ok) return withName({
|
|
25043
|
+
skipped: false,
|
|
25044
|
+
ok,
|
|
25045
|
+
reason
|
|
25046
|
+
});
|
|
25047
|
+
const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
|
|
25048
|
+
return askTheAccessibilityTree(action, resolve, sessionName) ?? {
|
|
25049
|
+
skipped: false,
|
|
25050
|
+
ok,
|
|
25051
|
+
reason
|
|
25052
|
+
};
|
|
25053
|
+
}
|
|
25054
|
+
const fallback = labelFallback(action);
|
|
25055
|
+
const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
|
|
25056
|
+
const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
|
|
25057
|
+
if (viaAlternate) {
|
|
25058
|
+
const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
|
|
25059
|
+
action.locator = fallback.locator;
|
|
25060
|
+
return {
|
|
25061
|
+
skipped: false,
|
|
25062
|
+
ok: true,
|
|
25063
|
+
reason: "",
|
|
25064
|
+
promoted
|
|
25065
|
+
};
|
|
25066
|
+
}
|
|
25067
|
+
if (result.status === 0) return withName({
|
|
25068
|
+
skipped: false,
|
|
25069
|
+
ok: true,
|
|
25070
|
+
reason: ""
|
|
25071
|
+
});
|
|
25072
|
+
return {
|
|
25073
|
+
skipped: false,
|
|
25074
|
+
ok: false,
|
|
25075
|
+
reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
|
|
25076
|
+
};
|
|
25044
25077
|
}
|
|
25045
25078
|
/**
|
|
25046
|
-
*
|
|
25047
|
-
*
|
|
25048
|
-
*
|
|
25049
|
-
* the
|
|
25079
|
+
* Replay actions in order against a live session and stop at the first that
|
|
25080
|
+
* fails. Unlike `validateActions` there is no cascade or rescue: the caller
|
|
25081
|
+
* hands what is left to someone who can look at the page. A wait this cannot
|
|
25082
|
+
* reproduce still settles the page, since the next action may depend on it;
|
|
25083
|
+
* a check it cannot run is returned as `unchecked`, never as passed.
|
|
25050
25084
|
*/
|
|
25051
|
-
function
|
|
25052
|
-
|
|
25053
|
-
|
|
25054
|
-
|
|
25055
|
-
|
|
25056
|
-
|
|
25057
|
-
|
|
25085
|
+
function replayUntilFailure(actions, opts) {
|
|
25086
|
+
const patience = { spent: false };
|
|
25087
|
+
const passed = [];
|
|
25088
|
+
const unchecked = [];
|
|
25089
|
+
for (const action of actions) {
|
|
25090
|
+
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25091
|
+
if (outcome.skipped && action.action === "wait") spawnAB([
|
|
25092
|
+
"--session",
|
|
25093
|
+
opts.sessionName,
|
|
25094
|
+
"wait",
|
|
25095
|
+
"--load",
|
|
25096
|
+
"networkidle"
|
|
25097
|
+
]);
|
|
25098
|
+
else if (outcome.skipped && action.action === "assert") {
|
|
25099
|
+
unchecked.push(action);
|
|
25100
|
+
continue;
|
|
25101
|
+
} else if (outcome.skipped) return {
|
|
25102
|
+
passed,
|
|
25103
|
+
unchecked,
|
|
25104
|
+
failed: {
|
|
25105
|
+
action,
|
|
25106
|
+
reason: "the recording does not say how to replay it"
|
|
25107
|
+
}
|
|
25108
|
+
};
|
|
25109
|
+
else if (!outcome.ok) return {
|
|
25110
|
+
passed,
|
|
25111
|
+
unchecked,
|
|
25112
|
+
failed: {
|
|
25113
|
+
action,
|
|
25114
|
+
reason: outcome.reason
|
|
25115
|
+
}
|
|
25116
|
+
};
|
|
25117
|
+
passed.push(action);
|
|
25118
|
+
}
|
|
25119
|
+
return {
|
|
25120
|
+
passed,
|
|
25121
|
+
unchecked
|
|
25122
|
+
};
|
|
25058
25123
|
}
|
|
25059
|
-
|
|
25060
|
-
|
|
25061
|
-
|
|
25062
|
-
|
|
25063
|
-
|
|
25064
|
-
|
|
25065
|
-
|
|
25066
|
-
|
|
25067
|
-
|
|
25068
|
-
|
|
25069
|
-
|
|
25070
|
-
|
|
25071
|
-
|
|
25072
|
-
|
|
25073
|
-
|
|
25074
|
-
|
|
25075
|
-
|
|
25076
|
-
|
|
25077
|
-
|
|
25124
|
+
function validateActions(actions, opts) {
|
|
25125
|
+
const kept = [];
|
|
25126
|
+
const dropped = [];
|
|
25127
|
+
const promoted = [];
|
|
25128
|
+
const patience = { spent: false };
|
|
25129
|
+
let skipFromStepId = null;
|
|
25130
|
+
for (let i = 0; i < actions.length; i++) {
|
|
25131
|
+
const action = actions[i];
|
|
25132
|
+
opts.onProgress?.(i, actions.length, action);
|
|
25133
|
+
const stepId = action.stepId ?? NO_STEP_ID;
|
|
25134
|
+
if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
|
|
25135
|
+
if (skipFromStepId !== null && isPassiveAction(action.action)) {
|
|
25136
|
+
dropped.push({
|
|
25137
|
+
index: i,
|
|
25138
|
+
action,
|
|
25139
|
+
reason: CASCADE_REASON
|
|
25140
|
+
});
|
|
25141
|
+
continue;
|
|
25142
|
+
}
|
|
25143
|
+
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25144
|
+
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25145
|
+
if (outcome.skipped) {
|
|
25146
|
+
kept.push(action);
|
|
25147
|
+
continue;
|
|
25148
|
+
}
|
|
25149
|
+
if (outcome.ok) {
|
|
25150
|
+
kept.push(action);
|
|
25151
|
+
if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
|
|
25152
|
+
continue;
|
|
25153
|
+
}
|
|
25154
|
+
dropped.push({
|
|
25155
|
+
index: i,
|
|
25156
|
+
action,
|
|
25157
|
+
reason: outcome.reason
|
|
25158
|
+
});
|
|
25159
|
+
if (!isPassiveAction(action.action)) skipFromStepId = stepId;
|
|
25160
|
+
}
|
|
25161
|
+
const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
|
|
25162
|
+
promoted.push(...afterRescue.promoted ?? []);
|
|
25163
|
+
return {
|
|
25164
|
+
...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
|
|
25165
|
+
promoted
|
|
25166
|
+
};
|
|
25078
25167
|
}
|
|
25079
25168
|
/**
|
|
25080
|
-
*
|
|
25081
|
-
*
|
|
25082
|
-
*
|
|
25083
|
-
*
|
|
25084
|
-
*
|
|
25085
|
-
* the one that was asked, and refused when two roles carry it.
|
|
25169
|
+
* Translate the internal `{ kept, dropped }` result of the rescue pass
|
|
25170
|
+
* into the public-facing shape. In strict mode the caller sees the same
|
|
25171
|
+
* shape as before (kept/dropped); in lenient mode the still-failed
|
|
25172
|
+
* actions move to `unstable` with `replayUnstable: true` tagged on, so
|
|
25173
|
+
* codegen can warn about them while still emitting the line.
|
|
25086
25174
|
*/
|
|
25087
|
-
function
|
|
25088
|
-
|
|
25089
|
-
|
|
25090
|
-
|
|
25091
|
-
|
|
25175
|
+
function splitByMode(originalActions, result, mode) {
|
|
25176
|
+
if (mode === "strict") return {
|
|
25177
|
+
kept: result.kept,
|
|
25178
|
+
unstable: [],
|
|
25179
|
+
dropped: result.dropped,
|
|
25180
|
+
rescuedSteps: result.rescuedSteps
|
|
25181
|
+
};
|
|
25182
|
+
const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
|
|
25183
|
+
const keptSet = new Set(result.kept);
|
|
25184
|
+
const finalKept = [];
|
|
25185
|
+
const unstable = [];
|
|
25186
|
+
for (let i = 0; i < originalActions.length; i++) {
|
|
25187
|
+
const action = originalActions[i];
|
|
25188
|
+
if (keptSet.has(action)) {
|
|
25189
|
+
finalKept.push(action);
|
|
25190
|
+
continue;
|
|
25191
|
+
}
|
|
25192
|
+
const drop = droppedByIndex.get(i);
|
|
25193
|
+
if (drop) {
|
|
25194
|
+
action.replayUnstable = true;
|
|
25195
|
+
action.replayReason = drop.reason;
|
|
25196
|
+
unstable.push(action);
|
|
25197
|
+
}
|
|
25092
25198
|
}
|
|
25093
|
-
return
|
|
25094
|
-
|
|
25095
|
-
|
|
25096
|
-
|
|
25097
|
-
|
|
25098
|
-
|
|
25099
|
-
}
|
|
25100
|
-
function runProbe(probe, sessionName) {
|
|
25101
|
-
return probe.kind === "state" ? runStateCheck(probe, sessionName) : runPollCheck(probe, sessionName);
|
|
25199
|
+
return {
|
|
25200
|
+
kept: finalKept,
|
|
25201
|
+
unstable,
|
|
25202
|
+
dropped: [],
|
|
25203
|
+
rescuedSteps: result.rescuedSteps
|
|
25204
|
+
};
|
|
25102
25205
|
}
|
|
25103
|
-
function
|
|
25104
|
-
|
|
25105
|
-
const
|
|
25106
|
-
|
|
25107
|
-
|
|
25108
|
-
|
|
25109
|
-
|
|
25110
|
-
|
|
25111
|
-
|
|
25112
|
-
|
|
25113
|
-
]);
|
|
25114
|
-
answer = r.status === 0 ? r.stdout.trim() : "";
|
|
25115
|
-
if (answer === String(check.expected)) return {
|
|
25116
|
-
ok: true,
|
|
25117
|
-
reason: ""
|
|
25118
|
-
};
|
|
25119
|
-
if (round + 1 < rounds) sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
25206
|
+
function rescueLostSteps(actions, kept, dropped, opts, patience) {
|
|
25207
|
+
const stepsWithSurvivors = /* @__PURE__ */ new Set();
|
|
25208
|
+
for (const a of kept) if (a.stepId) stepsWithSurvivors.add(a.stepId);
|
|
25209
|
+
const lostStepDrops = /* @__PURE__ */ new Map();
|
|
25210
|
+
for (const d of dropped) {
|
|
25211
|
+
const id = d.action.stepId;
|
|
25212
|
+
if (!id || stepsWithSurvivors.has(id)) continue;
|
|
25213
|
+
const list = lostStepDrops.get(id) ?? [];
|
|
25214
|
+
list.push(d);
|
|
25215
|
+
lostStepDrops.set(id, list);
|
|
25120
25216
|
}
|
|
25121
|
-
|
|
25122
|
-
|
|
25123
|
-
|
|
25124
|
-
} : {
|
|
25125
|
-
ok: false,
|
|
25126
|
-
reason: `is ${check.state} answered ${answer || "nothing"}`
|
|
25217
|
+
if (lostStepDrops.size === 0) return {
|
|
25218
|
+
kept,
|
|
25219
|
+
dropped
|
|
25127
25220
|
};
|
|
25128
|
-
|
|
25129
|
-
const
|
|
25130
|
-
|
|
25131
|
-
|
|
25132
|
-
const
|
|
25133
|
-
const
|
|
25134
|
-
|
|
25135
|
-
|
|
25136
|
-
|
|
25137
|
-
|
|
25138
|
-
|
|
25139
|
-
|
|
25140
|
-
|
|
25221
|
+
const promoted = [];
|
|
25222
|
+
const rescuedIndices = /* @__PURE__ */ new Set();
|
|
25223
|
+
const rescuedSteps = [];
|
|
25224
|
+
const strict = (opts.mode ?? "lenient") === "strict";
|
|
25225
|
+
for (const [stepId, drops] of lostStepDrops.entries()) {
|
|
25226
|
+
const replayed = [];
|
|
25227
|
+
const unverifiable = [];
|
|
25228
|
+
let stillFailing = false;
|
|
25229
|
+
for (const d of drops) {
|
|
25230
|
+
const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
|
|
25231
|
+
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25232
|
+
if (outcome.skipped) {
|
|
25233
|
+
unverifiable.push(d.index);
|
|
25234
|
+
continue;
|
|
25235
|
+
}
|
|
25236
|
+
if (outcome.ok) {
|
|
25237
|
+
replayed.push(d.index);
|
|
25238
|
+
continue;
|
|
25239
|
+
}
|
|
25240
|
+
stillFailing = true;
|
|
25241
|
+
if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
|
|
25242
|
+
}
|
|
25243
|
+
if (replayed.length === 0) continue;
|
|
25244
|
+
for (const i of replayed) rescuedIndices.add(i);
|
|
25245
|
+
if (!stillFailing) for (const i of unverifiable) rescuedIndices.add(i);
|
|
25246
|
+
rescuedSteps.push(stepId);
|
|
25247
|
+
}
|
|
25248
|
+
if (rescuedIndices.size === 0) return {
|
|
25249
|
+
kept,
|
|
25250
|
+
dropped,
|
|
25251
|
+
promoted
|
|
25141
25252
|
};
|
|
25142
|
-
const
|
|
25143
|
-
const
|
|
25144
|
-
let
|
|
25145
|
-
|
|
25146
|
-
|
|
25147
|
-
if (!Number.isNaN(count) && count > 0) return {
|
|
25148
|
-
ok: true,
|
|
25149
|
-
reason: ""
|
|
25150
|
-
};
|
|
25151
|
-
if (round + 1 >= rounds || Date.now() >= deadline) break;
|
|
25152
|
-
sleepSync(SELECTOR_POLL_INTERVAL_MS);
|
|
25253
|
+
const keptSet = new Set(kept);
|
|
25254
|
+
const newKept = [];
|
|
25255
|
+
for (let i = 0; i < actions.length; i++) {
|
|
25256
|
+
const action = actions[i];
|
|
25257
|
+
if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
|
|
25153
25258
|
}
|
|
25154
25259
|
return {
|
|
25155
|
-
|
|
25156
|
-
|
|
25260
|
+
kept: newKept,
|
|
25261
|
+
dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
|
|
25262
|
+
rescuedSteps,
|
|
25263
|
+
promoted
|
|
25157
25264
|
};
|
|
25158
25265
|
}
|
|
25159
|
-
/** What the validator records against an action it never replayed. */
|
|
25160
|
-
const CASCADE_REASON = "skipped after a preceding action failed";
|
|
25161
|
-
/** Whether a drop reason means "not replayed", rather than "replayed and failed". */
|
|
25162
|
-
function isCascadeReason(reason) {
|
|
25163
|
-
return reason === CASCADE_REASON;
|
|
25164
|
-
}
|
|
25165
|
-
const SHORT_TIMEOUT_MS = 5e3;
|
|
25166
|
-
const ASSERT_TIMEOUT_MS = 1e4;
|
|
25167
|
-
const INTERACTION_POLL_INTERVAL_MS = 250;
|
|
25168
|
-
const INTERACTION_ATTEMPTS = attemptsFor(ASSERT_TIMEOUT_MS, INTERACTION_POLL_INTERVAL_MS);
|
|
25169
|
-
/** Bounds a poll when `sleepSync` is stubbed and the clock never moves. */
|
|
25170
|
-
function attemptsFor(timeoutMs, intervalMs) {
|
|
25171
|
-
return Math.max(1, Math.ceil(timeoutMs / intervalMs));
|
|
25172
|
-
}
|
|
25173
25266
|
/**
|
|
25174
|
-
*
|
|
25175
|
-
*
|
|
25176
|
-
*
|
|
25177
|
-
* get validation-specific handling (poll checks, timeouts, unverifiable
|
|
25178
|
-
* skips). Returns `null` for actions that should not be validated (snapshot
|
|
25179
|
-
* has no side effect; assert types whose codegen forms aren't directly
|
|
25180
|
-
* verifiable here fall through to the caller's `unverifiable` fallback).
|
|
25267
|
+
* Passive (read-only) actions whose only effect is observation. When a
|
|
25268
|
+
* preceding action fails, dropping these too is the right move because
|
|
25269
|
+
* they were trying to observe state the failed action would have set up.
|
|
25181
25270
|
*/
|
|
25182
|
-
function
|
|
25183
|
-
|
|
25184
|
-
const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
|
|
25185
|
-
switch (action.action) {
|
|
25186
|
-
case "snapshot": return null;
|
|
25187
|
-
case "assert": return assertToAbArgs(action, sub, sessionName);
|
|
25188
|
-
case "wait": {
|
|
25189
|
-
const loc = action.locator;
|
|
25190
|
-
if (!loc) return null;
|
|
25191
|
-
if (loc.by === "text") return [
|
|
25192
|
-
...base,
|
|
25193
|
-
"wait",
|
|
25194
|
-
"--text",
|
|
25195
|
-
sub(loc.value),
|
|
25196
|
-
"--timeout",
|
|
25197
|
-
String(SHORT_TIMEOUT_MS)
|
|
25198
|
-
];
|
|
25199
|
-
const raw = sub(locatorToSelector(loc));
|
|
25200
|
-
if (!raw) return null;
|
|
25201
|
-
if (/^\d+$/.test(raw)) return null;
|
|
25202
|
-
if (raw.startsWith("--")) return null;
|
|
25203
|
-
if (raw.startsWith("text=")) return [
|
|
25204
|
-
...base,
|
|
25205
|
-
"wait",
|
|
25206
|
-
"--text",
|
|
25207
|
-
raw.slice(5),
|
|
25208
|
-
"--timeout",
|
|
25209
|
-
String(SHORT_TIMEOUT_MS)
|
|
25210
|
-
];
|
|
25211
|
-
return {
|
|
25212
|
-
kind: "poll-present",
|
|
25213
|
-
selector: raw,
|
|
25214
|
-
timeoutMs: SHORT_TIMEOUT_MS
|
|
25215
|
-
};
|
|
25216
|
-
}
|
|
25217
|
-
default: {
|
|
25218
|
-
const tokens = toAgentBrowserArgs(action);
|
|
25219
|
-
if (tokens === null) return null;
|
|
25220
|
-
const resolve = action.action === "navigate" ? (text) => collapseUrlPath(sub(text)) : sub;
|
|
25221
|
-
return [...base, ...tokens.map((t) => resolve(t.text))];
|
|
25222
|
-
}
|
|
25223
|
-
}
|
|
25224
|
-
}
|
|
25225
|
-
function roleProbe(base, role, name) {
|
|
25226
|
-
return [...base, ...roleProbeTokens(role, name).map((t) => t.text)];
|
|
25271
|
+
function isPassiveAction(action) {
|
|
25272
|
+
return action === "snapshot" || action === "wait" || action === "assert";
|
|
25227
25273
|
}
|
|
25274
|
+
//#endregion
|
|
25275
|
+
//#region src/prompts/previous-recording.ts
|
|
25228
25276
|
/**
|
|
25229
|
-
*
|
|
25230
|
-
*
|
|
25231
|
-
* parsed is not evidence.
|
|
25277
|
+
* The previous recording as the commands that perform it, for the trace prompt:
|
|
25278
|
+
* a re-record is usually one moved screen in an otherwise intact route.
|
|
25232
25279
|
*/
|
|
25233
|
-
function
|
|
25234
|
-
|
|
25235
|
-
|
|
25236
|
-
|
|
25237
|
-
|
|
25238
|
-
|
|
25239
|
-
|
|
25240
|
-
|
|
25241
|
-
|
|
25242
|
-
|
|
25243
|
-
|
|
25244
|
-
|
|
25245
|
-
if (
|
|
25246
|
-
|
|
25247
|
-
|
|
25248
|
-
|
|
25249
|
-
|
|
25250
|
-
|
|
25251
|
-
|
|
25252
|
-
|
|
25280
|
+
function renderPreviousRecording(actions, sessionName, currentStepIds) {
|
|
25281
|
+
const current = new Set([SETUP_STEP_ID, ...currentStepIds]);
|
|
25282
|
+
const brokenStep = actions.find(failedToReplay)?.stepId;
|
|
25283
|
+
const byStep = /* @__PURE__ */ new Map();
|
|
25284
|
+
for (const action of actions) {
|
|
25285
|
+
if (action.stepId === void 0 || !current.has(action.stepId)) continue;
|
|
25286
|
+
const command = previousCommand(action, sessionName, action.stepId === brokenStep);
|
|
25287
|
+
if (command === null) continue;
|
|
25288
|
+
const lines = byStep.get(action.stepId) ?? [];
|
|
25289
|
+
lines.push(command);
|
|
25290
|
+
byStep.set(action.stepId, lines);
|
|
25291
|
+
}
|
|
25292
|
+
if (byStep.size === 0) return "";
|
|
25293
|
+
return `
|
|
25294
|
+
## Previous recording
|
|
25295
|
+
|
|
25296
|
+
This case was recorded before. Below is what that recording did, step by step,
|
|
25297
|
+
as the commands that perform it. The application may have moved since, and the
|
|
25298
|
+
steps above may have been edited: **the steps above are the contract, this is a
|
|
25299
|
+
map.**
|
|
25300
|
+
|
|
25301
|
+
For a step listed here, this replaces the snapshot-first routine of the
|
|
25302
|
+
Execution Workflow:
|
|
25303
|
+
|
|
25304
|
+
1. While a step's previous commands still do what its instruction and
|
|
25305
|
+
\`Expected\` say, call the \`replay_step\` tool with the step id instead of
|
|
25306
|
+
running them yourself. It runs and records them in this session, checks
|
|
25307
|
+
included, and stops at the first that fails. Do not snapshot first.
|
|
25308
|
+
2. When it reports they all ran, emit \`STEP_DONE\` and go to the next step
|
|
25309
|
+
without a confirming snapshot.
|
|
25310
|
+
3. When it reports a failure, the commands before it are recorded and the
|
|
25311
|
+
page is where they left it: finish that step with the Execution Workflow —
|
|
25312
|
+
snapshot, find what works, record it — without re-running what was
|
|
25313
|
+
recorded. A step whose instruction or \`Expected\` asks for something these
|
|
25314
|
+
commands do not do skips \`replay_step\` and uses the Execution Workflow
|
|
25315
|
+
from the start. The next step goes back to \`replay_step\`.
|
|
25316
|
+
4. A step you record yourself takes its assertions from its current
|
|
25317
|
+
\`Expected\`: keep a previous check only while it verifies what
|
|
25318
|
+
\`Expected\` says now. A step whose \`Expected\` has changed is one to
|
|
25319
|
+
record yourself, since \`replay_step\` records the previous checks as
|
|
25320
|
+
they are.
|
|
25321
|
+
5. A command marked \`# did not replay last time\` is where the previous
|
|
25322
|
+
recording's own validation found the route broken: expect that step to
|
|
25323
|
+
need the Execution Workflow, and do not keep a form that only half works.
|
|
25324
|
+
|
|
25325
|
+
A step with no section here is recorded the usual way.
|
|
25326
|
+
|
|
25327
|
+
${[...byStep].map(([stepId, lines]) => `### ${stepId}\n\n\`\`\`bash\n${lines.join("\n")}\n\`\`\``).join("\n\n")}
|
|
25328
|
+
`;
|
|
25253
25329
|
}
|
|
25254
|
-
function
|
|
25255
|
-
|
|
25256
|
-
const
|
|
25257
|
-
const
|
|
25258
|
-
|
|
25259
|
-
|
|
25260
|
-
|
|
25261
|
-
|
|
25262
|
-
|
|
25263
|
-
|
|
25264
|
-
|
|
25265
|
-
|
|
25266
|
-
|
|
25267
|
-
|
|
25268
|
-
|
|
25269
|
-
case "text_not_visible": return null;
|
|
25270
|
-
case "element_visible": return sel ? presenceCheck(action.locator, sel, sub, base) : null;
|
|
25271
|
-
case "element_not_visible": return null;
|
|
25272
|
-
case "url_contains": return null;
|
|
25273
|
-
case "element_enabled":
|
|
25274
|
-
case "element_disabled":
|
|
25275
|
-
case "element_checked":
|
|
25276
|
-
case "element_unchecked":
|
|
25277
|
-
if (!sel || action.locator?.by !== "css") return null;
|
|
25278
|
-
if (parseNotation(sel) !== null || nameFromAttributeSelector(sel) !== null) return null;
|
|
25279
|
-
return {
|
|
25280
|
-
kind: "state",
|
|
25281
|
-
state: action.assert === "element_checked" || action.assert === "element_unchecked" ? "checked" : "enabled",
|
|
25282
|
-
selector: sel,
|
|
25283
|
-
expected: action.assert === "element_enabled" || action.assert === "element_checked"
|
|
25284
|
-
};
|
|
25285
|
-
default: return null;
|
|
25286
|
-
}
|
|
25330
|
+
function previousCommand(action, sessionName, inBrokenStep) {
|
|
25331
|
+
if (action.action === "wait" && action.locator?.by !== "text") return null;
|
|
25332
|
+
const marked = action.action === "assert" ? markedAssertArgs(action) : null;
|
|
25333
|
+
const tokens = marked?.tokens ?? (action.action === "assert" ? null : toAgentBrowserArgs(action));
|
|
25334
|
+
if (tokens === null) return null;
|
|
25335
|
+
const line = [
|
|
25336
|
+
`CCQA_STEP=${action.stepId}`,
|
|
25337
|
+
...marked ? [`CCQA_ASSERT=${shellWord(marked.marker)}`] : [],
|
|
25338
|
+
...action.secret ? ["CCQA_SECRET=1"] : [],
|
|
25339
|
+
"agent-browser",
|
|
25340
|
+
"--session",
|
|
25341
|
+
sessionName,
|
|
25342
|
+
renderTokens(tokens)
|
|
25343
|
+
].join(" ");
|
|
25344
|
+
return inBrokenStep && failedToReplay(action) ? `${line} # did not replay last time` : line;
|
|
25287
25345
|
}
|
|
25288
|
-
|
|
25289
|
-
|
|
25290
|
-
|
|
25291
|
-
*
|
|
25292
|
-
* `getByLabel` needs a real association — a `<label for>`, a wrapping label,
|
|
25293
|
-
* an `aria-labelledby`. A form that has none still shows the same string as
|
|
25294
|
-
* the field's accessible name, which is what the recorder read off the
|
|
25295
|
-
* snapshot, so the label it recorded matches nothing while the same text as a
|
|
25296
|
-
* role's name matches exactly. Only the actions whose element has one
|
|
25297
|
-
* unambiguous role are listed: a labelled `click` may be a button, a link or
|
|
25298
|
-
* a checkbox, and guessing there would replace a locator that failed with one
|
|
25299
|
-
* that finds the wrong element.
|
|
25300
|
-
*/
|
|
25301
|
-
const LABEL_FALLBACK_ROLE = {
|
|
25302
|
-
fill: "textbox",
|
|
25303
|
-
type: "textbox",
|
|
25304
|
-
check: "checkbox",
|
|
25305
|
-
uncheck: "checkbox"
|
|
25306
|
-
};
|
|
25307
|
-
/** Whether agent-browser's failure was "no element", the one the fallback answers. */
|
|
25308
|
-
function notFound(result) {
|
|
25309
|
-
return /not\s+found|no\s+element|no\s+such\s+element/i.test(`${result.stderr} ${result.stdout}`);
|
|
25346
|
+
/** Failed on its own, rather than skipped behind an earlier failure. */
|
|
25347
|
+
function failedToReplay(action) {
|
|
25348
|
+
return action.replayUnstable === true && !isCascadeReason(action.replayReason);
|
|
25310
25349
|
}
|
|
25311
|
-
|
|
25312
|
-
|
|
25313
|
-
return argv[2] === "open";
|
|
25350
|
+
function renderTokens(tokens) {
|
|
25351
|
+
return tokens.map((t) => t.expandsEnv ? quote(t.text) : t.text).join(" ");
|
|
25314
25352
|
}
|
|
25315
|
-
/**
|
|
25316
|
-
function
|
|
25317
|
-
return /
|
|
25353
|
+
/** Double-quoted with `${VAR}` refs left live, as the recording keeps them; any other `$` is literal text. */
|
|
25354
|
+
function quote(s) {
|
|
25355
|
+
return `"${s.replace(/(["\\`])|\$(?!\{[A-Za-z_]\w*\})/g, (m, c) => `\\${c ?? m}`)}"`;
|
|
25318
25356
|
}
|
|
25319
|
-
|
|
25320
|
-
|
|
25321
|
-
|
|
25357
|
+
function shellWord(s) {
|
|
25358
|
+
return /^[\w:/.,=-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`;
|
|
25359
|
+
}
|
|
25360
|
+
//#endregion
|
|
25361
|
+
//#region src/prompts/trace.ts
|
|
25362
|
+
function generateSessionName() {
|
|
25363
|
+
return `ccqa-trace-${buildRunId()}`;
|
|
25364
|
+
}
|
|
25365
|
+
/**
|
|
25366
|
+
* Build the trace system prompt. `input.steps` is a flat list with includes
|
|
25367
|
+
* already expanded (each step carries id / source / instruction / expected).
|
|
25368
|
+
* The spec opens URLs via explicit step instructions (e.g.
|
|
25369
|
+
* `instruction: "${APP_URL}/articles を開く"`).
|
|
25370
|
+
*
|
|
25371
|
+
* In v0.4 every spec is traced from scratch — block contents are inlined
|
|
25372
|
+
* into the spec's own step list at expand time, so the prompt has no
|
|
25373
|
+
* special "this is a block" mode. The `source` tag on each step still
|
|
25374
|
+
* distinguishes spec-native steps from inlined block steps for the
|
|
25375
|
+
* `// step:` comments in the eventual codegen output.
|
|
25376
|
+
*/
|
|
25377
|
+
function buildTraceSystemPrompt(input) {
|
|
25378
|
+
const sessionName = input.sessionName ?? generateSessionName();
|
|
25379
|
+
const firstStepId = input.steps[0]?.id ?? "step-01";
|
|
25380
|
+
const callerGuidance = input.instruction ? `## Caller Guidance
|
|
25381
|
+
|
|
25382
|
+
The caller provided extra guidance for this recording — for example, a drift
|
|
25383
|
+
audit's finding about what a previous recording of this spec got wrong. Treat
|
|
25384
|
+
it as advice on what to avoid or verify while recording; the spec's steps
|
|
25385
|
+
above remain the contract for what to do.
|
|
25386
|
+
|
|
25387
|
+
${input.instruction}
|
|
25388
|
+
|
|
25389
|
+
` : "";
|
|
25390
|
+
const stepsText = input.steps.map((step) => [
|
|
25391
|
+
`### ${step.id} [${step.source}]`,
|
|
25392
|
+
`- **Instruction**: ${step.instruction}`,
|
|
25393
|
+
...step.expected ? [`- **Expected**: ${step.expected}`] : []
|
|
25394
|
+
].join("\n")).join("\n\n");
|
|
25395
|
+
const expectationsText = renderExpectations(input.expectations ?? []) + renderCleanupExpectations(input.cleanupExpectations ?? []);
|
|
25396
|
+
const contextText = renderContext(input.context ?? []);
|
|
25397
|
+
const conventionsText = renderConventions(input.conventions ?? []);
|
|
25398
|
+
const previousText = renderPreviousRecording(input.previousRecording ?? [], sessionName, input.steps.map((s) => s.id));
|
|
25399
|
+
return `You are an expert QA engineer executing a browser E2E test. Execute each step precisely and record every browser action as a structured log line.
|
|
25400
|
+
|
|
25401
|
+
## Session
|
|
25402
|
+
|
|
25403
|
+
SESSION NAME: \`${sessionName}\`
|
|
25404
|
+
|
|
25405
|
+
Always pass \`--session ${sessionName}\` to every \`agent-browser\` command.
|
|
25406
|
+
|
|
25407
|
+
**Always prefix every \`agent-browser\` command with a \`CCQA_STEP=<step-id>\`
|
|
25408
|
+
env assignment naming the spec step it executes**, e.g.
|
|
25409
|
+
|
|
25410
|
+
\`\`\`bash
|
|
25411
|
+
CCQA_STEP=step-03 agent-browser --session ${sessionName} click "text=Submit"
|
|
25412
|
+
\`\`\`
|
|
25413
|
+
|
|
25414
|
+
This prefix is what ties each recorded action to its step in the generated
|
|
25415
|
+
test — a command without it loses its step attribution.
|
|
25416
|
+
|
|
25417
|
+
**Anything you do before the first step carries \`CCQA_STEP=${SETUP_STEP_ID}\`.** Signing
|
|
25418
|
+
in, opening the application, putting the account where the precondition says it
|
|
25419
|
+
starts — those are not step-01, and marking them as step-01 makes the first
|
|
25420
|
+
step of the case look like it does all of it. The next \`CCQA_STEP=\` names the
|
|
25421
|
+
first step the case actually lists.
|
|
25422
|
+
|
|
25423
|
+
Other env assignments may precede it (\`FOO=x CCQA_STEP=step-02 agent-browser ...\`); agent-browser
|
|
25424
|
+
itself ignores the variable.
|
|
25425
|
+
|
|
25426
|
+
Verification commands additionally carry a \`CCQA_ASSERT=<marker>\` env
|
|
25427
|
+
assignment (alongside \`CCQA_STEP\`) that records the check as a test
|
|
25428
|
+
assertion — see "Assertion Protocol".
|
|
25429
|
+
|
|
25430
|
+
**A command that types into a password field carries \`CCQA_SECRET=1\`** as
|
|
25431
|
+
well, e.g.
|
|
25432
|
+
|
|
25433
|
+
\`\`\`bash
|
|
25434
|
+
CCQA_STEP=step-01 CCQA_SECRET=1 agent-browser --session ${sessionName} find label "Password" fill "\${LOGIN_PASSWORD}"
|
|
25435
|
+
\`\`\`
|
|
25436
|
+
|
|
25437
|
+
A password field is an \`<input type="password">\` — the snapshot shows its
|
|
25438
|
+
value as dots, and nothing else about the page is a password. The prefix is
|
|
25439
|
+
what stops the typed value from being written to the recording: a value that
|
|
25440
|
+
came from a \`\${VAR}\` is kept as that reference, and one typed literally is
|
|
25441
|
+
dropped along with its command.
|
|
25442
|
+
|
|
25443
|
+
## Browser Commands
|
|
25444
|
+
|
|
25445
|
+
\`\`\`
|
|
25446
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION open <url>
|
|
25447
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -i -c # interactive elements only — the default for finding controls
|
|
25448
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION snapshot -c -s "<css>" # the page's text, scoped to one region
|
|
25449
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION click "<selector>"
|
|
25450
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION fill "<selector>" "<value>"
|
|
25451
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION check "<selector>"
|
|
25452
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION uncheck "<selector>"
|
|
25453
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION press <Key>
|
|
25454
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION select "<selector>" "<value>"
|
|
25455
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION hover "<selector>"
|
|
25456
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION wait --text "<text>" [--timeout <ms>]
|
|
25457
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION wait --load networkidle
|
|
25458
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION get count "<selector>" # element-existence check (returns a number, fast)
|
|
25459
|
+
agent-browser --session SESSION cookies clear # pre-step preamble — no CCQA_STEP prefix
|
|
25460
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION find <locator> <value> <action> [<input>] [--name "<n>"] [--exact]
|
|
25461
|
+
CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] selector>" <file> [<file> ...]
|
|
25462
|
+
# See "Selector Rules" for the full \`find\` subset.
|
|
25463
|
+
# IMPORTANT: do NOT use \`wait "<css-selector>"\`. agent-browser ignores --timeout on a
|
|
25464
|
+
# CSS-selector wait and blocks for ~150s when the selector never matches, killing the run.
|
|
25465
|
+
# Wait for readiness with \`wait --text\`, \`wait --load networkidle\`, or just use \`find\`
|
|
25466
|
+
# (which waits internally). To check an element exists, use \`get count "<selector>"\`.
|
|
25467
|
+
\`\`\`
|
|
25468
|
+
|
|
25469
|
+
## Selector Rules
|
|
25470
|
+
|
|
25471
|
+
**ALLOWED selector formats — use ONE of these everywhere a selector appears (click, fill, wait, assert, ...):**
|
|
25472
|
+
|
|
25473
|
+
| Format | Use when |
|
|
25474
|
+
|--------|----------|
|
|
25475
|
+
| \`[aria-label='label']\` | Element **has the \`aria-label\` attribute** — **FIRST CHOICE**. A name the snapshot shows is not proof of one: a name can come from a \`<label>\`, from the element's own text, or from \`aria-labelledby\`, and this selector then matches nothing. |
|
|
25476
|
+
| \`text=visible text\` | Unique visible text, no aria-label — for **actions**. Not for a \`get count\` probe: see step 3. |
|
|
25477
|
+
| \`[placeholder='text']\` | Input identified by placeholder |
|
|
25478
|
+
| \`[type='password']\` | Password inputs only |
|
|
25479
|
+
| \`a[href*='pattern']\` | Links where \`text=\` fails — use the URL pattern from the ARIA snapshot |
|
|
25480
|
+
| \`[data-testid='...']\`, \`[data-qa='...']\` | Specific attribute selectors when an aria-label is absent |
|
|
25481
|
+
|
|
25482
|
+
**FORBIDDEN — these break recorded tests and are rejected by the hook layer:**
|
|
25483
|
+
|
|
25484
|
+
- \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
|
|
25485
|
+
- **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
|
|
25486
|
+
- \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
|
|
25487
|
+
- **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them. They do not error — \`get count\` answers \`0\` and \`find\` reports no element — so a test using one fails while looking like the page lost the element. Use \`text=...\` or plain CSS instead.
|
|
25488
|
+
- JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
|
|
25489
|
+
|
|
25490
|
+
### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
|
|
25491
|
+
|
|
25492
|
+
When repeated aria-labels / visible text make ALLOWED selectors ambiguous (e.g. a chat client where every message row has the same "1 reply" button), use one of these — they record as structured actions and replay deterministically:
|
|
25493
|
+
|
|
25494
|
+
\`\`\`
|
|
25495
|
+
find role <role> <action> [--name "<n>"] [--exact]
|
|
25496
|
+
find text|label|placeholder|alt|title "<text>" <action> [--exact]
|
|
25497
|
+
find testid "<id>" <action>
|
|
25498
|
+
find first|last "<ALLOWED-css>" <action>
|
|
25499
|
+
find nth <index> "<ALLOWED-css>" <action>
|
|
25500
|
+
\`\`\`
|
|
25501
|
+
|
|
25502
|
+
\`<action>\` is one of \`click | dblclick | fill | type | hover | focus | check | uncheck\`. For \`fill\`/\`type\`, the input value follows the action: \`find label "Email" fill "user@example.com"\`.
|
|
25503
|
+
|
|
25504
|
+
**Rules for \`find\`:**
|
|
25505
|
+
|
|
25506
|
+
1. Try ALLOWED selectors first. Only reach for \`find\` when they demonstrably cannot uniquely target the element.
|
|
25507
|
+
2. **The inner selector for \`first/last/nth\` MUST be one of the ALLOWED formats above.** Never pass a bare tag — "the last button" is meaningless on replay.
|
|
25508
|
+
3. \`find last\` is reliable only when the layout guarantees "the target is the bottom-most match" (e.g. the most-recently-sent chat message). Be explicit in the AB_ACTION label.
|
|
25509
|
+
4. Argument order is \`<value> <action> [flags]\` — flags after the action. Putting \`--name\` / \`--exact\` before the action makes agent-browser fail with "Unknown subaction".
|
|
25510
|
+
5. \`--name "<n>"\` is **role-only**. Never pass it to \`find text\`, \`find label\`, etc.
|
|
25511
|
+
6. \`--name\` matches by substring. Add \`--exact\` whenever the name you read from the snapshot is the element's whole accessible name — without it a short name such as "Log in" also matches "Log in with Google" and "Log in with SSO", and the recording silently pins the first of them.
|
|
25512
|
+
7. \`find\` includes its own wait; do not chain a \`wait\` before it.
|
|
25513
|
+
8. **A step that names an element by position gets a positional locator.** "the first row", "the topmost item", "the latest message" identify by \`first\`/\`last\`/\`nth\` on a stable inner selector — never by the title, subject, or body text that record happened to carry. That text is data the environment supplies: pinning it makes the test pass only while that same record stays in that position, and the next run opens a different one.
|
|
25514
|
+
9. **A string read off a snapshot row is an accessible name, not a label.** A row like \`textbox "Email"\` says the field's accessible name is "Email" — which \`find role textbox --name "Email" --exact\` matches and \`find label "Email"\` matches only when the page also associates a \`<label>\` with the input. Many forms do not. Record \`find label\` only when you confirmed the association (there is a \`<label for=...>\`, or the input sits inside the label); otherwise use the role form with \`--name\`.
|
|
25515
|
+
|
|
25516
|
+
**Examples:**
|
|
25517
|
+
|
|
25518
|
+
- ✓ \`find last "[data-testid='reply-link']" click\` — specific attribute + layout-guaranteed last match
|
|
25519
|
+
- ✓ \`find role button click --name "Submit"\` — role + accessible name (flags after action)
|
|
25520
|
+
- ✓ \`find role button click --name "Log in" --exact\` — short name needs \`--exact\`, or "Log in with SSO" matches too
|
|
25521
|
+
- ✗ \`find role button --name "Submit" click\` — wrong order
|
|
25522
|
+
- ✗ \`find last "button" click\` — bare tag
|
|
25523
|
+
|
|
25524
|
+
### Selector workflow
|
|
25525
|
+
|
|
25526
|
+
1. Run \`snapshot\` and read the ARIA tree.
|
|
25527
|
+
2. Identify the element; note its exact \`aria-label\` if present.
|
|
25528
|
+
3. If the element really carries \`aria-label\` → use \`[aria-label='...']\`. Otherwise → use \`text=...\` **for the action**.
|
|
25529
|
+
For a **\`get count\` probe** neither is safe: \`get count\` takes plain CSS and answers \`0\` for anything else, and an \`aria-label\` selector for a name the element does not carry as that attribute counts \`0\` too — both report an element that is plainly there as missing. Probe with \`[data-testid='...']\`, or with \`find role <role> text --name "<accessible name>" --exact\`, which asks the accessibility tree the snapshot came from.
|
|
25530
|
+
4. For links where \`text=\` fails, find the link's URL in the snapshot and use \`a[href*='...']\` with a distinctive substring.
|
|
25531
|
+
5. For checkboxes: try \`check "text=Label"\` or \`check "[aria-label='Label']"\`.
|
|
25532
|
+
6. If repeated labels make every ALLOWED selector ambiguous → use the \`find\` subset above.
|
|
25533
|
+
7. Never guess. If a selector fails, take a fresh snapshot before retrying.
|
|
25534
|
+
8. **A click that reports success but changes nothing means the element sits outside the visible part of an inner scroll container** — the event dispatches, the app never reacts, and no error is printed. Run \`scrollintoview\` on the element and click again. Scroll to the element itself, addressed by a CSS selector (\`scrollintoview "button.primary-action"\`); \`scrollintoview "text=…"\` can land on a text node that leaves the control still clipped. Do not read this as a product bug or reach for a \`@ref\` — verify by checking that the DOM changed (e.g. \`get count\` on what the click should have produced).
|
|
25535
|
+
|
|
25536
|
+
### Special input types
|
|
25537
|
+
|
|
25538
|
+
**contenteditable / RichText editors**: \`fill "[contenteditable='true']" "<text>"\` works on contenteditable elements (chat composers, WYSIWYG bodies) — agent-browser sets the text directly. Use a single \`fill\`; do NOT just \`click\` the field and rely on \`keyboard inserttext\` (that keystroke command is not recorded as a structured action, so the text never makes it into the generated test and the field ends up empty on replay).
|
|
25539
|
+
|
|
25540
|
+
**combobox / select with a required marker (\`*\`)**: required form fields often include the marker in their accessible name. If \`find role combobox click --name "<label>"\` misses, prefer \`find label "<label>" click\` or \`click "[aria-label='<label> *']"\`.
|
|
25541
|
+
|
|
25542
|
+
**Starting the cleanup**: the undo's first command must be the navigation that puts the browser where the undo can act — the same \`navigate\` you would issue if you arrived with the page freshly opened. Do not begin it from wherever the steps happened to end. The undo exists for the runs that went wrong, and on those the browser is not where the last step left it; a cleanup recorded as "click the menu that is on screen" then fails, and what the case created stays in the environment. Record the navigation even when it changes nothing at recording time.
|
|
25543
|
+
|
|
25544
|
+
**Verifying cleanup / deletion**: assert the *absence* of the deleted thing, not the surrounding listing screen's text. Wait for the deletion to settle with \`wait --fn "!document.body.innerText.includes('<unique-label>')"\`, then record the assert with a marked probe: \`CCQA_ASSERT=element_not_visible ... get count "text=<unique-label>"\` (see Assertion Protocol) — never \`wait "<css-selector>" --state hidden\` (blocks the daemon) and never \`wait --text "<navbar label>"\` (passes regardless of the deletion).
|
|
25545
|
+
|
|
25546
|
+
**File inputs (\`<input type="file">\`) / OS file-picker dialogs**: do NOT \`click\` the input — that opens the OS picker, which agent-browser cannot drive. Use \`upload "<selector>" <path>\` instead. agent-browser sets the input's files directly via the underlying browser API, no native dialog ever opens. Use an ALLOWED selector to identify the input (\`[aria-label='…']\`, \`[data-testid='…']\`, \`[type='file']\` only when it's unique on the page). File paths must be plain shell args — wrap each in \`"\` for safety. Reference fixtures via \`\${CCQA_FIXTURES_DIR}/<name>\` so the same spec works locally and in CI; conventionally fixtures live under \`.ccqa/fixtures/\` and the env var resolves there. Multi-file inputs accept several positionals: \`upload "[aria-label='Attach']" "\${CCQA_FIXTURES_DIR}/a.pdf" "\${CCQA_FIXTURES_DIR}/b.pdf"\`.
|
|
25547
|
+
|
|
25548
|
+
## Test Specification
|
|
25549
|
+
|
|
25550
|
+
Title: ${input.title}
|
|
25551
|
+
|
|
25552
|
+
Each step's instruction names the URL to open directly (or via \`\${ENV_VAR}\`). Open exactly the URL the step says to open.
|
|
25553
|
+
|
|
25554
|
+
## Steps
|
|
25555
|
+
|
|
25556
|
+
${stepsText}
|
|
25557
|
+
${expectationsText}${contextText}${conventionsText}${previousText}
|
|
25558
|
+
## Execution Workflow
|
|
25559
|
+
|
|
25560
|
+
For each step:
|
|
25561
|
+
1. Emit \`STEP_START|<step-id>|<short description>\`.
|
|
25562
|
+
2. Run \`snapshot -i -c\` and identify selectors from the ARIA tree. A full \`snapshot\` is large; when you need the page's text, scope it with \`snapshot -c -s "<css>"\`.
|
|
25563
|
+
3. Execute the step's actions using ALLOWED selectors (see Selector Rules), prefixing each command with \`CCQA_STEP=<step-id>\` like every agent-browser command in the step. **Batch them:** put every command you can already write — the actions, their waits, the \`CCQA_ASSERT\` checks, and a closing \`snapshot -i -c\` when you need to see the result — into one \`run_commands\` call. Each turn costs far more than a command; use a single Bash call only when you must see one answer before you can write the next command.
|
|
25564
|
+
4. Emit \`AB_ACTION|...\` for every browser action (see AB_ACTION Protocol).
|
|
25565
|
+
5. Verify the outcome from the batch's output. When a batch stops at a failure, everything before it is recorded: look at the page and continue from there.
|
|
25566
|
+
6. Confirm at least **two independent signals** (URL change, element appearance, text change, ...). This is how *you* decide the step worked and it is safe to continue. It is not what gets recorded.
|
|
25567
|
+
7. Record as assertions only the signals the step's own \`expected\` asks about, by putting a \`CCQA_ASSERT=<marker>\` prefix on the verification command itself (see Assertion Protocol). Every assert type has a marker form; a printed \`AB_ACTION|assert|...\` line records nothing.
|
|
25568
|
+
8. Emit \`STEP_DONE\`, \`ASSERTION_FAILED\`, or \`STEP_SKIPPED\`.
|
|
25569
|
+
|
|
25570
|
+
**Protocol lines are recorded ONLY from your plain assistant text.** Every
|
|
25571
|
+
\`STEP_START\` / \`STEP_DONE\` / \`RUN_COMPLETED\` line MUST be printed as normal
|
|
25572
|
+
message text between tool calls — never through Bash (\`echo\` etc.) and never
|
|
25573
|
+
omitted. Step attribution of browser actions comes from the \`CCQA_STEP\`
|
|
25574
|
+
prefix on each command (that prefix is authoritative, not the \`STEP_START\`
|
|
25575
|
+
line). **Assertions are recorded only from the \`CCQA_ASSERT\` prefix on
|
|
25576
|
+
verification commands** — a command that ran, whose failure rolls the record
|
|
25577
|
+
back. A printed \`AB_ACTION|assert|...\` line performs nothing, so nothing
|
|
25578
|
+
proves it; it is reported and not recorded.
|
|
25579
|
+
|
|
25580
|
+
**After form submission or navigation:** take a fresh snapshot before continuing. If an intermediate screen appears (account selection, role picker, ...), complete it and emit AB_ACTION for each interaction.
|
|
25581
|
+
|
|
25582
|
+
## Guardrails
|
|
25583
|
+
|
|
25584
|
+
- **Stop after 3 consecutive failures on the same step** — emit \`ASSERTION_FAILED\` and report the blocker.
|
|
25585
|
+
- **Stay on a flow you have started.** Once a step is inside something whose state lives on the page — a conversation, a multi-step dialog, a form not yet saved — do not navigate away to investigate; that state does not come back. Look things up in the source with Read, Grep and Glob instead.
|
|
25586
|
+
- **Clean up before you stop.** When you end the run with \`ASSERTION_FAILED\` after the case has created something, still perform its cleanup steps, then emit \`RUN_COMPLETED|failed\`. What a failed recording leaves behind breaks the next run of every case that lists the same things.
|
|
25587
|
+
- **The step must match the application.** When what a step asks for is not there, or the application does it in another place or order, do not bend the recording to fit: emit \`ASSERTION_FAILED|<step-id>|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>\`. Read the source to ground the suggestion. A recording that works around the step hides the mismatch; the report gets the spec fixed. Redoing an earlier step's work to make this one possible — going back to refill or resubmit what an earlier step already did — is such a workaround.
|
|
25588
|
+
- **No workarounds.** If all ALLOWED selectors fail, emit \`ASSERTION_FAILED|...|selector-drift: ...\`. Do NOT fall back to coordinate clicks, mouse moves, or \`Tab\`+\`Enter\` keyboard navigation — they cannot be recorded as reliable test actions.
|
|
25589
|
+
- Do NOT retry a selector without taking a fresh snapshot first.
|
|
25590
|
+
- Do NOT work around blockers (login walls, missing data, captchas) — stop and report.
|
|
25591
|
+
- **Do NOT suppress errors.** Never use \`2>/dev/null\`, \`|| true\`, \`; true\`, or any technique that hides agent-browser failures. Each \`agent-browser\` invocation must be its own standalone Bash call. Chaining multiple agent-browser commands with \`&&\` / \`;\` / \`|\` is rejected by the hook layer.
|
|
25592
|
+
- **If \`agent-browser\` is not found, stop immediately.** Do not run \`which\`, \`find\`, \`npm ls\`, \`npm install\`, \`npx\`, \`brew\`, or any other discovery / installation command. Emit one line and terminate: \`ASSERTION_FAILED|step-XX|agent-browser binary not available in PATH\`.
|
|
25593
|
+
|
|
25594
|
+
## Source Code Reference
|
|
25595
|
+
|
|
25596
|
+
You have \`Read\`, \`Grep\`, and \`Glob\` to inspect the application source code. Use them proactively to find correct selectors — do NOT guess \`a[href*='...']\` patterns.
|
|
25597
|
+
|
|
25598
|
+
**When**: before clicking a link (find the \`href\`); before navigating to a new page (understand routing); when an ARIA element exists but no ALLOWED selector matches (find the actual HTML attributes).
|
|
25599
|
+
|
|
25600
|
+
**How**: \`Grep\` for UI text / component names / URL patterns → \`Read\` the JSX/TSX to find \`href\`, \`aria-label\`, \`data-testid\`, or class names → build a precise ALLOWED selector.
|
|
25601
|
+
|
|
25602
|
+
**Rules**: only READ source files, never modify them. Keep searches focused.
|
|
25603
|
+
|
|
25604
|
+
## Waiting for Async Operations
|
|
25605
|
+
|
|
25606
|
+
Prefer \`wait\` over polling:
|
|
25607
|
+
|
|
25608
|
+
\`\`\`bash
|
|
25609
|
+
CCQA_STEP=<step-id> agent-browser --session ${sessionName} wait --text "<completion text>"
|
|
25610
|
+
\`\`\`
|
|
25611
|
+
|
|
25612
|
+
If polling is required (e.g. waiting for a spinner to disappear):
|
|
25613
|
+
|
|
25614
|
+
\`\`\`bash
|
|
25615
|
+
for i in $(seq 1 18); do
|
|
25616
|
+
sleep 10
|
|
25617
|
+
result=$(agent-browser --session ${sessionName} snapshot 2>&1)
|
|
25618
|
+
echo "$result" | grep -q "<done indicator>" && break
|
|
25619
|
+
done
|
|
25620
|
+
CCQA_STEP=<step-id> agent-browser --session ${sessionName} snapshot
|
|
25621
|
+
\`\`\`
|
|
25622
|
+
|
|
25623
|
+
After waiting, always take a final snapshot. Emit \`AB_ACTION|wait|text=<text>|<label>\`.
|
|
25624
|
+
|
|
25625
|
+
## AB_ACTION Protocol
|
|
25626
|
+
|
|
25627
|
+
After **every** browser action, emit one line (outside any code block):
|
|
25628
|
+
|
|
25629
|
+
\`\`\`
|
|
25630
|
+
AB_ACTION|cookies_clear
|
|
25631
|
+
AB_ACTION|open|<url>
|
|
25632
|
+
AB_ACTION|click|<selector>|<visible label>
|
|
25633
|
+
AB_ACTION|dblclick|<selector>|<visible label>
|
|
25634
|
+
AB_ACTION|fill|<selector>|<value>|<aria label>
|
|
25635
|
+
AB_ACTION|check|<selector>|<visible label>
|
|
25636
|
+
AB_ACTION|uncheck|<selector>|<visible label>
|
|
25637
|
+
AB_ACTION|press|<Key>
|
|
25638
|
+
AB_ACTION|select|<selector>|<value>|<aria label>
|
|
25639
|
+
AB_ACTION|hover|<selector>|<visible label>
|
|
25640
|
+
AB_ACTION|scroll|<direction>|<pixels>
|
|
25641
|
+
AB_ACTION|drag|<source selector>|<target selector>|<source label>
|
|
25642
|
+
AB_ACTION|upload|<file-input selector>|<file1>[|<file2>...]
|
|
25643
|
+
AB_ACTION|wait|<selector or text>|<label>
|
|
25644
|
+
AB_ACTION|snapshot|<key observation, max 100 chars>
|
|
25645
|
+
|
|
25646
|
+
# find_* (semantic locator fallback). <extra> = role's --name OR nth's index OR "".
|
|
25647
|
+
# <exact> = literal "exact" if --exact was passed, "" otherwise. Keep empty pipe slots.
|
|
25648
|
+
AB_ACTION|find_click|<locator>|<value>|<extra>|<exact>|<label>
|
|
25649
|
+
AB_ACTION|find_dblclick|<locator>|<value>|<extra>|<exact>|<label>
|
|
25650
|
+
AB_ACTION|find_hover|<locator>|<value>|<extra>|<exact>|<label>
|
|
25651
|
+
AB_ACTION|find_focus|<locator>|<value>|<extra>|<exact>|<label>
|
|
25652
|
+
AB_ACTION|find_check|<locator>|<value>|<extra>|<exact>|<label>
|
|
25653
|
+
AB_ACTION|find_uncheck|<locator>|<value>|<extra>|<exact>|<label>
|
|
25654
|
+
AB_ACTION|find_fill|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
25655
|
+
AB_ACTION|find_type|<locator>|<value>|<extra>|<exact>|<input>|<label>
|
|
25656
|
+
\`\`\`
|
|
25657
|
+
|
|
25658
|
+
Selectors in AB_ACTION must follow Selector Rules. \`find_*\` lines use the locator + value pair instead of a separate selector. Do NOT include literal \`|\` inside any field — replace with a space if necessary.
|
|
25659
|
+
|
|
25660
|
+
**CRITICAL — record only successful actions.** The AB_ACTION stream is the canonical replay sequence: every line must be reproducible on a fresh browser session.
|
|
25661
|
+
|
|
25662
|
+
- A non-zero exit from agent-browser (selector not found, element not interactable, timeout) → **do NOT emit AB_ACTION** for that attempt. Switch selector and only emit the AB_ACTION for the call that finally succeeded.
|
|
25663
|
+
- If you tried several selectors / \`find_*\` locators for the same logical action, emit AB_ACTION for the **last working one only**. Multiple failed attempts in a row will all fail at replay validation and silently delete the step from the generated test.
|
|
25664
|
+
- An assertion follows the same rule: mark only the check that verified on the current page in the current snapshot.
|
|
25665
|
+
- **Open only a URL the step names.** A step's \`open\` takes the address its own instruction gives — a \`\${VAR}\` plus the literal tail after it. Never open an address the run produced: the id in \`/items/01H8XZ...\` belongs to the record this run created, and opening it next time visits a record that run does not have (or worse, one it does, silently testing the wrong thing). Reach such a page the way a person does — click through from where the run already is.
|
|
25666
|
+
|
|
25667
|
+
- **Environment-failure recovery is not part of the test.** If a session times out, a network blip drops you to login, or the app crashes and you re-login / re-navigate / re-fill to recover, do NOT emit AB_ACTION for the recovery operations.
|
|
25668
|
+
- If a step ultimately fails after retries: emit \`ASSERTION_FAILED\` and STOP. Do not leave half-recorded actions in the stream.
|
|
25669
|
+
|
|
25670
|
+
## Assertion Protocol
|
|
25671
|
+
|
|
25672
|
+
Record each signal you verified as an assertion. The **primary mechanism** is
|
|
25673
|
+
a \`CCQA_ASSERT=<marker>\` env prefix on the verification command itself — the
|
|
25674
|
+
marker rides the same channel as the command, so the assertion is captured
|
|
25675
|
+
even if you print no protocol text. The check you were going to run anyway
|
|
25676
|
+
becomes the recorded assertion:
|
|
25677
|
+
|
|
25678
|
+
\`\`\`bash
|
|
25679
|
+
# text_visible — the marked wait IS the assertion (no separate assert line)
|
|
25680
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "Submitted" --timeout 3000
|
|
25681
|
+
# element_visible / element_not_visible — mark the get-count probe
|
|
25682
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "[aria-label='Settings']"
|
|
25683
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "text=Deleted item"
|
|
25684
|
+
# url_contains — the substring rides in the marker; put it on \`get url\`
|
|
25685
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=url_contains:/dashboard agent-browser --session SESSION get url
|
|
25686
|
+
# element state — mark the \`is\` probe that asks for it
|
|
25687
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "#submit"
|
|
25688
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "#opt-in"
|
|
25689
|
+
\`\`\`
|
|
25690
|
+
|
|
25691
|
+
**Marker semantics:**
|
|
25692
|
+
|
|
25693
|
+
| Marker | On command | Records |
|
|
25694
|
+
|--------|------------|---------|
|
|
25695
|
+
| \`1\` (alias of \`text_visible\`) | \`wait --text "<text>"\` | \`assert text_visible <text>\` — replaces the wait (the recorded assert waits itself) |
|
|
25696
|
+
| \`element_visible\` | \`get count "<selector>"\` | \`assert element_visible <selector>\` |
|
|
25697
|
+
| \`element_not_visible\` | \`get count "<selector>"\` | \`assert element_not_visible <selector>\` (use \`text=<text>\` as the selector to assert text absence) |
|
|
25698
|
+
| \`url_contains:<substring>\` | any command | \`assert url_contains <substring>\` in addition to whatever the command records |
|
|
25699
|
+
| \`element_enabled\` / \`element_disabled\` | \`is enabled "<selector>"\` | \`assert <marker> <selector>\` |
|
|
25700
|
+
| \`element_checked\` / \`element_unchecked\` | \`is checked "<selector>"\` | \`assert <marker> <selector>\` |
|
|
25701
|
+
|
|
25702
|
+
- A marked command that exits non-zero records nothing — fix the check and re-run it.
|
|
25703
|
+
- A marker that doesn't match its command (e.g. \`CCQA_ASSERT=1\` on a \`click\`) is ignored with a warning — never do that.
|
|
25704
|
+
- \`get count\`, \`get url\` and \`is\` exit 0 regardless of what they print. **Read the output**: if the printed count / URL / \`true\`-\`false\` contradicts the marker, the signal did NOT verify — treat it as a failed verification (emit \`ASSERTION_FAILED\` if the step cannot be confirmed another way).
|
|
25705
|
+
- **Assert the thing, not how much of it there is.** A step that says a list, a table or a section is shown is satisfied by the list being there. Do not assert the count it happened to have, and never assert an empty-state message ("no items yet") unless the step's own \`expected\` asks for emptiness — the next run has one more record than this one and the test breaks for a reason nobody chose.
|
|
25706
|
+
|
|
25707
|
+
- **Assert what the step asks about, nothing else.** The \`expected\` is the contract; anything else you happened to see on the way is not. A nav item, a heading or a greeting that the step never mentions adds no coverage, differs between recordings of the same spec, and is the first thing to break on replay — so the next recording quietly drops it and the test gets weaker without anyone deciding that.
|
|
25708
|
+
|
|
25709
|
+
- **An \`expected\` that labels its parts is recorded down to the \`must:\` lines.** A \`when present:\` line names something the product does not produce on every run, so **nothing that fails on its absence may be recorded for it** — no assert, and no \`wait --text\` either: a recorded wait throws on the next run that legitimately lacks it, which is the same failure an assert would have caused. Look at it with \`snapshot\` or \`get count\` if you want to see it, and record nothing. When the step needs to wait for something asynchronous to arrive, anchor that wait on a \`must:\` signal — waiting on the optional one makes the whole step hinge on it. Every line of a labelled \`expected\` carries a label; text with no labels at all is all \`must:\`.
|
|
25710
|
+
|
|
25711
|
+
- **\`url_contains\` is opt-in, not a habit.** The same rule, in the form that gets broken most. Emit it ONLY when a step's own \`expected\` explicitly asks about the URL or path. Do NOT add a \`url_contains\` to "prove" a login succeeded, a page loaded, or a navigation happened — confirm those with \`text_visible\` / \`element_visible\` on something the destination page renders. An unrequested URL assertion adds no coverage the visible-content assert doesn't already give, and is the single most common way an environment gets baked into a test.
|
|
25712
|
+
- **When you do assert a URL, the substring may come from ONE place only:** a \`\${VAR}\` URL that *this step's own instruction* opened, written as that \`\${VAR}\` followed by the literal tail after it. If the step opens \`\${APP_URL}/todos\`, assert \`\${APP_URL}/todos\` (or the tail \`/todos\`); the recorder resolves \`\${APP_URL}\` per environment. Never assert on a URL you merely *observed* — login redirects, identity-provider pages, and OAuth callbacks all live on a **different, environment-named origin** than the app, so any substring of them (host, origin, OR path) names the environment.
|
|
25713
|
+
- **A leading slash does NOT make a substring safe.** \`/auth-staging\`, \`/env-qa\`, \`/tenant-acme\` look like paths but are environment labels — the first segment of an identity-provider or tenant URL, not an application route. If a substring contains an environment name, a stage token (\`dev\`, \`stg\`, \`prod\`), a tenant/org name, or any fragment of a hostname, it is forbidden even with a leading slash. The only safe path substrings are stable *application* routes off the app's own origin (\`/dashboard\`, \`/todos/new\`), taken from a \`\${VAR}\` you opened — not from a redirect you watched.
|
|
25714
|
+
|
|
25715
|
+
**Every assert type has a marker command.** There is no text form: a printed
|
|
25716
|
+
\`AB_ACTION|assert|...\` line performs nothing, so nothing proves it, and it is
|
|
25717
|
+
reported rather than recorded.
|
|
25718
|
+
|
|
25719
|
+
| assertType | The command that records it | Use when |
|
|
25720
|
+
|------------|------------------------------|----------|
|
|
25721
|
+
| \`text_visible\` | \`CCQA_ASSERT=1 ... wait --text "<text>"\` | stable text appears on the page |
|
|
25722
|
+
| \`element_visible\` | \`CCQA_ASSERT=element_visible ... get count "<selector>"\` | element is present |
|
|
25723
|
+
| \`element_not_visible\` | \`CCQA_ASSERT=element_not_visible ... get count "<selector>"\` | element is gone — and text absence too, as \`text=<text>\` |
|
|
25724
|
+
| \`url_contains\` | \`CCQA_ASSERT=url_contains:<substring> ... get url\` | the URL contains a pattern |
|
|
25725
|
+
| \`element_enabled\` / \`element_disabled\` | \`CCQA_ASSERT=<marker> ... is enabled "<selector>"\` | a control is usable, or is not |
|
|
25726
|
+
| \`element_checked\` / \`element_unchecked\` | \`CCQA_ASSERT=<marker> ... is checked "<selector>"\` | a checkbox is ticked, or is not |
|
|
25727
|
+
|
|
25728
|
+
**Stability rules — CRITICAL. NEVER assert on values that change run-to-run:**
|
|
25729
|
+
|
|
25730
|
+
- Timestamps, session IDs, exact numeric counts that vary between runs.
|
|
25731
|
+
- **Absolute dates / clock times**: \`12:34:56\`, \`2026-05-20\`, \`2026年5月20日\`, \`5月20日\`. These are scrubbed by post-trace literal-scrub anyway — avoid them at the source.
|
|
25732
|
+
- **Relative-time labels** — true only in the moment of the trace, stale by replay:
|
|
25733
|
+
- English: \`just now\`, \`5 minutes ago\`, \`2 hours ago\`, \`yesterday\`, \`last week\`.
|
|
25734
|
+
- Japanese: \`たった今\`, \`3分前\`, \`1時間前\`, \`昨日\`.
|
|
25735
|
+
- Dynamic counts like "42 results" → assert on the stable suffix ("results") only.
|
|
25736
|
+
- **PREFER**: status text, button labels, URL patterns, element enabled/disabled state.
|
|
25737
|
+
|
|
25738
|
+
**No tautological state asserts — CRITICAL for \`element_enabled\` / \`element_disabled\`:**
|
|
25739
|
+
|
|
25740
|
+
The selector must identify *which* element by something **other than the state you are asserting**. Selecting the element *by* its state and then asserting that state is a tautology that always passes and verifies nothing.
|
|
25741
|
+
|
|
25742
|
+
- ✗ \`element_disabled | button[disabled] |\` — picks an already-disabled button, then "confirms" it is disabled. Passes even if the button the spec cares about is missing or enabled.
|
|
25743
|
+
- ✗ \`element_enabled | button:enabled |\`, \`[aria-disabled='true']\`, \`input:disabled\` — same trap.
|
|
25744
|
+
- ✓ Name the element by a stable, state-independent selector and assert the state on it: e.g. the "Submit" button is \`find role button --name "Submit"\`; to assert it is disabled, give \`element_disabled\` a selector that targets *that* button (a stable \`id\` / \`data-testid\` / unique class), **not** \`[disabled]\`.
|
|
25745
|
+
- If you cannot target the specific element without a state pseudo-class/attribute, **do not emit the enabled/disabled assert** — assert a user-visible consequence instead (e.g. the action it gates does not happen, a "you don't have permission" message is shown), or rely on \`text_visible\` for the label plus \`text_not_visible\` for what an enabled control would have produced.
|
|
25746
|
+
|
|
25747
|
+
**Page-context and selector rules:**
|
|
25748
|
+
|
|
25749
|
+
- After a navigation, take a **fresh snapshot** before emitting any assertion. Don't assert on text from the previous page.
|
|
25750
|
+
- Assertion selectors follow the same Selector Rules as actions — never invent aria-label values; use the exact strings from the current snapshot.
|
|
25751
|
+
- When unsure, prefer \`text_visible\` — a \`CCQA_ASSERT=1\` marked \`wait --text\` verifies and records it in one command.
|
|
25752
|
+
|
|
25753
|
+
**MUST-VERIFY rule — STRICT (applies to every assert except \`url_contains\`):**
|
|
25754
|
+
|
|
25755
|
+
The \`snapshot\` output is the **accessibility tree**, but \`agent-browser\` queries the **real DOM**. They don't always agree. Two known traps:
|
|
25756
|
+
|
|
25757
|
+
1. *Selector trap*: a snapshot row like \`textbox "Email address"\` may be reachable via \`[placeholder='...']\` but **not** via \`[aria-label='...']\` if no aria-label attribute is actually set (the browser inferred the label from \`<label for=>\` / placeholder).
|
|
25758
|
+
2. *Text trap*: a snapshot row like \`link "Dashboard"\` may come from \`<a><img alt="Dashboard"></a>\` — the visible "text" is an \`alt\` attribute, not a text node. \`text_visible\` (which scans visible text nodes) will NOT find it.
|
|
25759
|
+
3. *Input-value trap*: after you \`fill\` an \`<input>\` / \`<textarea>\` / \`[contenteditable]\`, the text you typed lives in the element's **value**, not as a visible text node. **Do NOT assert the typed value with \`text_visible\`** — it will never match. The spec's "the field reflects X" expectation is implicitly confirmed when the form submits successfully and the value shows up on the *result* page (a list row, a detail page). Assert there, not on the input itself.
|
|
25760
|
+
|
|
25761
|
+
**Verify the assertion form actually resolves on the live page.** The
|
|
25762
|
+
verification and the recorded assertion are the same marked command:
|
|
25763
|
+
|
|
25764
|
+
\`\`\`bash
|
|
25765
|
+
# text_visible
|
|
25766
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=1 agent-browser --session SESSION wait --text "<text>" --timeout 3000
|
|
25767
|
+
# element_visible — >=1 printed means present
|
|
25768
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_visible agent-browser --session SESSION get count "<selector>"
|
|
25769
|
+
# element_not_visible — 0 printed means absent (works for text too: "text=<text>")
|
|
25770
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_not_visible agent-browser --session SESSION get count "<selector>"
|
|
25771
|
+
# element state — \`is\` prints true/false and exits 0 either way; read it
|
|
25772
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_disabled agent-browser --session SESSION is enabled "<selector>"
|
|
25773
|
+
CCQA_STEP=<step-id> CCQA_ASSERT=element_checked agent-browser --session SESSION is checked "<selector>"
|
|
25774
|
+
\`\`\`
|
|
25775
|
+
|
|
25776
|
+
When *no* form verifies — e.g. \`[aria-label='X']\`, \`[placeholder='X']\`, and \`text=X\` all timed out, or the visible text turned out to be an \`alt\` — **drop the assertion entirely**. Fewer real assertions beat invented ones that fail at replay. \`url_contains\` is exempt (it checks the URL string, not the DOM).
|
|
25777
|
+
|
|
25778
|
+
## Status Protocol
|
|
25779
|
+
|
|
25780
|
+
Emit exactly one status line per step (outside any code block):
|
|
25781
|
+
|
|
25782
|
+
\`\`\`
|
|
25783
|
+
STEP_START|<step-id>|<short description>
|
|
25784
|
+
STEP_DONE|<step-id>|<what was verified>
|
|
25785
|
+
ASSERTION_FAILED|<step-id>|<category: spec-mismatch|app-bug|env-issue|auth-blocked|missing-test-data|selector-drift|agent-misread>: <reason>
|
|
25786
|
+
STEP_SKIPPED|<step-id>|<reason>
|
|
25787
|
+
RUN_COMPLETED|passed|<summary>
|
|
25788
|
+
RUN_COMPLETED|failed|<summary>
|
|
25789
|
+
\`\`\`
|
|
25790
|
+
|
|
25791
|
+
**\`RUN_COMPLETED\` is what ends the run.** A session that stops without it is
|
|
25792
|
+
recorded as FAILED, however many steps went well: nothing else can tell a
|
|
25793
|
+
finished route from one you abandoned halfway.
|
|
25794
|
+
|
|
25795
|
+
${callerGuidance}## Start
|
|
25796
|
+
|
|
25797
|
+
Begin by clearing cookies, then proceed straight to the first step's instruction.
|
|
25798
|
+
|
|
25799
|
+
\`\`\`bash
|
|
25800
|
+
agent-browser --session ${sessionName} cookies clear
|
|
25801
|
+
\`\`\`
|
|
25802
|
+
|
|
25803
|
+
(The cookies-clear preamble belongs to no step, so it is the one command
|
|
25804
|
+
without a \`CCQA_STEP\` prefix.)
|
|
25805
|
+
|
|
25806
|
+
Emit:
|
|
25807
|
+
\`\`\`
|
|
25808
|
+
AB_ACTION|cookies_clear
|
|
25809
|
+
\`\`\`
|
|
25810
|
+
|
|
25811
|
+
Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
|
|
25812
|
+
every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
|
|
25813
|
+
step is responsible for opening the initial URL.
|
|
25814
|
+
`;
|
|
25322
25815
|
}
|
|
25323
|
-
|
|
25324
|
-
|
|
25325
|
-
const loc = action.locator;
|
|
25326
|
-
if (!loc || loc.by !== "label") return null;
|
|
25327
|
-
const role = LABEL_FALLBACK_ROLE[action.action];
|
|
25328
|
-
if (role === void 0) return null;
|
|
25329
|
-
return {
|
|
25330
|
-
...action,
|
|
25331
|
-
locator: {
|
|
25332
|
-
by: "role",
|
|
25333
|
-
value: role,
|
|
25334
|
-
name: loc.value,
|
|
25335
|
-
exact: true
|
|
25336
|
-
}
|
|
25337
|
-
};
|
|
25816
|
+
function buildTracePrompt(title) {
|
|
25817
|
+
return `Execute the test for "${title}". Each step's instruction includes the URL or selector context it needs.`;
|
|
25338
25818
|
}
|
|
25339
25819
|
/**
|
|
25340
|
-
*
|
|
25341
|
-
*
|
|
25342
|
-
*
|
|
25343
|
-
*
|
|
25344
|
-
*
|
|
25345
|
-
* Only a "no element" failure waits — anything else repeats identically. When
|
|
25346
|
-
* there is an `alternate` form it is tried on every attempt rather than after
|
|
25347
|
-
* the wait: a label the page never associates with its input will not start
|
|
25348
|
-
* matching, so waiting first spends the budget for nothing.
|
|
25820
|
+
* The case's expectations, plus the one rule that decides where each is
|
|
25821
|
+
* recorded. Without the rule a recorder either asserts everything at the end —
|
|
25822
|
+
* on a page the flow has already left — or asserts each item at every step it
|
|
25823
|
+
* might apply to, which is the over-assertion the selector rules exist to
|
|
25824
|
+
* prevent.
|
|
25349
25825
|
*/
|
|
25350
|
-
function
|
|
25351
|
-
|
|
25352
|
-
|
|
25353
|
-
|
|
25354
|
-
|
|
25355
|
-
|
|
25356
|
-
|
|
25357
|
-
|
|
25358
|
-
|
|
25359
|
-
|
|
25360
|
-
|
|
25361
|
-
"--session",
|
|
25362
|
-
sessionName,
|
|
25363
|
-
"wait",
|
|
25364
|
-
"--load",
|
|
25365
|
-
"networkidle"
|
|
25366
|
-
]);
|
|
25367
|
-
continue;
|
|
25368
|
-
}
|
|
25369
|
-
if (result.status === 0 || !notFound(result)) return {
|
|
25370
|
-
result,
|
|
25371
|
-
viaAlternate: false,
|
|
25372
|
-
waitedMs
|
|
25373
|
-
};
|
|
25374
|
-
if (alternate !== null) {
|
|
25375
|
-
const alt = spawnAB(alternate);
|
|
25376
|
-
if (alt.status === 0) return {
|
|
25377
|
-
result: alt,
|
|
25378
|
-
viaAlternate: true,
|
|
25379
|
-
waitedMs
|
|
25380
|
-
};
|
|
25381
|
-
}
|
|
25382
|
-
if (patience.spent || attempt + 1 >= INTERACTION_ATTEMPTS || Date.now() >= deadline) {
|
|
25383
|
-
patience.spent = true;
|
|
25384
|
-
return {
|
|
25385
|
-
result,
|
|
25386
|
-
viaAlternate: false,
|
|
25387
|
-
waitedMs: Date.now() - started
|
|
25388
|
-
};
|
|
25389
|
-
}
|
|
25390
|
-
sleepSync(INTERACTION_POLL_INTERVAL_MS);
|
|
25391
|
-
}
|
|
25826
|
+
function renderExpectations(expectations) {
|
|
25827
|
+
if (expectations.length === 0) return "";
|
|
25828
|
+
return `
|
|
25829
|
+
## Expected results
|
|
25830
|
+
|
|
25831
|
+
The case states these without saying which step delivers them:
|
|
25832
|
+
|
|
25833
|
+
${expectations.map((text) => `- ${text}`).join("\n")}
|
|
25834
|
+
|
|
25835
|
+
**Assign each to the step it first becomes true in, and assert it there and nowhere else.** Read the flow to decide: an expectation about what a form did belongs to the step that submitted it, not to a later step that happens to still show it. Assert each one exactly once — a repeat adds no coverage and breaks first. An expectation you cannot place is one you report in \`STEP_DONE\` text as unplaced; never invent a step for it, and never assert it "just in case".
|
|
25836
|
+
`;
|
|
25392
25837
|
}
|
|
25393
25838
|
/**
|
|
25394
|
-
*
|
|
25395
|
-
*
|
|
25396
|
-
*
|
|
25397
|
-
*
|
|
25398
|
-
* saved route carries the form that replays rather than the one that counts
|
|
25399
|
-
* nothing. Notation this cannot convert leaves the locator alone; the assert
|
|
25400
|
-
* path then treats it as unverifiable rather than absent.
|
|
25839
|
+
* What the cleanup must make true. Separate from the case's expectations
|
|
25840
|
+
* because its steps are: a deletion is confirmed by the thing being gone,
|
|
25841
|
+
* which is only true after the undo has run, and asserting it among the
|
|
25842
|
+
* case's own steps would check it while the thing still exists.
|
|
25401
25843
|
*/
|
|
25402
|
-
function
|
|
25403
|
-
|
|
25404
|
-
|
|
25405
|
-
|
|
25406
|
-
|
|
25407
|
-
|
|
25408
|
-
|
|
25844
|
+
function renderCleanupExpectations(expectations) {
|
|
25845
|
+
if (expectations.length === 0) return "";
|
|
25846
|
+
return `
|
|
25847
|
+
## Expected results of the cleanup
|
|
25848
|
+
|
|
25849
|
+
The case states these about its cleanup steps:
|
|
25850
|
+
|
|
25851
|
+
${expectations.map((text) => `- ${text}`).join("\n")}
|
|
25852
|
+
|
|
25853
|
+
**Assert each in the cleanup step that makes it true, and nowhere else.** They describe the state after the undo, so a cleanup step is not done when the command returns — it is done when what the case says here holds. Record the check as an assertion the same way a step's own is recorded.
|
|
25854
|
+
`;
|
|
25409
25855
|
}
|
|
25410
25856
|
/**
|
|
25411
|
-
*
|
|
25412
|
-
*
|
|
25413
|
-
*
|
|
25414
|
-
|
|
25415
|
-
|
|
25416
|
-
|
|
25417
|
-
|
|
25418
|
-
|
|
25419
|
-
|
|
25420
|
-
|
|
25421
|
-
|
|
25422
|
-
const snapshot = spawnAB([
|
|
25423
|
-
"--session",
|
|
25424
|
-
sessionName,
|
|
25425
|
-
"snapshot"
|
|
25426
|
-
]);
|
|
25427
|
-
if (snapshot.status !== 0) return null;
|
|
25428
|
-
const role = roleOfAccessibleName(snapshot.stdout, sub(name));
|
|
25429
|
-
if (role === null) return null;
|
|
25430
|
-
if (spawnAB(roleProbe(["--session", sessionName], role, sub(name))).status !== 0) return null;
|
|
25431
|
-
const promoted = `${action.locator.value} names an element rather than an attribute — confirmed as role=${role}`;
|
|
25432
|
-
action.locator = {
|
|
25433
|
-
by: "role",
|
|
25434
|
-
value: role,
|
|
25435
|
-
name,
|
|
25436
|
-
exact: true
|
|
25437
|
-
};
|
|
25438
|
-
return {
|
|
25439
|
-
skipped: false,
|
|
25440
|
-
ok: true,
|
|
25441
|
-
reason: "",
|
|
25442
|
-
promoted
|
|
25443
|
-
};
|
|
25857
|
+
* Sections of the case ccqa does not interpret — the precondition, notes the
|
|
25858
|
+
* author left. Passed through verbatim: they are written for whoever runs the
|
|
25859
|
+
* case, and the recorder is now that reader.
|
|
25860
|
+
*/
|
|
25861
|
+
function renderContext(context) {
|
|
25862
|
+
if (context.length === 0) return "";
|
|
25863
|
+
return `
|
|
25864
|
+
## About this case
|
|
25865
|
+
|
|
25866
|
+
${context.map((s) => `### ${s.heading}\n\n${s.body}`).join("\n\n")}
|
|
25867
|
+
`;
|
|
25444
25868
|
}
|
|
25445
25869
|
/**
|
|
25446
|
-
*
|
|
25447
|
-
*
|
|
25448
|
-
*
|
|
25870
|
+
* The project's own recording guidance. Where the case says what to do, this
|
|
25871
|
+
* says how this project is driven — which account signs in, what has to be
|
|
25872
|
+
* true before the first step. Prose, verbatim: it is written for whoever runs
|
|
25873
|
+
* the case, and this is now that reader.
|
|
25449
25874
|
*/
|
|
25450
|
-
function
|
|
25451
|
-
|
|
25452
|
-
|
|
25453
|
-
|
|
25454
|
-
|
|
25455
|
-
|
|
25456
|
-
|
|
25457
|
-
if (built === null) return {
|
|
25458
|
-
skipped: true,
|
|
25459
|
-
ok: false,
|
|
25460
|
-
reason: ""
|
|
25461
|
-
};
|
|
25462
|
-
if (isProbe(built)) {
|
|
25463
|
-
const { ok, reason } = runProbe(built, sessionName);
|
|
25464
|
-
if (ok) return withName({
|
|
25465
|
-
skipped: false,
|
|
25466
|
-
ok,
|
|
25467
|
-
reason
|
|
25468
|
-
});
|
|
25469
|
-
const resolve = (v) => v === void 0 ? "" : resolveEnvRefs(v, envOverrides);
|
|
25470
|
-
return askTheAccessibilityTree(action, resolve, sessionName) ?? {
|
|
25471
|
-
skipped: false,
|
|
25472
|
-
ok,
|
|
25473
|
-
reason
|
|
25474
|
-
};
|
|
25475
|
-
}
|
|
25476
|
-
const fallback = labelFallback(action);
|
|
25477
|
-
const alternate = fallback === null ? null : actionToAbArgs(fallback, sessionName, envOverrides);
|
|
25478
|
-
const { result, viaAlternate, waitedMs } = runInteraction(built, alternate !== null && !isProbe(alternate) ? alternate : null, patience, sessionName);
|
|
25479
|
-
if (viaAlternate) {
|
|
25480
|
-
const promoted = `${action.locator.by}=${action.locator.value} → role=${fallback.locator.value} name="${action.locator.value}"`;
|
|
25481
|
-
action.locator = fallback.locator;
|
|
25482
|
-
return {
|
|
25483
|
-
skipped: false,
|
|
25484
|
-
ok: true,
|
|
25485
|
-
reason: "",
|
|
25486
|
-
promoted
|
|
25487
|
-
};
|
|
25488
|
-
}
|
|
25489
|
-
if (result.status === 0) return withName({
|
|
25490
|
-
skipped: false,
|
|
25491
|
-
ok: true,
|
|
25492
|
-
reason: ""
|
|
25493
|
-
});
|
|
25494
|
-
return {
|
|
25495
|
-
skipped: false,
|
|
25496
|
-
ok: false,
|
|
25497
|
-
reason: `${(result.stderr.trim() || result.stdout.trim() || `agent-browser exit ${result.status ?? "?"}`).slice(0, 200)}${notFound(result) ? ` (waited ${waitedMs}ms)` : ""}`
|
|
25498
|
-
};
|
|
25499
|
-
}
|
|
25500
|
-
function validateActions(actions, opts) {
|
|
25501
|
-
const kept = [];
|
|
25502
|
-
const dropped = [];
|
|
25503
|
-
const promoted = [];
|
|
25504
|
-
const patience = { spent: false };
|
|
25505
|
-
let skipFromStepId = null;
|
|
25506
|
-
for (let i = 0; i < actions.length; i++) {
|
|
25507
|
-
const action = actions[i];
|
|
25508
|
-
opts.onProgress?.(i, actions.length, action);
|
|
25509
|
-
const stepId = action.stepId ?? NO_STEP_ID;
|
|
25510
|
-
if (skipFromStepId !== null && skipFromStepId !== stepId) skipFromStepId = null;
|
|
25511
|
-
if (skipFromStepId !== null && isPassiveAction(action.action)) {
|
|
25512
|
-
dropped.push({
|
|
25513
|
-
index: i,
|
|
25514
|
-
action,
|
|
25515
|
-
reason: CASCADE_REASON
|
|
25516
|
-
});
|
|
25517
|
-
continue;
|
|
25518
|
-
}
|
|
25519
|
-
const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides, patience);
|
|
25520
|
-
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25521
|
-
if (outcome.skipped) {
|
|
25522
|
-
kept.push(action);
|
|
25523
|
-
continue;
|
|
25524
|
-
}
|
|
25525
|
-
if (outcome.ok) {
|
|
25526
|
-
kept.push(action);
|
|
25527
|
-
if (skipFromStepId !== null && !isPassiveAction(action.action)) skipFromStepId = null;
|
|
25528
|
-
continue;
|
|
25529
|
-
}
|
|
25530
|
-
dropped.push({
|
|
25531
|
-
index: i,
|
|
25532
|
-
action,
|
|
25533
|
-
reason: outcome.reason
|
|
25534
|
-
});
|
|
25535
|
-
if (!isPassiveAction(action.action)) skipFromStepId = stepId;
|
|
25536
|
-
}
|
|
25537
|
-
const afterRescue = rescueLostSteps(actions, kept, dropped, opts, patience);
|
|
25538
|
-
promoted.push(...afterRescue.promoted ?? []);
|
|
25539
|
-
return {
|
|
25540
|
-
...splitByMode(actions, afterRescue, opts.mode ?? "lenient"),
|
|
25541
|
-
promoted
|
|
25542
|
-
};
|
|
25875
|
+
function renderConventions(conventions) {
|
|
25876
|
+
if (conventions.length === 0) return "";
|
|
25877
|
+
return `
|
|
25878
|
+
## How this project is driven
|
|
25879
|
+
|
|
25880
|
+
${conventions.map((c) => `### ${c.heading}\n\n${c.body}`).join("\n\n")}
|
|
25881
|
+
`;
|
|
25543
25882
|
}
|
|
25883
|
+
//#endregion
|
|
25884
|
+
//#region src/cli/trace-tools.ts
|
|
25885
|
+
const TRACE_TOOLS_SERVER = "ccqa";
|
|
25886
|
+
/** A wedged command must not hold the whole trace; agent-browser's own waits are far shorter. */
|
|
25887
|
+
const COMMAND_TIMEOUT_MS = 12e4;
|
|
25888
|
+
/** Enough of a snapshot or a `get` answer to act on, without flooding the turn. */
|
|
25889
|
+
const OUTPUT_CAP = 6e3;
|
|
25544
25890
|
/**
|
|
25545
|
-
*
|
|
25546
|
-
*
|
|
25547
|
-
*
|
|
25548
|
-
*
|
|
25549
|
-
*
|
|
25891
|
+
* The trace's own tools. Both exist to cut round trips: one turn per browser
|
|
25892
|
+
* command is what made a re-record cost hundreds of turns.
|
|
25893
|
+
*
|
|
25894
|
+
* - `run_commands` runs a batch the recorder wrote, recording each that passes.
|
|
25895
|
+
* - `replay_step` runs one step's previous commands.
|
|
25550
25896
|
*/
|
|
25551
|
-
function
|
|
25552
|
-
|
|
25553
|
-
kept: result.kept,
|
|
25554
|
-
unstable: [],
|
|
25555
|
-
dropped: result.dropped,
|
|
25556
|
-
rescuedSteps: result.rescuedSteps
|
|
25557
|
-
};
|
|
25558
|
-
const droppedByIndex = new Map(result.dropped.map((d) => [d.index, d]));
|
|
25559
|
-
const keptSet = new Set(result.kept);
|
|
25560
|
-
const finalKept = [];
|
|
25561
|
-
const unstable = [];
|
|
25562
|
-
for (let i = 0; i < originalActions.length; i++) {
|
|
25563
|
-
const action = originalActions[i];
|
|
25564
|
-
if (keptSet.has(action)) {
|
|
25565
|
-
finalKept.push(action);
|
|
25566
|
-
continue;
|
|
25567
|
-
}
|
|
25568
|
-
const drop = droppedByIndex.get(i);
|
|
25569
|
-
if (drop) {
|
|
25570
|
-
action.replayUnstable = true;
|
|
25571
|
-
action.replayReason = drop.reason;
|
|
25572
|
-
unstable.push(action);
|
|
25573
|
-
}
|
|
25574
|
-
}
|
|
25897
|
+
function buildTraceTools(input) {
|
|
25898
|
+
const offersReplay = input.previous.length > 0;
|
|
25575
25899
|
return {
|
|
25576
|
-
|
|
25577
|
-
|
|
25578
|
-
|
|
25579
|
-
|
|
25900
|
+
server: createSdkMcpServer({
|
|
25901
|
+
name: TRACE_TOOLS_SERVER,
|
|
25902
|
+
version: "1.0.0",
|
|
25903
|
+
tools: [tool("run_commands", "Run several agent-browser commands in order, each written exactly as you would run it in Bash (with its CCQA_STEP / CCQA_ASSERT / CCQA_SECRET prefixes). Each that succeeds is recorded; the batch stops at the first that fails. Returns each command's outcome and the output of the last one.", { commands: z.array(z.string()).min(1).describe("agent-browser command lines, one per entry") }, async ({ commands }) => ({ content: [{
|
|
25904
|
+
type: "text",
|
|
25905
|
+
text: await runCommands(commands, input)
|
|
25906
|
+
}] })), ...offersReplay ? [tool("replay_step", "Run one step's previous-recording commands in this browser session and record those that pass. Stops at the first that fails.", { step: z.string().describe("The step id, e.g. step-03") }, async ({ step }) => ({ content: [{
|
|
25907
|
+
type: "text",
|
|
25908
|
+
text: replayStep(step, input)
|
|
25909
|
+
}] }))] : []]
|
|
25910
|
+
}),
|
|
25911
|
+
allowedTools: ["run_commands", ...offersReplay ? ["replay_step"] : []].map((t) => `mcp__${TRACE_TOOLS_SERVER}__${t}`)
|
|
25580
25912
|
};
|
|
25581
25913
|
}
|
|
25582
|
-
function
|
|
25583
|
-
const
|
|
25584
|
-
for (const
|
|
25585
|
-
|
|
25586
|
-
|
|
25587
|
-
|
|
25588
|
-
|
|
25589
|
-
|
|
25590
|
-
|
|
25591
|
-
|
|
25592
|
-
|
|
25593
|
-
|
|
25594
|
-
|
|
25595
|
-
|
|
25596
|
-
|
|
25597
|
-
|
|
25598
|
-
|
|
25599
|
-
|
|
25600
|
-
|
|
25601
|
-
|
|
25602
|
-
|
|
25603
|
-
|
|
25604
|
-
|
|
25605
|
-
for (const d of drops) {
|
|
25606
|
-
const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides, patience);
|
|
25607
|
-
if (outcome.promoted) promoted.push(outcome.promoted);
|
|
25608
|
-
if (outcome.skipped) {
|
|
25609
|
-
unverifiable.push(d.index);
|
|
25610
|
-
continue;
|
|
25611
|
-
}
|
|
25612
|
-
if (outcome.ok) {
|
|
25613
|
-
replayed.push(d.index);
|
|
25614
|
-
continue;
|
|
25615
|
-
}
|
|
25616
|
-
stillFailing = true;
|
|
25617
|
-
if (strict && isCascadeReason(d.reason)) d.reason = outcome.reason;
|
|
25914
|
+
async function runCommands(commands, input) {
|
|
25915
|
+
const report = [];
|
|
25916
|
+
for (const [i, cmd] of commands.entries()) {
|
|
25917
|
+
const stop = (why) => [
|
|
25918
|
+
...report,
|
|
25919
|
+
`✗ ${cmd}\n${why}`,
|
|
25920
|
+
...i + 1 < commands.length ? [`The ${commands.length - i - 1} command(s) after it were not run.`] : []
|
|
25921
|
+
].join("\n");
|
|
25922
|
+
if (!isAgentBrowserCommand(cmd)) return stop("Only agent-browser commands can be batched; run anything else with Bash.");
|
|
25923
|
+
const blocked = abCommandBlockReason(cmd, input.envScrubMap);
|
|
25924
|
+
if (blocked !== null) return stop(blocked);
|
|
25925
|
+
const recorded = abActionEventFromCommand(cmd);
|
|
25926
|
+
const checkpoint = input.beforeAbCommand(extractCcqaStepFromBashCommand(cmd) ?? void 0);
|
|
25927
|
+
if (checkpoint) return stop(checkpoint);
|
|
25928
|
+
info(`$ ${scrubEnvValues(cmd, input.envScrubMap)}`);
|
|
25929
|
+
const result = await runShell(cmd, {
|
|
25930
|
+
...process.env,
|
|
25931
|
+
...input.env
|
|
25932
|
+
});
|
|
25933
|
+
const out = result.stdout.trim();
|
|
25934
|
+
if (!(result.status === 0 && (recorded?.holds ? recorded.holds(out) : true))) {
|
|
25935
|
+
const detail = (result.stderr || out || `exit ${result.status ?? "timeout"}`).trim().slice(0, 600);
|
|
25936
|
+
return stop(result.status === 0 ? `The check did not hold; the page answered: ${out.slice(0, 300)}` : detail);
|
|
25618
25937
|
}
|
|
25619
|
-
if (
|
|
25620
|
-
|
|
25621
|
-
|
|
25622
|
-
rescuedSteps.push(stepId);
|
|
25623
|
-
}
|
|
25624
|
-
if (rescuedIndices.size === 0) return {
|
|
25625
|
-
kept,
|
|
25626
|
-
dropped,
|
|
25627
|
-
promoted
|
|
25628
|
-
};
|
|
25629
|
-
const keptSet = new Set(kept);
|
|
25630
|
-
const newKept = [];
|
|
25631
|
-
for (let i = 0; i < actions.length; i++) {
|
|
25632
|
-
const action = actions[i];
|
|
25633
|
-
if (rescuedIndices.has(i) || keptSet.has(action)) newKept.push(action);
|
|
25938
|
+
if (recorded) input.onAbAction(recorded.event);
|
|
25939
|
+
const last = i === commands.length - 1;
|
|
25940
|
+
report.push(`✓ ${cmd}${out && (last || out.length < 200) ? `\n${cap(out)}` : ""}`);
|
|
25634
25941
|
}
|
|
25635
|
-
return
|
|
25636
|
-
kept: newKept,
|
|
25637
|
-
dropped: dropped.filter((d) => !rescuedIndices.has(d.index)),
|
|
25638
|
-
rescuedSteps,
|
|
25639
|
-
promoted
|
|
25640
|
-
};
|
|
25942
|
+
return report.join("\n");
|
|
25641
25943
|
}
|
|
25642
25944
|
/**
|
|
25643
|
-
*
|
|
25644
|
-
*
|
|
25645
|
-
* they were trying to observe state the failed action would have set up.
|
|
25945
|
+
* Async, so the trace's deadline and signal teardown still run while a batch
|
|
25946
|
+
* does. Settles on `exit` too: a daemon the command started may hold the pipes open.
|
|
25646
25947
|
*/
|
|
25647
|
-
function
|
|
25648
|
-
return
|
|
25948
|
+
function runShell(cmd, env) {
|
|
25949
|
+
return new Promise((resolve) => {
|
|
25950
|
+
const child = spawn("bash", ["-c", cmd], {
|
|
25951
|
+
env,
|
|
25952
|
+
timeout: COMMAND_TIMEOUT_MS
|
|
25953
|
+
});
|
|
25954
|
+
let stdout = "";
|
|
25955
|
+
let stderr = "";
|
|
25956
|
+
child.stdout.setEncoding("utf8").on("data", (d) => stdout += d);
|
|
25957
|
+
child.stderr.setEncoding("utf8").on("data", (d) => stderr += d);
|
|
25958
|
+
const done = (status, err) => resolve({
|
|
25959
|
+
status,
|
|
25960
|
+
stdout,
|
|
25961
|
+
stderr: stderr || (err ?? "")
|
|
25962
|
+
});
|
|
25963
|
+
child.on("error", (e) => done(null, e.message));
|
|
25964
|
+
child.on("close", (status) => done(status));
|
|
25965
|
+
child.on("exit", (status) => setTimeout(() => done(status), 1e3).unref());
|
|
25966
|
+
});
|
|
25967
|
+
}
|
|
25968
|
+
function cap(out) {
|
|
25969
|
+
return out.length <= OUTPUT_CAP ? out : `${out.slice(0, OUTPUT_CAP)}\n… (${out.length - OUTPUT_CAP} more characters; scope it with snapshot -s <selector>)`;
|
|
25970
|
+
}
|
|
25971
|
+
function replayStep(step, input) {
|
|
25972
|
+
const actions = input.previous.filter((a) => a.stepId === step && a.action !== "snapshot").map(({ replayUnstable: _u, replayReason: _r, ...a }) => a);
|
|
25973
|
+
if (actions.length === 0) return `No previous commands for ${step}. Record it with the Execution Workflow.`;
|
|
25974
|
+
const { passed, unchecked, failed } = replayUntilFailure(actions, {
|
|
25975
|
+
sessionName: input.sessionName,
|
|
25976
|
+
envOverrides: { CCQA_RUN_ID: input.env["CCQA_RUN_ID"] ?? input.sessionName }
|
|
25977
|
+
});
|
|
25978
|
+
input.onReplayed(step, passed);
|
|
25979
|
+
info(`replay_step ${step}: ${passed.length}/${actions.length} replayed${failed ? "" : " — all passed"}`);
|
|
25980
|
+
const notRun = unchecked.length === 0 ? "" : `\nThese checks could not be run here and were not recorded — perform the ones Expected still asks for yourself:\n` + unchecked.map((a) => `- ${describeStepAction(a)}`).join("\n");
|
|
25981
|
+
if (!failed) {
|
|
25982
|
+
const next = unchecked.length === 0 ? " Emit STEP_DONE and go on." : "";
|
|
25983
|
+
return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded, its checks included.${next}${notRun}`;
|
|
25984
|
+
}
|
|
25985
|
+
return `${passed.length} of ${actions.length} previous command(s) of ${step} ran and are recorded. The next one failed: ${describeStepAction(failed.action)} — ${failed.reason}\nThe page is where the recorded commands left it. Finish ${step} with the Execution Workflow (snapshot first); do not re-run the recorded commands.${notRun}`;
|
|
25649
25986
|
}
|
|
25650
25987
|
//#endregion
|
|
25651
25988
|
//#region src/cli/trace.ts
|
|
25989
|
+
/** The recorder's Bash runs agent-browser only: source is read with Read/Grep/Glob. */
|
|
25990
|
+
function traceBashGuard(cmd) {
|
|
25991
|
+
if (isAgentBrowserCommand(cmd)) return null;
|
|
25992
|
+
return "Bash here runs agent-browser commands only. Read, search and list the source with the Read, Grep and Glob tools; wait for the page with agent-browser wait, not sleep.";
|
|
25993
|
+
}
|
|
25994
|
+
/** Browser commands in one step between spec-mismatch checkpoints (see `beforeAbCommand`). */
|
|
25995
|
+
const STEP_CHECKPOINT = 20;
|
|
25652
25996
|
/**
|
|
25653
25997
|
* Step ids (in spec order) whose kept actions include no assertion. A step
|
|
25654
25998
|
* with none produced a test that performs the step but verifies nothing
|
|
@@ -25686,6 +26030,11 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25686
26030
|
}
|
|
25687
26031
|
const conventions = await loadConventions(opts.cwd ?? process.cwd(), { guides: opts.conventions ?? [] });
|
|
25688
26032
|
for (const w of conventions.warnings) warn(w);
|
|
26033
|
+
const caseRef = testCase.ref;
|
|
26034
|
+
const previous = await tryGetRecording(caseRef);
|
|
26035
|
+
const previousActions = previous ? [...previous.actions, ...previous.cleanup ?? []] : [];
|
|
26036
|
+
if (previousActions.length > 0) meta("previous", opts.freshIr ? "ignored (--fresh-ir)" : `${previousActions.length} action(s) offered as a map`);
|
|
26037
|
+
const offersMap = !opts.freshIr && previousActions.length > 0;
|
|
25689
26038
|
const baseSystemPrompt = buildTraceSystemPrompt({
|
|
25690
26039
|
title: testCase.title,
|
|
25691
26040
|
steps,
|
|
@@ -25697,7 +26046,8 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25697
26046
|
...testCase.expectations.length > 0 ? { expectations: testCase.expectations } : {},
|
|
25698
26047
|
...testCase.cleanupExpectations.length > 0 ? { cleanupExpectations: testCase.cleanupExpectations } : {},
|
|
25699
26048
|
...testCase.context.length > 0 ? { context: testCase.context } : {},
|
|
25700
|
-
...opts.instruction ? { instruction: opts.instruction } : {}
|
|
26049
|
+
...opts.instruction ? { instruction: opts.instruction } : {},
|
|
26050
|
+
...offersMap ? { previousRecording: previousActions } : {}
|
|
25701
26051
|
});
|
|
25702
26052
|
const promptBundle = await loadPromptBundle(opts.hubContext ?? null, "record", opts.cwd ?? process.cwd());
|
|
25703
26053
|
if (promptBundle !== null) meta("prompt", promptBundle.loaded.join(" + "));
|
|
@@ -25718,40 +26068,71 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25718
26068
|
} : action;
|
|
25719
26069
|
};
|
|
25720
26070
|
let lastCommandPushCount = 0;
|
|
26071
|
+
const onAbAction = ({ abAction, stepId, assertMarker, secret }) => {
|
|
26072
|
+
const stepForCommand = stepTracker.fromCommand(stepId);
|
|
26073
|
+
const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);
|
|
26074
|
+
let recorded = null;
|
|
26075
|
+
if (assertMarker !== void 0) {
|
|
26076
|
+
recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
|
|
26077
|
+
if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
|
|
26078
|
+
}
|
|
26079
|
+
if (recorded === null) {
|
|
26080
|
+
const parsed = line === null ? null : parseAbActionLine(line);
|
|
26081
|
+
recorded = parsed === null ? [] : [parsed];
|
|
26082
|
+
}
|
|
26083
|
+
let pushed = 0;
|
|
26084
|
+
for (const action of recorded) {
|
|
26085
|
+
const stamped = withStepId(action, stepForCommand);
|
|
26086
|
+
if (stamped) {
|
|
26087
|
+
traceActions.push(secret ? {
|
|
26088
|
+
...stamped,
|
|
26089
|
+
secret: true
|
|
26090
|
+
} : stamped);
|
|
26091
|
+
pushed += 1;
|
|
26092
|
+
}
|
|
26093
|
+
}
|
|
26094
|
+
lastCommandPushCount = pushed;
|
|
26095
|
+
};
|
|
26096
|
+
const commandsByStep = /* @__PURE__ */ new Map();
|
|
26097
|
+
const beforeAbCommand = (stepId) => {
|
|
26098
|
+
if (stepId === void 0) return null;
|
|
26099
|
+
const n = (commandsByStep.get(stepId) ?? 0) + 1;
|
|
26100
|
+
commandsByStep.set(stepId, n);
|
|
26101
|
+
if (n % STEP_CHECKPOINT !== 0) return null;
|
|
26102
|
+
return `${stepId} has taken ${n - 1} browser commands. If the application does not do what this step says — the control is somewhere else, things happen in a different order, the result never appears — stop working around it and report it: ASSERTION_FAILED|${stepId}|spec-mismatch: <what the application does instead> — suggested step: <the instruction / expected rewritten to match>. If the step does match and you are close, run this command again to continue.`;
|
|
26103
|
+
};
|
|
26104
|
+
const invokeBase = agentBrowserInvokeBase({
|
|
26105
|
+
sessionName,
|
|
26106
|
+
runId: sessionName
|
|
26107
|
+
});
|
|
26108
|
+
const traceTools = buildTraceTools({
|
|
26109
|
+
previous: offersMap ? previousActions : [],
|
|
26110
|
+
sessionName,
|
|
26111
|
+
env: invokeBase.env ?? {},
|
|
26112
|
+
envScrubMap,
|
|
26113
|
+
onReplayed: (stepId, passed) => {
|
|
26114
|
+
stepTracker.fromCommand(stepId);
|
|
26115
|
+
for (const action of passed) traceActions.push({
|
|
26116
|
+
...action,
|
|
26117
|
+
stepId
|
|
26118
|
+
});
|
|
26119
|
+
lastCommandPushCount = 0;
|
|
26120
|
+
},
|
|
26121
|
+
onAbAction,
|
|
26122
|
+
beforeAbCommand
|
|
26123
|
+
});
|
|
25721
26124
|
const { isError, errorDetail } = await invokeClaudeStreaming({
|
|
25722
26125
|
prompt,
|
|
25723
26126
|
systemPrompt,
|
|
25724
|
-
...
|
|
25725
|
-
|
|
25726
|
-
|
|
25727
|
-
}),
|
|
26127
|
+
...invokeBase,
|
|
26128
|
+
allowedTools: [...invokeBase.allowedTools, ...traceTools.allowedTools],
|
|
26129
|
+
mcpServers: { [TRACE_TOOLS_SERVER]: traceTools.server },
|
|
25728
26130
|
model,
|
|
25729
26131
|
envScrubMap,
|
|
25730
|
-
onAbAction
|
|
25731
|
-
|
|
25732
|
-
|
|
25733
|
-
|
|
25734
|
-
if (assertMarker !== void 0) {
|
|
25735
|
-
recorded = promoteMarkedAssert(line, scrubEnvValues(assertMarker, envScrubMap));
|
|
25736
|
-
if (recorded === null) warn(`CCQA_ASSERT=${assertMarker} does not match a promotable command (wait --text / get count / url_contains:<substring>) — recording the command without an assert`);
|
|
25737
|
-
}
|
|
25738
|
-
if (recorded === null) {
|
|
25739
|
-
const parsed = line === null ? null : parseAbActionLine(line);
|
|
25740
|
-
recorded = parsed === null ? [] : [parsed];
|
|
25741
|
-
}
|
|
25742
|
-
let pushed = 0;
|
|
25743
|
-
for (const action of recorded) {
|
|
25744
|
-
const stamped = withStepId(action, stepForCommand);
|
|
25745
|
-
if (stamped) {
|
|
25746
|
-
traceActions.push(secret ? {
|
|
25747
|
-
...stamped,
|
|
25748
|
-
secret: true
|
|
25749
|
-
} : stamped);
|
|
25750
|
-
pushed += 1;
|
|
25751
|
-
}
|
|
25752
|
-
}
|
|
25753
|
-
lastCommandPushCount = pushed;
|
|
25754
|
-
},
|
|
26132
|
+
onAbAction,
|
|
26133
|
+
beforeAbCommand,
|
|
26134
|
+
effort: "low",
|
|
26135
|
+
bashGuard: traceBashGuard,
|
|
25755
26136
|
onAbActionFailed: () => {
|
|
25756
26137
|
if (lastCommandPushCount > 0) traceActions.splice(-lastCommandPushCount);
|
|
25757
26138
|
lastCommandPushCount = 0;
|
|
@@ -25786,10 +26167,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25786
26167
|
});
|
|
25787
26168
|
const overallStatus = failureReason === null ? "passed" : "failed";
|
|
25788
26169
|
const dedupedActions = dedupAndReport(scrubAndReport(traceActions));
|
|
26170
|
+
const validationStart = Date.now();
|
|
25789
26171
|
const validatedActions = overallStatus === "passed" || opts.validateFailedTrace === true ? validateAndReport(dedupedActions, validationMode, { CCQA_RUN_ID: sessionName }, opts.teardown) : dedupedActions;
|
|
26172
|
+
if (validatedActions !== dedupedActions) meta("validation time", `${((Date.now() - validationStart) / 1e3).toFixed(1)}s`);
|
|
25790
26173
|
opts.teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
|
|
25791
|
-
const caseRef = testCase.ref;
|
|
25792
|
-
const previous = overallStatus === "passed" ? await tryGetRecording(caseRef) : null;
|
|
25793
26174
|
let recordingPath;
|
|
25794
26175
|
const cleanupIds = new Set(testCase.cleanup.map((s) => s.id));
|
|
25795
26176
|
const routeActions = validatedActions.filter((a) => !cleanupIds.has(a.stepId ?? ""));
|
|
@@ -25828,7 +26209,10 @@ async function runTrace(testCase, model, validationMode = "lenient", language, o
|
|
|
25828
26209
|
*/
|
|
25829
26210
|
function traceFailureReason(lines, session) {
|
|
25830
26211
|
const assertionFailed = lines.find((l) => l.type === "ASSERTION_FAILED");
|
|
25831
|
-
if (assertionFailed)
|
|
26212
|
+
if (assertionFailed) {
|
|
26213
|
+
const detail = assertionFailed.detail.trim();
|
|
26214
|
+
return `${assertionFailed.stepId || "(unnamed step)"} reported ASSERTION_FAILED${detail ? `: ${detail}` : ""}`;
|
|
26215
|
+
}
|
|
25832
26216
|
if (session.isError) return `the Claude session ended in an error: ${session.errorDetail ?? "no detail reported"}`;
|
|
25833
26217
|
const completed = lines.filter((l) => l.type === "RUN_COMPLETED").at(-1);
|
|
25834
26218
|
if (completed === void 0) return "the session ended without a RUN_COMPLETED line — the model stopped before it reported an outcome";
|
|
@@ -26481,6 +26865,7 @@ async function runGenerateLocked(resolved, opts, cwd) {
|
|
|
26481
26865
|
cleanup: testCase.cleanup,
|
|
26482
26866
|
expectations: testCase.expectations,
|
|
26483
26867
|
cleanupExpectations: testCase.cleanupExpectations,
|
|
26868
|
+
...testCase.context.length > 0 ? { context: testCase.context } : {},
|
|
26484
26869
|
fields: testCase.fields,
|
|
26485
26870
|
cwd,
|
|
26486
26871
|
testPath,
|
|
@@ -26655,7 +27040,7 @@ const VALIDATION_MODES = ["lenient", "strict"];
|
|
|
26655
27040
|
const recordCommand = addHubOptions(addProfileOption(addLanguageOption(new Command("record").argument("<case>", "The case to record: a spec id ('<feature>/<spec>'), or — for a target with a `cases` module — a case id or the path of its source file").description("Record a test from a case: run agent-browser to collect actions (trace), then compile them into runnable code via the case's target (generate) — a vitest test.spec.ts for agent-browser, a @playwright/test spec for playwright and for a target the project defines. Recording-backed targets only; spec-input targets like runn have no trace step (use `ccqa generate`), and agent-browser live specs need no recording.").optionsGroup("How to record:").option("-m, --model <name>", "Claude model alias ('sonnet'|'opus'|'haiku') or full ID. Overrides CCQA_MODEL.").option("--instruction <text>", "Extra guidance for the recording agent — e.g. the drift audit's finding when re-recording a drifted spec.").option("--trace-validation <mode>", "What to do with actions that fail post-trace validation: 'lenient' (default) tags them; 'strict' drops them.", (raw) => {
|
|
26656
27041
|
if (VALIDATION_MODES.includes(raw)) return raw;
|
|
26657
27042
|
throw new Error(`--trace-validation must be one of ${VALIDATION_MODES.join(" | ")}`);
|
|
26658
|
-
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
27043
|
+
}, "lenient").option("--auto-fix <mode>", "Auto-fix behaviour during script generation: 'interactive' (default, prompt y/N; declines on non-TTY), 'auto' (apply without prompt, for CI), 'skip' (agent-browser: apply only high-confidence fixes; external targets like playwright/runn: no fix pass at all).", parseAutoFixFlag, "interactive").option("--auto-fix-max-retries <n>", "Maximum number of auto-fix retries", "3").option("--timeout <seconds>", "Abort the recording after this many seconds, wherever it is (trace, generate, auto-fix): reap the browser session, seal the open hub run (--report-to-hub) with a 'timed out' note, and exit 124. Prefer this over wrapping the command in an external `timeout`, whose SIGTERM may never reach this process.", parseTimeoutSeconds).option("--fresh-ir", "Record from scratch. By default an existing ir.json is shown to the recorder as a map of the route, so steps that still work are not rediscovered.").option("--trace-only", "Stop after the trace step; do not generate test code").option("--no-session-pin", "Don't pin AGENT_BROWSER_SESSION / capture page snapshots after a failure (debug toggle)").optionsGroup("What to do with the result:").option("--report-to-hub", "Leave a run (kind: record) on the hub saying this spec was recorded and what the recording spent on Claude, so a budget summed over the hub's runs sees it. It advances no ledger: a recording verifies nothing.").optionsGroup("Learning:").option("--learn-hub-trace-prompt", "After the trace finishes, ask Claude to refresh the \"record.agent\" prompt on the hub from a summary of the run. Requires a hub connection.").optionsGroup("Environment and connection:").option("--cwd <path>", "Working directory containing the .ccqa/ tree (monorepo support). Defaults to the current directory.").option("--project <name>", "Project name for the hub. Defaults to the current directory's name.")))).action(withUsageErrors(async (specPath, opts) => {
|
|
26659
27044
|
await withCostReporting("record", () => runRecord(specPath, opts));
|
|
26660
27045
|
}));
|
|
26661
27046
|
async function runRecord(caseArgument, opts) {
|
|
@@ -26767,6 +27152,7 @@ async function runRecord(caseArgument, opts) {
|
|
|
26767
27152
|
...config.sessionState ? { sessionState: config.sessionState } : {},
|
|
26768
27153
|
...resolved.targetConfig.conventions.operate.length > 0 ? { conventions: resolved.targetConfig.conventions.operate } : {},
|
|
26769
27154
|
...opts.instruction ? { instruction: opts.instruction } : {},
|
|
27155
|
+
...opts.freshIr ? { freshIr: true } : {},
|
|
26770
27156
|
onStep: (stepId) => {
|
|
26771
27157
|
tracingStep = stepId;
|
|
26772
27158
|
}
|