ccqa 1.46.1 → 1.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/ccqa.mjs +336 -2281
- package/dist/diagnose-CSQzwS5f.mjs +2222 -0
- package/dist/package.json +5 -1
- package/dist/runtime/judge.d.mts +46 -0
- package/dist/runtime/judge.mjs +78 -0
- package/dist/runtime/step-evidence.mjs +1 -1
- package/dist/runtime/test-helpers.mjs +2 -2
- package/package.json +5 -1
- /package/dist/{evidence-constants-C425F7ZG.mjs → evidence-constants-Cm_S_5od.mjs} +0 -0
- /package/dist/{spawn-ab-Bm34WBui.mjs → spawn-ab-CR_Sr7wh.mjs} +0 -0
package/dist/bin/ccqa.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { $ as bracedRefsToJsExpression, A as progressEnd, B as isExpandedActionStep, C as error, D as info, E as hint, F as withBuffer, G as tryParseTestSpec, H as isJudgeBody, I as CREDENTIAL_ENV_KEYS, J as SessionNameSchema, K as AGENT_BROWSER_TARGET, L as collectIncludedBlockNames, M as step, N as timedPhase, O as meta, P as warn, Q as isParamRequired, R as expandActionSteps, S as emitRaw, T as header, U as parseBlockSpec, V as isExpandedJudgeByLlmStep, W as parseTestSpec, X as TargetIdSchema, Y as SpecModeSchema, Z as isIncludeStep, _ as buildSpecEnvScrub, a as truncate$2, b as promoteMarkedAssert, c as toAgentBrowserArgs, d as surfaceAxisAside, et as envRefsToJsExpression, f as surfaceDefinitionBlock, g as buildProseEnvScrubMap, h as withCostTally, i as isObject, j as run, k as progress, l as numberLines, m as readCostTally, n as diagnose, nt as resolveEnvRefs, o as describeLocator, p as invokeClaudeStreaming, q as DEFAULT_SPEC_MODE, r as extractJsonCandidates, s as locatorToSelector, t as clamp, tt as iterEnvRefNames, u as outputLanguageBlock, v as scrubEnvValues, w as fix, x as blank, y as parseAbActionLine, z as expandSpec } from "../diagnose-CSQzwS5f.mjs";
|
|
2
3
|
import { HubApiError, createHubClient, hubRequest } from "../hub-client/index.mjs";
|
|
3
|
-
import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-
|
|
4
|
-
import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-
|
|
4
|
+
import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-Cm_S_5od.mjs";
|
|
5
|
+
import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-CR_Sr7wh.mjs";
|
|
5
6
|
import { createRequire } from "node:module";
|
|
6
7
|
import { Command } from "commander";
|
|
7
8
|
import { accessSync, appendFileSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -13,8 +14,7 @@ import { basename, dirname, isAbsolute, join, normalize, posix, relative, resolv
|
|
|
13
14
|
import { parse, stringify } from "yaml";
|
|
14
15
|
import { ZodError, z } from "zod";
|
|
15
16
|
import { execFile, spawn, spawnSync } from "node:child_process";
|
|
16
|
-
import { createSdkMcpServer,
|
|
17
|
-
import { AsyncLocalStorage } from "node:async_hooks";
|
|
17
|
+
import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
|
|
18
18
|
import { promisify } from "node:util";
|
|
19
19
|
import { createInterface } from "node:readline/promises";
|
|
20
20
|
import { createServer } from "node:http";
|
|
@@ -62,354 +62,6 @@ function errMessage(err) {
|
|
|
62
62
|
return err instanceof Error ? err.message : String(err);
|
|
63
63
|
}
|
|
64
64
|
//#endregion
|
|
65
|
-
//#region src/runtime/env-vars.ts
|
|
66
|
-
const ENV_VAR_RE = /\$\{([A-Z_][A-Z0-9_]*)\}|\$([A-Z_][A-Z0-9_]*)/g;
|
|
67
|
-
const ANY_VAR_RE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
68
|
-
/**
|
|
69
|
-
* Replace every `$NAME` / `${NAME}` reference in `value` using `lookup`. When
|
|
70
|
-
* `lookup` returns `undefined`, the original reference text is preserved
|
|
71
|
-
* (callers that want empty-string substitution should wrap with `?? ""`).
|
|
72
|
-
*/
|
|
73
|
-
function substituteVars(value, lookup) {
|
|
74
|
-
ANY_VAR_RE.lastIndex = 0;
|
|
75
|
-
return value.replace(ANY_VAR_RE, (match, braced, plain) => {
|
|
76
|
-
const replacement = lookup(braced ?? plain ?? "");
|
|
77
|
-
return replacement === void 0 ? match : replacement;
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Iterate every `${NAME}` / `$NAME` reference name (case-insensitive form)
|
|
82
|
-
* appearing in `value`. Used by callers that want to enumerate refs without
|
|
83
|
-
* also substituting, e.g. the env-scrub map builder. The reference name
|
|
84
|
-
* grammar is the canonical one shared with `substituteVars`.
|
|
85
|
-
*/
|
|
86
|
-
function* iterEnvRefNames(value) {
|
|
87
|
-
ANY_VAR_RE.lastIndex = 0;
|
|
88
|
-
let m;
|
|
89
|
-
while ((m = ANY_VAR_RE.exec(value)) !== null) {
|
|
90
|
-
const name = m[1] ?? m[2];
|
|
91
|
-
if (name) yield name;
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Resolve every `$VAR` / `${VAR}` reference against `overrides`, then the
|
|
96
|
-
* current process env. `overrides` carries values an invoker injects into a
|
|
97
|
-
* child process (e.g. CCQA_RUN_ID), which beat the parent env there.
|
|
98
|
-
*
|
|
99
|
-
* Missing variables expand to the empty string, mirroring `sh` behaviour.
|
|
100
|
-
* Throwing would force ccqa to be invoked with every var set even for
|
|
101
|
-
* unused blocks, which is more user-hostile than letting the test fail
|
|
102
|
-
* downstream with a clearer message ("login form rejected: empty password").
|
|
103
|
-
*/
|
|
104
|
-
function resolveEnvRefs(value, overrides = {}) {
|
|
105
|
-
return value.replace(ENV_VAR_RE, (_, braced, plain) => {
|
|
106
|
-
const name = braced ?? plain ?? "";
|
|
107
|
-
return overrides[name] ?? process.env[name] ?? "";
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Embed `$VAR` / `${VAR}` as a JS template-literal expression that reads
|
|
112
|
-
* `process.env.VAR ?? ""` at runtime. Used by `ccqa generate` so the test
|
|
113
|
-
* script never bakes in the secret value.
|
|
114
|
-
*
|
|
115
|
-
* Returns a JavaScript string-literal expression (template literal when env
|
|
116
|
-
* refs are present, plain string literal otherwise).
|
|
117
|
-
*
|
|
118
|
-
* Examples:
|
|
119
|
-
* "${PASSWORD}" -> '`${process.env.PASSWORD ?? ""}`'
|
|
120
|
-
* "user-${SUFFIX}@x.com" -> '`user-${process.env.SUFFIX ?? ""}@x.com`'
|
|
121
|
-
* "literal value" -> '"literal value"'
|
|
122
|
-
*/
|
|
123
|
-
function envRefsToJsExpression(value) {
|
|
124
|
-
return refsToJsExpression(value, () => null);
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Generalised version of `envRefsToJsExpression`. Each `$NAME` / `${NAME}`
|
|
128
|
-
* reference in `value` is passed to `nameToExpr(name)` first:
|
|
129
|
-
*
|
|
130
|
-
* - If it returns a string, that string is interpolated as a JS expression
|
|
131
|
-
* (no quoting / no `?? ""` wrap — the caller decides the shape).
|
|
132
|
-
* - If it returns `null`, the reference is treated as a missing env var
|
|
133
|
-
* and expands to `process.env.<NAME> ?? ""` (the legacy behaviour).
|
|
134
|
-
*
|
|
135
|
-
* Used by the block codegen path: param names map to `params.<name>`,
|
|
136
|
-
* everything else falls through to `process.env.X ?? ""`.
|
|
137
|
-
*/
|
|
138
|
-
function refsToJsExpression(value, nameToExpr) {
|
|
139
|
-
ANY_VAR_RE.lastIndex = 0;
|
|
140
|
-
if (!ANY_VAR_RE.test(value)) return JSON.stringify(value);
|
|
141
|
-
const escaped = value.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, (_match, offset, source) => {
|
|
142
|
-
const probe = new RegExp(ANY_VAR_RE.source, "g");
|
|
143
|
-
let m;
|
|
144
|
-
while ((m = probe.exec(source)) !== null) if (m.index === offset) return "${";
|
|
145
|
-
return "\\${";
|
|
146
|
-
});
|
|
147
|
-
ANY_VAR_RE.lastIndex = 0;
|
|
148
|
-
return `\`${escaped.replace(ANY_VAR_RE, (_match, braced, plain) => {
|
|
149
|
-
const name = braced ?? plain ?? "";
|
|
150
|
-
const expr = nameToExpr(name);
|
|
151
|
-
return expr !== null ? `\${${expr}}` : `\${process.env.${name} ?? ""}`;
|
|
152
|
-
})}\``;
|
|
153
|
-
}
|
|
154
|
-
//#endregion
|
|
155
|
-
//#region src/spec/yaml-schema.ts
|
|
156
|
-
/**
|
|
157
|
-
* An action step: one user-facing browser interaction. `instruction` and
|
|
158
|
-
* `expected` are the natural-language description handed to Claude during
|
|
159
|
-
* `ccqa trace`. URLs live inside `instruction`, either verbatim or via
|
|
160
|
-
* `${ENV_VAR}` references (resolved at runtime).
|
|
161
|
-
*/
|
|
162
|
-
const ActionStepSchema = z.object({
|
|
163
|
-
instruction: z.string().min(1),
|
|
164
|
-
expected: z.string().min(1)
|
|
165
|
-
}).strict();
|
|
166
|
-
/**
|
|
167
|
-
* An include step: invokes a reusable block (`.ccqa/blocks/<name>/spec.yaml`).
|
|
168
|
-
* `params` values are plain strings; env refs (`${VAR}`) inside them are
|
|
169
|
-
* resolved at expand time the same way step instructions are.
|
|
170
|
-
*/
|
|
171
|
-
const IncludeStepSchema = z.object({
|
|
172
|
-
include: z.string().min(1),
|
|
173
|
-
params: z.record(z.string(), z.string()).optional()
|
|
174
|
-
}).strict();
|
|
175
|
-
/**
|
|
176
|
-
* A spec step is either an action step or an include step. The two are
|
|
177
|
-
* discriminated by the presence of the `include` key — see `isIncludeStep`.
|
|
178
|
-
*/
|
|
179
|
-
const StepSchema = z.union([ActionStepSchema, IncludeStepSchema]);
|
|
180
|
-
/**
|
|
181
|
-
* Execution mode for `ccqa run`:
|
|
182
|
-
* - `deterministic` (default): vitest replays the recorded `test.spec.ts`.
|
|
183
|
-
* - `live`: Claude drives agent-browser per step (for fragile UIs where
|
|
184
|
-
* codegen is impractical). Cost ~$0.5 per spec.
|
|
185
|
-
*/
|
|
186
|
-
const SpecModeSchema = z.enum(["deterministic", "live"]);
|
|
187
|
-
/**
|
|
188
|
-
* A name a spec chooses that ccqa resolves to a path or looks up in a
|
|
189
|
-
* registry. Restricted to a slug so it cannot escape a directory.
|
|
190
|
-
*/
|
|
191
|
-
function slug$1(what) {
|
|
192
|
-
return z.string().min(1).regex(/^[a-z0-9][a-z0-9._-]*$/i, `${what} must be a slug (letters, digits, '.', '_', '-'; no path separators)`);
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* A saved browser session (cookies + localStorage) to restore before the spec
|
|
196
|
-
* runs, resolved to `.ccqa/sessions/<profile>/<name>.json` at run time.
|
|
197
|
-
*/
|
|
198
|
-
const SessionNameSchema = slug$1("session name");
|
|
199
|
-
/**
|
|
200
|
-
* Sessions to restore before a `mode: live` spec runs: one name or a list,
|
|
201
|
-
* always read back as a list. Multiple names are merged (their cookies +
|
|
202
|
-
* localStorage are unioned) and restored together, so a spec can start
|
|
203
|
-
* signed-in to several providers at once.
|
|
204
|
-
*/
|
|
205
|
-
const SessionFieldSchema = z.union([SessionNameSchema, z.array(SessionNameSchema).min(1)]).transform((v) => Array.isArray(v) ? v : [v]);
|
|
206
|
-
/**
|
|
207
|
-
* A generation-target id: which plugin turns this spec into runnable tests
|
|
208
|
-
* (e.g. "agent-browser", "playwright", "runn"). Whether the id names a
|
|
209
|
-
* registered target is the registry's responsibility, so new targets don't
|
|
210
|
-
* require a schema change.
|
|
211
|
-
*/
|
|
212
|
-
const TargetIdSchema = slug$1("target");
|
|
213
|
-
/** The built-in recorder-backed target. `mode:` / `session:` only apply to it. */
|
|
214
|
-
const AGENT_BROWSER_TARGET = "agent-browser";
|
|
215
|
-
/**
|
|
216
|
-
* Top-level spec schema. `.strict()` rejects any unknown key.
|
|
217
|
-
*
|
|
218
|
-
* `mode:` and `session:` are agent-browser-only fields, enforced here when
|
|
219
|
-
* `target:` names another target. When `target:` is omitted the effective
|
|
220
|
-
* target comes from config (`defaultTarget`, falling back to agent-browser),
|
|
221
|
-
* which this schema can't see — so mode/session pass parsing and the
|
|
222
|
-
* post-resolution check is the target resolver's responsibility.
|
|
223
|
-
*/
|
|
224
|
-
const TestSpecSchema = z.object({
|
|
225
|
-
title: z.string().min(1),
|
|
226
|
-
disabled: z.boolean().optional(),
|
|
227
|
-
target: TargetIdSchema.optional(),
|
|
228
|
-
mode: SpecModeSchema.optional(),
|
|
229
|
-
session: SessionFieldSchema.optional(),
|
|
230
|
-
steps: z.array(StepSchema).min(1)
|
|
231
|
-
}).strict().superRefine((spec, ctx) => {
|
|
232
|
-
if (spec.target === void 0 || spec.target === "agent-browser") return;
|
|
233
|
-
for (const key of ["mode", "session"]) if (spec[key] !== void 0) ctx.addIssue({
|
|
234
|
-
code: "custom",
|
|
235
|
-
path: [key],
|
|
236
|
-
message: `\`${key}\` only applies to the agent-browser target — remove it or drop \`target: ${spec.target}\``
|
|
237
|
-
});
|
|
238
|
-
});
|
|
239
|
-
/** Default mode when `mode:` is absent. */
|
|
240
|
-
const DEFAULT_SPEC_MODE = "deterministic";
|
|
241
|
-
/**
|
|
242
|
-
* A block param declaration. `required` defaults to true; only explicit
|
|
243
|
-
* `required: false` makes it optional. `secret: true` flags the value as
|
|
244
|
-
* sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
|
|
245
|
-
* template literals so the secret never ends up baked into test.spec.ts.
|
|
246
|
-
*/
|
|
247
|
-
const BlockParamSchema = z.object({
|
|
248
|
-
name: z.string().min(1),
|
|
249
|
-
required: z.boolean().optional(),
|
|
250
|
-
secret: z.boolean().optional()
|
|
251
|
-
}).strict();
|
|
252
|
-
/**
|
|
253
|
-
* Block schema. Block steps are restricted to ActionStep — nested blocks are
|
|
254
|
-
* forbidden. Including a block from inside another block fails parsing here
|
|
255
|
-
* (the store layer maps the cryptic "Unrecognized key: 'include'" error into
|
|
256
|
-
* a targeted nested-block message).
|
|
257
|
-
*/
|
|
258
|
-
const BlockSpecSchema = z.object({
|
|
259
|
-
title: z.string().min(1),
|
|
260
|
-
params: z.array(BlockParamSchema).optional(),
|
|
261
|
-
steps: z.array(ActionStepSchema).min(1)
|
|
262
|
-
}).strict();
|
|
263
|
-
/** Runtime predicate for the StepSchema union. */
|
|
264
|
-
function isIncludeStep(step) {
|
|
265
|
-
return "include" in step;
|
|
266
|
-
}
|
|
267
|
-
/** Returns true if a block param is required (default: true). */
|
|
268
|
-
function isParamRequired(param) {
|
|
269
|
-
return param.required !== false;
|
|
270
|
-
}
|
|
271
|
-
//#endregion
|
|
272
|
-
//#region src/spec/parser.ts
|
|
273
|
-
/** The spec/block root (an `unrecognized_keys` issue there has an empty path). */
|
|
274
|
-
const atRoot = (path) => path.length === 0;
|
|
275
|
-
/** A block param entry — the issue path is `params.<index>`. */
|
|
276
|
-
const atBlockParam = (path) => path.length === 2 && path[0] === "params" && typeof path[1] === "number";
|
|
277
|
-
const UNREAD_PARAM_FIELD = "nothing reads it (a block param reaches the prompts as its name, required and secret only). Delete the line.";
|
|
278
|
-
const REMOVED_FIELDS = {
|
|
279
|
-
relatedPaths: {
|
|
280
|
-
at: atRoot,
|
|
281
|
-
message: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field."
|
|
282
|
-
},
|
|
283
|
-
dummy: {
|
|
284
|
-
at: atBlockParam,
|
|
285
|
-
message: UNREAD_PARAM_FIELD
|
|
286
|
-
},
|
|
287
|
-
description: {
|
|
288
|
-
at: atBlockParam,
|
|
289
|
-
message: UNREAD_PARAM_FIELD
|
|
290
|
-
}
|
|
291
|
-
};
|
|
292
|
-
/** Parse a spec.yaml. Schema rejections are rewritten with actionable messages. */
|
|
293
|
-
function parseTestSpec(content, source = "spec.yaml") {
|
|
294
|
-
const raw = parseYamlOrThrow(content, source);
|
|
295
|
-
try {
|
|
296
|
-
return TestSpecSchema.parse(raw);
|
|
297
|
-
} catch (e) {
|
|
298
|
-
throw enrichZodError$1(e, source, false);
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
/**
|
|
302
|
-
* Throw-suppressed sibling of `parseTestSpec`. Used by report-side helpers
|
|
303
|
-
* that derive cosmetic data (title, step descriptions) from spec.yaml and
|
|
304
|
-
* want a missing or malformed file to degrade silently rather than abort
|
|
305
|
-
* the report.
|
|
306
|
-
*/
|
|
307
|
-
function tryParseTestSpec(yaml) {
|
|
308
|
-
if (!yaml) return null;
|
|
309
|
-
try {
|
|
310
|
-
return parseTestSpec(yaml);
|
|
311
|
-
} catch {
|
|
312
|
-
return null;
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Parse a block's spec.yaml. Block-specific errors include the targeted
|
|
317
|
-
* nested-block message (the underlying zod failure on an `include` key
|
|
318
|
-
* inside a block step is hard to read).
|
|
319
|
-
*/
|
|
320
|
-
function parseBlockSpec(content, source = "block spec.yaml") {
|
|
321
|
-
const raw = parseYamlOrThrow(content, source);
|
|
322
|
-
try {
|
|
323
|
-
return BlockSpecSchema.parse(raw);
|
|
324
|
-
} catch (e) {
|
|
325
|
-
throw enrichZodError$1(e, source, true);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
function parseYamlOrThrow(content, source) {
|
|
329
|
-
try {
|
|
330
|
-
return parse(content);
|
|
331
|
-
} catch (e) {
|
|
332
|
-
throw new Error(`Failed to parse YAML (${source}): ${e.message}`);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
function enrichZodError$1(error, source, isBlock) {
|
|
336
|
-
if (!(error instanceof ZodError)) return error;
|
|
337
|
-
const lines = [`Invalid ${source}:`];
|
|
338
|
-
for (const issue of error.issues) {
|
|
339
|
-
const path = issue.path.join(".") || "(root)";
|
|
340
|
-
const message = humanizeIssue(issue, isBlock);
|
|
341
|
-
lines.push(` - ${path}: ${message}`);
|
|
342
|
-
}
|
|
343
|
-
return new Error(lines.join("\n"));
|
|
344
|
-
}
|
|
345
|
-
function humanizeIssue(issue, isBlock) {
|
|
346
|
-
if (issue.code === "unrecognized_keys") {
|
|
347
|
-
const keys = Array.isArray(issue.keys) ? issue.keys : [];
|
|
348
|
-
if (isBlock && keys.includes("include")) return `Nested blocks are not supported — flatten by inlining the included block's steps into this block.`;
|
|
349
|
-
const removed = keys.filter((k) => REMOVED_FIELDS[k]?.at(issue.path));
|
|
350
|
-
const stillUnknown = keys.filter((k) => !REMOVED_FIELDS[k]?.at(issue.path));
|
|
351
|
-
const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k].message}`);
|
|
352
|
-
if (stillUnknown.length > 0) parts.push(`Unknown keys: ${stillUnknown.join(", ")}`);
|
|
353
|
-
return parts.join(" ");
|
|
354
|
-
}
|
|
355
|
-
return issue.message;
|
|
356
|
-
}
|
|
357
|
-
//#endregion
|
|
358
|
-
//#region src/spec/expand.ts
|
|
359
|
-
/**
|
|
360
|
-
* Walk the spec's top-level steps, inlining any `- include: <block>` reference
|
|
361
|
-
* as the block's own steps in order. The result is a flat `step-NN`-numbered
|
|
362
|
-
* sequence — block boundaries survive only as the `source` tag, so trace and
|
|
363
|
-
* codegen never need a separate block code path.
|
|
364
|
-
*/
|
|
365
|
-
function expandSpec(spec, options) {
|
|
366
|
-
const out = [];
|
|
367
|
-
let counter = 0;
|
|
368
|
-
const allocId = () => {
|
|
369
|
-
counter += 1;
|
|
370
|
-
return `step-${String(counter).padStart(2, "0")}`;
|
|
371
|
-
};
|
|
372
|
-
for (const step of spec.steps) if (isIncludeStep(step)) {
|
|
373
|
-
const block = resolveBlock(step.include, step.params ?? {}, options.blocks);
|
|
374
|
-
for (const blockStep of block.steps) out.push({
|
|
375
|
-
id: allocId(),
|
|
376
|
-
source: step.include,
|
|
377
|
-
instruction: substituteVars(blockStep.instruction, block.lookup),
|
|
378
|
-
expected: substituteVars(blockStep.expected, block.lookup)
|
|
379
|
-
});
|
|
380
|
-
} else out.push({
|
|
381
|
-
id: allocId(),
|
|
382
|
-
source: "spec",
|
|
383
|
-
instruction: step.instruction,
|
|
384
|
-
expected: step.expected
|
|
385
|
-
});
|
|
386
|
-
return out;
|
|
387
|
-
}
|
|
388
|
-
function resolveBlock(blockName, rawParams, blocks) {
|
|
389
|
-
const block = blocks.get(blockName);
|
|
390
|
-
if (!block) throw new Error(`Unknown block: "${blockName}". Define it under .ccqa/blocks/${blockName}/spec.yaml.`);
|
|
391
|
-
const declaredParams = new Map((block.params ?? []).map((p) => [p.name, p]));
|
|
392
|
-
for (const key of Object.keys(rawParams)) if (!declaredParams.has(key)) throw new Error(`Block "${blockName}" received unknown param "${key}". Declared params: ${[...declaredParams.keys()].join(", ") || "(none)"}.`);
|
|
393
|
-
for (const [pname, def] of declaredParams) if (isParamRequired(def) && !(pname in rawParams)) throw new Error(`Block "${blockName}" is missing required param "${pname}".`);
|
|
394
|
-
const lookup = (name) => {
|
|
395
|
-
if (Object.prototype.hasOwnProperty.call(rawParams, name)) return rawParams[name];
|
|
396
|
-
};
|
|
397
|
-
return {
|
|
398
|
-
steps: block.steps,
|
|
399
|
-
lookup
|
|
400
|
-
};
|
|
401
|
-
}
|
|
402
|
-
/**
|
|
403
|
-
* Collect every block name referenced by a spec (top-level only — blocks
|
|
404
|
-
* cannot nest). Used by the store / drift layers to know which blocks to
|
|
405
|
-
* load or invalidate.
|
|
406
|
-
*/
|
|
407
|
-
function collectIncludedBlockNames(spec) {
|
|
408
|
-
const names = /* @__PURE__ */ new Set();
|
|
409
|
-
for (const step of spec.steps) if (isIncludeStep(step)) names.add(step.include);
|
|
410
|
-
return [...names];
|
|
411
|
-
}
|
|
412
|
-
//#endregion
|
|
413
65
|
//#region src/store/index.ts
|
|
414
66
|
const CCQA_DIR = ".ccqa";
|
|
415
67
|
const SPEC_FILE = "spec.yaml";
|
|
@@ -748,10 +400,10 @@ function bundledVitestConfigPath() {
|
|
|
748
400
|
}
|
|
749
401
|
//#endregion
|
|
750
402
|
//#region src/runtime/spawn-vitest.ts
|
|
751
|
-
const require$
|
|
403
|
+
const require$1 = createRequire(import.meta.url);
|
|
752
404
|
function resolveVitestBin() {
|
|
753
|
-
const pkgPath = require$
|
|
754
|
-
const pkg = require$
|
|
405
|
+
const pkgPath = require$1.resolve("vitest/package.json");
|
|
406
|
+
const pkg = require$1(pkgPath);
|
|
755
407
|
const binRel = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.vitest;
|
|
756
408
|
if (!binRel) throw new Error(`vitest package.json has no bin entry (resolved at ${pkgPath})`);
|
|
757
409
|
return resolve(dirname(pkgPath), binRel);
|
|
@@ -813,1902 +465,135 @@ function waitExit(child) {
|
|
|
813
465
|
* random suffix because the timestamp alone does not separate two specs.
|
|
814
466
|
*
|
|
815
467
|
* The pool launches specs back-to-back, so at `--concurrency > 1` two of them
|
|
816
|
-
* land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
|
|
817
|
-
* of something it creates would then share that name with its neighbour, and
|
|
818
|
-
* each would find — and delete — the other's row. Nothing fails; the
|
|
819
|
-
* assertions just read the wrong state.
|
|
820
|
-
*
|
|
821
|
-
* Caller is expected to mkdir the directory once and pass
|
|
822
|
-
* `runDir = <baseDir>/<runId>` to the path helpers below.
|
|
823
|
-
*/
|
|
824
|
-
function buildRunId() {
|
|
825
|
-
return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
|
|
826
|
-
}
|
|
827
|
-
/**
|
|
828
|
-
* Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
|
|
829
|
-
* Three files per step:
|
|
830
|
-
* - <stepId>.before.png : screenshot taken BEFORE Claude executes the step.
|
|
831
|
-
* - <stepId>.after.png : screenshot taken AFTER Claude executes the step.
|
|
832
|
-
* - <stepId>.log.txt : full assistant transcript for the step (judgement
|
|
833
|
-
* reasoning, any STEP_RESULT lines, raw tool output
|
|
834
|
-
* summaries the model chose to keep).
|
|
835
|
-
*/
|
|
836
|
-
function stepArtifactPaths(runDir, stepId) {
|
|
837
|
-
const dir = join(runDir, "steps");
|
|
838
|
-
return {
|
|
839
|
-
beforePng: join(dir, `${stepId}.before.png`),
|
|
840
|
-
afterPng: join(dir, `${stepId}.after.png`),
|
|
841
|
-
logTxt: join(dir, `${stepId}.log.txt`)
|
|
842
|
-
};
|
|
843
|
-
}
|
|
844
|
-
//#endregion
|
|
845
|
-
//#region src/runtime/pool.ts
|
|
846
|
-
async function runPool(items, concurrency, fn, opts = {}) {
|
|
847
|
-
const results = new Array(items.length);
|
|
848
|
-
const needs = items.map((item) => opts.resources?.(item) ?? []);
|
|
849
|
-
const busy = /* @__PURE__ */ new Set();
|
|
850
|
-
const queued = new Set(items.map((_, i) => i));
|
|
851
|
-
const inFlight = /* @__PURE__ */ new Map();
|
|
852
|
-
const limit = Math.max(1, Math.min(concurrency, items.length));
|
|
853
|
-
const failures = [];
|
|
854
|
-
const start = (idx) => {
|
|
855
|
-
queued.delete(idx);
|
|
856
|
-
for (const name of needs[idx]) busy.add(name);
|
|
857
|
-
inFlight.set(idx, Promise.resolve().then(async () => {
|
|
858
|
-
try {
|
|
859
|
-
results[idx] = await fn(items[idx], idx);
|
|
860
|
-
} catch (err) {
|
|
861
|
-
failures.push(err);
|
|
862
|
-
} finally {
|
|
863
|
-
for (const name of needs[idx]) busy.delete(name);
|
|
864
|
-
inFlight.delete(idx);
|
|
865
|
-
}
|
|
866
|
-
}));
|
|
867
|
-
};
|
|
868
|
-
while (queued.size > 0 || inFlight.size > 0) {
|
|
869
|
-
if (failures.length === 0) {
|
|
870
|
-
for (const idx of queued) {
|
|
871
|
-
if (inFlight.size >= limit) break;
|
|
872
|
-
if (needs[idx].some((name) => busy.has(name))) continue;
|
|
873
|
-
start(idx);
|
|
874
|
-
}
|
|
875
|
-
if (inFlight.size === 0) throw new Error(`runPool: ${queued.size} item(s) unrunnable with nothing in flight`);
|
|
876
|
-
}
|
|
877
|
-
if (inFlight.size === 0) break;
|
|
878
|
-
await Promise.race(inFlight.values());
|
|
879
|
-
}
|
|
880
|
-
if (failures.length === 1) throw failures[0];
|
|
881
|
-
if (failures.length > 1) throw new AggregateError(failures, `${failures.length} items failed`);
|
|
882
|
-
return results;
|
|
883
|
-
}
|
|
884
|
-
//#endregion
|
|
885
|
-
//#region src/claude/env-keys.ts
|
|
886
|
-
/**
|
|
887
|
-
* Variables that carry a credential the Claude Code process can use on its
|
|
888
|
-
* own, with no login on the host: an API key, a gateway bearer token, or a
|
|
889
|
-
* subscription token from `claude setup-token`.
|
|
890
|
-
*/
|
|
891
|
-
const CREDENTIAL_ENV_KEYS = [
|
|
892
|
-
"ANTHROPIC_API_KEY",
|
|
893
|
-
"ANTHROPIC_AUTH_TOKEN",
|
|
894
|
-
"CLAUDE_CODE_OAUTH_TOKEN"
|
|
895
|
-
];
|
|
896
|
-
/**
|
|
897
|
-
* Standard Claude Code environment variables that select the API endpoint and
|
|
898
|
-
* credentials. ccqa forwards whichever of these are set to the underlying
|
|
899
|
-
* Claude Code process; it does not read or interpret their values.
|
|
900
|
-
*
|
|
901
|
-
* - `ANTHROPIC_BASE_URL` — the API endpoint to send requests to.
|
|
902
|
-
* - `ANTHROPIC_AUTH_TOKEN` — sent as `Authorization: Bearer <token>`.
|
|
903
|
-
* - `ANTHROPIC_API_KEY` — API key, when used instead of a token.
|
|
904
|
-
* - `ANTHROPIC_CUSTOM_HEADERS` — extra request headers.
|
|
905
|
-
* - `CLAUDE_CODE_OAUTH_TOKEN` — long-lived subscription token from
|
|
906
|
-
* `claude setup-token`, the headless-CI counterpart of a login.
|
|
907
|
-
*/
|
|
908
|
-
const ENDPOINT_ENV_KEYS = [
|
|
909
|
-
"ANTHROPIC_BASE_URL",
|
|
910
|
-
"ANTHROPIC_CUSTOM_HEADERS",
|
|
911
|
-
...CREDENTIAL_ENV_KEYS
|
|
912
|
-
];
|
|
913
|
-
//#endregion
|
|
914
|
-
//#region src/drift/auth.ts
|
|
915
|
-
/**
|
|
916
|
-
* Claude Code can also run against AWS Bedrock / Google Vertex AI, selected by
|
|
917
|
-
* these env toggles. Credentials then come from the cloud SDK's own chain
|
|
918
|
-
* (instance/task roles, gcloud auth, …), so none of the Anthropic-side probes
|
|
919
|
-
* below apply — a set toggle counts as auth being available. ccqa forwards the
|
|
920
|
-
* toggle verbatim; only "0"/"false" (any case) is treated as explicitly off.
|
|
921
|
-
*/
|
|
922
|
-
const CLOUD_PROVIDER_ENV_KEYS = ["CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX"];
|
|
923
|
-
function cloudProviderEnabled() {
|
|
924
|
-
return CLOUD_PROVIDER_ENV_KEYS.some((key) => {
|
|
925
|
-
const value = process.env[key]?.trim().toLowerCase();
|
|
926
|
-
return value !== void 0 && value !== "" && value !== "0" && value !== "false";
|
|
927
|
-
});
|
|
928
|
-
}
|
|
929
|
-
/**
|
|
930
|
-
* Probe whether the host has any credential the Anthropic SDK can pick up:
|
|
931
|
-
* - one of CREDENTIAL_ENV_KEYS (API key, gateway bearer token, or the
|
|
932
|
-
* subscription token from `claude setup-token`)
|
|
933
|
-
* - CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
|
|
934
|
-
* endpoints authenticated by the cloud SDK's credential chain)
|
|
935
|
-
* - ~/.claude/.credentials.json (Claude Code login, file-based platforms)
|
|
936
|
-
* - macOS Keychain item "Claude Code-credentials" (Claude Code login on
|
|
937
|
-
* darwin stores the OAuth credentials in the Keychain, not on disk)
|
|
938
|
-
*
|
|
939
|
-
* Claude-driven hooks are opt-in, so the caller only consults this after the
|
|
940
|
-
* user has asked for analysis. We never throw — auth absence is a normal flow
|
|
941
|
-
* that surfaces as "analysis skipped".
|
|
942
|
-
*/
|
|
943
|
-
function driftAuthAvailable() {
|
|
944
|
-
for (const key of CREDENTIAL_ENV_KEYS) {
|
|
945
|
-
const value = process.env[key];
|
|
946
|
-
if (typeof value === "string" && value.length > 0) return { ok: true };
|
|
947
|
-
}
|
|
948
|
-
if (cloudProviderEnabled()) return { ok: true };
|
|
949
|
-
if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
|
|
950
|
-
if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
|
|
951
|
-
return {
|
|
952
|
-
ok: false,
|
|
953
|
-
reason: `no ${CREDENTIAL_ENV_KEYS.join(" / ")} / Bedrock or Vertex env / claude login`
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
/**
|
|
957
|
-
* `security find-generic-password` without `-w` only checks the item's
|
|
958
|
-
* existence (exit 0) — it never reads the secret, so no Keychain unlock
|
|
959
|
-
* prompt is triggered. Resolved via PATH so tests can stub the binary.
|
|
960
|
-
*/
|
|
961
|
-
function keychainHasClaudeCredentials() {
|
|
962
|
-
try {
|
|
963
|
-
return spawnSync("security", [
|
|
964
|
-
"find-generic-password",
|
|
965
|
-
"-s",
|
|
966
|
-
"Claude Code-credentials"
|
|
967
|
-
], {
|
|
968
|
-
stdio: "ignore",
|
|
969
|
-
timeout: 3e3
|
|
970
|
-
}).status === 0;
|
|
971
|
-
} catch {
|
|
972
|
-
return false;
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
//#endregion
|
|
976
|
-
//#region src/cli/logger.ts
|
|
977
|
-
const STEP_ICONS = {
|
|
978
|
-
STEP_START: "▶",
|
|
979
|
-
STEP_DONE: "✓",
|
|
980
|
-
ASSERTION_FAILED: "✗",
|
|
981
|
-
STEP_SKIPPED: "⊘",
|
|
982
|
-
RUN_COMPLETED: "■"
|
|
983
|
-
};
|
|
984
|
-
/**
|
|
985
|
-
* When a `withBuffer` scope is active, every log line (stdout and stderr) is
|
|
986
|
-
* appended to its buffer instead of being written immediately. Parallel spec
|
|
987
|
-
* runs use this so each spec's narration — including logs emitted deep inside
|
|
988
|
-
* the live executor — flushes as one contiguous block, not interleaved.
|
|
989
|
-
*/
|
|
990
|
-
const bufferStore = new AsyncLocalStorage();
|
|
991
|
-
const sinkStore = new AsyncLocalStorage();
|
|
992
|
-
/** True while inside a `withBuffer` scope: progress lines avoid TTY cursor tricks. */
|
|
993
|
-
function isBuffered() {
|
|
994
|
-
return bufferStore.getStore() !== void 0;
|
|
995
|
-
}
|
|
996
|
-
function emit(text, sink = process.stdout) {
|
|
997
|
-
const store = bufferStore.getStore();
|
|
998
|
-
if (store) {
|
|
999
|
-
store.out.push(text);
|
|
1000
|
-
return;
|
|
1001
|
-
}
|
|
1002
|
-
const activeSink = sinkStore.getStore();
|
|
1003
|
-
if (activeSink) {
|
|
1004
|
-
activeSink.write(text);
|
|
1005
|
-
return;
|
|
1006
|
-
}
|
|
1007
|
-
sink.write(text);
|
|
1008
|
-
}
|
|
1009
|
-
/**
|
|
1010
|
-
* Write raw text to the active `withBuffer` scope, or straight to stdout when
|
|
1011
|
-
* none is active. Lets a runner redirect sub-process output (e.g. a child's
|
|
1012
|
-
* stdout) into the same buffer as its `log.*` lines so they flush together.
|
|
1013
|
-
*/
|
|
1014
|
-
function emitRaw(text) {
|
|
1015
|
-
emit(text);
|
|
1016
|
-
}
|
|
1017
|
-
/**
|
|
1018
|
-
* Run `fn` with all its log output captured into a buffer, then flush the
|
|
1019
|
-
* buffer in one shot under `label`. Used by parallel runners to keep each
|
|
1020
|
-
* spec's output legible. Output is flushed even when `fn` throws.
|
|
1021
|
-
*
|
|
1022
|
-
* When `buffered` is false, `fn` runs with no buffer so its output streams
|
|
1023
|
-
* live — this is the sequential (concurrency 1) path, unchanged from before.
|
|
1024
|
-
*/
|
|
1025
|
-
async function withBuffer(label, buffered, fn) {
|
|
1026
|
-
if (!buffered) return fn();
|
|
1027
|
-
const store = { out: [] };
|
|
1028
|
-
try {
|
|
1029
|
-
return await bufferStore.run(store, fn);
|
|
1030
|
-
} finally {
|
|
1031
|
-
emit(`\n──── ${label} ────\n${store.out.join("")}`);
|
|
1032
|
-
}
|
|
1033
|
-
}
|
|
1034
|
-
function header(command, target) {
|
|
1035
|
-
emit(`\nccqa ${command}${target ? ` ${target}` : ""}\n\n`);
|
|
1036
|
-
}
|
|
1037
|
-
function write(scope, message, sink = process.stdout) {
|
|
1038
|
-
emit(`[${scope}] ${message}\n`, sink);
|
|
1039
|
-
}
|
|
1040
|
-
function meta(key, value) {
|
|
1041
|
-
write("meta", `${key}: ${value}`);
|
|
1042
|
-
}
|
|
1043
|
-
function blank() {
|
|
1044
|
-
emit("\n");
|
|
1045
|
-
}
|
|
1046
|
-
function info(message) {
|
|
1047
|
-
write("info", message);
|
|
1048
|
-
}
|
|
1049
|
-
function step(type, stepId, detail) {
|
|
1050
|
-
emit(` ${STEP_ICONS[type]} [${stepId}] ${detail}\n`);
|
|
1051
|
-
}
|
|
1052
|
-
function bash(command) {
|
|
1053
|
-
emit(` $ ${command.slice(0, 120)}\n`);
|
|
1054
|
-
}
|
|
1055
|
-
function error(message) {
|
|
1056
|
-
write("error", message, process.stderr);
|
|
1057
|
-
}
|
|
1058
|
-
function warn(message) {
|
|
1059
|
-
write("warn", message, process.stderr);
|
|
1060
|
-
}
|
|
1061
|
-
function hint(message) {
|
|
1062
|
-
emit("\n");
|
|
1063
|
-
write("hint", message);
|
|
1064
|
-
}
|
|
1065
|
-
function fix(message) {
|
|
1066
|
-
write("fix", message);
|
|
1067
|
-
}
|
|
1068
|
-
function run(message) {
|
|
1069
|
-
write("run", message);
|
|
1070
|
-
}
|
|
1071
|
-
/**
|
|
1072
|
-
* Render a single-line progress indicator for a step-by-step loop.
|
|
1073
|
-
*
|
|
1074
|
-
* On a TTY the line is rewritten in place via `\r` so the terminal stays
|
|
1075
|
-
* uncluttered. In a non-TTY environment (CI, piped runs) we fall back to
|
|
1076
|
-
* a regular `[info]` line every PROGRESS_NONTTY_STRIDE steps to avoid
|
|
1077
|
-
* spamming the log with one line per action.
|
|
1078
|
-
*
|
|
1079
|
-
* Callers MUST call `progressEnd()` when the loop finishes (or aborts) so
|
|
1080
|
-
* the carriage-return line gets a final newline; otherwise the next log
|
|
1081
|
-
* line lands on the same physical row.
|
|
1082
|
-
*/
|
|
1083
|
-
const PROGRESS_NONTTY_STRIDE = 5;
|
|
1084
|
-
let lastProgressNonTtyEmit = -1;
|
|
1085
|
-
function progress(current, total, label) {
|
|
1086
|
-
const text = `[info] ${current + 1}/${total} ${label}`;
|
|
1087
|
-
if (process.stdout.isTTY && !isBuffered()) {
|
|
1088
|
-
process.stdout.write(`\r${text}\x1b[K`);
|
|
1089
|
-
return;
|
|
1090
|
-
}
|
|
1091
|
-
if (current === 0 || current - lastProgressNonTtyEmit >= PROGRESS_NONTTY_STRIDE) {
|
|
1092
|
-
emit(`${text}\n`);
|
|
1093
|
-
lastProgressNonTtyEmit = current;
|
|
1094
|
-
}
|
|
1095
|
-
}
|
|
1096
|
-
function progressEnd() {
|
|
1097
|
-
if (process.stdout.isTTY && !isBuffered()) process.stdout.write(`\r\x1b[K`);
|
|
1098
|
-
lastProgressNonTtyEmit = -1;
|
|
1099
|
-
}
|
|
1100
|
-
/**
|
|
1101
|
-
* Time a long-running step under the given scope, emitting `started` and
|
|
1102
|
-
* `finished in N.Ns` markers. Scope must be a tag the user wants to grep
|
|
1103
|
-
* for — typically "run" for vitest and "fix" for diagnose-loop steps.
|
|
1104
|
-
*/
|
|
1105
|
-
async function timedPhase(label, fn, scope = "fix") {
|
|
1106
|
-
const startedAt = Date.now();
|
|
1107
|
-
write(scope, `${label} started`);
|
|
1108
|
-
try {
|
|
1109
|
-
const result = await fn();
|
|
1110
|
-
write(scope, `${label} finished in ${((Date.now() - startedAt) / 1e3).toFixed(1)}s`);
|
|
1111
|
-
return result;
|
|
1112
|
-
} catch (err) {
|
|
1113
|
-
write(scope, `${label} threw after ${((Date.now() - startedAt) / 1e3).toFixed(1)}s`);
|
|
1114
|
-
throw err;
|
|
1115
|
-
}
|
|
1116
|
-
}
|
|
1117
|
-
//#endregion
|
|
1118
|
-
//#region src/ir/from-agent-browser.ts
|
|
1119
|
-
/**
|
|
1120
|
-
* Normalization from the agent-browser side of the recorder into the IR.
|
|
1121
|
-
* The trace protocol emits one pipe-delimited `AB_ACTION|...` line per
|
|
1122
|
-
* browser action (see `src/prompts/trace.ts` and
|
|
1123
|
-
* `claude/invoke.ts:extractAbActionFromBashCommand`); `parseAbActionLine`
|
|
1124
|
-
* turns each line into a `RecordedAction`.
|
|
1125
|
-
*
|
|
1126
|
-
* The mapping is a deterministic re-encoding: `to-agent-browser.ts` is its
|
|
1127
|
-
* inverse, and the round-trip identity (ab argv → wire → IR → ab argv) is
|
|
1128
|
-
* pinned by `roundtrip.test.ts`.
|
|
1129
|
-
*/
|
|
1130
|
-
/**
|
|
1131
|
-
* Semantic locator strategies exposed by `agent-browser find`. Used by the
|
|
1132
|
-
* `find_*` wire commands when a target cannot be uniquely picked out by the
|
|
1133
|
-
* ALLOWED CSS forms (e.g. repeated `aria-label='1 reply'` rows where only
|
|
1134
|
-
* "the last one" is meaningful).
|
|
1135
|
-
*
|
|
1136
|
-
* `first` / `last` / `nth` are positional helpers whose value carries an
|
|
1137
|
-
* inner CSS selector (`nth` additionally needs an index); they normalize to
|
|
1138
|
-
* a `css` Locator plus `index`. The remaining strategies read the value as
|
|
1139
|
-
* the human-visible text/id and normalize to the matching `Locator.by`.
|
|
1140
|
-
*/
|
|
1141
|
-
const FIND_LOCATORS = [
|
|
1142
|
-
"role",
|
|
1143
|
-
"text",
|
|
1144
|
-
"label",
|
|
1145
|
-
"placeholder",
|
|
1146
|
-
"alt",
|
|
1147
|
-
"title",
|
|
1148
|
-
"testid",
|
|
1149
|
-
"first",
|
|
1150
|
-
"last",
|
|
1151
|
-
"nth"
|
|
1152
|
-
];
|
|
1153
|
-
/**
|
|
1154
|
-
* Actions reachable via `agent-browser find <locator> ... <action>`. Kept
|
|
1155
|
-
* here next to the locator list so all `find` wire knowledge lives in one
|
|
1156
|
-
* place — `claude/invoke.ts` imports these instead of redefining its own sets.
|
|
1157
|
-
*/
|
|
1158
|
-
const FIND_ACTIONS = [
|
|
1159
|
-
"click",
|
|
1160
|
-
"dblclick",
|
|
1161
|
-
"fill",
|
|
1162
|
-
"type",
|
|
1163
|
-
"hover",
|
|
1164
|
-
"focus",
|
|
1165
|
-
"check",
|
|
1166
|
-
"uncheck"
|
|
1167
|
-
];
|
|
1168
|
-
const css = (value) => ({
|
|
1169
|
-
by: "css",
|
|
1170
|
-
value
|
|
1171
|
-
});
|
|
1172
|
-
function parseAbActionLine(line) {
|
|
1173
|
-
if (!line.startsWith("AB_ACTION|")) return null;
|
|
1174
|
-
const parts = line.split("|");
|
|
1175
|
-
const command = parts[1];
|
|
1176
|
-
switch (command) {
|
|
1177
|
-
case "cookies_clear": return { action: "cookies_clear" };
|
|
1178
|
-
case "open": return {
|
|
1179
|
-
action: "navigate",
|
|
1180
|
-
value: (parts[2] ?? "").replace(/^["']|["']$/g, "")
|
|
1181
|
-
};
|
|
1182
|
-
case "press": return {
|
|
1183
|
-
action: "press",
|
|
1184
|
-
...opt("value", parts[2])
|
|
1185
|
-
};
|
|
1186
|
-
case "scroll": return {
|
|
1187
|
-
action: "scroll",
|
|
1188
|
-
...opt("direction", parts[2]),
|
|
1189
|
-
...opt("pixels", parts[3])
|
|
1190
|
-
};
|
|
1191
|
-
case "snapshot": return {
|
|
1192
|
-
action: "snapshot",
|
|
1193
|
-
...opt("observation", parts[2])
|
|
1194
|
-
};
|
|
1195
|
-
case "assert": return {
|
|
1196
|
-
action: "assert",
|
|
1197
|
-
assert: parts[2],
|
|
1198
|
-
...parts[3] ? { locator: css(parts[3]) } : {},
|
|
1199
|
-
...parts[4] ? { value: parts[4] } : {},
|
|
1200
|
-
...parts[5] ? { observation: parts[5] } : {}
|
|
1201
|
-
};
|
|
1202
|
-
case "click":
|
|
1203
|
-
case "dblclick":
|
|
1204
|
-
case "check":
|
|
1205
|
-
case "uncheck":
|
|
1206
|
-
case "hover":
|
|
1207
|
-
if (!parts[2]) return null;
|
|
1208
|
-
return {
|
|
1209
|
-
action: command,
|
|
1210
|
-
locator: css(parts[2]),
|
|
1211
|
-
...opt("label", parts[3])
|
|
1212
|
-
};
|
|
1213
|
-
case "wait":
|
|
1214
|
-
if (parts[2] === "--text") {
|
|
1215
|
-
if (!parts[3]) return null;
|
|
1216
|
-
return {
|
|
1217
|
-
action: "wait",
|
|
1218
|
-
locator: {
|
|
1219
|
-
by: "text",
|
|
1220
|
-
value: parts[3]
|
|
1221
|
-
},
|
|
1222
|
-
...opt("label", parts[4])
|
|
1223
|
-
};
|
|
1224
|
-
}
|
|
1225
|
-
if (!parts[2]) return null;
|
|
1226
|
-
return {
|
|
1227
|
-
action: "wait",
|
|
1228
|
-
locator: css(parts[2]),
|
|
1229
|
-
...opt("label", parts[3])
|
|
1230
|
-
};
|
|
1231
|
-
case "fill":
|
|
1232
|
-
case "type":
|
|
1233
|
-
case "select":
|
|
1234
|
-
if (!parts[2]) return null;
|
|
1235
|
-
return {
|
|
1236
|
-
action: command,
|
|
1237
|
-
locator: css(parts[2]),
|
|
1238
|
-
...parts[3] !== void 0 ? { value: parts[3] } : {},
|
|
1239
|
-
...opt("label", parts[4])
|
|
1240
|
-
};
|
|
1241
|
-
case "drag":
|
|
1242
|
-
if (!parts[2] || !parts[3]) return null;
|
|
1243
|
-
return {
|
|
1244
|
-
action: "drag",
|
|
1245
|
-
locator: css(parts[2]),
|
|
1246
|
-
target: css(parts[3]),
|
|
1247
|
-
...opt("label", parts[4])
|
|
1248
|
-
};
|
|
1249
|
-
case "upload": {
|
|
1250
|
-
const selector = parts[2];
|
|
1251
|
-
const files = parts.slice(3).filter((f) => f !== "");
|
|
1252
|
-
if (!selector || files.length === 0) return null;
|
|
1253
|
-
return {
|
|
1254
|
-
action: "upload",
|
|
1255
|
-
locator: css(selector),
|
|
1256
|
-
files
|
|
1257
|
-
};
|
|
1258
|
-
}
|
|
1259
|
-
case "find_click":
|
|
1260
|
-
case "find_dblclick":
|
|
1261
|
-
case "find_hover":
|
|
1262
|
-
case "find_focus":
|
|
1263
|
-
case "find_check":
|
|
1264
|
-
case "find_uncheck": return parseFindAction(command.slice(5), parts, false);
|
|
1265
|
-
case "find_fill":
|
|
1266
|
-
case "find_type": return parseFindAction(command.slice(5), parts, true);
|
|
1267
|
-
case "get_count":
|
|
1268
|
-
case "get_url": return null;
|
|
1269
|
-
default: return null;
|
|
1270
|
-
}
|
|
1271
|
-
}
|
|
1272
|
-
/**
|
|
1273
|
-
* Promote a `CCQA_ASSERT=<marker>` env marker on an agent-browser command
|
|
1274
|
-
* into recorded assert action(s). The marker travels on the same channel as
|
|
1275
|
-
* the command itself (see `claude/invoke.ts`), so a verification the model
|
|
1276
|
-
* performs anyway (`wait --text`, `get count`) becomes a recorded assert
|
|
1277
|
-
* without relying on the `AB_ACTION|assert|...` text protocol.
|
|
1278
|
-
*
|
|
1279
|
-
* `abAction` is the wire line for the marked command (null when the command
|
|
1280
|
-
* has no wire form at all). Mapping — anything else returns null and the
|
|
1281
|
-
* caller warns and records the command unpromoted:
|
|
1282
|
-
*
|
|
1283
|
-
* - `wait --text "X"` + `1` (or `text_visible`) → `assert text_visible X`,
|
|
1284
|
-
* REPLACING the wait: the emitted abAssert is itself a timed wait, so
|
|
1285
|
-
* keeping both would wait twice.
|
|
1286
|
-
* - `get count "<sel>"` + `element_visible` / `element_not_visible`
|
|
1287
|
-
* → `assert <marker> <sel>` (the probe records nothing by itself).
|
|
1288
|
-
* - any command + `url_contains:<substring>` → the command's own action (if
|
|
1289
|
-
* it records one) followed by `assert url_contains <substring>`.
|
|
1290
|
-
*/
|
|
1291
|
-
function promoteMarkedAssert(abAction, marker) {
|
|
1292
|
-
if (marker.startsWith("url_contains:")) {
|
|
1293
|
-
const substring = marker.slice(13);
|
|
1294
|
-
if (!substring) return null;
|
|
1295
|
-
const assert = {
|
|
1296
|
-
action: "assert",
|
|
1297
|
-
assert: "url_contains",
|
|
1298
|
-
value: substring
|
|
1299
|
-
};
|
|
1300
|
-
const base = abAction === null ? null : parseAbActionLine(abAction);
|
|
1301
|
-
return base === null ? [assert] : [base, assert];
|
|
1302
|
-
}
|
|
1303
|
-
const parts = abAction === null ? [] : abAction.split("|");
|
|
1304
|
-
if (marker === "1" || marker === "text_visible") {
|
|
1305
|
-
if (parts[1] === "wait" && parts[2] === "--text" && parts[3]) return [{
|
|
1306
|
-
action: "assert",
|
|
1307
|
-
assert: "text_visible",
|
|
1308
|
-
value: parts[3]
|
|
1309
|
-
}];
|
|
1310
|
-
return null;
|
|
1311
|
-
}
|
|
1312
|
-
if (marker === "element_visible" || marker === "element_not_visible") {
|
|
1313
|
-
if (parts[1] === "get_count" && parts[2]) return [{
|
|
1314
|
-
action: "assert",
|
|
1315
|
-
assert: marker,
|
|
1316
|
-
locator: css(parts[2])
|
|
1317
|
-
}];
|
|
1318
|
-
return null;
|
|
1319
|
-
}
|
|
1320
|
-
return null;
|
|
1321
|
-
}
|
|
1322
|
-
/**
|
|
1323
|
-
* Common parser for the `find_*` wire family. `<extra>` carries `--name` for
|
|
1324
|
-
* `role`, the integer index for `nth`, and is empty otherwise. We accept a
|
|
1325
|
-
* literally empty `<extra>` (the LLM emits a placeholder `|` so the
|
|
1326
|
-
* positional layout stays stable across locators).
|
|
1327
|
-
*/
|
|
1328
|
-
function parseFindAction(action, parts, hasFillValue) {
|
|
1329
|
-
const locatorToken = parts[2];
|
|
1330
|
-
const findValue = parts[3];
|
|
1331
|
-
const extra = parts[4] ?? "";
|
|
1332
|
-
const exact = (parts[5] ?? "") === "exact";
|
|
1333
|
-
if (!locatorToken || !FIND_LOCATORS.includes(locatorToken) || !findValue) return null;
|
|
1334
|
-
let locator;
|
|
1335
|
-
let index;
|
|
1336
|
-
if (locatorToken === "first" || locatorToken === "last") {
|
|
1337
|
-
locator = css(findValue);
|
|
1338
|
-
index = locatorToken;
|
|
1339
|
-
} else if (locatorToken === "nth") {
|
|
1340
|
-
const parsed = extra ? Number.parseInt(extra, 10) : NaN;
|
|
1341
|
-
if (Number.isNaN(parsed)) return null;
|
|
1342
|
-
locator = css(findValue);
|
|
1343
|
-
index = parsed;
|
|
1344
|
-
} else if (locatorToken === "role") locator = {
|
|
1345
|
-
by: "role",
|
|
1346
|
-
value: findValue,
|
|
1347
|
-
...extra ? { name: extra } : {},
|
|
1348
|
-
...exact ? { exact: true } : {}
|
|
1349
|
-
};
|
|
1350
|
-
else locator = {
|
|
1351
|
-
by: locatorToken,
|
|
1352
|
-
value: findValue,
|
|
1353
|
-
...exact ? { exact: true } : {}
|
|
1354
|
-
};
|
|
1355
|
-
return {
|
|
1356
|
-
action,
|
|
1357
|
-
locator,
|
|
1358
|
-
...index !== void 0 ? { index } : {},
|
|
1359
|
-
...hasFillValue ? {
|
|
1360
|
-
...parts[6] !== void 0 ? { value: parts[6] } : {},
|
|
1361
|
-
...opt("label", parts[7])
|
|
1362
|
-
} : opt("label", parts[6])
|
|
1363
|
-
};
|
|
1364
|
-
}
|
|
1365
|
-
/** Include an optional string field only when it is non-empty. */
|
|
1366
|
-
function opt(key, value) {
|
|
1367
|
-
return value ? { [key]: value } : {};
|
|
1368
|
-
}
|
|
1369
|
-
//#endregion
|
|
1370
|
-
//#region src/runtime/env-scrub.ts
|
|
1371
|
-
/**
|
|
1372
|
-
* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
|
|
1373
|
-
* mentioned in the spec OR in any of its expanded (block-inlined) steps.
|
|
1374
|
-
* Used at trace time to scrub recorded Claude-text outputs so a value the
|
|
1375
|
-
* spec author intentionally threaded through `process.env` is preserved as
|
|
1376
|
-
* `${VAR}` in `ir.json` rather than baked in as the concrete
|
|
1377
|
-
* trace-time value.
|
|
1378
|
-
*
|
|
1379
|
-
* Why we walk `spec.steps` AND `expanded`:
|
|
1380
|
-
* - `spec.steps` carries the spec's own `instruction` / `expected` + each
|
|
1381
|
-
* include's raw `params` (which may themselves be `${ENV}` refs).
|
|
1382
|
-
* - `expanded` carries the inlined block-internal steps, whose
|
|
1383
|
-
* `instruction` / `expected` may *also* contain `${ENV}` refs that
|
|
1384
|
-
* don't go through include params.
|
|
1385
|
-
*
|
|
1386
|
-
* Each ref resolves against `overrides` first, then `process.env` —
|
|
1387
|
-
* `overrides` carries values the invoker injects into the child process,
|
|
1388
|
-
* which beat the parent env there. Only refs that resolve non-empty land in
|
|
1389
|
-
* the map — scrubbing against an empty string would corrupt unrelated empty
|
|
1390
|
-
* strings in the action stream; the rest are returned via `unresolved` so
|
|
1391
|
-
* the caller can warn the user.
|
|
1392
|
-
*
|
|
1393
|
-
* Longer values sort first so a `${SHORT}` whose value is a substring of a
|
|
1394
|
-
* `${LONG}` value doesn't clobber the longer one.
|
|
1395
|
-
*
|
|
1396
|
-
* `title` is deliberately NOT scanned — it never reaches the recorded action
|
|
1397
|
-
* stream.
|
|
1398
|
-
*/
|
|
1399
|
-
function buildSpecEnvScrub(spec, expanded, overrides = {}) {
|
|
1400
|
-
const refNames = /* @__PURE__ */ new Set();
|
|
1401
|
-
for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
|
|
1402
|
-
else {
|
|
1403
|
-
collect(step.instruction, refNames);
|
|
1404
|
-
collect(step.expected, refNames);
|
|
1405
|
-
}
|
|
1406
|
-
for (const step of expanded) {
|
|
1407
|
-
collect(step.instruction, refNames);
|
|
1408
|
-
collect(step.expected, refNames);
|
|
1409
|
-
}
|
|
1410
|
-
const map = [];
|
|
1411
|
-
const unresolved = [];
|
|
1412
|
-
for (const name of refNames) {
|
|
1413
|
-
const value = overrides[name] ?? process.env[name];
|
|
1414
|
-
if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
|
|
1415
|
-
else unresolved.push(name);
|
|
1416
|
-
}
|
|
1417
|
-
map.sort((a, b) => b[0].length - a[0].length);
|
|
1418
|
-
return {
|
|
1419
|
-
map,
|
|
1420
|
-
unresolved
|
|
1421
|
-
};
|
|
1422
|
-
}
|
|
1423
|
-
function collect(value, into) {
|
|
1424
|
-
for (const name of iterEnvRefNames(value)) into.add(name);
|
|
1425
|
-
}
|
|
1426
|
-
/** Shorter than this, a value is no secret and matches inside ordinary words. */
|
|
1427
|
-
const MIN_PROSE_SCRUB_LENGTH = 4;
|
|
1428
|
-
/** Long enough to clear the length bar, still ordinary prose / JSON. */
|
|
1429
|
-
const COMMON_PROSE_VALUES = new Set([
|
|
1430
|
-
"true",
|
|
1431
|
-
"false",
|
|
1432
|
-
"null",
|
|
1433
|
-
"none",
|
|
1434
|
-
"undefined"
|
|
1435
|
-
]);
|
|
1436
|
-
/**
|
|
1437
|
-
* Scrub map for model output, built like {@link buildSpecEnvScrub} but
|
|
1438
|
-
* without the values that read as ordinary text (`"1"`, `"true"`): prose
|
|
1439
|
-
* runs to paragraphs, where replacing every occurrence of such a value
|
|
1440
|
-
* costs more meaning than it protects. Record's own scrub keeps them for
|
|
1441
|
-
* its single command lines; the live path reuses this one map for its Bash
|
|
1442
|
-
* command log too, trading that short-value coverage for not building a
|
|
1443
|
-
* second map.
|
|
1444
|
-
*/
|
|
1445
|
-
function buildProseEnvScrubMap(spec, expanded, overrides = {}) {
|
|
1446
|
-
return buildSpecEnvScrub(spec, expanded, overrides).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
|
|
1447
|
-
}
|
|
1448
|
-
/**
|
|
1449
|
-
* Replace every occurrence of an env value with its `${VAR}` placeholder in
|
|
1450
|
-
* `text`. **Caller invariant**: the map must be sorted longest-value-first
|
|
1451
|
-
* so a shorter value doesn't shadow a longer one that contains it as a
|
|
1452
|
-
* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
|
|
1453
|
-
*/
|
|
1454
|
-
function scrubEnvValues(text, scrubMap) {
|
|
1455
|
-
if (scrubMap.length === 0) return text;
|
|
1456
|
-
let out = text;
|
|
1457
|
-
for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
|
|
1458
|
-
return out;
|
|
1459
|
-
}
|
|
1460
|
-
//#endregion
|
|
1461
|
-
//#region src/claude/native-binary.ts
|
|
1462
|
-
const require$1 = createRequire(import.meta.url);
|
|
1463
|
-
/**
|
|
1464
|
-
* The agent SDK launches Claude through a native `claude` binary that ships in
|
|
1465
|
-
* a per-platform package (`@anthropic-ai/claude-agent-sdk-<platform>-<cpu>`),
|
|
1466
|
-
* declared as an *optional* dependency of the SDK. Optional means a consumer's
|
|
1467
|
-
* lockfile can omit it without any install-time error — and then every Claude
|
|
1468
|
-
* call fails at runtime with a message that never reaches our logs. Resolving
|
|
1469
|
-
* the package up front lets us say so once, in a line that names the fix.
|
|
1470
|
-
*
|
|
1471
|
-
* ccqa's own package.json repeats these packages in `optionalDependencies` for
|
|
1472
|
-
* the same reason: a second declaration gives the resolver another chance to
|
|
1473
|
-
* record them. Keep that list's version range in step with the SDK's.
|
|
1474
|
-
*/
|
|
1475
|
-
function nativeBinaryPackage(platform = process.platform, arch = process.arch, musl = isMusl(platform)) {
|
|
1476
|
-
return `@anthropic-ai/claude-agent-sdk-${platform}-${arch === "arm64" ? "arm64" : "x64"}${platform === "linux" && musl ? "-musl" : ""}`;
|
|
1477
|
-
}
|
|
1478
|
-
/**
|
|
1479
|
-
* musl builds (Alpine and friends) need their own binary. Node doesn't expose
|
|
1480
|
-
* the libc flavour directly; the absence of `glibcVersionRuntime` in the
|
|
1481
|
-
* process report is the usual proxy.
|
|
1482
|
-
*/
|
|
1483
|
-
function isMusl(platform) {
|
|
1484
|
-
if (platform !== "linux") return false;
|
|
1485
|
-
return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
|
|
1486
|
-
}
|
|
1487
|
-
/**
|
|
1488
|
-
* Name of the platform package this host needs, or `null` when it resolves.
|
|
1489
|
-
* The per-platform packages have no `exports`, so the manifest is reachable.
|
|
1490
|
-
*/
|
|
1491
|
-
function missingNativeBinaryPackage(resolve = require$1.resolve) {
|
|
1492
|
-
const pkg = nativeBinaryPackage();
|
|
1493
|
-
try {
|
|
1494
|
-
resolve(`${pkg}/package.json`);
|
|
1495
|
-
return null;
|
|
1496
|
-
} catch {
|
|
1497
|
-
return pkg;
|
|
1498
|
-
}
|
|
1499
|
-
}
|
|
1500
|
-
/** Advice shown when the binary is absent — the package name plus how to fix it. */
|
|
1501
|
-
function missingNativeBinaryMessage(pkg) {
|
|
1502
|
-
return `${pkg} is not installed. The Claude Agent SDK needs it to start Claude on this platform, so every Claude-backed command (run in live mode, drift, diagnose) will fail. It ships as an optional dependency of the SDK, which a lockfile can drop silently: reinstall without omitting optional dependencies, or add it to your project as a direct dependency pinned to the same version as @anthropic-ai/claude-agent-sdk.`;
|
|
1503
|
-
}
|
|
1504
|
-
//#endregion
|
|
1505
|
-
//#region src/claude/cost-tally.ts
|
|
1506
|
-
/**
|
|
1507
|
-
* Sum every Claude invocation made inside a scope.
|
|
1508
|
-
*
|
|
1509
|
-
* A command like `record` calls Claude several times — the browser trace, the
|
|
1510
|
-
* codegen cleanup, one diagnosis per auto-fix retry — and the caller wants one
|
|
1511
|
-
* number for the whole command. Threading a cost out of each of those return
|
|
1512
|
-
* types would touch every layer in between, so the tally is scoped instead:
|
|
1513
|
-
* `invokeClaudeStreaming` adds to whichever scope is active, and nothing
|
|
1514
|
-
* between the two has to know.
|
|
1515
|
-
*
|
|
1516
|
-
* Scoped rather than module-global because commands run specs concurrently
|
|
1517
|
-
* (`drift` uses a pool). Two scopes must not fold into each other.
|
|
1518
|
-
*/
|
|
1519
|
-
const tallyStore = new AsyncLocalStorage();
|
|
1520
|
-
/** Record one invocation against the active scope. No-op outside one. */
|
|
1521
|
-
function tallyInvocation(cost) {
|
|
1522
|
-
tallyStore.getStore()?.push(cost);
|
|
1523
|
-
}
|
|
1524
|
-
/** Run `fn` with a fresh tally. Read the total from inside with `readCostTally`. */
|
|
1525
|
-
async function withCostTally(fn) {
|
|
1526
|
-
return tallyStore.run([], fn);
|
|
1527
|
-
}
|
|
1528
|
-
/**
|
|
1529
|
-
* The active scope's total so far, or null outside one.
|
|
1530
|
-
*
|
|
1531
|
-
* Read rather than pushed at the caller because commands end in
|
|
1532
|
-
* `process.exit`, which never reaches a `finally`; whoever opened the scope
|
|
1533
|
-
* reads the total on the way out (see `withCostReporting`).
|
|
1534
|
-
*
|
|
1535
|
-
* Fields stay `null` when no invocation reported them, so a caller can tell
|
|
1536
|
-
* "nothing was billed" from "the SDK didn't say" (mock runs, SDK errors).
|
|
1537
|
-
*/
|
|
1538
|
-
function readCostTally() {
|
|
1539
|
-
const collected = tallyStore.getStore();
|
|
1540
|
-
return collected === void 0 ? null : sum(collected);
|
|
1541
|
-
}
|
|
1542
|
-
function sum(costs) {
|
|
1543
|
-
const add = (pick) => {
|
|
1544
|
-
const present = costs.map(pick).filter((v) => v !== null);
|
|
1545
|
-
return present.length === 0 ? null : present.reduce((a, b) => a + b, 0);
|
|
1546
|
-
};
|
|
1547
|
-
return {
|
|
1548
|
-
totalCostUsd: add((c) => c.totalCostUsd),
|
|
1549
|
-
durationMs: add((c) => c.durationMs),
|
|
1550
|
-
durationApiMs: add((c) => c.durationApiMs),
|
|
1551
|
-
numTurns: add((c) => c.numTurns),
|
|
1552
|
-
inputTokens: add((c) => c.inputTokens),
|
|
1553
|
-
cacheCreationInputTokens: add((c) => c.cacheCreationInputTokens),
|
|
1554
|
-
cacheReadInputTokens: add((c) => c.cacheReadInputTokens),
|
|
1555
|
-
outputTokens: add((c) => c.outputTokens),
|
|
1556
|
-
models: [...new Set(costs.flatMap((c) => c.models))]
|
|
1557
|
-
};
|
|
1558
|
-
}
|
|
1559
|
-
//#endregion
|
|
1560
|
-
//#region src/claude/invoke.ts
|
|
1561
|
-
/** The built-in tools an allow-list names: `Bash(*)` is `Bash`; `mcp__*` are not built-ins. */
|
|
1562
|
-
function builtinToolNames(allowedTools) {
|
|
1563
|
-
const names = allowedTools.map((entry) => entry.replace(/\(.*\)$/, "")).filter((name) => !name.startsWith("mcp__"));
|
|
1564
|
-
return [...new Set(names)];
|
|
1565
|
-
}
|
|
1566
|
-
function resolveModel(explicit) {
|
|
1567
|
-
if (explicit) return explicit;
|
|
1568
|
-
const envModel = process.env["CCQA_MODEL"];
|
|
1569
|
-
return envModel && envModel.length > 0 ? envModel : void 0;
|
|
1570
|
-
}
|
|
1571
|
-
/**
|
|
1572
|
-
* When both credentials are present the OAuth token wins and the API key is
|
|
1573
|
-
* dropped. Left to the CLI the API key would win, which makes "switch a CI
|
|
1574
|
-
* job to the subscription token" require unwiring the key everywhere; with
|
|
1575
|
-
* this rule, adding the one variable is the whole switch, and removing it is
|
|
1576
|
-
* the whole rollback. The one place the rule lives — both the resolved view
|
|
1577
|
-
* and the env the SDK receives apply it through here.
|
|
1578
|
-
*/
|
|
1579
|
-
function preferOauthToken(env) {
|
|
1580
|
-
if (env["CLAUDE_CODE_OAUTH_TOKEN"]) delete env["ANTHROPIC_API_KEY"];
|
|
1581
|
-
}
|
|
1582
|
-
/**
|
|
1583
|
-
* Drop endpoint variables that are present but empty, so an empty value never
|
|
1584
|
-
* reaches the Claude Code process as an override. "Set to nothing" is how a
|
|
1585
|
-
* caller that cannot omit the key says "use the default" — a CI job wiring
|
|
1586
|
-
* `ANTHROPIC_BASE_URL` from an unset repository variable, most of all.
|
|
1587
|
-
*/
|
|
1588
|
-
function withoutEmptyEndpointVars(env) {
|
|
1589
|
-
const out = { ...env };
|
|
1590
|
-
for (const key of ENDPOINT_ENV_KEYS) if (out[key] === "") delete out[key];
|
|
1591
|
-
return out;
|
|
1592
|
-
}
|
|
1593
|
-
/**
|
|
1594
|
-
* The environment actually handed to the Claude Code process: the full process
|
|
1595
|
-
* environment with the caller's overrides on top, empty endpoint variables
|
|
1596
|
-
* dropped, and — when both credentials survive the merge — the API key removed
|
|
1597
|
-
* so the OAuth token wins.
|
|
1598
|
-
*
|
|
1599
|
-
* That removal MUST happen on the env the SDK receives, not only on the
|
|
1600
|
-
* resolved view: left to the CLI the API key would win, silently moving every
|
|
1601
|
-
* call from the subscription to metered billing when a CI job wires both
|
|
1602
|
-
* (which is exactly what happened before this function existed).
|
|
1603
|
-
*/
|
|
1604
|
-
function buildInvocationEnv(env) {
|
|
1605
|
-
const merged = withoutEmptyEndpointVars({
|
|
1606
|
-
...process.env,
|
|
1607
|
-
...env
|
|
1608
|
-
});
|
|
1609
|
-
preferOauthToken(merged);
|
|
1610
|
-
merged["CLAUDE_CODE_DISABLE_AUTO_MEMORY"] = "1";
|
|
1611
|
-
return merged;
|
|
1612
|
-
}
|
|
1613
|
-
let nativeBinaryWarned = false;
|
|
1614
|
-
/**
|
|
1615
|
-
* Warn once per process when the SDK's per-platform native binary is missing:
|
|
1616
|
-
* every Claude call is about to fail, and the opaque per-step errors alone are
|
|
1617
|
-
* expensive to trace back to a lockfile that dropped an optional dependency.
|
|
1618
|
-
*/
|
|
1619
|
-
function warnOnceIfNativeBinaryMissing() {
|
|
1620
|
-
if (nativeBinaryWarned) return;
|
|
1621
|
-
nativeBinaryWarned = true;
|
|
1622
|
-
const missing = missingNativeBinaryPackage();
|
|
1623
|
-
if (missing) warn(missingNativeBinaryMessage(missing));
|
|
1624
|
-
}
|
|
1625
|
-
/** Whole minutes read best, but the ceiling is set in ms and may be seconds. */
|
|
1626
|
-
function formatDuration$1(ms) {
|
|
1627
|
-
if (ms < 6e4 || ms % 6e4 !== 0) return `${Math.round(ms / 1e3)}s`;
|
|
1628
|
-
const minutes = ms / 6e4;
|
|
1629
|
-
return `${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
1630
|
-
}
|
|
1631
|
-
async function invokeClaudeStreaming(options, onEvent) {
|
|
1632
|
-
const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false } = options;
|
|
1633
|
-
const resolvedModel = resolveModel(model);
|
|
1634
|
-
const mergedEnv = buildInvocationEnv(env);
|
|
1635
|
-
const abortController = new AbortController();
|
|
1636
|
-
let lastAbToolUseId = null;
|
|
1637
|
-
const claimAbToolUse = (toolUseId) => {
|
|
1638
|
-
if (toolUseId !== lastAbToolUseId) return false;
|
|
1639
|
-
lastAbToolUseId = null;
|
|
1640
|
-
return true;
|
|
1641
|
-
};
|
|
1642
|
-
const sdkOptions = {
|
|
1643
|
-
systemPrompt,
|
|
1644
|
-
maxTurns,
|
|
1645
|
-
allowedTools,
|
|
1646
|
-
tools: builtinToolNames(allowedTools),
|
|
1647
|
-
strictMcpConfig: true,
|
|
1648
|
-
settingSources: [],
|
|
1649
|
-
permissionMode: "bypassPermissions",
|
|
1650
|
-
allowDangerouslySkipPermissions: true,
|
|
1651
|
-
abortController,
|
|
1652
|
-
...resolvedModel ? { model: resolvedModel } : {},
|
|
1653
|
-
...cwd ? { cwd } : {},
|
|
1654
|
-
env: mergedEnv,
|
|
1655
|
-
...mcpServers ? { mcpServers } : {},
|
|
1656
|
-
...disableThinking ? { thinking: { type: "disabled" } } : {},
|
|
1657
|
-
hooks: onAbAction || onAbActionFailed ? {
|
|
1658
|
-
PreToolUse: [{ hooks: [async (input) => {
|
|
1659
|
-
if (input.hook_event_name !== "PreToolUse") return {};
|
|
1660
|
-
if (input.tool_name !== "Bash") return {};
|
|
1661
|
-
const cmd = input.tool_input?.["command"];
|
|
1662
|
-
if (typeof cmd !== "string") return {};
|
|
1663
|
-
if (!relaxAbConstraints) {
|
|
1664
|
-
if (isBlockedAbSubcommand(cmd)) return {
|
|
1665
|
-
decision: "block",
|
|
1666
|
-
reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
|
|
1667
|
-
};
|
|
1668
|
-
if (hasRefSelector(cmd)) return {
|
|
1669
|
-
decision: "block",
|
|
1670
|
-
reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
|
|
1671
|
-
};
|
|
1672
|
-
const bareTag = findPositionalBareTag(cmd);
|
|
1673
|
-
if (bareTag !== null) return {
|
|
1674
|
-
decision: "block",
|
|
1675
|
-
reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
|
|
1676
|
-
};
|
|
1677
|
-
if (hasMultipleAbInvocations(cmd)) return {
|
|
1678
|
-
decision: "block",
|
|
1679
|
-
reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
|
|
1680
|
-
};
|
|
1681
|
-
if (hasErrorSuppression(cmd)) return {
|
|
1682
|
-
decision: "block",
|
|
1683
|
-
reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
|
|
1684
|
-
};
|
|
1685
|
-
}
|
|
1686
|
-
const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
|
|
1687
|
-
const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
|
|
1688
|
-
if ((ab !== null || assertMarker !== null) && onAbAction) {
|
|
1689
|
-
lastAbToolUseId = input.tool_use_id;
|
|
1690
|
-
const stepId = extractCcqaStepFromBashCommand(cmd);
|
|
1691
|
-
onAbAction({
|
|
1692
|
-
...ab !== null ? { abAction: ab } : {},
|
|
1693
|
-
...stepId ? { stepId } : {},
|
|
1694
|
-
...assertMarker !== null ? { assertMarker } : {}
|
|
1695
|
-
});
|
|
1696
|
-
} else lastAbToolUseId = null;
|
|
1697
|
-
return {};
|
|
1698
|
-
}] }],
|
|
1699
|
-
PostToolUse: [{ hooks: [async (input) => {
|
|
1700
|
-
if (input.hook_event_name !== "PostToolUse") return {};
|
|
1701
|
-
if (input.tool_name !== "Bash") return {};
|
|
1702
|
-
if (!isBashToolResponseError(input.tool_response)) return {};
|
|
1703
|
-
if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
|
|
1704
|
-
return {};
|
|
1705
|
-
}] }],
|
|
1706
|
-
PostToolUseFailure: [{ hooks: [async (input) => {
|
|
1707
|
-
if (input.hook_event_name !== "PostToolUseFailure") return {};
|
|
1708
|
-
if (input.tool_name !== "Bash") return {};
|
|
1709
|
-
if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
|
|
1710
|
-
return {};
|
|
1711
|
-
}] }]
|
|
1712
|
-
} : void 0
|
|
1713
|
-
};
|
|
1714
|
-
warnOnceIfNativeBinaryMissing();
|
|
1715
|
-
const capTimer = timeoutMs === void 0 ? null : setTimeout(() => abortController.abort(), timeoutMs);
|
|
1716
|
-
capTimer?.unref?.();
|
|
1717
|
-
let result = "";
|
|
1718
|
-
let answered = false;
|
|
1719
|
-
let isError = false;
|
|
1720
|
-
let errorDetail = null;
|
|
1721
|
-
let cost = {
|
|
1722
|
-
totalCostUsd: null,
|
|
1723
|
-
durationMs: null,
|
|
1724
|
-
durationApiMs: null,
|
|
1725
|
-
numTurns: null,
|
|
1726
|
-
inputTokens: null,
|
|
1727
|
-
cacheCreationInputTokens: null,
|
|
1728
|
-
cacheReadInputTokens: null,
|
|
1729
|
-
outputTokens: null,
|
|
1730
|
-
models: []
|
|
1731
|
-
};
|
|
1732
|
-
const q = await buildMessageStream(prompt, sdkOptions);
|
|
1733
|
-
try {
|
|
1734
|
-
for await (const msg of q) {
|
|
1735
|
-
onEvent(msg);
|
|
1736
|
-
if (msg.type === "assistant" && !silenceBashLog) {
|
|
1737
|
-
for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
|
|
1738
|
-
const cmd = block.input?.["command"];
|
|
1739
|
-
if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
|
|
1740
|
-
}
|
|
1741
|
-
}
|
|
1742
|
-
if (msg.type === "result") {
|
|
1743
|
-
answered = true;
|
|
1744
|
-
isError = msg.is_error ?? false;
|
|
1745
|
-
if (msg.subtype === "success") result = msg.result;
|
|
1746
|
-
else {
|
|
1747
|
-
result = "";
|
|
1748
|
-
errorDetail = `SDK reported ${msg.subtype}`;
|
|
1749
|
-
}
|
|
1750
|
-
cost = extractInvocationCost(msg);
|
|
1751
|
-
}
|
|
1752
|
-
}
|
|
1753
|
-
} catch (err) {
|
|
1754
|
-
isError = true;
|
|
1755
|
-
errorDetail = err instanceof Error ? err.message : String(err);
|
|
1756
|
-
if (!result) result = errorDetail;
|
|
1757
|
-
} finally {
|
|
1758
|
-
if (capTimer) clearTimeout(capTimer);
|
|
1759
|
-
}
|
|
1760
|
-
if (abortController.signal.aborted && timeoutMs !== void 0 && !answered) {
|
|
1761
|
-
isError = true;
|
|
1762
|
-
errorDetail = `stopped after ${formatDuration$1(timeoutMs)} (host time limit)`;
|
|
1763
|
-
result = errorDetail;
|
|
1764
|
-
}
|
|
1765
|
-
tallyInvocation(cost);
|
|
1766
|
-
return {
|
|
1767
|
-
result,
|
|
1768
|
-
isError,
|
|
1769
|
-
errorDetail,
|
|
1770
|
-
cost
|
|
1771
|
-
};
|
|
1772
|
-
}
|
|
1773
|
-
/**
|
|
1774
|
-
* Pull the cost / usage / turn / duration fields off the SDK `result` message.
|
|
1775
|
-
* The SDK's success and error result shapes share these fields, so we read
|
|
1776
|
-
* them defensively as `unknown` and coerce — newer SDK versions may rename a
|
|
1777
|
-
* field without breaking our extraction.
|
|
1778
|
-
*/
|
|
1779
|
-
function extractInvocationCost(msg) {
|
|
1780
|
-
const m = msg;
|
|
1781
|
-
const usage = m["usage"];
|
|
1782
|
-
const modelUsage = m["modelUsage"];
|
|
1783
|
-
const num = (v) => typeof v === "number" && Number.isFinite(v) ? v : null;
|
|
1784
|
-
const models = modelUsage && typeof modelUsage === "object" ? Object.keys(modelUsage) : [];
|
|
1785
|
-
return {
|
|
1786
|
-
totalCostUsd: pricedForClaude(models) ? num(m["total_cost_usd"]) : null,
|
|
1787
|
-
durationMs: num(m["duration_ms"]),
|
|
1788
|
-
durationApiMs: num(m["duration_api_ms"]),
|
|
1789
|
-
numTurns: num(m["num_turns"]),
|
|
1790
|
-
inputTokens: num(usage?.["input_tokens"]),
|
|
1791
|
-
cacheCreationInputTokens: num(usage?.["cache_creation_input_tokens"]),
|
|
1792
|
-
cacheReadInputTokens: num(usage?.["cache_read_input_tokens"]),
|
|
1793
|
-
outputTokens: num(usage?.["output_tokens"]),
|
|
1794
|
-
models
|
|
1795
|
-
};
|
|
1796
|
-
}
|
|
1797
|
-
/**
|
|
1798
|
-
* The SDK prices an unknown model id at a default Claude rate rather than
|
|
1799
|
-
* returning null, so a self-hosted model would report dollars nobody is billed.
|
|
1800
|
-
*/
|
|
1801
|
-
function pricedForClaude(models) {
|
|
1802
|
-
return models.every((id) => /claude/i.test(id));
|
|
1803
|
-
}
|
|
1804
|
-
const BLOCKED_AB_SUBCOMMANDS = new Set([
|
|
1805
|
-
"eval",
|
|
1806
|
-
"js",
|
|
1807
|
-
"label",
|
|
1808
|
-
"textbox"
|
|
1809
|
-
]);
|
|
1810
|
-
/**
|
|
1811
|
-
* Shell-aware tokenizer: splits a command string into tokens respecting single/double quotes.
|
|
1812
|
-
* e.g. `click "[role='dialog'] button:last-child"` → ["click", "[role='dialog'] button:last-child"]
|
|
1813
|
-
*/
|
|
1814
|
-
function shellTokenize(s) {
|
|
1815
|
-
const tokens = [];
|
|
1816
|
-
let cur = "";
|
|
1817
|
-
let quote = null;
|
|
1818
|
-
for (let i = 0; i < s.length; i++) {
|
|
1819
|
-
const ch = s[i];
|
|
1820
|
-
if (quote) if (ch === quote) quote = null;
|
|
1821
|
-
else cur += ch;
|
|
1822
|
-
else if (ch === "\"" || ch === "'") quote = ch;
|
|
1823
|
-
else if (ch === " " || ch === " ") {
|
|
1824
|
-
if (cur) {
|
|
1825
|
-
tokens.push(cur);
|
|
1826
|
-
cur = "";
|
|
1827
|
-
}
|
|
1828
|
-
} else cur += ch;
|
|
1829
|
-
}
|
|
1830
|
-
if (cur) tokens.push(cur);
|
|
1831
|
-
return tokens;
|
|
1832
|
-
}
|
|
1833
|
-
/** Extracts the subcommand from an `agent-browser [flags] <subcommand> [args...]` command string. */
|
|
1834
|
-
function extractAbSubcommand(cmd) {
|
|
1835
|
-
const abIdx = cmd.indexOf("agent-browser");
|
|
1836
|
-
if (abIdx === -1) return null;
|
|
1837
|
-
const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
|
|
1838
|
-
let i = 0;
|
|
1839
|
-
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
1840
|
-
return parts[i] ?? null;
|
|
1841
|
-
}
|
|
1842
|
-
/** Returns true if the agent-browser subcommand is blocked (eval/js/find/etc). */
|
|
1843
|
-
function isBlockedAbSubcommand(cmd) {
|
|
1844
|
-
const sub = extractAbSubcommand(cmd);
|
|
1845
|
-
return sub !== null && BLOCKED_AB_SUBCOMMANDS.has(sub);
|
|
1846
|
-
}
|
|
1847
|
-
/**
|
|
1848
|
-
* Detects "the Bash tool returned an error" from a SDK PostToolUse hook's
|
|
1849
|
-
* `tool_response`. The SDK can shape this two ways depending on how Claude
|
|
1850
|
-
* Code reports Bash failures:
|
|
1851
|
-
*
|
|
1852
|
-
* - `{ is_error: true, ... }` — the canonical Bash failure shape
|
|
1853
|
-
* - `{ output, exitCode, killed?, ... }` — the BashOutput shape; treat
|
|
1854
|
-
* non-zero exit / kill as error
|
|
1855
|
-
*
|
|
1856
|
-
* We accept either. Anything else (including missing fields) is treated as a
|
|
1857
|
-
* successful response so we never roll back over an unrelated tool call.
|
|
1858
|
-
*/
|
|
1859
|
-
function isBashToolResponseError(tool_response) {
|
|
1860
|
-
if (tool_response === null || typeof tool_response !== "object") return false;
|
|
1861
|
-
const r = tool_response;
|
|
1862
|
-
if (r["is_error"] === true) return true;
|
|
1863
|
-
if (typeof r["exitCode"] === "number" && r["exitCode"] !== 0) return true;
|
|
1864
|
-
if (r["killed"] === true) return true;
|
|
1865
|
-
return false;
|
|
1866
|
-
}
|
|
1867
|
-
/**
|
|
1868
|
-
* Detect `agent-browser ... find first|last|nth <bare-tag> <action>`. A bare
|
|
1869
|
-
* tag inside a *positional* finder matches every element of that tag on the
|
|
1870
|
-
* page, so "the last button" picks a different element whenever the page
|
|
1871
|
-
* shape shifts — recorded tests built on top are flaky by construction. The
|
|
1872
|
-
* check is narrow on purpose: `find role button --name X` is fine because
|
|
1873
|
-
* role + accessible name stays stable.
|
|
1874
|
-
*/
|
|
1875
|
-
function findPositionalBareTag(cmd) {
|
|
1876
|
-
if (extractAbSubcommand(cmd) !== "find") return null;
|
|
1877
|
-
const abIdx = cmd.indexOf("agent-browser");
|
|
1878
|
-
const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
|
|
1879
|
-
let i = 0;
|
|
1880
|
-
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
1881
|
-
const locator = parts[i + 1];
|
|
1882
|
-
if (locator !== "first" && locator !== "last" && locator !== "nth") return null;
|
|
1883
|
-
const innerIdx = locator === "nth" ? i + 3 : i + 2;
|
|
1884
|
-
const inner = parts[innerIdx];
|
|
1885
|
-
const action = parts[innerIdx + 1] ?? "";
|
|
1886
|
-
if (!inner) return null;
|
|
1887
|
-
if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(inner)) return null;
|
|
1888
|
-
return {
|
|
1889
|
-
locator,
|
|
1890
|
-
selector: inner,
|
|
1891
|
-
action
|
|
1892
|
-
};
|
|
1893
|
-
}
|
|
1894
|
-
/** Returns true if any argument to an agent-browser command uses a @ref selector (e.g. @e14). */
|
|
1895
|
-
function hasRefSelector(cmd) {
|
|
1896
|
-
const abIdx = cmd.indexOf("agent-browser");
|
|
1897
|
-
if (abIdx === -1) return false;
|
|
1898
|
-
const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
|
|
1899
|
-
let i = 0;
|
|
1900
|
-
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
1901
|
-
i++;
|
|
1902
|
-
for (; i < parts.length; i++) if (/^@/.test(parts[i])) return true;
|
|
1903
|
-
return false;
|
|
1904
|
-
}
|
|
1905
|
-
/**
|
|
1906
|
-
* Split `cmd` into shell statements at unquoted separators (`;`, `|`, `&`,
|
|
1907
|
-
* newline; consecutive separator chars like `&&` count once). String
|
|
1908
|
-
* literals are honoured so `fill "a;b"` stays a single statement. This is a
|
|
1909
|
-
* heuristic split (no subshell grammar), shared by the compound-invocation
|
|
1910
|
-
* guard and the CCQA_STEP prefix extraction so both agree on what "one
|
|
1911
|
-
* command" means.
|
|
1912
|
-
*/
|
|
1913
|
-
function splitShellStatements(cmd) {
|
|
1914
|
-
const statements = [];
|
|
1915
|
-
let start = 0;
|
|
1916
|
-
let quote = null;
|
|
1917
|
-
for (let i = 0; i < cmd.length; i++) {
|
|
1918
|
-
const ch = cmd[i];
|
|
1919
|
-
if (quote) {
|
|
1920
|
-
if (ch === quote) quote = null;
|
|
1921
|
-
continue;
|
|
1922
|
-
}
|
|
1923
|
-
if (ch === "\"" || ch === "'" || ch === "`") {
|
|
1924
|
-
quote = ch;
|
|
1925
|
-
continue;
|
|
1926
|
-
}
|
|
1927
|
-
if (ch === ";" || ch === "|" || ch === "&" || ch === "\n") {
|
|
1928
|
-
statements.push(cmd.slice(start, i));
|
|
1929
|
-
while (i + 1 < cmd.length && (cmd[i + 1] === "|" || cmd[i + 1] === "&" || cmd[i + 1] === ";" || cmd[i + 1] === "\n")) i++;
|
|
1930
|
-
start = i + 1;
|
|
1931
|
-
}
|
|
1932
|
-
}
|
|
1933
|
-
statements.push(cmd.slice(start));
|
|
1934
|
-
return statements;
|
|
1935
|
-
}
|
|
1936
|
-
/** One leading `KEY=value` env assignment; value may be single/double-quoted. */
|
|
1937
|
-
const ENV_ASSIGN_HEAD_RE = /^([A-Za-z_][A-Za-z0-9_]*)=(?:"([^"]*)"|'([^']*)'|(\S*))(?:\s+|$)/;
|
|
1938
|
-
/**
|
|
1939
|
-
* Split a statement into its leading `KEY=value` env assignments and the
|
|
1940
|
-
* command they prefix. An assignment whose value is a command substitution
|
|
1941
|
-
* (`$(...)` / backticks) is NOT treated as a prefix — `result=$(agent-browser
|
|
1942
|
-
* ... snapshot)` is an assignment statement, not an agent-browser invocation,
|
|
1943
|
-
* and must stay invisible to the guards below.
|
|
1944
|
-
*/
|
|
1945
|
-
function splitLeadingEnvAssignments(statement) {
|
|
1946
|
-
const env = /* @__PURE__ */ new Map();
|
|
1947
|
-
let command = statement.trimStart();
|
|
1948
|
-
for (;;) {
|
|
1949
|
-
const m = ENV_ASSIGN_HEAD_RE.exec(command);
|
|
1950
|
-
if (!m) break;
|
|
1951
|
-
const value = m[2] ?? m[3] ?? m[4] ?? "";
|
|
1952
|
-
if (value.startsWith("$(") || value.startsWith("`")) return {
|
|
1953
|
-
env: /* @__PURE__ */ new Map(),
|
|
1954
|
-
command: statement.trimStart()
|
|
1955
|
-
};
|
|
1956
|
-
env.set(m[1], value);
|
|
1957
|
-
command = command.slice(m[0].length);
|
|
1958
|
-
}
|
|
1959
|
-
return {
|
|
1960
|
-
env,
|
|
1961
|
-
command
|
|
1962
|
-
};
|
|
1963
|
-
}
|
|
1964
|
-
/** True when `command` starts with `agent-browser` as the command word. */
|
|
1965
|
-
function isAgentBrowserHead(command) {
|
|
1966
|
-
if (!command.startsWith("agent-browser")) return false;
|
|
1967
|
-
const after = command[13];
|
|
1968
|
-
return after === void 0 || !/[A-Za-z0-9_\-]/.test(after);
|
|
1969
|
-
}
|
|
1970
|
-
/** Step ids passed via `CCQA_STEP=<step-id>` must be a plain slug. */
|
|
1971
|
-
const STEP_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
|
|
1972
|
-
/**
|
|
1973
|
-
* Extract the step id from the `CCQA_STEP=<step-id>` env prefix on the
|
|
1974
|
-
* agent-browser invocation in `cmd` (e.g. `CCQA_STEP=step-03 agent-browser
|
|
1975
|
-
* --session s click "text=Submit"`). The prefix may sit anywhere in the
|
|
1976
|
-
* leading env-assignment run (`FOO=x CCQA_STEP=step-02 agent-browser ...`),
|
|
1977
|
-
* and the invocation may be a later statement of a compound command
|
|
1978
|
-
* (`cd app && CCQA_STEP=step-01 agent-browser ...`). Returns null when the
|
|
1979
|
-
* prefix is absent or its value is not a valid slug — callers then fall back
|
|
1980
|
-
* to the STEP_START text protocol.
|
|
1981
|
-
*/
|
|
1982
|
-
function extractCcqaStepFromBashCommand(cmd) {
|
|
1983
|
-
for (const statement of splitShellStatements(cmd)) {
|
|
1984
|
-
const { env, command } = splitLeadingEnvAssignments(statement);
|
|
1985
|
-
if (!isAgentBrowserHead(command)) continue;
|
|
1986
|
-
const value = env.get("CCQA_STEP");
|
|
1987
|
-
return value !== void 0 && STEP_SLUG_RE.test(value) ? value : null;
|
|
1988
|
-
}
|
|
1989
|
-
return null;
|
|
1990
|
-
}
|
|
1991
|
-
/**
|
|
1992
|
-
* Extract the assert marker from the `CCQA_ASSERT=<marker>` env prefix on
|
|
1993
|
-
* the agent-browser invocation in `cmd`, e.g. `CCQA_STEP=step-03
|
|
1994
|
-
* CCQA_ASSERT=1 agent-browser --session s wait --text "Submitted" --timeout
|
|
1995
|
-
* 3000`. The marker declares that the command verifies a step signal;
|
|
1996
|
-
* `promoteMarkedAssert` maps it onto recorded assert action(s). Returns the
|
|
1997
|
-
* raw value — semantic validation (which markers combine with which
|
|
1998
|
-
* commands) happens at promotion time so mismatches surface as warnings
|
|
1999
|
-
* instead of being silently dropped here. Returns null when the prefix is
|
|
2000
|
-
* absent or empty.
|
|
2001
|
-
*/
|
|
2002
|
-
function extractCcqaAssertFromBashCommand(cmd) {
|
|
2003
|
-
for (const statement of splitShellStatements(cmd)) {
|
|
2004
|
-
const { env, command } = splitLeadingEnvAssignments(statement);
|
|
2005
|
-
if (!isAgentBrowserHead(command)) continue;
|
|
2006
|
-
const value = env.get("CCQA_ASSERT");
|
|
2007
|
-
return value !== void 0 && value.length > 0 ? value : null;
|
|
2008
|
-
}
|
|
2009
|
-
return null;
|
|
2010
|
-
}
|
|
2011
|
-
/**
|
|
2012
|
-
* Returns true when `cmd` contains more than one `agent-browser` invocation
|
|
2013
|
-
* chained together via shell operators (`&&`, `||`, `;`, `|`, newline). The
|
|
2014
|
-
* PreToolUse hook only records ONE AB_ACTION per Bash call, so chained
|
|
2015
|
-
* invocations would silently drop every intermediate failure — turning
|
|
2016
|
-
* "I tried four selectors before one worked" into a clean-looking trace
|
|
2017
|
-
* with five orphaned actions that later fail at replay.
|
|
2018
|
-
*
|
|
2019
|
-
* Counts statements whose command word is `agent-browser`, skipping any
|
|
2020
|
-
* leading env assignments (the trace protocol prefixes every invocation
|
|
2021
|
-
* with `CCQA_STEP=<step-id>`). String literals are honoured so
|
|
2022
|
-
* `agent-browser fill 'agent-browser'` doesn't false-fire.
|
|
2023
|
-
*/
|
|
2024
|
-
function hasMultipleAbInvocations(cmd) {
|
|
2025
|
-
let count = 0;
|
|
2026
|
-
for (const statement of splitShellStatements(cmd)) {
|
|
2027
|
-
if (!isAgentBrowserHead(splitLeadingEnvAssignments(statement).command)) continue;
|
|
2028
|
-
count++;
|
|
2029
|
-
if (count > 1) return true;
|
|
2030
|
-
}
|
|
2031
|
-
return false;
|
|
2032
|
-
}
|
|
2033
|
-
/**
|
|
2034
|
-
* Returns true when an `agent-browser` command in `cmd` has its exit
|
|
2035
|
-
* status hidden by a shell decorator that would prevent ccqa from rolling
|
|
2036
|
-
* back a failed attempt:
|
|
2037
|
-
*
|
|
2038
|
-
* - trailing `|| true` / `|| :` / `; true` (force exit 0)
|
|
2039
|
-
* - `2>/dev/null` and friends (drop stderr, sometimes paired with `|| true`)
|
|
2040
|
-
*
|
|
2041
|
-
* The agent-browser command itself returns exit 1 on selector miss, so
|
|
2042
|
-
* once one of these is present the PostToolUse hook sees `is_error=false`
|
|
2043
|
-
* and the bad attempt sneaks into ir.json.
|
|
2044
|
-
*/
|
|
2045
|
-
function hasErrorSuppression(cmd) {
|
|
2046
|
-
if (cmd.indexOf("agent-browser") === -1) return false;
|
|
2047
|
-
if (/\|\|\s*(true|:|\s*$|#)/.test(cmd)) return true;
|
|
2048
|
-
if (/;\s*(true|:)\b/.test(cmd)) return true;
|
|
2049
|
-
if (/2\s*>\s*\/dev\/null/.test(cmd)) return true;
|
|
2050
|
-
if (/&\s*>\s*\/dev\/null/.test(cmd)) return true;
|
|
2051
|
-
return false;
|
|
2052
|
-
}
|
|
2053
|
-
/**
|
|
2054
|
-
* Parse an `agent-browser --session <name> <cmd> [args...]` bash command
|
|
2055
|
-
* and return the corresponding AB_ACTION line, or null if not an agent-browser call.
|
|
2056
|
-
*/
|
|
2057
|
-
function extractAbActionFromBashCommand(cmd) {
|
|
2058
|
-
const subCmd = extractAbSubcommand(cmd);
|
|
2059
|
-
if (!subCmd) return null;
|
|
2060
|
-
const abIdx = cmd.indexOf("agent-browser");
|
|
2061
|
-
const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
|
|
2062
|
-
let i = 0;
|
|
2063
|
-
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
2064
|
-
const args = parts.slice(i + 1);
|
|
2065
|
-
switch (subCmd) {
|
|
2066
|
-
case "cookies":
|
|
2067
|
-
if (args[0] === "clear") return "AB_ACTION|cookies_clear";
|
|
2068
|
-
return null;
|
|
2069
|
-
case "open": return `AB_ACTION|open|${args[0] ?? ""}`;
|
|
2070
|
-
case "press": return `AB_ACTION|press|${args[0] ?? ""}`;
|
|
2071
|
-
case "scroll": return `AB_ACTION|scroll|${args.join("|")}`;
|
|
2072
|
-
case "click":
|
|
2073
|
-
case "dblclick":
|
|
2074
|
-
case "check":
|
|
2075
|
-
case "uncheck":
|
|
2076
|
-
case "hover":
|
|
2077
|
-
case "wait": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}`;
|
|
2078
|
-
case "fill":
|
|
2079
|
-
case "type":
|
|
2080
|
-
case "select": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
|
|
2081
|
-
case "drag": return `AB_ACTION|drag|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
|
|
2082
|
-
case "upload": {
|
|
2083
|
-
const sel = args[0] ?? "";
|
|
2084
|
-
const files = args.slice(1);
|
|
2085
|
-
if (!sel || files.length === 0) return null;
|
|
2086
|
-
return `AB_ACTION|upload|${sel}|${files.join("|")}`;
|
|
2087
|
-
}
|
|
2088
|
-
case "snapshot": return null;
|
|
2089
|
-
case "find": return extractFindAbAction(args);
|
|
2090
|
-
default: return null;
|
|
2091
|
-
}
|
|
2092
|
-
}
|
|
2093
|
-
/**
|
|
2094
|
-
* Wire lines for the observation-only probes `get count <sel>` / `get url`.
|
|
2095
|
-
* These commands read state without mutating it, so they have no place in
|
|
2096
|
-
* the replay sequence and `extractAbActionFromBashCommand` ignores them.
|
|
2097
|
-
* They matter only when a `CCQA_ASSERT=<marker>` env prefix declares the
|
|
2098
|
-
* probe verifies a step signal — the hook layer then surfaces them via this
|
|
2099
|
-
* function so `promoteMarkedAssert` can turn them into recorded asserts.
|
|
2100
|
-
* Only consulted when a marker is present; unmarked `get` commands stay
|
|
2101
|
-
* unobserved as before.
|
|
2102
|
-
*/
|
|
2103
|
-
function extractObservationAbAction(cmd) {
|
|
2104
|
-
if (extractAbSubcommand(cmd) !== "get") return null;
|
|
2105
|
-
const abIdx = cmd.indexOf("agent-browser");
|
|
2106
|
-
const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
|
|
2107
|
-
let i = 0;
|
|
2108
|
-
while (i < parts.length && parts[i].startsWith("-")) i += 2;
|
|
2109
|
-
const args = parts.slice(i + 1);
|
|
2110
|
-
if (args[0] === "count" && args[1]) return `AB_ACTION|get_count|${args[1]}`;
|
|
2111
|
-
if (args[0] === "url") return "AB_ACTION|get_url";
|
|
2112
|
-
return null;
|
|
2113
|
-
}
|
|
2114
|
-
const FIND_ACTION_SET = new Set(FIND_ACTIONS);
|
|
2115
|
-
const FIND_LOCATOR_SET = new Set(FIND_LOCATORS);
|
|
2116
|
-
/**
|
|
2117
|
-
* Parse the positional tokens of `agent-browser find <locator> <value> [...]
|
|
2118
|
-
* <action> [fillValue]` and produce a canonical
|
|
2119
|
-
* `AB_ACTION|find_<action>|<locator>|<value>|<extra>|<exact>|...|<label>`
|
|
2120
|
-
* line. The wire format keeps a fixed positional layout across locators so
|
|
2121
|
-
* downstream `parseAbActionLine` in `ir/from-agent-browser.ts` can split on
|
|
2122
|
-
* `|` alone:
|
|
2123
|
-
*
|
|
2124
|
-
* <extra> is `--name` value for role, integer index for nth, "" otherwise.
|
|
2125
|
-
* <exact> is the literal "exact" if --exact was passed, "" otherwise.
|
|
2126
|
-
*
|
|
2127
|
-
* Returns null for malformed invocations — the caller treats null as "not a
|
|
2128
|
-
* structured action" and the Bash command still runs unobserved.
|
|
2129
|
-
*/
|
|
2130
|
-
function extractFindAbAction(args) {
|
|
2131
|
-
const locator = args[0];
|
|
2132
|
-
if (!locator || !FIND_LOCATOR_SET.has(locator)) return null;
|
|
2133
|
-
let i = 1;
|
|
2134
|
-
let value = args[i] ?? "";
|
|
2135
|
-
i++;
|
|
2136
|
-
let extra = "";
|
|
2137
|
-
if (locator === "nth") {
|
|
2138
|
-
extra = value;
|
|
2139
|
-
value = args[i] ?? "";
|
|
2140
|
-
i++;
|
|
2141
|
-
}
|
|
2142
|
-
let action = "";
|
|
2143
|
-
let name = "";
|
|
2144
|
-
let exact = "";
|
|
2145
|
-
let fillValue = "";
|
|
2146
|
-
for (; i < args.length; i++) {
|
|
2147
|
-
const tok = args[i];
|
|
2148
|
-
if (tok === "--name") {
|
|
2149
|
-
name = args[i + 1] ?? "";
|
|
2150
|
-
i++;
|
|
2151
|
-
} else if (tok === "--exact") exact = "exact";
|
|
2152
|
-
else if (FIND_ACTION_SET.has(tok)) action = tok;
|
|
2153
|
-
else if (action) fillValue = tok;
|
|
2154
|
-
}
|
|
2155
|
-
if (!action) return null;
|
|
2156
|
-
if (locator === "role") extra = name;
|
|
2157
|
-
const command = `find_${action}`;
|
|
2158
|
-
if (action === "fill" || action === "type") return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|${fillValue}|`;
|
|
2159
|
-
return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|`;
|
|
2160
|
-
}
|
|
2161
|
-
async function buildMessageStream(prompt, options) {
|
|
2162
|
-
const mockFile = process.env["CCQA_CLAUDE_MOCK_FILE"];
|
|
2163
|
-
if (mockFile) return replayMockMessages(mockFile, options);
|
|
2164
|
-
return query({
|
|
2165
|
-
prompt,
|
|
2166
|
-
options
|
|
2167
|
-
});
|
|
2168
|
-
}
|
|
2169
|
-
async function* replayMockMessages(path, options) {
|
|
2170
|
-
const raw = await readFile(path, "utf8");
|
|
2171
|
-
for (const line of raw.split("\n")) {
|
|
2172
|
-
const trimmed = line.trim();
|
|
2173
|
-
if (!trimmed) continue;
|
|
2174
|
-
const msg = JSON.parse(trimmed);
|
|
2175
|
-
await fireMockPreToolUseHooks(msg, options);
|
|
2176
|
-
yield msg;
|
|
2177
|
-
}
|
|
2178
|
-
}
|
|
2179
|
-
/**
|
|
2180
|
-
* The real SDK fires PreToolUse hooks as it executes tool calls; the JSONL
|
|
2181
|
-
* replay approximates that by invoking the configured PreToolUse hooks for
|
|
2182
|
-
* every Bash tool_use block before yielding its message, so e2e stubs
|
|
2183
|
-
* exercise the AB_ACTION recording path (including CCQA_STEP step
|
|
2184
|
-
* attribution). Hook decisions are ignored and post-tool hooks are not
|
|
2185
|
-
* simulated — the replay runs no tools, so there is nothing to block or fail.
|
|
2186
|
-
*/
|
|
2187
|
-
async function fireMockPreToolUseHooks(msg, options) {
|
|
2188
|
-
const matchers = options.hooks?.PreToolUse;
|
|
2189
|
-
if (!matchers || msg.type !== "assistant") return;
|
|
2190
|
-
for (const block of msg.message.content ?? []) {
|
|
2191
|
-
if (block.type !== "tool_use" || block.name !== "Bash") continue;
|
|
2192
|
-
const input = {
|
|
2193
|
-
hook_event_name: "PreToolUse",
|
|
2194
|
-
tool_name: "Bash",
|
|
2195
|
-
tool_input: block.input,
|
|
2196
|
-
tool_use_id: block.id,
|
|
2197
|
-
session_id: "mock",
|
|
2198
|
-
transcript_path: "",
|
|
2199
|
-
cwd: process.cwd()
|
|
2200
|
-
};
|
|
2201
|
-
for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
|
|
2202
|
-
}
|
|
2203
|
-
}
|
|
2204
|
-
//#endregion
|
|
2205
|
-
//#region src/prompts/format.ts
|
|
2206
|
-
/**
|
|
2207
|
-
* Formatting helpers shared by the Claude prompt builders (diagnose, report,
|
|
2208
|
-
* drift). Centralised so the prompts cannot drift apart on mechanics that
|
|
2209
|
-
* must stay consistent across commands.
|
|
2210
|
-
*/
|
|
2211
|
-
/** Prefix every line with its 1-based number, the form fix suggestions cite. */
|
|
2212
|
-
function numberLines(script) {
|
|
2213
|
-
return script.split("\n").map((l, i) => `${i + 1}: ${l}`).join("\n");
|
|
2214
|
-
}
|
|
2215
|
-
/**
|
|
2216
|
-
* The "## Output language" prompt section. Empty for "auto" so the prompt
|
|
2217
|
-
* stays byte-identical to the no-flag baseline. `fields` names the
|
|
2218
|
-
* human-readable JSON fields to translate; `verbatimNames` names the
|
|
2219
|
-
* enum-like values that must never be translated.
|
|
2220
|
-
*/
|
|
2221
|
-
function outputLanguageBlock(outputLanguage, fields, verbatimNames) {
|
|
2222
|
-
if (outputLanguage === "auto") return "";
|
|
2223
|
-
return `## Output language
|
|
2224
|
-
|
|
2225
|
-
Write all human-readable fields (${fields}) in **${outputLanguage}** (BCP-47 tag).
|
|
2226
|
-
Selectors, file paths, identifiers, ${verbatimNames}, JSON keys, and quoted strings stay verbatim regardless of language.
|
|
2227
|
-
|
|
2228
|
-
`;
|
|
2229
|
-
}
|
|
2230
|
-
/**
|
|
2231
|
-
* The `spec`/`generated` surface-axis definitions, plus the "if both are
|
|
2232
|
-
* stale, answer spec" tie-break rule. Shared verbatim by the audit prompt
|
|
2233
|
-
* (`prompts/drift.ts`) and the run's failure-classification prompt
|
|
2234
|
-
* (`report/prompt.ts`): both fill the same wire field (`DriftSurfaceSchema`),
|
|
2235
|
-
* so a diverging definition in one would make the two paths disagree on what
|
|
2236
|
-
* a spec's own field means.
|
|
2237
|
-
*/
|
|
2238
|
-
function surfaceDefinitionBlock() {
|
|
2239
|
-
return `- **\`spec\`** — spec.yaml asks about something the source no longer has. It has to be rewritten, and the code regenerated after.
|
|
2240
|
-
- **\`generated\`** — the spec still describes the product correctly, but the generated code reaches for a selector or string the source no longer has. Only a regeneration is needed; nobody has to rewrite the spec.
|
|
2241
|
-
|
|
2242
|
-
If both are stale, answer \`spec\`: it is the root, and fixing it regenerates the code.`;
|
|
2243
|
-
}
|
|
2244
|
-
/**
|
|
2245
|
-
* "surface is a separate axis from the label" clarifying example, shared for
|
|
2246
|
-
* the same reason as {@link surfaceDefinitionBlock}. `labelToken` is the
|
|
2247
|
-
* exact text to name the label by (e.g. `"TEST_DRIFT"` or `` "`TEST_DRIFT`" ``)
|
|
2248
|
-
* so each caller keeps its own backtick/bold convention.
|
|
2249
|
-
*/
|
|
2250
|
-
function surfaceAxisAside(labelToken) {
|
|
2251
|
-
return `This is a separate axis from the label. A renamed selector that only the generated code names is ${labelToken} on the \`generated\` surface; a spec whose \`expected\` quotes a string the product renamed is ${labelToken} on the \`spec\` surface.`;
|
|
2252
|
-
}
|
|
2253
|
-
//#endregion
|
|
2254
|
-
//#region src/ir/to-agent-browser.ts
|
|
2255
|
-
const lit = (text) => ({
|
|
2256
|
-
text,
|
|
2257
|
-
expandsEnv: false
|
|
2258
|
-
});
|
|
2259
|
-
const val = (text) => ({
|
|
2260
|
-
text,
|
|
2261
|
-
expandsEnv: true
|
|
2262
|
-
});
|
|
2263
|
-
/**
|
|
2264
|
-
* Render a locator as the selector string a plain agent-browser command
|
|
2265
|
-
* accepts. Only `css` (verbatim) and `text` (`text=` engine form) have a
|
|
2266
|
-
* plain-selector form; other strategies are reachable via `find` only and
|
|
2267
|
-
* fall back to their raw value (callers guard against that case).
|
|
2268
|
-
*/
|
|
2269
|
-
function locatorToSelector(locator) {
|
|
2270
|
-
return locator.by === "text" ? `text=${locator.value}` : locator.value;
|
|
2271
|
-
}
|
|
2272
|
-
/**
|
|
2273
|
-
* Compact human-readable locator form for logs and LLM-prompt summaries: the
|
|
2274
|
-
* raw selector for `css`, `by=value` otherwise. Distinct from
|
|
2275
|
-
* `locatorToSelector` (which produces a selector agent-browser can execute) —
|
|
2276
|
-
* this one is for display only and never round-trips.
|
|
2277
|
-
*/
|
|
2278
|
-
function describeLocator(locator) {
|
|
2279
|
-
return locator.by === "css" ? locator.value : `${locator.by}=${locator.value}`;
|
|
2280
|
-
}
|
|
2281
|
-
/**
|
|
2282
|
-
* Canonical agent-browser argv (sans `--session`) for one action. Returns
|
|
2283
|
-
* null for actions with no direct argv form: observation-only `snapshot`,
|
|
2284
|
-
* `assert` (validation probes / abAssert* helpers live in the consumers),
|
|
2285
|
-
* and structurally incomplete actions (e.g. a missing locator).
|
|
2286
|
-
*/
|
|
2287
|
-
function toAgentBrowserArgs(action) {
|
|
2288
|
-
switch (action.action) {
|
|
2289
|
-
case "cookies_clear": return [lit("cookies"), lit("clear")];
|
|
2290
|
-
case "navigate": return [lit("open"), val(action.value ?? "")];
|
|
2291
|
-
case "press": return [lit("press"), val(action.value ?? "")];
|
|
2292
|
-
case "scroll": return [
|
|
2293
|
-
lit("scroll"),
|
|
2294
|
-
lit(action.direction ?? "down"),
|
|
2295
|
-
...action.pixels ? [lit(action.pixels)] : []
|
|
2296
|
-
];
|
|
2297
|
-
case "select":
|
|
2298
|
-
if (!action.locator) return null;
|
|
2299
|
-
return [
|
|
2300
|
-
lit("select"),
|
|
2301
|
-
val(locatorToSelector(action.locator)),
|
|
2302
|
-
val(action.value ?? "")
|
|
2303
|
-
];
|
|
2304
|
-
case "drag":
|
|
2305
|
-
if (!action.locator || !action.target) return null;
|
|
2306
|
-
return [
|
|
2307
|
-
lit("drag"),
|
|
2308
|
-
val(locatorToSelector(action.locator)),
|
|
2309
|
-
val(locatorToSelector(action.target))
|
|
2310
|
-
];
|
|
2311
|
-
case "upload": {
|
|
2312
|
-
const files = action.files ?? [];
|
|
2313
|
-
if (!action.locator || files.length === 0) return null;
|
|
2314
|
-
return [
|
|
2315
|
-
lit("upload"),
|
|
2316
|
-
val(locatorToSelector(action.locator)),
|
|
2317
|
-
...files.map(val)
|
|
2318
|
-
];
|
|
2319
|
-
}
|
|
2320
|
-
case "wait": {
|
|
2321
|
-
const loc = action.locator;
|
|
2322
|
-
if (!loc) return null;
|
|
2323
|
-
if (loc.by === "text") return [
|
|
2324
|
-
lit("wait"),
|
|
2325
|
-
lit("--text"),
|
|
2326
|
-
val(loc.value)
|
|
2327
|
-
];
|
|
2328
|
-
return [lit("wait"), val(locatorToSelector(loc))];
|
|
2329
|
-
}
|
|
2330
|
-
case "click":
|
|
2331
|
-
case "dblclick":
|
|
2332
|
-
case "check":
|
|
2333
|
-
case "uncheck":
|
|
2334
|
-
case "hover":
|
|
2335
|
-
case "focus":
|
|
2336
|
-
case "fill":
|
|
2337
|
-
case "type": return interactionToArgs(action);
|
|
2338
|
-
case "snapshot":
|
|
2339
|
-
case "assert": return null;
|
|
2340
|
-
}
|
|
2341
|
-
}
|
|
2342
|
-
/**
|
|
2343
|
-
* Element interactions come in two argv shapes: the plain command
|
|
2344
|
-
* (`click "<css>"`) when the locator is a raw selector string, and the
|
|
2345
|
-
* `find <locator> <value> <action> [input] [--name <n>] [--exact]` form for
|
|
2346
|
-
* semantic locators and positional (`index`) picks. `type` is a ccqa-side
|
|
2347
|
-
* alias of `fill` in both shapes. Flags MUST follow the action token —
|
|
2348
|
-
* putting them before it makes agent-browser fail with "Unknown subaction".
|
|
2349
|
-
*/
|
|
2350
|
-
function interactionToArgs(action) {
|
|
2351
|
-
const loc = action.locator;
|
|
2352
|
-
if (!loc) return null;
|
|
2353
|
-
const abAction = action.action === "type" ? "fill" : action.action;
|
|
2354
|
-
const takesInput = action.action === "fill" || action.action === "type";
|
|
2355
|
-
if (!(loc.by !== "css" || action.index !== void 0 || action.action === "focus")) {
|
|
2356
|
-
const args = [lit(abAction), val(loc.value)];
|
|
2357
|
-
if (takesInput) args.push(val(action.value ?? ""));
|
|
2358
|
-
return args;
|
|
2359
|
-
}
|
|
2360
|
-
if (!loc.value) return null;
|
|
2361
|
-
const out = [lit("find")];
|
|
2362
|
-
if (action.index !== void 0) {
|
|
2363
|
-
if (loc.by !== "css") return null;
|
|
2364
|
-
if (action.index === "first" || action.index === "last") out.push(lit(action.index));
|
|
2365
|
-
else out.push(lit("nth"), lit(String(action.index)));
|
|
2366
|
-
out.push(val(loc.value));
|
|
2367
|
-
} else {
|
|
2368
|
-
if (loc.by === "css") return null;
|
|
2369
|
-
out.push(lit(loc.by), val(loc.value));
|
|
2370
|
-
}
|
|
2371
|
-
out.push(lit(abAction));
|
|
2372
|
-
if (takesInput) out.push(val(action.value ?? ""));
|
|
2373
|
-
if (loc.by === "role" && loc.name) out.push(lit("--name"), val(loc.name));
|
|
2374
|
-
if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
|
|
2375
|
-
return out;
|
|
2376
|
-
}
|
|
2377
|
-
//#endregion
|
|
2378
|
-
//#region src/diagnose/prompt.ts
|
|
2379
|
-
function buildDiagnosePrompt(input) {
|
|
2380
|
-
const { script, specYaml, actions, failureLog, pageSnapshot, outputLanguage = "auto" } = input;
|
|
2381
|
-
const numbered = numberLines(script);
|
|
2382
|
-
const actionsSummary = actions.map((a, i) => {
|
|
2383
|
-
const parts = [`${i + 1}. ${a.action}`];
|
|
2384
|
-
if (a.assert) parts.push(`assert="${a.assert}"`);
|
|
2385
|
-
if (a.locator) parts.push(`locator="${describeLocator(a.locator)}"`);
|
|
2386
|
-
if (a.index !== void 0) parts.push(`index=${a.index}`);
|
|
2387
|
-
if (a.value) parts.push(`value="${a.value}"`);
|
|
2388
|
-
if (a.observation) parts.push(`→ ${a.observation}`);
|
|
2389
|
-
return parts.join(" ");
|
|
2390
|
-
}).join("\n");
|
|
2391
|
-
return `You are diagnosing a failing E2E test. The test was generated from a recorded trace of the original interaction. Compare the failing run against the original spec and recorded actions to determine WHY the test failed and what the right fix is.
|
|
2392
|
-
|
|
2393
|
-
${outputLanguageBlock(outputLanguage, "`reasoning`, `reason`", "code, type names (TIMING_ISSUE, etc.)")}## You have read-only filesystem tools
|
|
2394
|
-
|
|
2395
|
-
You can call \`Grep\`, \`Glob\`, and \`Read\` against the current repository before producing the JSON.
|
|
2396
|
-
|
|
2397
|
-
For SELECTOR_DRIFT specifically the failure log is usually NOT enough on its own — the runner only reports "selector X not visible". To confirm a rename, search the application source for the *type* of selector that's failing:
|
|
2398
|
-
|
|
2399
|
-
- For \`[aria-label='OLD']\` failures: \`Grep\` for \`aria-label=\` (or i18n key \`OLD\`) in the app source. If you find a near-miss like \`aria-label="NEW"\` whose text is a superset/rephrase of the failing label, that is your evidence.
|
|
2400
|
-
- For \`[placeholder='OLD']\` failures: \`Grep\` for \`placeholder=\`.
|
|
2401
|
-
- For \`[role='OLD']\` or \`[data-testid='OLD']\`: same pattern.
|
|
2402
|
-
- For \`text=OLD\` failures: \`Grep\` the source / i18n bundles for \`OLD\`. Locale files (\`*.json\`, \`*.yml\`, \`messages.ts\`, etc.) often hold the canonical strings.
|
|
2403
|
-
|
|
2404
|
-
You have **up to 10 tool turns**. Spend them on grep/read; do not loop. Only when you have concrete file:line evidence should you emit SELECTOR_DRIFT — otherwise prefer UNKNOWN with confidence < 0.4 and let the human decide.
|
|
2405
|
-
|
|
2406
|
-
Do NOT attempt to write, edit, run shell commands, or hit the network. Only Grep/Glob/Read.
|
|
2407
|
-
|
|
2408
|
-
## Diagnosis categories
|
|
2409
|
-
|
|
2410
|
-
Pick exactly ONE category. The output JSON must follow the shape for that category.
|
|
2411
|
-
|
|
2412
|
-
1. TIMING_ISSUE — element not yet present because the page hasn't loaded / navigated. Fix by inserting or extending sleeps.
|
|
2413
|
-
{
|
|
2414
|
-
"diagnosis": {
|
|
2415
|
-
"type": "TIMING_ISSUE",
|
|
2416
|
-
"fixes": [
|
|
2417
|
-
{ "kind": "insert", "line": <1-based>, "seconds": <int>, "reason": "<short>" },
|
|
2418
|
-
{ "kind": "increase", "line": <1-based of existing sleep>, "increase_to": <int>, "reason": "<short>" }
|
|
2419
|
-
]
|
|
2420
|
-
},
|
|
2421
|
-
"confidence": <0.0-1.0>,
|
|
2422
|
-
"reasoning": "<why timing is the cause>"
|
|
2423
|
-
}
|
|
2424
|
-
|
|
2425
|
-
2. OVER_ASSERTION — the test is asserting something the spec never required, OR a recorded assertion that is environment-dependent (e.g. a placeholder text that varies). The right fix is to remove those lines from the test.
|
|
2426
|
-
{
|
|
2427
|
-
"diagnosis": {
|
|
2428
|
-
"type": "OVER_ASSERTION",
|
|
2429
|
-
"lines": [<1-based line numbers to remove>],
|
|
2430
|
-
"reason": "<short>"
|
|
2431
|
-
},
|
|
2432
|
-
"confidence": <0.0-1.0>,
|
|
2433
|
-
"reasoning": "<why this assertion isn't required by the spec>"
|
|
2434
|
-
}
|
|
2435
|
-
|
|
2436
|
-
3. SELECTOR_DRIFT — the page is healthy but a selector has been renamed/refined since the trace was recorded. The failure log will typically contain a snapshot showing the new selector. ONLY use this when you can name the exact replacement selector.
|
|
2437
|
-
{
|
|
2438
|
-
"diagnosis": {
|
|
2439
|
-
"type": "SELECTOR_DRIFT",
|
|
2440
|
-
"line": <1-based>,
|
|
2441
|
-
"oldSelector": "<exact string in current line>",
|
|
2442
|
-
"newSelector": "<exact replacement>",
|
|
2443
|
-
"reason": "<short>"
|
|
2444
|
-
},
|
|
2445
|
-
"confidence": <0.0-1.0>,
|
|
2446
|
-
"reasoning": "<evidence from failure log>"
|
|
2447
|
-
}
|
|
2448
|
-
|
|
2449
|
-
4. DATA_MISSING — the test depends on data (a record, a setup, a logged-in state) that no longer exists. Not auto-fixable; the human must reseed or update the spec.
|
|
2450
|
-
{
|
|
2451
|
-
"diagnosis": { "type": "DATA_MISSING", "reason": "<what is missing>" },
|
|
2452
|
-
"confidence": <0.0-1.0>,
|
|
2453
|
-
"reasoning": "<evidence>"
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
5. UNKNOWN — none of the above fit, or evidence is too weak to choose.
|
|
2457
|
-
{
|
|
2458
|
-
"diagnosis": { "type": "UNKNOWN", "reason": "<short>" },
|
|
2459
|
-
"confidence": <0.0-1.0>,
|
|
2460
|
-
"reasoning": "<what you saw and why you can't classify>"
|
|
2461
|
-
}
|
|
2462
|
-
|
|
2463
|
-
## Confidence guidance
|
|
2464
|
-
|
|
2465
|
-
- 0.9-1.0: failure log directly shows the cause (e.g. "selector X not found, snapshot lists Y" → SELECTOR_DRIFT)
|
|
2466
|
-
- 0.7-0.9: strong indirect evidence (e.g. timing pattern after navigation, or assertion text that doesn't appear in spec)
|
|
2467
|
-
- 0.4-0.7: plausible classification but multiple categories could explain it
|
|
2468
|
-
- < 0.4: prefer UNKNOWN over guessing
|
|
2469
|
-
|
|
2470
|
-
## Rules
|
|
2471
|
-
|
|
2472
|
-
- Your **final** assistant message must start with \`{\` and end with \`}\` — a single JSON object, nothing before or after. No prose preamble like "Confirmed: ...", no markdown fences, no commentary, no tool calls in the same turn. If you have an analysis sentence, put it in the \`reasoning\` field.
|
|
2473
|
-
- Line numbers refer to the numbered test script below (1-based).
|
|
2474
|
-
- For SELECTOR_DRIFT, \`oldSelector\` must match a substring of the script at that line; \`newSelector\` must be backed by a concrete file:line you read with Grep/Read (do not invent). Cite the evidence in \`reasoning\`.
|
|
2475
|
-
- For OVER_ASSERTION, only include lines that contain assert calls (\`abAssert*\`) or existence-checking waits (\`abWait\`); a recorded \`abWait("[selector]")\` is an implicit existence assertion and a valid removal candidate when the spec never required that element to be present.
|
|
2476
|
-
- Cross-check assertions against the spec YAML. If the spec doesn't require the assertion, OVER_ASSERTION is the better diagnosis than SELECTOR_DRIFT.
|
|
2477
|
-
|
|
2478
|
-
## Test Spec (spec.yaml)
|
|
2479
|
-
${specYaml}
|
|
2480
|
-
|
|
2481
|
-
## Recorded Actions (ir.json summary)
|
|
2482
|
-
${actionsSummary}
|
|
2483
|
-
|
|
2484
|
-
## Test Script (with line numbers)
|
|
2485
|
-
${numbered}
|
|
2486
|
-
|
|
2487
|
-
## Failure Log
|
|
2488
|
-
${failureLog.slice(0, 4e3)}${pageSnapshot ? formatPageSnapshot(pageSnapshot) : ""}`;
|
|
468
|
+
* land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
|
|
469
|
+
* of something it creates would then share that name with its neighbour, and
|
|
470
|
+
* each would find — and delete — the other's row. Nothing fails; the
|
|
471
|
+
* assertions just read the wrong state.
|
|
472
|
+
*
|
|
473
|
+
* Caller is expected to mkdir the directory once and pass
|
|
474
|
+
* `runDir = <baseDir>/<runId>` to the path helpers below.
|
|
475
|
+
*/
|
|
476
|
+
function buildRunId() {
|
|
477
|
+
return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
|
|
2489
478
|
}
|
|
2490
479
|
/**
|
|
2491
|
-
*
|
|
2492
|
-
*
|
|
2493
|
-
*
|
|
2494
|
-
*
|
|
2495
|
-
*
|
|
2496
|
-
*
|
|
480
|
+
* Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
|
|
481
|
+
* Three files per step:
|
|
482
|
+
* - <stepId>.before.png : screenshot taken BEFORE Claude executes the step.
|
|
483
|
+
* - <stepId>.after.png : screenshot taken AFTER Claude executes the step.
|
|
484
|
+
* - <stepId>.log.txt : full assistant transcript for the step (judgement
|
|
485
|
+
* reasoning, any STEP_RESULT lines, raw tool output
|
|
486
|
+
* summaries the model chose to keep).
|
|
2497
487
|
*/
|
|
2498
|
-
function
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
- If a near-miss of the failing selector appears here (e.g. failing \`[aria-label='A']\` and snapshot contains \`aria-label="A-prime"\`), that is direct evidence of SELECTOR_DRIFT — propose the snapshot's value as \`newSelector\`.
|
|
2506
|
-
- If the failing locator is genuinely absent and no near-miss exists, the page may be still loading (TIMING_ISSUE) or the spec is asserting something not on this page (OVER_ASSERTION / DATA_MISSING).
|
|
2507
|
-
- If the snapshot looks unrelated to the spec (e.g. error page, login wall), DATA_MISSING is likely.
|
|
2508
|
-
|
|
2509
|
-
\`\`\`
|
|
2510
|
-
${snapshot}
|
|
2511
|
-
\`\`\``;
|
|
488
|
+
function stepArtifactPaths(runDir, stepId) {
|
|
489
|
+
const dir = join(runDir, "steps");
|
|
490
|
+
return {
|
|
491
|
+
beforePng: join(dir, `${stepId}.before.png`),
|
|
492
|
+
afterPng: join(dir, `${stepId}.after.png`),
|
|
493
|
+
logTxt: join(dir, `${stepId}.log.txt`)
|
|
494
|
+
};
|
|
2512
495
|
}
|
|
2513
496
|
//#endregion
|
|
2514
|
-
//#region src/
|
|
2515
|
-
async function
|
|
2516
|
-
const
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
497
|
+
//#region src/runtime/pool.ts
|
|
498
|
+
async function runPool(items, concurrency, fn, opts = {}) {
|
|
499
|
+
const results = new Array(items.length);
|
|
500
|
+
const needs = items.map((item) => opts.resources?.(item) ?? []);
|
|
501
|
+
const busy = /* @__PURE__ */ new Set();
|
|
502
|
+
const queued = new Set(items.map((_, i) => i));
|
|
503
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
504
|
+
const limit = Math.max(1, Math.min(concurrency, items.length));
|
|
505
|
+
const failures = [];
|
|
506
|
+
const start = (idx) => {
|
|
507
|
+
queued.delete(idx);
|
|
508
|
+
for (const name of needs[idx]) busy.add(name);
|
|
509
|
+
inFlight.set(idx, Promise.resolve().then(async () => {
|
|
510
|
+
try {
|
|
511
|
+
results[idx] = await fn(items[idx], idx);
|
|
512
|
+
} catch (err) {
|
|
513
|
+
failures.push(err);
|
|
514
|
+
} finally {
|
|
515
|
+
for (const name of needs[idx]) busy.delete(name);
|
|
516
|
+
inFlight.delete(idx);
|
|
517
|
+
}
|
|
518
|
+
}));
|
|
2535
519
|
};
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
520
|
+
while (queued.size > 0 || inFlight.size > 0) {
|
|
521
|
+
if (failures.length === 0) {
|
|
522
|
+
for (const idx of queued) {
|
|
523
|
+
if (inFlight.size >= limit) break;
|
|
524
|
+
if (needs[idx].some((name) => busy.has(name))) continue;
|
|
525
|
+
start(idx);
|
|
526
|
+
}
|
|
527
|
+
if (inFlight.size === 0) throw new Error(`runPool: ${queued.size} item(s) unrunnable with nothing in flight`);
|
|
2543
528
|
}
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
result: normalised,
|
|
2547
|
-
raw,
|
|
2548
|
-
sdkError: false
|
|
2549
|
-
};
|
|
529
|
+
if (inFlight.size === 0) break;
|
|
530
|
+
await Promise.race(inFlight.values());
|
|
2550
531
|
}
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
532
|
+
if (failures.length === 1) throw failures[0];
|
|
533
|
+
if (failures.length > 1) throw new AggregateError(failures, `${failures.length} items failed`);
|
|
534
|
+
return results;
|
|
535
|
+
}
|
|
536
|
+
//#endregion
|
|
537
|
+
//#region src/drift/auth.ts
|
|
538
|
+
/**
|
|
539
|
+
* Claude Code can also run against AWS Bedrock / Google Vertex AI, selected by
|
|
540
|
+
* these env toggles. Credentials then come from the cloud SDK's own chain
|
|
541
|
+
* (instance/task roles, gcloud auth, …), so none of the Anthropic-side probes
|
|
542
|
+
* below apply — a set toggle counts as auth being available. ccqa forwards the
|
|
543
|
+
* toggle verbatim; only "0"/"false" (any case) is treated as explicitly off.
|
|
544
|
+
*/
|
|
545
|
+
const CLOUD_PROVIDER_ENV_KEYS = ["CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX"];
|
|
546
|
+
function cloudProviderEnabled() {
|
|
547
|
+
return CLOUD_PROVIDER_ENV_KEYS.some((key) => {
|
|
548
|
+
const value = process.env[key]?.trim().toLowerCase();
|
|
549
|
+
return value !== void 0 && value !== "" && value !== "0" && value !== "false";
|
|
550
|
+
});
|
|
2563
551
|
}
|
|
2564
552
|
/**
|
|
2565
|
-
*
|
|
2566
|
-
*
|
|
2567
|
-
*
|
|
2568
|
-
*
|
|
2569
|
-
*
|
|
2570
|
-
*
|
|
2571
|
-
*
|
|
2572
|
-
*
|
|
553
|
+
* Probe whether the host has any credential the Anthropic SDK can pick up:
|
|
554
|
+
* - one of CREDENTIAL_ENV_KEYS (API key, gateway bearer token, or the
|
|
555
|
+
* subscription token from `claude setup-token`)
|
|
556
|
+
* - CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
|
|
557
|
+
* endpoints authenticated by the cloud SDK's credential chain)
|
|
558
|
+
* - ~/.claude/.credentials.json (Claude Code login, file-based platforms)
|
|
559
|
+
* - macOS Keychain item "Claude Code-credentials" (Claude Code login on
|
|
560
|
+
* darwin stores the OAuth credentials in the Keychain, not on disk)
|
|
2573
561
|
*
|
|
2574
|
-
*
|
|
2575
|
-
*
|
|
562
|
+
* Claude-driven hooks are opt-in, so the caller only consults this after the
|
|
563
|
+
* user has asked for analysis. We never throw — auth absence is a normal flow
|
|
564
|
+
* that surfaces as "analysis skipped".
|
|
2576
565
|
*/
|
|
2577
|
-
function
|
|
2578
|
-
const
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
const blocks = [];
|
|
2582
|
-
let depth = 0;
|
|
2583
|
-
let start = -1;
|
|
2584
|
-
let inString = false;
|
|
2585
|
-
let escaped = false;
|
|
2586
|
-
for (let i = 0; i < raw.length; i++) {
|
|
2587
|
-
const ch = raw[i];
|
|
2588
|
-
if (inString) {
|
|
2589
|
-
if (escaped) escaped = false;
|
|
2590
|
-
else if (ch === "\\") escaped = true;
|
|
2591
|
-
else if (ch === "\"") inString = false;
|
|
2592
|
-
continue;
|
|
2593
|
-
}
|
|
2594
|
-
if (ch === "\"") {
|
|
2595
|
-
inString = true;
|
|
2596
|
-
continue;
|
|
2597
|
-
}
|
|
2598
|
-
if (ch === "{") {
|
|
2599
|
-
if (depth === 0) start = i;
|
|
2600
|
-
depth++;
|
|
2601
|
-
} else if (ch === "}") {
|
|
2602
|
-
depth--;
|
|
2603
|
-
if (depth === 0 && start >= 0) {
|
|
2604
|
-
blocks.push(raw.slice(start, i + 1));
|
|
2605
|
-
start = -1;
|
|
2606
|
-
}
|
|
2607
|
-
}
|
|
2608
|
-
}
|
|
2609
|
-
for (let i = blocks.length - 1; i >= 0; i--) {
|
|
2610
|
-
const block = blocks[i];
|
|
2611
|
-
if (!out.includes(block)) out.push(block);
|
|
566
|
+
function driftAuthAvailable() {
|
|
567
|
+
for (const key of CREDENTIAL_ENV_KEYS) {
|
|
568
|
+
const value = process.env[key];
|
|
569
|
+
if (typeof value === "string" && value.length > 0) return { ok: true };
|
|
2612
570
|
}
|
|
2613
|
-
return
|
|
2614
|
-
}
|
|
2615
|
-
|
|
2616
|
-
return s.length <= max ? s : `${s.slice(0, max)}... [truncated, ${s.length - max} more chars]`;
|
|
2617
|
-
}
|
|
2618
|
-
function stripFence(raw) {
|
|
2619
|
-
return raw.trim().replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
|
|
2620
|
-
}
|
|
2621
|
-
function normaliseResult(parsed) {
|
|
2622
|
-
if (!isObject(parsed)) return null;
|
|
2623
|
-
const diagnosis = normaliseDiagnosis(parsed["diagnosis"]);
|
|
2624
|
-
if (!diagnosis) return null;
|
|
571
|
+
if (cloudProviderEnabled()) return { ok: true };
|
|
572
|
+
if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
|
|
573
|
+
if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
|
|
2625
574
|
return {
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
reasoning: typeof parsed["reasoning"] === "string" ? parsed["reasoning"] : ""
|
|
575
|
+
ok: false,
|
|
576
|
+
reason: `no ${CREDENTIAL_ENV_KEYS.join(" / ")} / Bedrock or Vertex env / claude login`
|
|
2629
577
|
};
|
|
2630
578
|
}
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
|
|
2649
|
-
};
|
|
2650
|
-
}
|
|
2651
|
-
case "SELECTOR_DRIFT": {
|
|
2652
|
-
const line = typeof raw["line"] === "number" ? raw["line"] : null;
|
|
2653
|
-
const oldSelector = typeof raw["oldSelector"] === "string" ? raw["oldSelector"] : null;
|
|
2654
|
-
const newSelector = typeof raw["newSelector"] === "string" ? raw["newSelector"] : null;
|
|
2655
|
-
if (line === null || !oldSelector || !newSelector) return null;
|
|
2656
|
-
return {
|
|
2657
|
-
type: "SELECTOR_DRIFT",
|
|
2658
|
-
line,
|
|
2659
|
-
oldSelector,
|
|
2660
|
-
newSelector,
|
|
2661
|
-
reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
|
|
2662
|
-
};
|
|
2663
|
-
}
|
|
2664
|
-
case "DATA_MISSING": return {
|
|
2665
|
-
type: "DATA_MISSING",
|
|
2666
|
-
reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
|
|
2667
|
-
};
|
|
2668
|
-
case "UNKNOWN": return {
|
|
2669
|
-
type: "UNKNOWN",
|
|
2670
|
-
reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
|
|
2671
|
-
};
|
|
2672
|
-
default: return null;
|
|
2673
|
-
}
|
|
2674
|
-
}
|
|
2675
|
-
function normaliseSleepFixes(raw) {
|
|
2676
|
-
if (!Array.isArray(raw)) return [];
|
|
2677
|
-
const out = [];
|
|
2678
|
-
for (const item of raw) {
|
|
2679
|
-
if (!isObject(item)) continue;
|
|
2680
|
-
const line = typeof item["line"] === "number" ? item["line"] : null;
|
|
2681
|
-
if (line === null) continue;
|
|
2682
|
-
const reason = typeof item["reason"] === "string" ? item["reason"] : "";
|
|
2683
|
-
if (item["kind"] === "insert") {
|
|
2684
|
-
const seconds = typeof item["seconds"] === "number" ? item["seconds"] : null;
|
|
2685
|
-
if (seconds === null) continue;
|
|
2686
|
-
out.push({
|
|
2687
|
-
kind: "insert",
|
|
2688
|
-
line,
|
|
2689
|
-
seconds,
|
|
2690
|
-
reason
|
|
2691
|
-
});
|
|
2692
|
-
} else if (item["kind"] === "increase") {
|
|
2693
|
-
const increaseTo = typeof item["increase_to"] === "number" ? item["increase_to"] : null;
|
|
2694
|
-
if (increaseTo === null) continue;
|
|
2695
|
-
out.push({
|
|
2696
|
-
kind: "increase",
|
|
2697
|
-
line,
|
|
2698
|
-
increase_to: increaseTo,
|
|
2699
|
-
reason
|
|
2700
|
-
});
|
|
2701
|
-
}
|
|
579
|
+
/**
|
|
580
|
+
* `security find-generic-password` without `-w` only checks the item's
|
|
581
|
+
* existence (exit 0) — it never reads the secret, so no Keychain unlock
|
|
582
|
+
* prompt is triggered. Resolved via PATH so tests can stub the binary.
|
|
583
|
+
*/
|
|
584
|
+
function keychainHasClaudeCredentials() {
|
|
585
|
+
try {
|
|
586
|
+
return spawnSync("security", [
|
|
587
|
+
"find-generic-password",
|
|
588
|
+
"-s",
|
|
589
|
+
"Claude Code-credentials"
|
|
590
|
+
], {
|
|
591
|
+
stdio: "ignore",
|
|
592
|
+
timeout: 3e3
|
|
593
|
+
}).status === 0;
|
|
594
|
+
} catch {
|
|
595
|
+
return false;
|
|
2702
596
|
}
|
|
2703
|
-
return out;
|
|
2704
|
-
}
|
|
2705
|
-
function isObject(v) {
|
|
2706
|
-
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2707
|
-
}
|
|
2708
|
-
function clamp(n, lo, hi) {
|
|
2709
|
-
if (n < lo) return lo;
|
|
2710
|
-
if (n > hi) return hi;
|
|
2711
|
-
return n;
|
|
2712
597
|
}
|
|
2713
598
|
//#endregion
|
|
2714
599
|
//#region src/prompts/custom-prompt.ts
|
|
@@ -4078,10 +1963,22 @@ When you are done exploring, reply with ONLY a JSON object (no explanation, no m
|
|
|
4078
1963
|
function retryNote(error) {
|
|
4079
1964
|
return `\n\n## Previous attempt rejected\n\nYour previous reply violated the output contract: ${error}\nReply again with ONLY the JSON object described in "Output format".`;
|
|
4080
1965
|
}
|
|
1966
|
+
function formatStep(step) {
|
|
1967
|
+
if (isExpandedActionStep(step)) return `- ${step.id}: ${body(step.instruction)}\n expected: ${body(step.expected)}`;
|
|
1968
|
+
const source = step.from ? ` (read from \`${step.from}\`)` : "";
|
|
1969
|
+
return `- ${step.id}: judge by LLM${source}\n claim: ${body(step.judgeByLlm)}`;
|
|
1970
|
+
}
|
|
1971
|
+
/**
|
|
1972
|
+
* Step text is usually a block scalar, so it arrives with newlines. Left as-is
|
|
1973
|
+
* its later lines sit flush against the list and read as further steps.
|
|
1974
|
+
*/
|
|
1975
|
+
function body(text) {
|
|
1976
|
+
return text.trim().split("\n").join("\n ");
|
|
1977
|
+
}
|
|
4081
1978
|
function buildLlmGenPrompt(input) {
|
|
4082
1979
|
const sections = [];
|
|
4083
1980
|
sections.push(input.taskInstructions);
|
|
4084
|
-
const steps = input.steps.map(
|
|
1981
|
+
const steps = input.steps.map(formatStep).join("\n");
|
|
4085
1982
|
sections.push(`## Test spec\n\nTitle: ${input.specTitle}\n\nSteps:\n${steps}`);
|
|
4086
1983
|
if (input.draft) sections.push(`## Mechanical draft (recorded ground truth)\n\nPath: ${input.draft.path}\n\n\`\`\`
|
|
4087
1984
|
` + input.draft.contents + "\n```");
|
|
@@ -4941,13 +2838,14 @@ const PerspectiveStatusSchema = z.object({
|
|
|
4941
2838
|
* One step of a spec, transcribed verbatim for display. Exactly one shape per
|
|
4942
2839
|
* step: an include step carries the block name it invokes (its params stay in
|
|
4943
2840
|
* the spec — they are wiring, not procedure), an action step carries the
|
|
4944
|
-
* instruction and what it expects. Never authored or reworded here: like
|
|
2841
|
+
* instruction and what it expects, a judge step carries the claim it asserts. Never authored or reworded here: like
|
|
4945
2842
|
* `title`, this is a mechanical copy the next `ccqa perspectives` rewrites.
|
|
4946
2843
|
*/
|
|
4947
2844
|
const PerspectiveStepSchema = z.object({
|
|
4948
2845
|
include: z.string().min(1).optional(),
|
|
4949
2846
|
instruction: z.string().optional(),
|
|
4950
|
-
expected: z.string().optional()
|
|
2847
|
+
expected: z.string().optional(),
|
|
2848
|
+
judgeByLlm: z.string().optional()
|
|
4951
2849
|
}).strip();
|
|
4952
2850
|
/**
|
|
4953
2851
|
* One test case in the inventory.
|
|
@@ -8109,7 +6007,7 @@ function buildStepDescriptions(spec, blocks) {
|
|
|
8109
6007
|
if (!spec) return /* @__PURE__ */ new Map();
|
|
8110
6008
|
try {
|
|
8111
6009
|
const expanded = expandSpec(spec, { blocks });
|
|
8112
|
-
return new Map(expanded.map((s) => [s.id, s.expected.trim()]));
|
|
6010
|
+
return new Map(expanded.map((s) => [s.id, (isExpandedJudgeByLlmStep(s) ? s.judgeByLlm : s.expected).trim()]));
|
|
8113
6011
|
} catch {
|
|
8114
6012
|
return /* @__PURE__ */ new Map();
|
|
8115
6013
|
}
|
|
@@ -8703,8 +6601,6 @@ async function generateWithLlmEngine(req) {
|
|
|
8703
6601
|
meta("conventions", conventions.sections.length);
|
|
8704
6602
|
const bundle = await loadPromptBundleFromHub(ctx.hub, req.target);
|
|
8705
6603
|
if (bundle) meta("prompt-bundle", bundle.loaded.join(", "));
|
|
8706
|
-
const blocks = await loadAllBlocks(ctx.cwd);
|
|
8707
|
-
const steps = expandSpec(ctx.spec, { blocks });
|
|
8708
6604
|
const policy = {
|
|
8709
6605
|
cwd: ctx.cwd,
|
|
8710
6606
|
outDirAbs: resolve(ctx.cwd, outDir),
|
|
@@ -8714,7 +6610,7 @@ async function generateWithLlmEngine(req) {
|
|
|
8714
6610
|
const prompt = buildLlmGenPrompt({
|
|
8715
6611
|
taskInstructions: req.taskInstructions,
|
|
8716
6612
|
specTitle: ctx.spec.title,
|
|
8717
|
-
steps,
|
|
6613
|
+
steps: req.steps,
|
|
8718
6614
|
draft: req.draft,
|
|
8719
6615
|
...req.draftInvariant ? { draftInvariant: req.draftInvariant } : {},
|
|
8720
6616
|
resources: resources.map(toPromptResource),
|
|
@@ -10621,6 +8517,17 @@ function resolveSpecsModes(specs, catalog) {
|
|
|
10621
8517
|
}));
|
|
10622
8518
|
}
|
|
10623
8519
|
//#endregion
|
|
8520
|
+
//#region src/targets/agent-browser/judge-steps.ts
|
|
8521
|
+
/**
|
|
8522
|
+
* Lives apart from the plugin so the paths that refuse a claim can read the
|
|
8523
|
+
* same declaration the plugin publishes, without importing the plugin they
|
|
8524
|
+
* are part of.
|
|
8525
|
+
*/
|
|
8526
|
+
const AGENT_BROWSER_JUDGE_STEPS = {
|
|
8527
|
+
supported: false,
|
|
8528
|
+
reason: "it drives a browser through a model, whose `expected` already decides each step"
|
|
8529
|
+
};
|
|
8530
|
+
//#endregion
|
|
10624
8531
|
//#region src/cli/stale-blocks.ts
|
|
10625
8532
|
/**
|
|
10626
8533
|
* Hint when stale per-block artifacts (`test.spec.ts`, `actions.json`)
|
|
@@ -12199,7 +10106,7 @@ async function loadSpecInventory(cwd) {
|
|
|
12199
10106
|
*/
|
|
12200
10107
|
function describeSteps(spec, blocks, specKey) {
|
|
12201
10108
|
try {
|
|
12202
|
-
return expandSpec(spec, { blocks }).map(
|
|
10109
|
+
return expandSpec(spec, { blocks }).map(describeStepBody);
|
|
12203
10110
|
} catch (e) {
|
|
12204
10111
|
warn(`${specKey}: could not expand include steps (${e.message}) — showing block names instead`);
|
|
12205
10112
|
return spec.steps.map(describeStep);
|
|
@@ -12207,6 +10114,10 @@ function describeSteps(spec, blocks, specKey) {
|
|
|
12207
10114
|
}
|
|
12208
10115
|
function describeStep(step) {
|
|
12209
10116
|
if (isIncludeStep(step)) return `include block: ${step.include}`;
|
|
10117
|
+
return describeStepBody(step);
|
|
10118
|
+
}
|
|
10119
|
+
function describeStepBody(step) {
|
|
10120
|
+
if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
|
|
12210
10121
|
return `${oneLine$1(step.instruction)} → ${oneLine$1(step.expected)}`;
|
|
12211
10122
|
}
|
|
12212
10123
|
function oneLine$1(text) {
|
|
@@ -14299,9 +12210,12 @@ async function runOneSpec(args) {
|
|
|
14299
12210
|
};
|
|
14300
12211
|
}
|
|
14301
12212
|
const spec = parseTestSpec(specContent);
|
|
14302
|
-
const
|
|
12213
|
+
const steps = expandActionSteps(spec, { blocks: await loadAllBlocks(cwd) }, `${featureName}/${specName}`, {
|
|
12214
|
+
id: AGENT_BROWSER_TARGET,
|
|
12215
|
+
reason: AGENT_BROWSER_JUDGE_STEPS.reason
|
|
12216
|
+
});
|
|
14303
12217
|
meta("spec", spec.title);
|
|
14304
|
-
meta("steps",
|
|
12218
|
+
meta("steps", steps.length);
|
|
14305
12219
|
const includes = collectIncludedBlockNames(spec);
|
|
14306
12220
|
if (includes.length > 0) meta("blocks", includes.join(", "));
|
|
14307
12221
|
const sessionName = generateLiveSessionName();
|
|
@@ -14346,13 +12260,13 @@ async function runOneSpec(args) {
|
|
|
14346
12260
|
}
|
|
14347
12261
|
try {
|
|
14348
12262
|
const runId = buildRunId();
|
|
14349
|
-
const envScrubMap = buildProseEnvScrubMap(spec,
|
|
12263
|
+
const envScrubMap = buildProseEnvScrubMap(spec, steps, { CCQA_RUN_ID: runId });
|
|
14350
12264
|
const runDir = opts.out ?? join(specDir, "runs", runId);
|
|
14351
12265
|
await mkdir(runDir, { recursive: true });
|
|
14352
12266
|
meta("runDir", runDir);
|
|
14353
12267
|
const result = await runLiveExecutor({
|
|
14354
12268
|
spec: { title: spec.title },
|
|
14355
|
-
steps
|
|
12269
|
+
steps,
|
|
14356
12270
|
runId,
|
|
14357
12271
|
runDir,
|
|
14358
12272
|
sessionName,
|
|
@@ -15386,15 +13300,18 @@ async function generateAgentBrowserTest(ctx) {
|
|
|
15386
13300
|
const { spec, featureName, specName, cwd, fix } = ctx;
|
|
15387
13301
|
const actions = ctx.recording;
|
|
15388
13302
|
if (!actions) throw new Error(`the agent-browser target needs a recording — run \`ccqa record ${featureName}/${specName}\` first`);
|
|
15389
|
-
const
|
|
15390
|
-
|
|
13303
|
+
const steps = expandActionSteps(spec, { blocks: await loadAllBlocks(cwd) }, `${featureName}/${specName}`, {
|
|
13304
|
+
id: AGENT_BROWSER_TARGET,
|
|
13305
|
+
reason: AGENT_BROWSER_JUDGE_STEPS.reason
|
|
13306
|
+
});
|
|
13307
|
+
meta("steps", steps.length);
|
|
15391
13308
|
meta("fix-mode", fix.mode);
|
|
15392
13309
|
meta("language", ctx.language);
|
|
15393
13310
|
blank();
|
|
15394
13311
|
const cleanedActions = await cleanupActions(actions, ctx.model);
|
|
15395
13312
|
if (cleanedActions.length !== actions.length) meta("cleaned", cleanedActions.length);
|
|
15396
|
-
const markers = buildStepMarkers(
|
|
15397
|
-
const emptySteps = findEmptySteps(
|
|
13313
|
+
const markers = buildStepMarkers(steps, cleanedActions);
|
|
13314
|
+
const emptySteps = findEmptySteps(steps, cleanedActions);
|
|
15398
13315
|
const warnings = emptySteps.map((e) => `step ${e.stepId} has no kept actions — generated test will skip it (notice comment inserted).`);
|
|
15399
13316
|
for (const w of warnings) warn(w);
|
|
15400
13317
|
const scriptPath = await saveTestScript(featureName, specName, actionsToScript({
|
|
@@ -15467,6 +13384,15 @@ function buildStepMarkers(steps, actions) {
|
|
|
15467
13384
|
}
|
|
15468
13385
|
return markers;
|
|
15469
13386
|
}
|
|
13387
|
+
/** The last action index each step owns — where anything that follows it belongs. */
|
|
13388
|
+
function lastActionIndexPerStep(actions) {
|
|
13389
|
+
const last = /* @__PURE__ */ new Map();
|
|
13390
|
+
for (let i = 0; i < actions.length; i++) {
|
|
13391
|
+
const id = actions[i].stepId;
|
|
13392
|
+
if (id) last.set(id, i);
|
|
13393
|
+
}
|
|
13394
|
+
return last;
|
|
13395
|
+
}
|
|
15470
13396
|
/**
|
|
15471
13397
|
* Spec steps that lost every action by the time the trace finished its
|
|
15472
13398
|
* cleanup + validation passes. `actionsToScript` uses these to splice a
|
|
@@ -15482,11 +13408,7 @@ function buildStepMarkers(steps, actions) {
|
|
|
15482
13408
|
function findEmptySteps(steps, cleanedActions) {
|
|
15483
13409
|
const presentStepIds = /* @__PURE__ */ new Set();
|
|
15484
13410
|
for (const a of cleanedActions) if (a.stepId) presentStepIds.add(a.stepId);
|
|
15485
|
-
const lastActionIndexByStep =
|
|
15486
|
-
for (let i = 0; i < cleanedActions.length; i++) {
|
|
15487
|
-
const id = cleanedActions[i].stepId;
|
|
15488
|
-
if (id) lastActionIndexByStep.set(id, i);
|
|
15489
|
-
}
|
|
13411
|
+
const lastActionIndexByStep = lastActionIndexPerStep(cleanedActions);
|
|
15490
13412
|
const notices = [];
|
|
15491
13413
|
let lastSeenSurvivorIndex = -1;
|
|
15492
13414
|
for (const step of steps) {
|
|
@@ -15598,18 +13520,42 @@ const agentBrowserTarget = {
|
|
|
15598
13520
|
browserCoverage: {
|
|
15599
13521
|
browser: "cdp",
|
|
15600
13522
|
cdpEndpoint: acquireAgentBrowserEndpoint
|
|
15601
|
-
}
|
|
13523
|
+
},
|
|
13524
|
+
judgeSteps: AGENT_BROWSER_JUDGE_STEPS
|
|
15602
13525
|
};
|
|
15603
13526
|
//#endregion
|
|
15604
13527
|
//#region src/targets/playwright/emit-mechanical.ts
|
|
15605
13528
|
/** Module the emitted step-boundary capture calls import from. */
|
|
15606
13529
|
const STEP_EVIDENCE_MODULE = "ccqa/step-evidence";
|
|
13530
|
+
/** Module the emitted judge calls import from, and the call they make. */
|
|
13531
|
+
const JUDGE_MODULE = "ccqa/judge";
|
|
13532
|
+
const JUDGE_CALL = "judgeByLlm";
|
|
15607
13533
|
/** Capture call emitted when a step is entered / closed. Exported for the generation gate. */
|
|
15608
13534
|
const STEP_EVIDENCE_BEFORE = "ccqaStepBefore";
|
|
15609
13535
|
const STEP_EVIDENCE_AFTER = "ccqaStepAfter";
|
|
15610
13536
|
/** The exact boundary call for one step, as emitted and as the gate greps for it. */
|
|
15611
13537
|
function stepEvidenceCall(fn, marker) {
|
|
15612
|
-
return
|
|
13538
|
+
return injectedCall(fn, [j(marker.stepId), j(marker.source)]);
|
|
13539
|
+
}
|
|
13540
|
+
/**
|
|
13541
|
+
* A receiver expression: anything up to the argument's comma, allowing one
|
|
13542
|
+
* level of parentheses so `await ctx.newPage()` and `page.context().pages()[1]`
|
|
13543
|
+
* — how a rewrite names a tab a click opened — count as receivers.
|
|
13544
|
+
*/
|
|
13545
|
+
const RECEIVER = String.raw`(?:[^,()]|\([^()]*\))+`;
|
|
13546
|
+
/**
|
|
13547
|
+
* The call as emitted, plus the pattern that accepts it back on any page. The
|
|
13548
|
+
* receiver is the one part a rewrite is right to change: a click that opens a
|
|
13549
|
+
* new tab has to act on that tab. Everything else is required verbatim, the
|
|
13550
|
+
* `await` and `;` included — an unawaited capture races the end of the test,
|
|
13551
|
+
* and a mention in a comment is not a call.
|
|
13552
|
+
*/
|
|
13553
|
+
function injectedCall(name, args) {
|
|
13554
|
+
const tail = args.map(escapeRegExp).join(String.raw`\s*,\s*`);
|
|
13555
|
+
return {
|
|
13556
|
+
code: `await ${name}(page, ${args.join(", ")});`,
|
|
13557
|
+
pattern: new RegExp(String.raw`await\s+${name}\s*\(\s*${RECEIVER}\s*,\s*${tail}\s*,?\s*\)\s*;`)
|
|
13558
|
+
};
|
|
15613
13559
|
}
|
|
15614
13560
|
/**
|
|
15615
13561
|
* The "preserve the step-evidence calls" rule the library-rewrite prompt must
|
|
@@ -15619,35 +13565,58 @@ function stepEvidenceCall(fn, marker) {
|
|
|
15619
13565
|
* engine, and the prompt then omits the rule entirely.
|
|
15620
13566
|
*/
|
|
15621
13567
|
function stepEvidencePreserveRule() {
|
|
15622
|
-
return `**Keep the \`${STEP_EVIDENCE_MODULE}\` calls.** The draft's \`await ${STEP_EVIDENCE_BEFORE}(page, ...)\` / \`await ${STEP_EVIDENCE_AFTER}(page, ...)\` lines are load-bearing: ccqa run reads the per-step screenshots they capture. Keep both calls for every step, in place around that step's actions, with their exact \`
|
|
13568
|
+
return `**Keep the \`${STEP_EVIDENCE_MODULE}\` calls.** The draft's \`await ${STEP_EVIDENCE_BEFORE}(page, ...)\` / \`await ${STEP_EVIDENCE_AFTER}(page, ...)\` lines are load-bearing: ccqa run reads the per-step screenshots they capture. Keep both calls for every step, in place around that step's actions, with their exact \`"<stepId>", "<source>"\` arguments — and keep the import. Pass a different page only when the step genuinely acts on one (a click that opens a new tab). If you move a step's actions into a page-object method, leave these two calls in the test body around the call to that method; do NOT move them inside the page object. Never wrap a step in a closure to hold them.`;
|
|
13569
|
+
}
|
|
13570
|
+
/**
|
|
13571
|
+
* Told to the rewrite, because the alternative failure is silent: a claim
|
|
13572
|
+
* turned into a text match passes on the wording of one run, which is the
|
|
13573
|
+
* assertion the judge exists to replace.
|
|
13574
|
+
*/
|
|
13575
|
+
function judgePreserveRule() {
|
|
13576
|
+
return `**Keep the \`${JUDGE_MODULE}\` calls.** The draft's \`await ${JUDGE_CALL}(page, "<claim>")\` lines assert a claim a model decides at run time, for output whose wording changes every run. Keep each call where it is, with its claim text unchanged, and keep the import. Do NOT replace one with \`toContainText\`, \`toHaveText\` or any other match on the answer's wording, and do not move it into a page object.`;
|
|
15623
13577
|
}
|
|
15624
13578
|
function emitPlaywrightDraft(input) {
|
|
15625
|
-
const { actions, testName, stepMarkers = [] } = input;
|
|
13579
|
+
const { actions, testName, stepMarkers = [], judgements = [] } = input;
|
|
15626
13580
|
const markerByIndex = new Map(stepMarkers.map((m) => [m.actionIndex, m]));
|
|
15627
13581
|
const lines = [];
|
|
15628
13582
|
let prevLine = null;
|
|
15629
13583
|
let openMarker = null;
|
|
13584
|
+
const flushJudgements = (afterActionIndex) => {
|
|
13585
|
+
for (const { step } of judgements.filter((j) => j.afterActionIndex === afterActionIndex)) {
|
|
13586
|
+
if (openMarker) {
|
|
13587
|
+
lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker).code);
|
|
13588
|
+
openMarker = null;
|
|
13589
|
+
}
|
|
13590
|
+
if (lines.length > 0) lines.push("");
|
|
13591
|
+
lines.push(`// step: ${step.id} [${step.source}]`);
|
|
13592
|
+
lines.push(judgeCall(step).code);
|
|
13593
|
+
}
|
|
13594
|
+
};
|
|
13595
|
+
flushJudgements(-1);
|
|
15630
13596
|
for (let i = 0; i < actions.length; i++) {
|
|
15631
13597
|
const marker = markerByIndex.get(i);
|
|
15632
13598
|
if (marker) {
|
|
15633
|
-
if (openMarker) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker));
|
|
13599
|
+
if (openMarker) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker).code);
|
|
15634
13600
|
if (lines.length > 0) lines.push("");
|
|
15635
13601
|
lines.push(`// step: ${marker.stepId} [${marker.source}]`);
|
|
15636
|
-
lines.push(stepEvidenceCall(STEP_EVIDENCE_BEFORE, marker));
|
|
13602
|
+
lines.push(stepEvidenceCall(STEP_EVIDENCE_BEFORE, marker).code);
|
|
15637
13603
|
openMarker = marker;
|
|
15638
13604
|
}
|
|
15639
13605
|
const action = actions[i];
|
|
15640
13606
|
const line = actionToLine(action);
|
|
15641
|
-
if (line
|
|
15642
|
-
|
|
15643
|
-
|
|
15644
|
-
|
|
15645
|
-
|
|
13607
|
+
if (line !== null && line !== prevLine) {
|
|
13608
|
+
if (action.replayUnstable) lines.push(`// [warn] replay-unstable: ${action.replayReason ?? "(no reason recorded)"}`);
|
|
13609
|
+
lines.push(line);
|
|
13610
|
+
prevLine = line;
|
|
13611
|
+
}
|
|
13612
|
+
flushJudgements(i);
|
|
15646
13613
|
}
|
|
15647
|
-
if (openMarker) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker));
|
|
13614
|
+
if (openMarker) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker).code);
|
|
13615
|
+
if (judgements.length > 0) lines.unshift("test.slow();", "");
|
|
15648
13616
|
const body = lines.map((l) => l === "" ? "" : ` ${l}`).join("\n");
|
|
15649
13617
|
return [
|
|
15650
13618
|
`import { test, expect } from "@playwright/test";`,
|
|
13619
|
+
...judgements.length > 0 ? [`import { ${JUDGE_CALL} } from ${j(JUDGE_MODULE)};`] : [],
|
|
15651
13620
|
...stepMarkers.length > 0 ? [`import { ${STEP_EVIDENCE_BEFORE}, ${STEP_EVIDENCE_AFTER} } from ${j(STEP_EVIDENCE_MODULE)};`] : [],
|
|
15652
13621
|
"",
|
|
15653
13622
|
`test(${j(testName)}, async ({ page }) => {`,
|
|
@@ -15828,6 +13797,12 @@ const j = (s) => JSON.stringify(s);
|
|
|
15828
13797
|
* applies to user-supplied values).
|
|
15829
13798
|
*/
|
|
15830
13799
|
const jExpr = (s) => envRefsToJsExpression(s);
|
|
13800
|
+
/** One claim, asserted through the judge. Exported so the generation gate can require it back. */
|
|
13801
|
+
function judgeCall(step) {
|
|
13802
|
+
const args = [bracedRefsToJsExpression(step.judgeByLlm.trim())];
|
|
13803
|
+
if (step.from !== void 0) args.push(jExpr(step.from));
|
|
13804
|
+
return injectedCall(JUDGE_CALL, args);
|
|
13805
|
+
}
|
|
15831
13806
|
//#endregion
|
|
15832
13807
|
//#region src/targets/playwright/browser-server.ts
|
|
15833
13808
|
/**
|
|
@@ -16006,6 +13981,7 @@ const playwrightTarget = {
|
|
|
16006
13981
|
existingOutput: existingPlaywrightOutput,
|
|
16007
13982
|
runner: runCommandRunner,
|
|
16008
13983
|
stepEvidence: { supported: true },
|
|
13984
|
+
judgeSteps: { supported: true },
|
|
16009
13985
|
browserCoverage: {
|
|
16010
13986
|
browser: "cdp",
|
|
16011
13987
|
cdpEndpoint: acquirePlaywrightBrowser
|
|
@@ -16016,11 +13992,15 @@ async function generatePlaywrightTest(ctx) {
|
|
|
16016
13992
|
const actions = ctx.recording;
|
|
16017
13993
|
if (!actions) throw new Error(`the playwright target needs a recording — run \`ccqa record ${ctx.featureName}/${ctx.specName}\` first`);
|
|
16018
13994
|
const blocks = await loadAllBlocks(ctx.cwd);
|
|
16019
|
-
const
|
|
13995
|
+
const expanded = expandSpec(ctx.spec, { blocks });
|
|
13996
|
+
const stepMarkers = buildStepMarkers(expanded.filter(isExpandedActionStep), actions);
|
|
13997
|
+
const { judgements, warnings: judgeWarnings } = placeJudgements(expanded, actions, `${ctx.featureName}/${ctx.specName}`);
|
|
13998
|
+
for (const w of judgeWarnings) warn(w);
|
|
16020
13999
|
const draft = emitPlaywrightDraft({
|
|
16021
14000
|
actions,
|
|
16022
14001
|
testName: ctx.spec.title,
|
|
16023
|
-
stepMarkers
|
|
14002
|
+
stepMarkers,
|
|
14003
|
+
judgements
|
|
16024
14004
|
});
|
|
16025
14005
|
const outDir = ctx.targetConfig.outDir;
|
|
16026
14006
|
const draftPath = outDir ? `${outDir}/${ctx.featureName}/${ctx.specName}.spec.ts` : `${specDirRel(ctx)}/test.spec.ts`;
|
|
@@ -16031,12 +14011,13 @@ async function generatePlaywrightTest(ctx) {
|
|
|
16031
14011
|
const result = ctx.resources.length > 0 ? await generateWithLlmEngine({
|
|
16032
14012
|
ctx,
|
|
16033
14013
|
target: PLAYWRIGHT_TARGET,
|
|
14014
|
+
steps: expanded,
|
|
16034
14015
|
taskInstructions: playwrightTaskInstructions(draftPath),
|
|
16035
14016
|
draft: {
|
|
16036
14017
|
path: draftPath,
|
|
16037
14018
|
contents: draft
|
|
16038
14019
|
},
|
|
16039
|
-
draftInvariant: stepMarkers.length > 0 ? stepEvidencePreserveRule() : ""
|
|
14020
|
+
draftInvariant: [stepMarkers.length > 0 ? stepEvidencePreserveRule() : "", judgements.length > 0 ? judgePreserveRule() : ""].filter(Boolean).join("\n\n")
|
|
16040
14021
|
}) : await finalizePreparedFiles({
|
|
16041
14022
|
ctx,
|
|
16042
14023
|
target: PLAYWRIGHT_TARGET,
|
|
@@ -16048,11 +14029,15 @@ async function generatePlaywrightTest(ctx) {
|
|
|
16048
14029
|
summary: `Playwright spec compiled from ${actions.length} recorded action(s)`,
|
|
16049
14030
|
warnings: []
|
|
16050
14031
|
});
|
|
16051
|
-
const missing = await missingInjectedCalls(result, stepMarkers);
|
|
14032
|
+
const missing = await missingInjectedCalls(result, stepMarkers, judgements);
|
|
16052
14033
|
for (const w of missing) warn(w);
|
|
16053
14034
|
return {
|
|
16054
14035
|
...result,
|
|
16055
|
-
warnings: [
|
|
14036
|
+
warnings: [
|
|
14037
|
+
...result.warnings,
|
|
14038
|
+
...judgeWarnings,
|
|
14039
|
+
...missing
|
|
14040
|
+
]
|
|
16056
14041
|
};
|
|
16057
14042
|
}
|
|
16058
14043
|
/**
|
|
@@ -16063,14 +14048,18 @@ async function generatePlaywrightTest(ctx) {
|
|
|
16063
14048
|
* them); a file that can't be read is reported as missing everything rather
|
|
16064
14049
|
* than passing silently.
|
|
16065
14050
|
*/
|
|
16066
|
-
async function missingInjectedCalls(result, markers) {
|
|
14051
|
+
async function missingInjectedCalls(result, markers, judgements) {
|
|
16067
14052
|
const corpus = (await Promise.all(result.files.filter((f) => f.kind === "test").map((f) => readFile(f.path, "utf8").catch(() => "")))).join("\n");
|
|
16068
14053
|
const warnings = [];
|
|
16069
14054
|
for (const m of markers) {
|
|
16070
|
-
const hasBefore =
|
|
16071
|
-
const hasAfter =
|
|
14055
|
+
const hasBefore = stepEvidenceCall(STEP_EVIDENCE_BEFORE, m).pattern.test(corpus);
|
|
14056
|
+
const hasAfter = stepEvidenceCall(STEP_EVIDENCE_AFTER, m).pattern.test(corpus);
|
|
16072
14057
|
if (!hasBefore || !hasAfter) warnings.push(`step ${m.stepId}: generated test is missing its ${STEP_EVIDENCE_BEFORE}/${STEP_EVIDENCE_AFTER} call(s) — that step will have no report screenshots. A rewrite pass must not drop them.`);
|
|
16073
14058
|
}
|
|
14059
|
+
for (const { step } of judgements) {
|
|
14060
|
+
if (judgeCall(step).pattern.test(corpus)) continue;
|
|
14061
|
+
warnings.push(`step ${step.id}: generated test is missing its ${JUDGE_CALL} call — that claim is never decided and the spec passes without testing it. A rewrite pass must not drop or reword it.`);
|
|
14062
|
+
}
|
|
16074
14063
|
return warnings;
|
|
16075
14064
|
}
|
|
16076
14065
|
/**
|
|
@@ -16087,9 +14076,53 @@ async function existingPlaywrightOutput(ref, cwd) {
|
|
|
16087
14076
|
const specTest = resolve(cwd, `${specDirRel(ref)}/test.spec.ts`);
|
|
16088
14077
|
return stat(specTest).then(() => specTest, () => null);
|
|
16089
14078
|
}
|
|
14079
|
+
/**
|
|
14080
|
+
* Each claim paired with the action index it is asserted after: the last
|
|
14081
|
+
* action of the nearest preceding step. A claim reads what the run has
|
|
14082
|
+
* produced so far, so emitting it at the end of the test would judge a page
|
|
14083
|
+
* later steps have already navigated away from.
|
|
14084
|
+
*
|
|
14085
|
+
* A claim whose preceding steps recorded nothing has no position the
|
|
14086
|
+
* recording can justify. It goes last and says so, because the alternative —
|
|
14087
|
+
* the previous claim's index, or the start — judges a page the spec never
|
|
14088
|
+
* meant, and a negative claim would pass there without being tested.
|
|
14089
|
+
*/
|
|
14090
|
+
function placeJudgements(expanded, actions, specKey) {
|
|
14091
|
+
const lastIndex = lastActionIndexPerStep(actions);
|
|
14092
|
+
const judgements = [];
|
|
14093
|
+
const warnings = [];
|
|
14094
|
+
let afterActionIndex = null;
|
|
14095
|
+
for (const step of expanded) {
|
|
14096
|
+
if (!isExpandedJudgeByLlmStep(step)) {
|
|
14097
|
+
afterActionIndex = lastIndex.get(step.id) ?? afterActionIndex;
|
|
14098
|
+
continue;
|
|
14099
|
+
}
|
|
14100
|
+
if (afterActionIndex === null) {
|
|
14101
|
+
if (expanded.indexOf(step) === 0) throw new Error(`${specKey}: step ${step.id} uses \`judgeByLlm\` as the first step — there is nothing on the page to judge yet.`);
|
|
14102
|
+
warnings.push(`step ${step.id}: no recorded action belongs to the steps before this claim, so it is asserted at the end of the test instead of where the spec puts it. Re-record if the claim reads something a later step navigates away from.`);
|
|
14103
|
+
judgements.push({
|
|
14104
|
+
step,
|
|
14105
|
+
afterActionIndex: actions.length - 1
|
|
14106
|
+
});
|
|
14107
|
+
continue;
|
|
14108
|
+
}
|
|
14109
|
+
judgements.push({
|
|
14110
|
+
step,
|
|
14111
|
+
afterActionIndex
|
|
14112
|
+
});
|
|
14113
|
+
}
|
|
14114
|
+
return {
|
|
14115
|
+
judgements,
|
|
14116
|
+
warnings
|
|
14117
|
+
};
|
|
14118
|
+
}
|
|
16090
14119
|
//#endregion
|
|
16091
14120
|
//#region src/targets/runn/index.ts
|
|
16092
14121
|
const RUNN_TARGET = "runn";
|
|
14122
|
+
const JUDGE_STEPS = {
|
|
14123
|
+
supported: false,
|
|
14124
|
+
reason: "runn runs API scenarios; there is no page to read a claim off"
|
|
14125
|
+
};
|
|
16093
14126
|
/**
|
|
16094
14127
|
* The runn target (input: "spec"): no record phase — `ccqa generate` compiles
|
|
16095
14128
|
* the spec directly into a runn runbook (YAML) via the shared LLM engine. The
|
|
@@ -16111,14 +14144,20 @@ const runnTarget = {
|
|
|
16111
14144
|
browser: "none",
|
|
16112
14145
|
reason: "runn runs API scenarios; there is no browser to measure"
|
|
16113
14146
|
},
|
|
14147
|
+
judgeSteps: JUDGE_STEPS,
|
|
16114
14148
|
guidanceKind: RUNN_TARGET
|
|
16115
14149
|
};
|
|
16116
14150
|
/** Exported with the engine's invoke seam so unit tests can stub Claude. */
|
|
16117
14151
|
async function generateRunnRunbook(ctx, invoke) {
|
|
16118
14152
|
const outDir = ctx.targetConfig.outDir;
|
|
14153
|
+
const blocks = await loadAllBlocks(ctx.cwd);
|
|
16119
14154
|
return generateWithLlmEngine({
|
|
16120
14155
|
ctx,
|
|
16121
14156
|
target: RUNN_TARGET,
|
|
14157
|
+
steps: expandActionSteps(ctx.spec, { blocks }, `${ctx.featureName}/${ctx.specName}`, {
|
|
14158
|
+
id: RUNN_TARGET,
|
|
14159
|
+
reason: JUDGE_STEPS.reason
|
|
14160
|
+
}),
|
|
16122
14161
|
taskInstructions: runnTaskInstructions(outDir ? `${outDir}/${ctx.featureName}/${ctx.specName}.yaml` : `${specDirRel(ctx)}/runbook.yaml`),
|
|
16123
14162
|
validateFile: validateRunnFile,
|
|
16124
14163
|
invoke
|
|
@@ -18306,6 +16345,7 @@ function generateSessionName() {
|
|
|
18306
16345
|
*/
|
|
18307
16346
|
function buildTraceSystemPrompt(input) {
|
|
18308
16347
|
const sessionName = input.sessionName ?? generateSessionName();
|
|
16348
|
+
const firstStepId = input.steps[0]?.id ?? "step-01";
|
|
18309
16349
|
const callerGuidance = input.instruction ? `## Caller Guidance
|
|
18310
16350
|
|
|
18311
16351
|
The caller provided extra guidance for this recording — for example, a drift
|
|
@@ -18723,8 +16763,8 @@ Emit:
|
|
|
18723
16763
|
AB_ACTION|cookies_clear
|
|
18724
16764
|
\`\`\`
|
|
18725
16765
|
|
|
18726
|
-
Then emit \`STEP_START
|
|
18727
|
-
every one of its agent-browser commands with \`CCQA_STEP
|
|
16766
|
+
Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
|
|
16767
|
+
every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
|
|
18728
16768
|
step is responsible for opening the initial URL.
|
|
18729
16769
|
`;
|
|
18730
16770
|
}
|
|
@@ -19172,6 +17212,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
|
|
|
19172
17212
|
await preflightAgentBrowserCommand();
|
|
19173
17213
|
const spec = parseTestSpec(await readSpecFile(featureName, specName, opts.cwd));
|
|
19174
17214
|
const expanded = expandSpec(spec, { blocks: await loadAllBlocks(opts.cwd) });
|
|
17215
|
+
const steps = expanded.filter(isExpandedActionStep);
|
|
19175
17216
|
const sessionName = generateSessionName();
|
|
19176
17217
|
const envScrub = buildSpecEnvScrub(spec, expanded, { CCQA_RUN_ID: sessionName });
|
|
19177
17218
|
const envScrubMap = envScrub.map;
|
|
@@ -19181,14 +17222,14 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
|
|
|
19181
17222
|
hint("load these vars before recording — pass --profile <name> (hub) or define them in .env — then re-record.");
|
|
19182
17223
|
}
|
|
19183
17224
|
meta("spec", spec.title);
|
|
19184
|
-
meta("steps",
|
|
17225
|
+
meta("steps", steps.length);
|
|
19185
17226
|
const includes = collectIncludedBlockNames(spec);
|
|
19186
17227
|
if (includes.length > 0) meta("blocks", includes.join(", "));
|
|
19187
17228
|
blank();
|
|
19188
17229
|
opts.teardown?.trackSession(sessionName);
|
|
19189
17230
|
const baseSystemPrompt = buildTraceSystemPrompt({
|
|
19190
17231
|
title: spec.title,
|
|
19191
|
-
steps
|
|
17232
|
+
steps,
|
|
19192
17233
|
sessionName,
|
|
19193
17234
|
...opts.instruction ? { instruction: opts.instruction } : {}
|
|
19194
17235
|
});
|
|
@@ -19277,7 +17318,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
|
|
|
19277
17318
|
meta("actions", validatedActions.length);
|
|
19278
17319
|
meta("status", overallStatus.toUpperCase());
|
|
19279
17320
|
if (overallStatus === "passed") {
|
|
19280
|
-
for (const stepId of stepsWithoutAsserts(
|
|
17321
|
+
for (const stepId of stepsWithoutAsserts(steps.map((s) => s.id), validatedActions)) warn(`${stepId} recorded no assertion — nothing in the generated test verifies its 'expected'`);
|
|
19281
17322
|
hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
|
|
19282
17323
|
} else warn("trace FAILED — the recorded actions were saved beside the spec for diagnosis; the previous ir.json and generated code are left untouched");
|
|
19283
17324
|
return {
|
|
@@ -19842,6 +17883,18 @@ async function runRecord(specPath, opts) {
|
|
|
19842
17883
|
error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${featureName}/${specName}' instead`);
|
|
19843
17884
|
process.exit(2);
|
|
19844
17885
|
}
|
|
17886
|
+
if (!target.judgeSteps.supported) {
|
|
17887
|
+
const blocks = await loadAllBlocks(cwdForProfile);
|
|
17888
|
+
try {
|
|
17889
|
+
expandActionSteps(spec, { blocks }, `${featureName}/${specName}`, {
|
|
17890
|
+
id: target.id,
|
|
17891
|
+
reason: target.judgeSteps.reason
|
|
17892
|
+
});
|
|
17893
|
+
} catch (e) {
|
|
17894
|
+
error(e instanceof Error ? e.message : String(e));
|
|
17895
|
+
process.exit(2);
|
|
17896
|
+
}
|
|
17897
|
+
}
|
|
19845
17898
|
if (spec.mode === "live") {
|
|
19846
17899
|
error(`this spec is 'mode: live' — a live spec runs without a recording. Run 'ccqa run ${featureName}/${specName}' instead`);
|
|
19847
17900
|
process.exit(2);
|
|
@@ -21572,7 +19625,8 @@ function readSpecMeta(specName, specYaml) {
|
|
|
21572
19625
|
/**
|
|
21573
19626
|
* The spec's procedure, copied verbatim for the inventory: an include step
|
|
21574
19627
|
* keeps only the block name (its params are wiring, not procedure), an
|
|
21575
|
-
* action step keeps its instruction/expected text
|
|
19628
|
+
* action step keeps its instruction/expected text, a judge step its claim.
|
|
19629
|
+
* Anything malformed is
|
|
21576
19630
|
* skipped — the inventory never fails over one bad step, matching how the
|
|
21577
19631
|
* rest of this sweep treats a broken spec.
|
|
21578
19632
|
*/
|
|
@@ -21583,6 +19637,7 @@ function transcribeSteps(raw) {
|
|
|
21583
19637
|
if (typeof step !== "object" || step === null) continue;
|
|
21584
19638
|
const s = step;
|
|
21585
19639
|
if (typeof s.include === "string" && s.include.length > 0) steps.push({ include: s.include });
|
|
19640
|
+
else if (typeof s.judgeByLlm === "string" && s.judgeByLlm.length > 0) steps.push({ judgeByLlm: s.judgeByLlm });
|
|
21586
19641
|
else if (typeof s.instruction === "string" && s.instruction.length > 0) steps.push({
|
|
21587
19642
|
instruction: s.instruction,
|
|
21588
19643
|
...typeof s.expected === "string" && s.expected.length > 0 ? { expected: s.expected } : {}
|