ccqa 1.46.1 → 1.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bin/ccqa.mjs CHANGED
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env node
2
+ import { $ as bracedRefsToJsExpression, A as progressEnd, B as isExpandedActionStep, C as error, D as info, E as hint, F as withBuffer, G as tryParseTestSpec, H as isJudgeBody, I as CREDENTIAL_ENV_KEYS, J as SessionNameSchema, K as AGENT_BROWSER_TARGET, L as collectIncludedBlockNames, M as step, N as timedPhase, O as meta, P as warn, Q as isParamRequired, R as expandActionSteps, S as emitRaw, T as header, U as parseBlockSpec, V as isExpandedJudgeByLlmStep, W as parseTestSpec, X as TargetIdSchema, Y as SpecModeSchema, Z as isIncludeStep, _ as buildSpecEnvScrub, a as truncate$2, b as promoteMarkedAssert, c as toAgentBrowserArgs, d as surfaceAxisAside, et as envRefsToJsExpression, f as surfaceDefinitionBlock, g as buildProseEnvScrubMap, h as withCostTally, i as isObject, j as run, k as progress, l as numberLines, m as readCostTally, n as diagnose, nt as resolveEnvRefs, o as describeLocator, p as invokeClaudeStreaming, q as DEFAULT_SPEC_MODE, r as extractJsonCandidates, s as locatorToSelector, t as clamp, tt as iterEnvRefNames, u as outputLanguageBlock, v as scrubEnvValues, w as fix, x as blank, y as parseAbActionLine, z as expandSpec } from "../diagnose-CSQzwS5f.mjs";
2
3
  import { HubApiError, createHubClient, hubRequest } from "../hub-client/index.mjs";
3
- import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-C425F7ZG.mjs";
4
- import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-Bm34WBui.mjs";
4
+ import { t as EVIDENCE_DIR_ENV } from "../evidence-constants-Cm_S_5od.mjs";
5
+ import { a as formatAgentBrowserUnavailableMessage, i as assertAgentBrowserAvailable, n as spawnAB, o as pathWithAgentBrowserShim, r as AgentBrowserUnavailableError, s as resolveAgentBrowserBin$1, t as sleepSync } from "../spawn-ab-CR_Sr7wh.mjs";
5
6
  import { createRequire } from "node:module";
6
7
  import { Command } from "commander";
7
8
  import { accessSync, appendFileSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
@@ -13,8 +14,7 @@ import { basename, dirname, isAbsolute, join, normalize, posix, relative, resolv
13
14
  import { parse, stringify } from "yaml";
14
15
  import { ZodError, z } from "zod";
15
16
  import { execFile, spawn, spawnSync } from "node:child_process";
16
- import { createSdkMcpServer, query, tool } from "@anthropic-ai/claude-agent-sdk";
17
- import { AsyncLocalStorage } from "node:async_hooks";
17
+ import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
18
18
  import { promisify } from "node:util";
19
19
  import { createInterface } from "node:readline/promises";
20
20
  import { createServer } from "node:http";
@@ -62,354 +62,6 @@ function errMessage(err) {
62
62
  return err instanceof Error ? err.message : String(err);
63
63
  }
64
64
  //#endregion
65
- //#region src/runtime/env-vars.ts
66
- const ENV_VAR_RE = /\$\{([A-Z_][A-Z0-9_]*)\}|\$([A-Z_][A-Z0-9_]*)/g;
67
- const ANY_VAR_RE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)/g;
68
- /**
69
- * Replace every `$NAME` / `${NAME}` reference in `value` using `lookup`. When
70
- * `lookup` returns `undefined`, the original reference text is preserved
71
- * (callers that want empty-string substitution should wrap with `?? ""`).
72
- */
73
- function substituteVars(value, lookup) {
74
- ANY_VAR_RE.lastIndex = 0;
75
- return value.replace(ANY_VAR_RE, (match, braced, plain) => {
76
- const replacement = lookup(braced ?? plain ?? "");
77
- return replacement === void 0 ? match : replacement;
78
- });
79
- }
80
- /**
81
- * Iterate every `${NAME}` / `$NAME` reference name (case-insensitive form)
82
- * appearing in `value`. Used by callers that want to enumerate refs without
83
- * also substituting, e.g. the env-scrub map builder. The reference name
84
- * grammar is the canonical one shared with `substituteVars`.
85
- */
86
- function* iterEnvRefNames(value) {
87
- ANY_VAR_RE.lastIndex = 0;
88
- let m;
89
- while ((m = ANY_VAR_RE.exec(value)) !== null) {
90
- const name = m[1] ?? m[2];
91
- if (name) yield name;
92
- }
93
- }
94
- /**
95
- * Resolve every `$VAR` / `${VAR}` reference against `overrides`, then the
96
- * current process env. `overrides` carries values an invoker injects into a
97
- * child process (e.g. CCQA_RUN_ID), which beat the parent env there.
98
- *
99
- * Missing variables expand to the empty string, mirroring `sh` behaviour.
100
- * Throwing would force ccqa to be invoked with every var set even for
101
- * unused blocks, which is more user-hostile than letting the test fail
102
- * downstream with a clearer message ("login form rejected: empty password").
103
- */
104
- function resolveEnvRefs(value, overrides = {}) {
105
- return value.replace(ENV_VAR_RE, (_, braced, plain) => {
106
- const name = braced ?? plain ?? "";
107
- return overrides[name] ?? process.env[name] ?? "";
108
- });
109
- }
110
- /**
111
- * Embed `$VAR` / `${VAR}` as a JS template-literal expression that reads
112
- * `process.env.VAR ?? ""` at runtime. Used by `ccqa generate` so the test
113
- * script never bakes in the secret value.
114
- *
115
- * Returns a JavaScript string-literal expression (template literal when env
116
- * refs are present, plain string literal otherwise).
117
- *
118
- * Examples:
119
- * "${PASSWORD}" -> '`${process.env.PASSWORD ?? ""}`'
120
- * "user-${SUFFIX}@x.com" -> '`user-${process.env.SUFFIX ?? ""}@x.com`'
121
- * "literal value" -> '"literal value"'
122
- */
123
- function envRefsToJsExpression(value) {
124
- return refsToJsExpression(value, () => null);
125
- }
126
- /**
127
- * Generalised version of `envRefsToJsExpression`. Each `$NAME` / `${NAME}`
128
- * reference in `value` is passed to `nameToExpr(name)` first:
129
- *
130
- * - If it returns a string, that string is interpolated as a JS expression
131
- * (no quoting / no `?? ""` wrap — the caller decides the shape).
132
- * - If it returns `null`, the reference is treated as a missing env var
133
- * and expands to `process.env.<NAME> ?? ""` (the legacy behaviour).
134
- *
135
- * Used by the block codegen path: param names map to `params.<name>`,
136
- * everything else falls through to `process.env.X ?? ""`.
137
- */
138
- function refsToJsExpression(value, nameToExpr) {
139
- ANY_VAR_RE.lastIndex = 0;
140
- if (!ANY_VAR_RE.test(value)) return JSON.stringify(value);
141
- const escaped = value.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, (_match, offset, source) => {
142
- const probe = new RegExp(ANY_VAR_RE.source, "g");
143
- let m;
144
- while ((m = probe.exec(source)) !== null) if (m.index === offset) return "${";
145
- return "\\${";
146
- });
147
- ANY_VAR_RE.lastIndex = 0;
148
- return `\`${escaped.replace(ANY_VAR_RE, (_match, braced, plain) => {
149
- const name = braced ?? plain ?? "";
150
- const expr = nameToExpr(name);
151
- return expr !== null ? `\${${expr}}` : `\${process.env.${name} ?? ""}`;
152
- })}\``;
153
- }
154
- //#endregion
155
- //#region src/spec/yaml-schema.ts
156
- /**
157
- * An action step: one user-facing browser interaction. `instruction` and
158
- * `expected` are the natural-language description handed to Claude during
159
- * `ccqa trace`. URLs live inside `instruction`, either verbatim or via
160
- * `${ENV_VAR}` references (resolved at runtime).
161
- */
162
- const ActionStepSchema = z.object({
163
- instruction: z.string().min(1),
164
- expected: z.string().min(1)
165
- }).strict();
166
- /**
167
- * An include step: invokes a reusable block (`.ccqa/blocks/<name>/spec.yaml`).
168
- * `params` values are plain strings; env refs (`${VAR}`) inside them are
169
- * resolved at expand time the same way step instructions are.
170
- */
171
- const IncludeStepSchema = z.object({
172
- include: z.string().min(1),
173
- params: z.record(z.string(), z.string()).optional()
174
- }).strict();
175
- /**
176
- * A spec step is either an action step or an include step. The two are
177
- * discriminated by the presence of the `include` key — see `isIncludeStep`.
178
- */
179
- const StepSchema = z.union([ActionStepSchema, IncludeStepSchema]);
180
- /**
181
- * Execution mode for `ccqa run`:
182
- * - `deterministic` (default): vitest replays the recorded `test.spec.ts`.
183
- * - `live`: Claude drives agent-browser per step (for fragile UIs where
184
- * codegen is impractical). Cost ~$0.5 per spec.
185
- */
186
- const SpecModeSchema = z.enum(["deterministic", "live"]);
187
- /**
188
- * A name a spec chooses that ccqa resolves to a path or looks up in a
189
- * registry. Restricted to a slug so it cannot escape a directory.
190
- */
191
- function slug$1(what) {
192
- return z.string().min(1).regex(/^[a-z0-9][a-z0-9._-]*$/i, `${what} must be a slug (letters, digits, '.', '_', '-'; no path separators)`);
193
- }
194
- /**
195
- * A saved browser session (cookies + localStorage) to restore before the spec
196
- * runs, resolved to `.ccqa/sessions/<profile>/<name>.json` at run time.
197
- */
198
- const SessionNameSchema = slug$1("session name");
199
- /**
200
- * Sessions to restore before a `mode: live` spec runs: one name or a list,
201
- * always read back as a list. Multiple names are merged (their cookies +
202
- * localStorage are unioned) and restored together, so a spec can start
203
- * signed-in to several providers at once.
204
- */
205
- const SessionFieldSchema = z.union([SessionNameSchema, z.array(SessionNameSchema).min(1)]).transform((v) => Array.isArray(v) ? v : [v]);
206
- /**
207
- * A generation-target id: which plugin turns this spec into runnable tests
208
- * (e.g. "agent-browser", "playwright", "runn"). Whether the id names a
209
- * registered target is the registry's responsibility, so new targets don't
210
- * require a schema change.
211
- */
212
- const TargetIdSchema = slug$1("target");
213
- /** The built-in recorder-backed target. `mode:` / `session:` only apply to it. */
214
- const AGENT_BROWSER_TARGET = "agent-browser";
215
- /**
216
- * Top-level spec schema. `.strict()` rejects any unknown key.
217
- *
218
- * `mode:` and `session:` are agent-browser-only fields, enforced here when
219
- * `target:` names another target. When `target:` is omitted the effective
220
- * target comes from config (`defaultTarget`, falling back to agent-browser),
221
- * which this schema can't see — so mode/session pass parsing and the
222
- * post-resolution check is the target resolver's responsibility.
223
- */
224
- const TestSpecSchema = z.object({
225
- title: z.string().min(1),
226
- disabled: z.boolean().optional(),
227
- target: TargetIdSchema.optional(),
228
- mode: SpecModeSchema.optional(),
229
- session: SessionFieldSchema.optional(),
230
- steps: z.array(StepSchema).min(1)
231
- }).strict().superRefine((spec, ctx) => {
232
- if (spec.target === void 0 || spec.target === "agent-browser") return;
233
- for (const key of ["mode", "session"]) if (spec[key] !== void 0) ctx.addIssue({
234
- code: "custom",
235
- path: [key],
236
- message: `\`${key}\` only applies to the agent-browser target — remove it or drop \`target: ${spec.target}\``
237
- });
238
- });
239
- /** Default mode when `mode:` is absent. */
240
- const DEFAULT_SPEC_MODE = "deterministic";
241
- /**
242
- * A block param declaration. `required` defaults to true; only explicit
243
- * `required: false` makes it optional. `secret: true` flags the value as
244
- * sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
245
- * template literals so the secret never ends up baked into test.spec.ts.
246
- */
247
- const BlockParamSchema = z.object({
248
- name: z.string().min(1),
249
- required: z.boolean().optional(),
250
- secret: z.boolean().optional()
251
- }).strict();
252
- /**
253
- * Block schema. Block steps are restricted to ActionStep — nested blocks are
254
- * forbidden. Including a block from inside another block fails parsing here
255
- * (the store layer maps the cryptic "Unrecognized key: 'include'" error into
256
- * a targeted nested-block message).
257
- */
258
- const BlockSpecSchema = z.object({
259
- title: z.string().min(1),
260
- params: z.array(BlockParamSchema).optional(),
261
- steps: z.array(ActionStepSchema).min(1)
262
- }).strict();
263
- /** Runtime predicate for the StepSchema union. */
264
- function isIncludeStep(step) {
265
- return "include" in step;
266
- }
267
- /** Returns true if a block param is required (default: true). */
268
- function isParamRequired(param) {
269
- return param.required !== false;
270
- }
271
- //#endregion
272
- //#region src/spec/parser.ts
273
- /** The spec/block root (an `unrecognized_keys` issue there has an empty path). */
274
- const atRoot = (path) => path.length === 0;
275
- /** A block param entry — the issue path is `params.<index>`. */
276
- const atBlockParam = (path) => path.length === 2 && path[0] === "params" && typeof path[1] === "number";
277
- const UNREAD_PARAM_FIELD = "nothing reads it (a block param reaches the prompts as its name, required and secret only). Delete the line.";
278
- const REMOVED_FIELDS = {
279
- relatedPaths: {
280
- at: atRoot,
281
- message: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field."
282
- },
283
- dummy: {
284
- at: atBlockParam,
285
- message: UNREAD_PARAM_FIELD
286
- },
287
- description: {
288
- at: atBlockParam,
289
- message: UNREAD_PARAM_FIELD
290
- }
291
- };
292
- /** Parse a spec.yaml. Schema rejections are rewritten with actionable messages. */
293
- function parseTestSpec(content, source = "spec.yaml") {
294
- const raw = parseYamlOrThrow(content, source);
295
- try {
296
- return TestSpecSchema.parse(raw);
297
- } catch (e) {
298
- throw enrichZodError$1(e, source, false);
299
- }
300
- }
301
- /**
302
- * Throw-suppressed sibling of `parseTestSpec`. Used by report-side helpers
303
- * that derive cosmetic data (title, step descriptions) from spec.yaml and
304
- * want a missing or malformed file to degrade silently rather than abort
305
- * the report.
306
- */
307
- function tryParseTestSpec(yaml) {
308
- if (!yaml) return null;
309
- try {
310
- return parseTestSpec(yaml);
311
- } catch {
312
- return null;
313
- }
314
- }
315
- /**
316
- * Parse a block's spec.yaml. Block-specific errors include the targeted
317
- * nested-block message (the underlying zod failure on an `include` key
318
- * inside a block step is hard to read).
319
- */
320
- function parseBlockSpec(content, source = "block spec.yaml") {
321
- const raw = parseYamlOrThrow(content, source);
322
- try {
323
- return BlockSpecSchema.parse(raw);
324
- } catch (e) {
325
- throw enrichZodError$1(e, source, true);
326
- }
327
- }
328
- function parseYamlOrThrow(content, source) {
329
- try {
330
- return parse(content);
331
- } catch (e) {
332
- throw new Error(`Failed to parse YAML (${source}): ${e.message}`);
333
- }
334
- }
335
- function enrichZodError$1(error, source, isBlock) {
336
- if (!(error instanceof ZodError)) return error;
337
- const lines = [`Invalid ${source}:`];
338
- for (const issue of error.issues) {
339
- const path = issue.path.join(".") || "(root)";
340
- const message = humanizeIssue(issue, isBlock);
341
- lines.push(` - ${path}: ${message}`);
342
- }
343
- return new Error(lines.join("\n"));
344
- }
345
- function humanizeIssue(issue, isBlock) {
346
- if (issue.code === "unrecognized_keys") {
347
- const keys = Array.isArray(issue.keys) ? issue.keys : [];
348
- if (isBlock && keys.includes("include")) return `Nested blocks are not supported — flatten by inlining the included block's steps into this block.`;
349
- const removed = keys.filter((k) => REMOVED_FIELDS[k]?.at(issue.path));
350
- const stillUnknown = keys.filter((k) => !REMOVED_FIELDS[k]?.at(issue.path));
351
- const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k].message}`);
352
- if (stillUnknown.length > 0) parts.push(`Unknown keys: ${stillUnknown.join(", ")}`);
353
- return parts.join(" ");
354
- }
355
- return issue.message;
356
- }
357
- //#endregion
358
- //#region src/spec/expand.ts
359
- /**
360
- * Walk the spec's top-level steps, inlining any `- include: <block>` reference
361
- * as the block's own steps in order. The result is a flat `step-NN`-numbered
362
- * sequence — block boundaries survive only as the `source` tag, so trace and
363
- * codegen never need a separate block code path.
364
- */
365
- function expandSpec(spec, options) {
366
- const out = [];
367
- let counter = 0;
368
- const allocId = () => {
369
- counter += 1;
370
- return `step-${String(counter).padStart(2, "0")}`;
371
- };
372
- for (const step of spec.steps) if (isIncludeStep(step)) {
373
- const block = resolveBlock(step.include, step.params ?? {}, options.blocks);
374
- for (const blockStep of block.steps) out.push({
375
- id: allocId(),
376
- source: step.include,
377
- instruction: substituteVars(blockStep.instruction, block.lookup),
378
- expected: substituteVars(blockStep.expected, block.lookup)
379
- });
380
- } else out.push({
381
- id: allocId(),
382
- source: "spec",
383
- instruction: step.instruction,
384
- expected: step.expected
385
- });
386
- return out;
387
- }
388
- function resolveBlock(blockName, rawParams, blocks) {
389
- const block = blocks.get(blockName);
390
- if (!block) throw new Error(`Unknown block: "${blockName}". Define it under .ccqa/blocks/${blockName}/spec.yaml.`);
391
- const declaredParams = new Map((block.params ?? []).map((p) => [p.name, p]));
392
- for (const key of Object.keys(rawParams)) if (!declaredParams.has(key)) throw new Error(`Block "${blockName}" received unknown param "${key}". Declared params: ${[...declaredParams.keys()].join(", ") || "(none)"}.`);
393
- for (const [pname, def] of declaredParams) if (isParamRequired(def) && !(pname in rawParams)) throw new Error(`Block "${blockName}" is missing required param "${pname}".`);
394
- const lookup = (name) => {
395
- if (Object.prototype.hasOwnProperty.call(rawParams, name)) return rawParams[name];
396
- };
397
- return {
398
- steps: block.steps,
399
- lookup
400
- };
401
- }
402
- /**
403
- * Collect every block name referenced by a spec (top-level only — blocks
404
- * cannot nest). Used by the store / drift layers to know which blocks to
405
- * load or invalidate.
406
- */
407
- function collectIncludedBlockNames(spec) {
408
- const names = /* @__PURE__ */ new Set();
409
- for (const step of spec.steps) if (isIncludeStep(step)) names.add(step.include);
410
- return [...names];
411
- }
412
- //#endregion
413
65
  //#region src/store/index.ts
414
66
  const CCQA_DIR = ".ccqa";
415
67
  const SPEC_FILE = "spec.yaml";
@@ -748,10 +400,10 @@ function bundledVitestConfigPath() {
748
400
  }
749
401
  //#endregion
750
402
  //#region src/runtime/spawn-vitest.ts
751
- const require$2 = createRequire(import.meta.url);
403
+ const require$1 = createRequire(import.meta.url);
752
404
  function resolveVitestBin() {
753
- const pkgPath = require$2.resolve("vitest/package.json");
754
- const pkg = require$2(pkgPath);
405
+ const pkgPath = require$1.resolve("vitest/package.json");
406
+ const pkg = require$1(pkgPath);
755
407
  const binRel = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.vitest;
756
408
  if (!binRel) throw new Error(`vitest package.json has no bin entry (resolved at ${pkgPath})`);
757
409
  return resolve(dirname(pkgPath), binRel);
@@ -806,1909 +458,142 @@ function waitExit(child) {
806
458
  });
807
459
  }
808
460
  //#endregion
809
- //#region src/runtime/live-artifacts.ts
810
- /**
811
- * Build a sortable, unique run id. ISO8601 with `:` / `.` replaced so it's
812
- * filename-safe, timestamp first so run directories still sort by time, and a
813
- * random suffix because the timestamp alone does not separate two specs.
814
- *
815
- * The pool launches specs back-to-back, so at `--concurrency > 1` two of them
816
- * land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
817
- * of something it creates would then share that name with its neighbour, and
818
- * each would find — and delete — the other's row. Nothing fails; the
819
- * assertions just read the wrong state.
820
- *
821
- * Caller is expected to mkdir the directory once and pass
822
- * `runDir = <baseDir>/<runId>` to the path helpers below.
823
- */
824
- function buildRunId() {
825
- return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
826
- }
827
- /**
828
- * Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
829
- * Three files per step:
830
- * - <stepId>.before.png : screenshot taken BEFORE Claude executes the step.
831
- * - <stepId>.after.png : screenshot taken AFTER Claude executes the step.
832
- * - <stepId>.log.txt : full assistant transcript for the step (judgement
833
- * reasoning, any STEP_RESULT lines, raw tool output
834
- * summaries the model chose to keep).
835
- */
836
- function stepArtifactPaths(runDir, stepId) {
837
- const dir = join(runDir, "steps");
838
- return {
839
- beforePng: join(dir, `${stepId}.before.png`),
840
- afterPng: join(dir, `${stepId}.after.png`),
841
- logTxt: join(dir, `${stepId}.log.txt`)
842
- };
843
- }
844
- //#endregion
845
- //#region src/runtime/pool.ts
846
- async function runPool(items, concurrency, fn, opts = {}) {
847
- const results = new Array(items.length);
848
- const needs = items.map((item) => opts.resources?.(item) ?? []);
849
- const busy = /* @__PURE__ */ new Set();
850
- const queued = new Set(items.map((_, i) => i));
851
- const inFlight = /* @__PURE__ */ new Map();
852
- const limit = Math.max(1, Math.min(concurrency, items.length));
853
- const failures = [];
854
- const start = (idx) => {
855
- queued.delete(idx);
856
- for (const name of needs[idx]) busy.add(name);
857
- inFlight.set(idx, Promise.resolve().then(async () => {
858
- try {
859
- results[idx] = await fn(items[idx], idx);
860
- } catch (err) {
861
- failures.push(err);
862
- } finally {
863
- for (const name of needs[idx]) busy.delete(name);
864
- inFlight.delete(idx);
865
- }
866
- }));
867
- };
868
- while (queued.size > 0 || inFlight.size > 0) {
869
- if (failures.length === 0) {
870
- for (const idx of queued) {
871
- if (inFlight.size >= limit) break;
872
- if (needs[idx].some((name) => busy.has(name))) continue;
873
- start(idx);
874
- }
875
- if (inFlight.size === 0) throw new Error(`runPool: ${queued.size} item(s) unrunnable with nothing in flight`);
876
- }
877
- if (inFlight.size === 0) break;
878
- await Promise.race(inFlight.values());
879
- }
880
- if (failures.length === 1) throw failures[0];
881
- if (failures.length > 1) throw new AggregateError(failures, `${failures.length} items failed`);
882
- return results;
883
- }
884
- //#endregion
885
- //#region src/claude/env-keys.ts
886
- /**
887
- * Variables that carry a credential the Claude Code process can use on its
888
- * own, with no login on the host: an API key, a gateway bearer token, or a
889
- * subscription token from `claude setup-token`.
890
- */
891
- const CREDENTIAL_ENV_KEYS = [
892
- "ANTHROPIC_API_KEY",
893
- "ANTHROPIC_AUTH_TOKEN",
894
- "CLAUDE_CODE_OAUTH_TOKEN"
895
- ];
896
- /**
897
- * Standard Claude Code environment variables that select the API endpoint and
898
- * credentials. ccqa forwards whichever of these are set to the underlying
899
- * Claude Code process; it does not read or interpret their values.
900
- *
901
- * - `ANTHROPIC_BASE_URL` — the API endpoint to send requests to.
902
- * - `ANTHROPIC_AUTH_TOKEN` — sent as `Authorization: Bearer <token>`.
903
- * - `ANTHROPIC_API_KEY` — API key, when used instead of a token.
904
- * - `ANTHROPIC_CUSTOM_HEADERS` — extra request headers.
905
- * - `CLAUDE_CODE_OAUTH_TOKEN` — long-lived subscription token from
906
- * `claude setup-token`, the headless-CI counterpart of a login.
907
- */
908
- const ENDPOINT_ENV_KEYS = [
909
- "ANTHROPIC_BASE_URL",
910
- "ANTHROPIC_CUSTOM_HEADERS",
911
- ...CREDENTIAL_ENV_KEYS
912
- ];
913
- //#endregion
914
- //#region src/drift/auth.ts
915
- /**
916
- * Claude Code can also run against AWS Bedrock / Google Vertex AI, selected by
917
- * these env toggles. Credentials then come from the cloud SDK's own chain
918
- * (instance/task roles, gcloud auth, …), so none of the Anthropic-side probes
919
- * below apply — a set toggle counts as auth being available. ccqa forwards the
920
- * toggle verbatim; only "0"/"false" (any case) is treated as explicitly off.
921
- */
922
- const CLOUD_PROVIDER_ENV_KEYS = ["CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX"];
923
- function cloudProviderEnabled() {
924
- return CLOUD_PROVIDER_ENV_KEYS.some((key) => {
925
- const value = process.env[key]?.trim().toLowerCase();
926
- return value !== void 0 && value !== "" && value !== "0" && value !== "false";
927
- });
928
- }
929
- /**
930
- * Probe whether the host has any credential the Anthropic SDK can pick up:
931
- * - one of CREDENTIAL_ENV_KEYS (API key, gateway bearer token, or the
932
- * subscription token from `claude setup-token`)
933
- * - CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
934
- * endpoints authenticated by the cloud SDK's credential chain)
935
- * - ~/.claude/.credentials.json (Claude Code login, file-based platforms)
936
- * - macOS Keychain item "Claude Code-credentials" (Claude Code login on
937
- * darwin stores the OAuth credentials in the Keychain, not on disk)
938
- *
939
- * Claude-driven hooks are opt-in, so the caller only consults this after the
940
- * user has asked for analysis. We never throw — auth absence is a normal flow
941
- * that surfaces as "analysis skipped".
942
- */
943
- function driftAuthAvailable() {
944
- for (const key of CREDENTIAL_ENV_KEYS) {
945
- const value = process.env[key];
946
- if (typeof value === "string" && value.length > 0) return { ok: true };
947
- }
948
- if (cloudProviderEnabled()) return { ok: true };
949
- if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
950
- if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
951
- return {
952
- ok: false,
953
- reason: `no ${CREDENTIAL_ENV_KEYS.join(" / ")} / Bedrock or Vertex env / claude login`
954
- };
955
- }
956
- /**
957
- * `security find-generic-password` without `-w` only checks the item's
958
- * existence (exit 0) — it never reads the secret, so no Keychain unlock
959
- * prompt is triggered. Resolved via PATH so tests can stub the binary.
960
- */
961
- function keychainHasClaudeCredentials() {
962
- try {
963
- return spawnSync("security", [
964
- "find-generic-password",
965
- "-s",
966
- "Claude Code-credentials"
967
- ], {
968
- stdio: "ignore",
969
- timeout: 3e3
970
- }).status === 0;
971
- } catch {
972
- return false;
973
- }
974
- }
975
- //#endregion
976
- //#region src/cli/logger.ts
977
- const STEP_ICONS = {
978
- STEP_START: "▶",
979
- STEP_DONE: "✓",
980
- ASSERTION_FAILED: "✗",
981
- STEP_SKIPPED: "⊘",
982
- RUN_COMPLETED: "■"
983
- };
984
- /**
985
- * When a `withBuffer` scope is active, every log line (stdout and stderr) is
986
- * appended to its buffer instead of being written immediately. Parallel spec
987
- * runs use this so each spec's narration — including logs emitted deep inside
988
- * the live executor — flushes as one contiguous block, not interleaved.
989
- */
990
- const bufferStore = new AsyncLocalStorage();
991
- const sinkStore = new AsyncLocalStorage();
992
- /** True while inside a `withBuffer` scope: progress lines avoid TTY cursor tricks. */
993
- function isBuffered() {
994
- return bufferStore.getStore() !== void 0;
995
- }
996
- function emit(text, sink = process.stdout) {
997
- const store = bufferStore.getStore();
998
- if (store) {
999
- store.out.push(text);
1000
- return;
1001
- }
1002
- const activeSink = sinkStore.getStore();
1003
- if (activeSink) {
1004
- activeSink.write(text);
1005
- return;
1006
- }
1007
- sink.write(text);
1008
- }
1009
- /**
1010
- * Write raw text to the active `withBuffer` scope, or straight to stdout when
1011
- * none is active. Lets a runner redirect sub-process output (e.g. a child's
1012
- * stdout) into the same buffer as its `log.*` lines so they flush together.
1013
- */
1014
- function emitRaw(text) {
1015
- emit(text);
1016
- }
1017
- /**
1018
- * Run `fn` with all its log output captured into a buffer, then flush the
1019
- * buffer in one shot under `label`. Used by parallel runners to keep each
1020
- * spec's output legible. Output is flushed even when `fn` throws.
1021
- *
1022
- * When `buffered` is false, `fn` runs with no buffer so its output streams
1023
- * live — this is the sequential (concurrency 1) path, unchanged from before.
1024
- */
1025
- async function withBuffer(label, buffered, fn) {
1026
- if (!buffered) return fn();
1027
- const store = { out: [] };
1028
- try {
1029
- return await bufferStore.run(store, fn);
1030
- } finally {
1031
- emit(`\n──── ${label} ────\n${store.out.join("")}`);
1032
- }
1033
- }
1034
- function header(command, target) {
1035
- emit(`\nccqa ${command}${target ? ` ${target}` : ""}\n\n`);
1036
- }
1037
- function write(scope, message, sink = process.stdout) {
1038
- emit(`[${scope}] ${message}\n`, sink);
1039
- }
1040
- function meta(key, value) {
1041
- write("meta", `${key}: ${value}`);
1042
- }
1043
- function blank() {
1044
- emit("\n");
1045
- }
1046
- function info(message) {
1047
- write("info", message);
1048
- }
1049
- function step(type, stepId, detail) {
1050
- emit(` ${STEP_ICONS[type]} [${stepId}] ${detail}\n`);
1051
- }
1052
- function bash(command) {
1053
- emit(` $ ${command.slice(0, 120)}\n`);
1054
- }
1055
- function error(message) {
1056
- write("error", message, process.stderr);
1057
- }
1058
- function warn(message) {
1059
- write("warn", message, process.stderr);
1060
- }
1061
- function hint(message) {
1062
- emit("\n");
1063
- write("hint", message);
1064
- }
1065
- function fix(message) {
1066
- write("fix", message);
1067
- }
1068
- function run(message) {
1069
- write("run", message);
1070
- }
1071
- /**
1072
- * Render a single-line progress indicator for a step-by-step loop.
1073
- *
1074
- * On a TTY the line is rewritten in place via `\r` so the terminal stays
1075
- * uncluttered. In a non-TTY environment (CI, piped runs) we fall back to
1076
- * a regular `[info]` line every PROGRESS_NONTTY_STRIDE steps to avoid
1077
- * spamming the log with one line per action.
1078
- *
1079
- * Callers MUST call `progressEnd()` when the loop finishes (or aborts) so
1080
- * the carriage-return line gets a final newline; otherwise the next log
1081
- * line lands on the same physical row.
1082
- */
1083
- const PROGRESS_NONTTY_STRIDE = 5;
1084
- let lastProgressNonTtyEmit = -1;
1085
- function progress(current, total, label) {
1086
- const text = `[info] ${current + 1}/${total} ${label}`;
1087
- if (process.stdout.isTTY && !isBuffered()) {
1088
- process.stdout.write(`\r${text}\x1b[K`);
1089
- return;
1090
- }
1091
- if (current === 0 || current - lastProgressNonTtyEmit >= PROGRESS_NONTTY_STRIDE) {
1092
- emit(`${text}\n`);
1093
- lastProgressNonTtyEmit = current;
1094
- }
1095
- }
1096
- function progressEnd() {
1097
- if (process.stdout.isTTY && !isBuffered()) process.stdout.write(`\r\x1b[K`);
1098
- lastProgressNonTtyEmit = -1;
1099
- }
1100
- /**
1101
- * Time a long-running step under the given scope, emitting `started` and
1102
- * `finished in N.Ns` markers. Scope must be a tag the user wants to grep
1103
- * for — typically "run" for vitest and "fix" for diagnose-loop steps.
1104
- */
1105
- async function timedPhase(label, fn, scope = "fix") {
1106
- const startedAt = Date.now();
1107
- write(scope, `${label} started`);
1108
- try {
1109
- const result = await fn();
1110
- write(scope, `${label} finished in ${((Date.now() - startedAt) / 1e3).toFixed(1)}s`);
1111
- return result;
1112
- } catch (err) {
1113
- write(scope, `${label} threw after ${((Date.now() - startedAt) / 1e3).toFixed(1)}s`);
1114
- throw err;
1115
- }
1116
- }
1117
- //#endregion
1118
- //#region src/ir/from-agent-browser.ts
1119
- /**
1120
- * Normalization from the agent-browser side of the recorder into the IR.
1121
- * The trace protocol emits one pipe-delimited `AB_ACTION|...` line per
1122
- * browser action (see `src/prompts/trace.ts` and
1123
- * `claude/invoke.ts:extractAbActionFromBashCommand`); `parseAbActionLine`
1124
- * turns each line into a `RecordedAction`.
1125
- *
1126
- * The mapping is a deterministic re-encoding: `to-agent-browser.ts` is its
1127
- * inverse, and the round-trip identity (ab argv → wire → IR → ab argv) is
1128
- * pinned by `roundtrip.test.ts`.
1129
- */
1130
- /**
1131
- * Semantic locator strategies exposed by `agent-browser find`. Used by the
1132
- * `find_*` wire commands when a target cannot be uniquely picked out by the
1133
- * ALLOWED CSS forms (e.g. repeated `aria-label='1 reply'` rows where only
1134
- * "the last one" is meaningful).
1135
- *
1136
- * `first` / `last` / `nth` are positional helpers whose value carries an
1137
- * inner CSS selector (`nth` additionally needs an index); they normalize to
1138
- * a `css` Locator plus `index`. The remaining strategies read the value as
1139
- * the human-visible text/id and normalize to the matching `Locator.by`.
1140
- */
1141
- const FIND_LOCATORS = [
1142
- "role",
1143
- "text",
1144
- "label",
1145
- "placeholder",
1146
- "alt",
1147
- "title",
1148
- "testid",
1149
- "first",
1150
- "last",
1151
- "nth"
1152
- ];
1153
- /**
1154
- * Actions reachable via `agent-browser find <locator> ... <action>`. Kept
1155
- * here next to the locator list so all `find` wire knowledge lives in one
1156
- * place — `claude/invoke.ts` imports these instead of redefining its own sets.
1157
- */
1158
- const FIND_ACTIONS = [
1159
- "click",
1160
- "dblclick",
1161
- "fill",
1162
- "type",
1163
- "hover",
1164
- "focus",
1165
- "check",
1166
- "uncheck"
1167
- ];
1168
- const css = (value) => ({
1169
- by: "css",
1170
- value
1171
- });
1172
- function parseAbActionLine(line) {
1173
- if (!line.startsWith("AB_ACTION|")) return null;
1174
- const parts = line.split("|");
1175
- const command = parts[1];
1176
- switch (command) {
1177
- case "cookies_clear": return { action: "cookies_clear" };
1178
- case "open": return {
1179
- action: "navigate",
1180
- value: (parts[2] ?? "").replace(/^["']|["']$/g, "")
1181
- };
1182
- case "press": return {
1183
- action: "press",
1184
- ...opt("value", parts[2])
1185
- };
1186
- case "scroll": return {
1187
- action: "scroll",
1188
- ...opt("direction", parts[2]),
1189
- ...opt("pixels", parts[3])
1190
- };
1191
- case "snapshot": return {
1192
- action: "snapshot",
1193
- ...opt("observation", parts[2])
1194
- };
1195
- case "assert": return {
1196
- action: "assert",
1197
- assert: parts[2],
1198
- ...parts[3] ? { locator: css(parts[3]) } : {},
1199
- ...parts[4] ? { value: parts[4] } : {},
1200
- ...parts[5] ? { observation: parts[5] } : {}
1201
- };
1202
- case "click":
1203
- case "dblclick":
1204
- case "check":
1205
- case "uncheck":
1206
- case "hover":
1207
- if (!parts[2]) return null;
1208
- return {
1209
- action: command,
1210
- locator: css(parts[2]),
1211
- ...opt("label", parts[3])
1212
- };
1213
- case "wait":
1214
- if (parts[2] === "--text") {
1215
- if (!parts[3]) return null;
1216
- return {
1217
- action: "wait",
1218
- locator: {
1219
- by: "text",
1220
- value: parts[3]
1221
- },
1222
- ...opt("label", parts[4])
1223
- };
1224
- }
1225
- if (!parts[2]) return null;
1226
- return {
1227
- action: "wait",
1228
- locator: css(parts[2]),
1229
- ...opt("label", parts[3])
1230
- };
1231
- case "fill":
1232
- case "type":
1233
- case "select":
1234
- if (!parts[2]) return null;
1235
- return {
1236
- action: command,
1237
- locator: css(parts[2]),
1238
- ...parts[3] !== void 0 ? { value: parts[3] } : {},
1239
- ...opt("label", parts[4])
1240
- };
1241
- case "drag":
1242
- if (!parts[2] || !parts[3]) return null;
1243
- return {
1244
- action: "drag",
1245
- locator: css(parts[2]),
1246
- target: css(parts[3]),
1247
- ...opt("label", parts[4])
1248
- };
1249
- case "upload": {
1250
- const selector = parts[2];
1251
- const files = parts.slice(3).filter((f) => f !== "");
1252
- if (!selector || files.length === 0) return null;
1253
- return {
1254
- action: "upload",
1255
- locator: css(selector),
1256
- files
1257
- };
1258
- }
1259
- case "find_click":
1260
- case "find_dblclick":
1261
- case "find_hover":
1262
- case "find_focus":
1263
- case "find_check":
1264
- case "find_uncheck": return parseFindAction(command.slice(5), parts, false);
1265
- case "find_fill":
1266
- case "find_type": return parseFindAction(command.slice(5), parts, true);
1267
- case "get_count":
1268
- case "get_url": return null;
1269
- default: return null;
1270
- }
1271
- }
1272
- /**
1273
- * Promote a `CCQA_ASSERT=<marker>` env marker on an agent-browser command
1274
- * into recorded assert action(s). The marker travels on the same channel as
1275
- * the command itself (see `claude/invoke.ts`), so a verification the model
1276
- * performs anyway (`wait --text`, `get count`) becomes a recorded assert
1277
- * without relying on the `AB_ACTION|assert|...` text protocol.
1278
- *
1279
- * `abAction` is the wire line for the marked command (null when the command
1280
- * has no wire form at all). Mapping — anything else returns null and the
1281
- * caller warns and records the command unpromoted:
1282
- *
1283
- * - `wait --text "X"` + `1` (or `text_visible`) → `assert text_visible X`,
1284
- * REPLACING the wait: the emitted abAssert is itself a timed wait, so
1285
- * keeping both would wait twice.
1286
- * - `get count "<sel>"` + `element_visible` / `element_not_visible`
1287
- * → `assert <marker> <sel>` (the probe records nothing by itself).
1288
- * - any command + `url_contains:<substring>` → the command's own action (if
1289
- * it records one) followed by `assert url_contains <substring>`.
1290
- */
1291
- function promoteMarkedAssert(abAction, marker) {
1292
- if (marker.startsWith("url_contains:")) {
1293
- const substring = marker.slice(13);
1294
- if (!substring) return null;
1295
- const assert = {
1296
- action: "assert",
1297
- assert: "url_contains",
1298
- value: substring
1299
- };
1300
- const base = abAction === null ? null : parseAbActionLine(abAction);
1301
- return base === null ? [assert] : [base, assert];
1302
- }
1303
- const parts = abAction === null ? [] : abAction.split("|");
1304
- if (marker === "1" || marker === "text_visible") {
1305
- if (parts[1] === "wait" && parts[2] === "--text" && parts[3]) return [{
1306
- action: "assert",
1307
- assert: "text_visible",
1308
- value: parts[3]
1309
- }];
1310
- return null;
1311
- }
1312
- if (marker === "element_visible" || marker === "element_not_visible") {
1313
- if (parts[1] === "get_count" && parts[2]) return [{
1314
- action: "assert",
1315
- assert: marker,
1316
- locator: css(parts[2])
1317
- }];
1318
- return null;
1319
- }
1320
- return null;
1321
- }
1322
- /**
1323
- * Common parser for the `find_*` wire family. `<extra>` carries `--name` for
1324
- * `role`, the integer index for `nth`, and is empty otherwise. We accept a
1325
- * literally empty `<extra>` (the LLM emits a placeholder `|` so the
1326
- * positional layout stays stable across locators).
1327
- */
1328
- function parseFindAction(action, parts, hasFillValue) {
1329
- const locatorToken = parts[2];
1330
- const findValue = parts[3];
1331
- const extra = parts[4] ?? "";
1332
- const exact = (parts[5] ?? "") === "exact";
1333
- if (!locatorToken || !FIND_LOCATORS.includes(locatorToken) || !findValue) return null;
1334
- let locator;
1335
- let index;
1336
- if (locatorToken === "first" || locatorToken === "last") {
1337
- locator = css(findValue);
1338
- index = locatorToken;
1339
- } else if (locatorToken === "nth") {
1340
- const parsed = extra ? Number.parseInt(extra, 10) : NaN;
1341
- if (Number.isNaN(parsed)) return null;
1342
- locator = css(findValue);
1343
- index = parsed;
1344
- } else if (locatorToken === "role") locator = {
1345
- by: "role",
1346
- value: findValue,
1347
- ...extra ? { name: extra } : {},
1348
- ...exact ? { exact: true } : {}
1349
- };
1350
- else locator = {
1351
- by: locatorToken,
1352
- value: findValue,
1353
- ...exact ? { exact: true } : {}
1354
- };
1355
- return {
1356
- action,
1357
- locator,
1358
- ...index !== void 0 ? { index } : {},
1359
- ...hasFillValue ? {
1360
- ...parts[6] !== void 0 ? { value: parts[6] } : {},
1361
- ...opt("label", parts[7])
1362
- } : opt("label", parts[6])
1363
- };
1364
- }
1365
- /** Include an optional string field only when it is non-empty. */
1366
- function opt(key, value) {
1367
- return value ? { [key]: value } : {};
1368
- }
1369
- //#endregion
1370
- //#region src/runtime/env-scrub.ts
1371
- /**
1372
- * Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
1373
- * mentioned in the spec OR in any of its expanded (block-inlined) steps.
1374
- * Used at trace time to scrub recorded Claude-text outputs so a value the
1375
- * spec author intentionally threaded through `process.env` is preserved as
1376
- * `${VAR}` in `ir.json` rather than baked in as the concrete
1377
- * trace-time value.
1378
- *
1379
- * Why we walk `spec.steps` AND `expanded`:
1380
- * - `spec.steps` carries the spec's own `instruction` / `expected` + each
1381
- * include's raw `params` (which may themselves be `${ENV}` refs).
1382
- * - `expanded` carries the inlined block-internal steps, whose
1383
- * `instruction` / `expected` may *also* contain `${ENV}` refs that
1384
- * don't go through include params.
1385
- *
1386
- * Each ref resolves against `overrides` first, then `process.env` —
1387
- * `overrides` carries values the invoker injects into the child process,
1388
- * which beat the parent env there. Only refs that resolve non-empty land in
1389
- * the map — scrubbing against an empty string would corrupt unrelated empty
1390
- * strings in the action stream; the rest are returned via `unresolved` so
1391
- * the caller can warn the user.
1392
- *
1393
- * Longer values sort first so a `${SHORT}` whose value is a substring of a
1394
- * `${LONG}` value doesn't clobber the longer one.
1395
- *
1396
- * `title` is deliberately NOT scanned — it never reaches the recorded action
1397
- * stream.
1398
- */
1399
- function buildSpecEnvScrub(spec, expanded, overrides = {}) {
1400
- const refNames = /* @__PURE__ */ new Set();
1401
- for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
1402
- else {
1403
- collect(step.instruction, refNames);
1404
- collect(step.expected, refNames);
1405
- }
1406
- for (const step of expanded) {
1407
- collect(step.instruction, refNames);
1408
- collect(step.expected, refNames);
1409
- }
1410
- const map = [];
1411
- const unresolved = [];
1412
- for (const name of refNames) {
1413
- const value = overrides[name] ?? process.env[name];
1414
- if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
1415
- else unresolved.push(name);
1416
- }
1417
- map.sort((a, b) => b[0].length - a[0].length);
1418
- return {
1419
- map,
1420
- unresolved
1421
- };
1422
- }
1423
- function collect(value, into) {
1424
- for (const name of iterEnvRefNames(value)) into.add(name);
1425
- }
1426
- /** Shorter than this, a value is no secret and matches inside ordinary words. */
1427
- const MIN_PROSE_SCRUB_LENGTH = 4;
1428
- /** Long enough to clear the length bar, still ordinary prose / JSON. */
1429
- const COMMON_PROSE_VALUES = new Set([
1430
- "true",
1431
- "false",
1432
- "null",
1433
- "none",
1434
- "undefined"
1435
- ]);
1436
- /**
1437
- * Scrub map for model output, built like {@link buildSpecEnvScrub} but
1438
- * without the values that read as ordinary text (`"1"`, `"true"`): prose
1439
- * runs to paragraphs, where replacing every occurrence of such a value
1440
- * costs more meaning than it protects. Record's own scrub keeps them for
1441
- * its single command lines; the live path reuses this one map for its Bash
1442
- * command log too, trading that short-value coverage for not building a
1443
- * second map.
1444
- */
1445
- function buildProseEnvScrubMap(spec, expanded, overrides = {}) {
1446
- return buildSpecEnvScrub(spec, expanded, overrides).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
1447
- }
1448
- /**
1449
- * Replace every occurrence of an env value with its `${VAR}` placeholder in
1450
- * `text`. **Caller invariant**: the map must be sorted longest-value-first
1451
- * so a shorter value doesn't shadow a longer one that contains it as a
1452
- * substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
1453
- */
1454
- function scrubEnvValues(text, scrubMap) {
1455
- if (scrubMap.length === 0) return text;
1456
- let out = text;
1457
- for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
1458
- return out;
1459
- }
1460
- //#endregion
1461
- //#region src/claude/native-binary.ts
1462
- const require$1 = createRequire(import.meta.url);
1463
- /**
1464
- * The agent SDK launches Claude through a native `claude` binary that ships in
1465
- * a per-platform package (`@anthropic-ai/claude-agent-sdk-<platform>-<cpu>`),
1466
- * declared as an *optional* dependency of the SDK. Optional means a consumer's
1467
- * lockfile can omit it without any install-time error — and then every Claude
1468
- * call fails at runtime with a message that never reaches our logs. Resolving
1469
- * the package up front lets us say so once, in a line that names the fix.
1470
- *
1471
- * ccqa's own package.json repeats these packages in `optionalDependencies` for
1472
- * the same reason: a second declaration gives the resolver another chance to
1473
- * record them. Keep that list's version range in step with the SDK's.
1474
- */
1475
- function nativeBinaryPackage(platform = process.platform, arch = process.arch, musl = isMusl(platform)) {
1476
- return `@anthropic-ai/claude-agent-sdk-${platform}-${arch === "arm64" ? "arm64" : "x64"}${platform === "linux" && musl ? "-musl" : ""}`;
1477
- }
1478
- /**
1479
- * musl builds (Alpine and friends) need their own binary. Node doesn't expose
1480
- * the libc flavour directly; the absence of `glibcVersionRuntime` in the
1481
- * process report is the usual proxy.
1482
- */
1483
- function isMusl(platform) {
1484
- if (platform !== "linux") return false;
1485
- return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
1486
- }
1487
- /**
1488
- * Name of the platform package this host needs, or `null` when it resolves.
1489
- * The per-platform packages have no `exports`, so the manifest is reachable.
1490
- */
1491
- function missingNativeBinaryPackage(resolve = require$1.resolve) {
1492
- const pkg = nativeBinaryPackage();
1493
- try {
1494
- resolve(`${pkg}/package.json`);
1495
- return null;
1496
- } catch {
1497
- return pkg;
1498
- }
1499
- }
1500
- /** Advice shown when the binary is absent — the package name plus how to fix it. */
1501
- function missingNativeBinaryMessage(pkg) {
1502
- return `${pkg} is not installed. The Claude Agent SDK needs it to start Claude on this platform, so every Claude-backed command (run in live mode, drift, diagnose) will fail. It ships as an optional dependency of the SDK, which a lockfile can drop silently: reinstall without omitting optional dependencies, or add it to your project as a direct dependency pinned to the same version as @anthropic-ai/claude-agent-sdk.`;
1503
- }
1504
- //#endregion
1505
- //#region src/claude/cost-tally.ts
1506
- /**
1507
- * Sum every Claude invocation made inside a scope.
1508
- *
1509
- * A command like `record` calls Claude several times — the browser trace, the
1510
- * codegen cleanup, one diagnosis per auto-fix retry — and the caller wants one
1511
- * number for the whole command. Threading a cost out of each of those return
1512
- * types would touch every layer in between, so the tally is scoped instead:
1513
- * `invokeClaudeStreaming` adds to whichever scope is active, and nothing
1514
- * between the two has to know.
1515
- *
1516
- * Scoped rather than module-global because commands run specs concurrently
1517
- * (`drift` uses a pool). Two scopes must not fold into each other.
1518
- */
1519
- const tallyStore = new AsyncLocalStorage();
1520
- /** Record one invocation against the active scope. No-op outside one. */
1521
- function tallyInvocation(cost) {
1522
- tallyStore.getStore()?.push(cost);
1523
- }
1524
- /** Run `fn` with a fresh tally. Read the total from inside with `readCostTally`. */
1525
- async function withCostTally(fn) {
1526
- return tallyStore.run([], fn);
1527
- }
1528
- /**
1529
- * The active scope's total so far, or null outside one.
1530
- *
1531
- * Read rather than pushed at the caller because commands end in
1532
- * `process.exit`, which never reaches a `finally`; whoever opened the scope
1533
- * reads the total on the way out (see `withCostReporting`).
1534
- *
1535
- * Fields stay `null` when no invocation reported them, so a caller can tell
1536
- * "nothing was billed" from "the SDK didn't say" (mock runs, SDK errors).
1537
- */
1538
- function readCostTally() {
1539
- const collected = tallyStore.getStore();
1540
- return collected === void 0 ? null : sum(collected);
1541
- }
1542
- function sum(costs) {
1543
- const add = (pick) => {
1544
- const present = costs.map(pick).filter((v) => v !== null);
1545
- return present.length === 0 ? null : present.reduce((a, b) => a + b, 0);
1546
- };
1547
- return {
1548
- totalCostUsd: add((c) => c.totalCostUsd),
1549
- durationMs: add((c) => c.durationMs),
1550
- durationApiMs: add((c) => c.durationApiMs),
1551
- numTurns: add((c) => c.numTurns),
1552
- inputTokens: add((c) => c.inputTokens),
1553
- cacheCreationInputTokens: add((c) => c.cacheCreationInputTokens),
1554
- cacheReadInputTokens: add((c) => c.cacheReadInputTokens),
1555
- outputTokens: add((c) => c.outputTokens),
1556
- models: [...new Set(costs.flatMap((c) => c.models))]
1557
- };
1558
- }
1559
- //#endregion
1560
- //#region src/claude/invoke.ts
1561
- /** The built-in tools an allow-list names: `Bash(*)` is `Bash`; `mcp__*` are not built-ins. */
1562
- function builtinToolNames(allowedTools) {
1563
- const names = allowedTools.map((entry) => entry.replace(/\(.*\)$/, "")).filter((name) => !name.startsWith("mcp__"));
1564
- return [...new Set(names)];
1565
- }
1566
- function resolveModel(explicit) {
1567
- if (explicit) return explicit;
1568
- const envModel = process.env["CCQA_MODEL"];
1569
- return envModel && envModel.length > 0 ? envModel : void 0;
1570
- }
1571
- /**
1572
- * When both credentials are present the OAuth token wins and the API key is
1573
- * dropped. Left to the CLI the API key would win, which makes "switch a CI
1574
- * job to the subscription token" require unwiring the key everywhere; with
1575
- * this rule, adding the one variable is the whole switch, and removing it is
1576
- * the whole rollback. The one place the rule lives — both the resolved view
1577
- * and the env the SDK receives apply it through here.
1578
- */
1579
- function preferOauthToken(env) {
1580
- if (env["CLAUDE_CODE_OAUTH_TOKEN"]) delete env["ANTHROPIC_API_KEY"];
1581
- }
1582
- /**
1583
- * Drop endpoint variables that are present but empty, so an empty value never
1584
- * reaches the Claude Code process as an override. "Set to nothing" is how a
1585
- * caller that cannot omit the key says "use the default" — a CI job wiring
1586
- * `ANTHROPIC_BASE_URL` from an unset repository variable, most of all.
1587
- */
1588
- function withoutEmptyEndpointVars(env) {
1589
- const out = { ...env };
1590
- for (const key of ENDPOINT_ENV_KEYS) if (out[key] === "") delete out[key];
1591
- return out;
1592
- }
1593
- /**
1594
- * The environment actually handed to the Claude Code process: the full process
1595
- * environment with the caller's overrides on top, empty endpoint variables
1596
- * dropped, and — when both credentials survive the merge — the API key removed
1597
- * so the OAuth token wins.
1598
- *
1599
- * That removal MUST happen on the env the SDK receives, not only on the
1600
- * resolved view: left to the CLI the API key would win, silently moving every
1601
- * call from the subscription to metered billing when a CI job wires both
1602
- * (which is exactly what happened before this function existed).
1603
- */
1604
- function buildInvocationEnv(env) {
1605
- const merged = withoutEmptyEndpointVars({
1606
- ...process.env,
1607
- ...env
1608
- });
1609
- preferOauthToken(merged);
1610
- merged["CLAUDE_CODE_DISABLE_AUTO_MEMORY"] = "1";
1611
- return merged;
1612
- }
1613
- let nativeBinaryWarned = false;
1614
- /**
1615
- * Warn once per process when the SDK's per-platform native binary is missing:
1616
- * every Claude call is about to fail, and the opaque per-step errors alone are
1617
- * expensive to trace back to a lockfile that dropped an optional dependency.
1618
- */
1619
- function warnOnceIfNativeBinaryMissing() {
1620
- if (nativeBinaryWarned) return;
1621
- nativeBinaryWarned = true;
1622
- const missing = missingNativeBinaryPackage();
1623
- if (missing) warn(missingNativeBinaryMessage(missing));
1624
- }
1625
- /** Whole minutes read best, but the ceiling is set in ms and may be seconds. */
1626
- function formatDuration$1(ms) {
1627
- if (ms < 6e4 || ms % 6e4 !== 0) return `${Math.round(ms / 1e3)}s`;
1628
- const minutes = ms / 6e4;
1629
- return `${minutes} minute${minutes === 1 ? "" : "s"}`;
1630
- }
1631
- async function invokeClaudeStreaming(options, onEvent) {
1632
- const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false } = options;
1633
- const resolvedModel = resolveModel(model);
1634
- const mergedEnv = buildInvocationEnv(env);
1635
- const abortController = new AbortController();
1636
- let lastAbToolUseId = null;
1637
- const claimAbToolUse = (toolUseId) => {
1638
- if (toolUseId !== lastAbToolUseId) return false;
1639
- lastAbToolUseId = null;
1640
- return true;
1641
- };
1642
- const sdkOptions = {
1643
- systemPrompt,
1644
- maxTurns,
1645
- allowedTools,
1646
- tools: builtinToolNames(allowedTools),
1647
- strictMcpConfig: true,
1648
- settingSources: [],
1649
- permissionMode: "bypassPermissions",
1650
- allowDangerouslySkipPermissions: true,
1651
- abortController,
1652
- ...resolvedModel ? { model: resolvedModel } : {},
1653
- ...cwd ? { cwd } : {},
1654
- env: mergedEnv,
1655
- ...mcpServers ? { mcpServers } : {},
1656
- ...disableThinking ? { thinking: { type: "disabled" } } : {},
1657
- hooks: onAbAction || onAbActionFailed ? {
1658
- PreToolUse: [{ hooks: [async (input) => {
1659
- if (input.hook_event_name !== "PreToolUse") return {};
1660
- if (input.tool_name !== "Bash") return {};
1661
- const cmd = input.tool_input?.["command"];
1662
- if (typeof cmd !== "string") return {};
1663
- if (!relaxAbConstraints) {
1664
- if (isBlockedAbSubcommand(cmd)) return {
1665
- decision: "block",
1666
- reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
1667
- };
1668
- if (hasRefSelector(cmd)) return {
1669
- decision: "block",
1670
- reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
1671
- };
1672
- const bareTag = findPositionalBareTag(cmd);
1673
- if (bareTag !== null) return {
1674
- decision: "block",
1675
- reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
1676
- };
1677
- if (hasMultipleAbInvocations(cmd)) return {
1678
- decision: "block",
1679
- reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
1680
- };
1681
- if (hasErrorSuppression(cmd)) return {
1682
- decision: "block",
1683
- reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
1684
- };
1685
- }
1686
- const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
1687
- const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
1688
- if ((ab !== null || assertMarker !== null) && onAbAction) {
1689
- lastAbToolUseId = input.tool_use_id;
1690
- const stepId = extractCcqaStepFromBashCommand(cmd);
1691
- onAbAction({
1692
- ...ab !== null ? { abAction: ab } : {},
1693
- ...stepId ? { stepId } : {},
1694
- ...assertMarker !== null ? { assertMarker } : {}
1695
- });
1696
- } else lastAbToolUseId = null;
1697
- return {};
1698
- }] }],
1699
- PostToolUse: [{ hooks: [async (input) => {
1700
- if (input.hook_event_name !== "PostToolUse") return {};
1701
- if (input.tool_name !== "Bash") return {};
1702
- if (!isBashToolResponseError(input.tool_response)) return {};
1703
- if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
1704
- return {};
1705
- }] }],
1706
- PostToolUseFailure: [{ hooks: [async (input) => {
1707
- if (input.hook_event_name !== "PostToolUseFailure") return {};
1708
- if (input.tool_name !== "Bash") return {};
1709
- if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
1710
- return {};
1711
- }] }]
1712
- } : void 0
1713
- };
1714
- warnOnceIfNativeBinaryMissing();
1715
- const capTimer = timeoutMs === void 0 ? null : setTimeout(() => abortController.abort(), timeoutMs);
1716
- capTimer?.unref?.();
1717
- let result = "";
1718
- let answered = false;
1719
- let isError = false;
1720
- let errorDetail = null;
1721
- let cost = {
1722
- totalCostUsd: null,
1723
- durationMs: null,
1724
- durationApiMs: null,
1725
- numTurns: null,
1726
- inputTokens: null,
1727
- cacheCreationInputTokens: null,
1728
- cacheReadInputTokens: null,
1729
- outputTokens: null,
1730
- models: []
1731
- };
1732
- const q = await buildMessageStream(prompt, sdkOptions);
1733
- try {
1734
- for await (const msg of q) {
1735
- onEvent(msg);
1736
- if (msg.type === "assistant" && !silenceBashLog) {
1737
- for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
1738
- const cmd = block.input?.["command"];
1739
- if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
1740
- }
1741
- }
1742
- if (msg.type === "result") {
1743
- answered = true;
1744
- isError = msg.is_error ?? false;
1745
- if (msg.subtype === "success") result = msg.result;
1746
- else {
1747
- result = "";
1748
- errorDetail = `SDK reported ${msg.subtype}`;
1749
- }
1750
- cost = extractInvocationCost(msg);
1751
- }
1752
- }
1753
- } catch (err) {
1754
- isError = true;
1755
- errorDetail = err instanceof Error ? err.message : String(err);
1756
- if (!result) result = errorDetail;
1757
- } finally {
1758
- if (capTimer) clearTimeout(capTimer);
1759
- }
1760
- if (abortController.signal.aborted && timeoutMs !== void 0 && !answered) {
1761
- isError = true;
1762
- errorDetail = `stopped after ${formatDuration$1(timeoutMs)} (host time limit)`;
1763
- result = errorDetail;
1764
- }
1765
- tallyInvocation(cost);
1766
- return {
1767
- result,
1768
- isError,
1769
- errorDetail,
1770
- cost
1771
- };
1772
- }
1773
- /**
1774
- * Pull the cost / usage / turn / duration fields off the SDK `result` message.
1775
- * The SDK's success and error result shapes share these fields, so we read
1776
- * them defensively as `unknown` and coerce — newer SDK versions may rename a
1777
- * field without breaking our extraction.
1778
- */
1779
- function extractInvocationCost(msg) {
1780
- const m = msg;
1781
- const usage = m["usage"];
1782
- const modelUsage = m["modelUsage"];
1783
- const num = (v) => typeof v === "number" && Number.isFinite(v) ? v : null;
1784
- const models = modelUsage && typeof modelUsage === "object" ? Object.keys(modelUsage) : [];
1785
- return {
1786
- totalCostUsd: pricedForClaude(models) ? num(m["total_cost_usd"]) : null,
1787
- durationMs: num(m["duration_ms"]),
1788
- durationApiMs: num(m["duration_api_ms"]),
1789
- numTurns: num(m["num_turns"]),
1790
- inputTokens: num(usage?.["input_tokens"]),
1791
- cacheCreationInputTokens: num(usage?.["cache_creation_input_tokens"]),
1792
- cacheReadInputTokens: num(usage?.["cache_read_input_tokens"]),
1793
- outputTokens: num(usage?.["output_tokens"]),
1794
- models
1795
- };
1796
- }
1797
- /**
1798
- * The SDK prices an unknown model id at a default Claude rate rather than
1799
- * returning null, so a self-hosted model would report dollars nobody is billed.
1800
- */
1801
- function pricedForClaude(models) {
1802
- return models.every((id) => /claude/i.test(id));
1803
- }
1804
- const BLOCKED_AB_SUBCOMMANDS = new Set([
1805
- "eval",
1806
- "js",
1807
- "label",
1808
- "textbox"
1809
- ]);
1810
- /**
1811
- * Shell-aware tokenizer: splits a command string into tokens respecting single/double quotes.
1812
- * e.g. `click "[role='dialog'] button:last-child"` → ["click", "[role='dialog'] button:last-child"]
1813
- */
1814
- function shellTokenize(s) {
1815
- const tokens = [];
1816
- let cur = "";
1817
- let quote = null;
1818
- for (let i = 0; i < s.length; i++) {
1819
- const ch = s[i];
1820
- if (quote) if (ch === quote) quote = null;
1821
- else cur += ch;
1822
- else if (ch === "\"" || ch === "'") quote = ch;
1823
- else if (ch === " " || ch === " ") {
1824
- if (cur) {
1825
- tokens.push(cur);
1826
- cur = "";
1827
- }
1828
- } else cur += ch;
1829
- }
1830
- if (cur) tokens.push(cur);
1831
- return tokens;
1832
- }
1833
- /** Extracts the subcommand from an `agent-browser [flags] <subcommand> [args...]` command string. */
1834
- function extractAbSubcommand(cmd) {
1835
- const abIdx = cmd.indexOf("agent-browser");
1836
- if (abIdx === -1) return null;
1837
- const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
1838
- let i = 0;
1839
- while (i < parts.length && parts[i].startsWith("-")) i += 2;
1840
- return parts[i] ?? null;
1841
- }
1842
- /** Returns true if the agent-browser subcommand is blocked (eval/js/find/etc). */
1843
- function isBlockedAbSubcommand(cmd) {
1844
- const sub = extractAbSubcommand(cmd);
1845
- return sub !== null && BLOCKED_AB_SUBCOMMANDS.has(sub);
1846
- }
1847
- /**
1848
- * Detects "the Bash tool returned an error" from a SDK PostToolUse hook's
1849
- * `tool_response`. The SDK can shape this two ways depending on how Claude
1850
- * Code reports Bash failures:
1851
- *
1852
- * - `{ is_error: true, ... }` — the canonical Bash failure shape
1853
- * - `{ output, exitCode, killed?, ... }` — the BashOutput shape; treat
1854
- * non-zero exit / kill as error
1855
- *
1856
- * We accept either. Anything else (including missing fields) is treated as a
1857
- * successful response so we never roll back over an unrelated tool call.
1858
- */
1859
- function isBashToolResponseError(tool_response) {
1860
- if (tool_response === null || typeof tool_response !== "object") return false;
1861
- const r = tool_response;
1862
- if (r["is_error"] === true) return true;
1863
- if (typeof r["exitCode"] === "number" && r["exitCode"] !== 0) return true;
1864
- if (r["killed"] === true) return true;
1865
- return false;
1866
- }
1867
- /**
1868
- * Detect `agent-browser ... find first|last|nth <bare-tag> <action>`. A bare
1869
- * tag inside a *positional* finder matches every element of that tag on the
1870
- * page, so "the last button" picks a different element whenever the page
1871
- * shape shifts — recorded tests built on top are flaky by construction. The
1872
- * check is narrow on purpose: `find role button --name X` is fine because
1873
- * role + accessible name stays stable.
1874
- */
1875
- function findPositionalBareTag(cmd) {
1876
- if (extractAbSubcommand(cmd) !== "find") return null;
1877
- const abIdx = cmd.indexOf("agent-browser");
1878
- const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
1879
- let i = 0;
1880
- while (i < parts.length && parts[i].startsWith("-")) i += 2;
1881
- const locator = parts[i + 1];
1882
- if (locator !== "first" && locator !== "last" && locator !== "nth") return null;
1883
- const innerIdx = locator === "nth" ? i + 3 : i + 2;
1884
- const inner = parts[innerIdx];
1885
- const action = parts[innerIdx + 1] ?? "";
1886
- if (!inner) return null;
1887
- if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(inner)) return null;
1888
- return {
1889
- locator,
1890
- selector: inner,
1891
- action
1892
- };
1893
- }
1894
- /** Returns true if any argument to an agent-browser command uses a @ref selector (e.g. @e14). */
1895
- function hasRefSelector(cmd) {
1896
- const abIdx = cmd.indexOf("agent-browser");
1897
- if (abIdx === -1) return false;
1898
- const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
1899
- let i = 0;
1900
- while (i < parts.length && parts[i].startsWith("-")) i += 2;
1901
- i++;
1902
- for (; i < parts.length; i++) if (/^@/.test(parts[i])) return true;
1903
- return false;
1904
- }
1905
- /**
1906
- * Split `cmd` into shell statements at unquoted separators (`;`, `|`, `&`,
1907
- * newline; consecutive separator chars like `&&` count once). String
1908
- * literals are honoured so `fill "a;b"` stays a single statement. This is a
1909
- * heuristic split (no subshell grammar), shared by the compound-invocation
1910
- * guard and the CCQA_STEP prefix extraction so both agree on what "one
1911
- * command" means.
1912
- */
1913
- function splitShellStatements(cmd) {
1914
- const statements = [];
1915
- let start = 0;
1916
- let quote = null;
1917
- for (let i = 0; i < cmd.length; i++) {
1918
- const ch = cmd[i];
1919
- if (quote) {
1920
- if (ch === quote) quote = null;
1921
- continue;
1922
- }
1923
- if (ch === "\"" || ch === "'" || ch === "`") {
1924
- quote = ch;
1925
- continue;
1926
- }
1927
- if (ch === ";" || ch === "|" || ch === "&" || ch === "\n") {
1928
- statements.push(cmd.slice(start, i));
1929
- while (i + 1 < cmd.length && (cmd[i + 1] === "|" || cmd[i + 1] === "&" || cmd[i + 1] === ";" || cmd[i + 1] === "\n")) i++;
1930
- start = i + 1;
1931
- }
1932
- }
1933
- statements.push(cmd.slice(start));
1934
- return statements;
1935
- }
1936
- /** One leading `KEY=value` env assignment; value may be single/double-quoted. */
1937
- const ENV_ASSIGN_HEAD_RE = /^([A-Za-z_][A-Za-z0-9_]*)=(?:"([^"]*)"|'([^']*)'|(\S*))(?:\s+|$)/;
1938
- /**
1939
- * Split a statement into its leading `KEY=value` env assignments and the
1940
- * command they prefix. An assignment whose value is a command substitution
1941
- * (`$(...)` / backticks) is NOT treated as a prefix — `result=$(agent-browser
1942
- * ... snapshot)` is an assignment statement, not an agent-browser invocation,
1943
- * and must stay invisible to the guards below.
1944
- */
1945
- function splitLeadingEnvAssignments(statement) {
1946
- const env = /* @__PURE__ */ new Map();
1947
- let command = statement.trimStart();
1948
- for (;;) {
1949
- const m = ENV_ASSIGN_HEAD_RE.exec(command);
1950
- if (!m) break;
1951
- const value = m[2] ?? m[3] ?? m[4] ?? "";
1952
- if (value.startsWith("$(") || value.startsWith("`")) return {
1953
- env: /* @__PURE__ */ new Map(),
1954
- command: statement.trimStart()
1955
- };
1956
- env.set(m[1], value);
1957
- command = command.slice(m[0].length);
1958
- }
1959
- return {
1960
- env,
1961
- command
1962
- };
1963
- }
1964
- /** True when `command` starts with `agent-browser` as the command word. */
1965
- function isAgentBrowserHead(command) {
1966
- if (!command.startsWith("agent-browser")) return false;
1967
- const after = command[13];
1968
- return after === void 0 || !/[A-Za-z0-9_\-]/.test(after);
1969
- }
1970
- /** Step ids passed via `CCQA_STEP=<step-id>` must be a plain slug. */
1971
- const STEP_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
1972
- /**
1973
- * Extract the step id from the `CCQA_STEP=<step-id>` env prefix on the
1974
- * agent-browser invocation in `cmd` (e.g. `CCQA_STEP=step-03 agent-browser
1975
- * --session s click "text=Submit"`). The prefix may sit anywhere in the
1976
- * leading env-assignment run (`FOO=x CCQA_STEP=step-02 agent-browser ...`),
1977
- * and the invocation may be a later statement of a compound command
1978
- * (`cd app && CCQA_STEP=step-01 agent-browser ...`). Returns null when the
1979
- * prefix is absent or its value is not a valid slug — callers then fall back
1980
- * to the STEP_START text protocol.
1981
- */
1982
- function extractCcqaStepFromBashCommand(cmd) {
1983
- for (const statement of splitShellStatements(cmd)) {
1984
- const { env, command } = splitLeadingEnvAssignments(statement);
1985
- if (!isAgentBrowserHead(command)) continue;
1986
- const value = env.get("CCQA_STEP");
1987
- return value !== void 0 && STEP_SLUG_RE.test(value) ? value : null;
1988
- }
1989
- return null;
1990
- }
1991
- /**
1992
- * Extract the assert marker from the `CCQA_ASSERT=<marker>` env prefix on
1993
- * the agent-browser invocation in `cmd`, e.g. `CCQA_STEP=step-03
1994
- * CCQA_ASSERT=1 agent-browser --session s wait --text "Submitted" --timeout
1995
- * 3000`. The marker declares that the command verifies a step signal;
1996
- * `promoteMarkedAssert` maps it onto recorded assert action(s). Returns the
1997
- * raw value — semantic validation (which markers combine with which
1998
- * commands) happens at promotion time so mismatches surface as warnings
1999
- * instead of being silently dropped here. Returns null when the prefix is
2000
- * absent or empty.
2001
- */
2002
- function extractCcqaAssertFromBashCommand(cmd) {
2003
- for (const statement of splitShellStatements(cmd)) {
2004
- const { env, command } = splitLeadingEnvAssignments(statement);
2005
- if (!isAgentBrowserHead(command)) continue;
2006
- const value = env.get("CCQA_ASSERT");
2007
- return value !== void 0 && value.length > 0 ? value : null;
2008
- }
2009
- return null;
2010
- }
2011
- /**
2012
- * Returns true when `cmd` contains more than one `agent-browser` invocation
2013
- * chained together via shell operators (`&&`, `||`, `;`, `|`, newline). The
2014
- * PreToolUse hook only records ONE AB_ACTION per Bash call, so chained
2015
- * invocations would silently drop every intermediate failure — turning
2016
- * "I tried four selectors before one worked" into a clean-looking trace
2017
- * with five orphaned actions that later fail at replay.
2018
- *
2019
- * Counts statements whose command word is `agent-browser`, skipping any
2020
- * leading env assignments (the trace protocol prefixes every invocation
2021
- * with `CCQA_STEP=<step-id>`). String literals are honoured so
2022
- * `agent-browser fill 'agent-browser'` doesn't false-fire.
2023
- */
2024
- function hasMultipleAbInvocations(cmd) {
2025
- let count = 0;
2026
- for (const statement of splitShellStatements(cmd)) {
2027
- if (!isAgentBrowserHead(splitLeadingEnvAssignments(statement).command)) continue;
2028
- count++;
2029
- if (count > 1) return true;
2030
- }
2031
- return false;
2032
- }
2033
- /**
2034
- * Returns true when an `agent-browser` command in `cmd` has its exit
2035
- * status hidden by a shell decorator that would prevent ccqa from rolling
2036
- * back a failed attempt:
2037
- *
2038
- * - trailing `|| true` / `|| :` / `; true` (force exit 0)
2039
- * - `2>/dev/null` and friends (drop stderr, sometimes paired with `|| true`)
2040
- *
2041
- * The agent-browser command itself returns exit 1 on selector miss, so
2042
- * once one of these is present the PostToolUse hook sees `is_error=false`
2043
- * and the bad attempt sneaks into ir.json.
2044
- */
2045
- function hasErrorSuppression(cmd) {
2046
- if (cmd.indexOf("agent-browser") === -1) return false;
2047
- if (/\|\|\s*(true|:|\s*$|#)/.test(cmd)) return true;
2048
- if (/;\s*(true|:)\b/.test(cmd)) return true;
2049
- if (/2\s*>\s*\/dev\/null/.test(cmd)) return true;
2050
- if (/&\s*>\s*\/dev\/null/.test(cmd)) return true;
2051
- return false;
2052
- }
2053
- /**
2054
- * Parse an `agent-browser --session <name> <cmd> [args...]` bash command
2055
- * and return the corresponding AB_ACTION line, or null if not an agent-browser call.
2056
- */
2057
- function extractAbActionFromBashCommand(cmd) {
2058
- const subCmd = extractAbSubcommand(cmd);
2059
- if (!subCmd) return null;
2060
- const abIdx = cmd.indexOf("agent-browser");
2061
- const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
2062
- let i = 0;
2063
- while (i < parts.length && parts[i].startsWith("-")) i += 2;
2064
- const args = parts.slice(i + 1);
2065
- switch (subCmd) {
2066
- case "cookies":
2067
- if (args[0] === "clear") return "AB_ACTION|cookies_clear";
2068
- return null;
2069
- case "open": return `AB_ACTION|open|${args[0] ?? ""}`;
2070
- case "press": return `AB_ACTION|press|${args[0] ?? ""}`;
2071
- case "scroll": return `AB_ACTION|scroll|${args.join("|")}`;
2072
- case "click":
2073
- case "dblclick":
2074
- case "check":
2075
- case "uncheck":
2076
- case "hover":
2077
- case "wait": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}`;
2078
- case "fill":
2079
- case "type":
2080
- case "select": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
2081
- case "drag": return `AB_ACTION|drag|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
2082
- case "upload": {
2083
- const sel = args[0] ?? "";
2084
- const files = args.slice(1);
2085
- if (!sel || files.length === 0) return null;
2086
- return `AB_ACTION|upload|${sel}|${files.join("|")}`;
2087
- }
2088
- case "snapshot": return null;
2089
- case "find": return extractFindAbAction(args);
2090
- default: return null;
2091
- }
2092
- }
2093
- /**
2094
- * Wire lines for the observation-only probes `get count <sel>` / `get url`.
2095
- * These commands read state without mutating it, so they have no place in
2096
- * the replay sequence and `extractAbActionFromBashCommand` ignores them.
2097
- * They matter only when a `CCQA_ASSERT=<marker>` env prefix declares the
2098
- * probe verifies a step signal — the hook layer then surfaces them via this
2099
- * function so `promoteMarkedAssert` can turn them into recorded asserts.
2100
- * Only consulted when a marker is present; unmarked `get` commands stay
2101
- * unobserved as before.
2102
- */
2103
- function extractObservationAbAction(cmd) {
2104
- if (extractAbSubcommand(cmd) !== "get") return null;
2105
- const abIdx = cmd.indexOf("agent-browser");
2106
- const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
2107
- let i = 0;
2108
- while (i < parts.length && parts[i].startsWith("-")) i += 2;
2109
- const args = parts.slice(i + 1);
2110
- if (args[0] === "count" && args[1]) return `AB_ACTION|get_count|${args[1]}`;
2111
- if (args[0] === "url") return "AB_ACTION|get_url";
2112
- return null;
2113
- }
2114
- const FIND_ACTION_SET = new Set(FIND_ACTIONS);
2115
- const FIND_LOCATOR_SET = new Set(FIND_LOCATORS);
2116
- /**
2117
- * Parse the positional tokens of `agent-browser find <locator> <value> [...]
2118
- * <action> [fillValue]` and produce a canonical
2119
- * `AB_ACTION|find_<action>|<locator>|<value>|<extra>|<exact>|...|<label>`
2120
- * line. The wire format keeps a fixed positional layout across locators so
2121
- * downstream `parseAbActionLine` in `ir/from-agent-browser.ts` can split on
2122
- * `|` alone:
2123
- *
2124
- * <extra> is `--name` value for role, integer index for nth, "" otherwise.
2125
- * <exact> is the literal "exact" if --exact was passed, "" otherwise.
2126
- *
2127
- * Returns null for malformed invocations — the caller treats null as "not a
2128
- * structured action" and the Bash command still runs unobserved.
2129
- */
2130
- function extractFindAbAction(args) {
2131
- const locator = args[0];
2132
- if (!locator || !FIND_LOCATOR_SET.has(locator)) return null;
2133
- let i = 1;
2134
- let value = args[i] ?? "";
2135
- i++;
2136
- let extra = "";
2137
- if (locator === "nth") {
2138
- extra = value;
2139
- value = args[i] ?? "";
2140
- i++;
2141
- }
2142
- let action = "";
2143
- let name = "";
2144
- let exact = "";
2145
- let fillValue = "";
2146
- for (; i < args.length; i++) {
2147
- const tok = args[i];
2148
- if (tok === "--name") {
2149
- name = args[i + 1] ?? "";
2150
- i++;
2151
- } else if (tok === "--exact") exact = "exact";
2152
- else if (FIND_ACTION_SET.has(tok)) action = tok;
2153
- else if (action) fillValue = tok;
2154
- }
2155
- if (!action) return null;
2156
- if (locator === "role") extra = name;
2157
- const command = `find_${action}`;
2158
- if (action === "fill" || action === "type") return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|${fillValue}|`;
2159
- return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|`;
2160
- }
2161
- async function buildMessageStream(prompt, options) {
2162
- const mockFile = process.env["CCQA_CLAUDE_MOCK_FILE"];
2163
- if (mockFile) return replayMockMessages(mockFile, options);
2164
- return query({
2165
- prompt,
2166
- options
2167
- });
2168
- }
2169
- async function* replayMockMessages(path, options) {
2170
- const raw = await readFile(path, "utf8");
2171
- for (const line of raw.split("\n")) {
2172
- const trimmed = line.trim();
2173
- if (!trimmed) continue;
2174
- const msg = JSON.parse(trimmed);
2175
- await fireMockPreToolUseHooks(msg, options);
2176
- yield msg;
2177
- }
2178
- }
2179
- /**
2180
- * The real SDK fires PreToolUse hooks as it executes tool calls; the JSONL
2181
- * replay approximates that by invoking the configured PreToolUse hooks for
2182
- * every Bash tool_use block before yielding its message, so e2e stubs
2183
- * exercise the AB_ACTION recording path (including CCQA_STEP step
2184
- * attribution). Hook decisions are ignored and post-tool hooks are not
2185
- * simulated — the replay runs no tools, so there is nothing to block or fail.
2186
- */
2187
- async function fireMockPreToolUseHooks(msg, options) {
2188
- const matchers = options.hooks?.PreToolUse;
2189
- if (!matchers || msg.type !== "assistant") return;
2190
- for (const block of msg.message.content ?? []) {
2191
- if (block.type !== "tool_use" || block.name !== "Bash") continue;
2192
- const input = {
2193
- hook_event_name: "PreToolUse",
2194
- tool_name: "Bash",
2195
- tool_input: block.input,
2196
- tool_use_id: block.id,
2197
- session_id: "mock",
2198
- transcript_path: "",
2199
- cwd: process.cwd()
2200
- };
2201
- for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
2202
- }
2203
- }
2204
- //#endregion
2205
- //#region src/prompts/format.ts
2206
- /**
2207
- * Formatting helpers shared by the Claude prompt builders (diagnose, report,
2208
- * drift). Centralised so the prompts cannot drift apart on mechanics that
2209
- * must stay consistent across commands.
2210
- */
2211
- /** Prefix every line with its 1-based number, the form fix suggestions cite. */
2212
- function numberLines(script) {
2213
- return script.split("\n").map((l, i) => `${i + 1}: ${l}`).join("\n");
2214
- }
2215
- /**
2216
- * The "## Output language" prompt section. Empty for "auto" so the prompt
2217
- * stays byte-identical to the no-flag baseline. `fields` names the
2218
- * human-readable JSON fields to translate; `verbatimNames` names the
2219
- * enum-like values that must never be translated.
2220
- */
2221
- function outputLanguageBlock(outputLanguage, fields, verbatimNames) {
2222
- if (outputLanguage === "auto") return "";
2223
- return `## Output language
2224
-
2225
- Write all human-readable fields (${fields}) in **${outputLanguage}** (BCP-47 tag).
2226
- Selectors, file paths, identifiers, ${verbatimNames}, JSON keys, and quoted strings stay verbatim regardless of language.
2227
-
2228
- `;
2229
- }
2230
- /**
2231
- * The `spec`/`generated` surface-axis definitions, plus the "if both are
2232
- * stale, answer spec" tie-break rule. Shared verbatim by the audit prompt
2233
- * (`prompts/drift.ts`) and the run's failure-classification prompt
2234
- * (`report/prompt.ts`): both fill the same wire field (`DriftSurfaceSchema`),
2235
- * so a diverging definition in one would make the two paths disagree on what
2236
- * a spec's own field means.
2237
- */
2238
- function surfaceDefinitionBlock() {
2239
- return `- **\`spec\`** — spec.yaml asks about something the source no longer has. It has to be rewritten, and the code regenerated after.
2240
- - **\`generated\`** — the spec still describes the product correctly, but the generated code reaches for a selector or string the source no longer has. Only a regeneration is needed; nobody has to rewrite the spec.
2241
-
2242
- If both are stale, answer \`spec\`: it is the root, and fixing it regenerates the code.`;
2243
- }
2244
- /**
2245
- * "surface is a separate axis from the label" clarifying example, shared for
2246
- * the same reason as {@link surfaceDefinitionBlock}. `labelToken` is the
2247
- * exact text to name the label by (e.g. `"TEST_DRIFT"` or `` "`TEST_DRIFT`" ``)
2248
- * so each caller keeps its own backtick/bold convention.
2249
- */
2250
- function surfaceAxisAside(labelToken) {
2251
- return `This is a separate axis from the label. A renamed selector that only the generated code names is ${labelToken} on the \`generated\` surface; a spec whose \`expected\` quotes a string the product renamed is ${labelToken} on the \`spec\` surface.`;
2252
- }
2253
- //#endregion
2254
- //#region src/ir/to-agent-browser.ts
2255
- const lit = (text) => ({
2256
- text,
2257
- expandsEnv: false
2258
- });
2259
- const val = (text) => ({
2260
- text,
2261
- expandsEnv: true
2262
- });
2263
- /**
2264
- * Render a locator as the selector string a plain agent-browser command
2265
- * accepts. Only `css` (verbatim) and `text` (`text=` engine form) have a
2266
- * plain-selector form; other strategies are reachable via `find` only and
2267
- * fall back to their raw value (callers guard against that case).
2268
- */
2269
- function locatorToSelector(locator) {
2270
- return locator.by === "text" ? `text=${locator.value}` : locator.value;
2271
- }
2272
- /**
2273
- * Compact human-readable locator form for logs and LLM-prompt summaries: the
2274
- * raw selector for `css`, `by=value` otherwise. Distinct from
2275
- * `locatorToSelector` (which produces a selector agent-browser can execute) —
2276
- * this one is for display only and never round-trips.
2277
- */
2278
- function describeLocator(locator) {
2279
- return locator.by === "css" ? locator.value : `${locator.by}=${locator.value}`;
2280
- }
2281
- /**
2282
- * Canonical agent-browser argv (sans `--session`) for one action. Returns
2283
- * null for actions with no direct argv form: observation-only `snapshot`,
2284
- * `assert` (validation probes / abAssert* helpers live in the consumers),
2285
- * and structurally incomplete actions (e.g. a missing locator).
2286
- */
2287
- function toAgentBrowserArgs(action) {
2288
- switch (action.action) {
2289
- case "cookies_clear": return [lit("cookies"), lit("clear")];
2290
- case "navigate": return [lit("open"), val(action.value ?? "")];
2291
- case "press": return [lit("press"), val(action.value ?? "")];
2292
- case "scroll": return [
2293
- lit("scroll"),
2294
- lit(action.direction ?? "down"),
2295
- ...action.pixels ? [lit(action.pixels)] : []
2296
- ];
2297
- case "select":
2298
- if (!action.locator) return null;
2299
- return [
2300
- lit("select"),
2301
- val(locatorToSelector(action.locator)),
2302
- val(action.value ?? "")
2303
- ];
2304
- case "drag":
2305
- if (!action.locator || !action.target) return null;
2306
- return [
2307
- lit("drag"),
2308
- val(locatorToSelector(action.locator)),
2309
- val(locatorToSelector(action.target))
2310
- ];
2311
- case "upload": {
2312
- const files = action.files ?? [];
2313
- if (!action.locator || files.length === 0) return null;
2314
- return [
2315
- lit("upload"),
2316
- val(locatorToSelector(action.locator)),
2317
- ...files.map(val)
2318
- ];
2319
- }
2320
- case "wait": {
2321
- const loc = action.locator;
2322
- if (!loc) return null;
2323
- if (loc.by === "text") return [
2324
- lit("wait"),
2325
- lit("--text"),
2326
- val(loc.value)
2327
- ];
2328
- return [lit("wait"), val(locatorToSelector(loc))];
2329
- }
2330
- case "click":
2331
- case "dblclick":
2332
- case "check":
2333
- case "uncheck":
2334
- case "hover":
2335
- case "focus":
2336
- case "fill":
2337
- case "type": return interactionToArgs(action);
2338
- case "snapshot":
2339
- case "assert": return null;
2340
- }
2341
- }
2342
- /**
2343
- * Element interactions come in two argv shapes: the plain command
2344
- * (`click "<css>"`) when the locator is a raw selector string, and the
2345
- * `find <locator> <value> <action> [input] [--name <n>] [--exact]` form for
2346
- * semantic locators and positional (`index`) picks. `type` is a ccqa-side
2347
- * alias of `fill` in both shapes. Flags MUST follow the action token —
2348
- * putting them before it makes agent-browser fail with "Unknown subaction".
2349
- */
2350
- function interactionToArgs(action) {
2351
- const loc = action.locator;
2352
- if (!loc) return null;
2353
- const abAction = action.action === "type" ? "fill" : action.action;
2354
- const takesInput = action.action === "fill" || action.action === "type";
2355
- if (!(loc.by !== "css" || action.index !== void 0 || action.action === "focus")) {
2356
- const args = [lit(abAction), val(loc.value)];
2357
- if (takesInput) args.push(val(action.value ?? ""));
2358
- return args;
2359
- }
2360
- if (!loc.value) return null;
2361
- const out = [lit("find")];
2362
- if (action.index !== void 0) {
2363
- if (loc.by !== "css") return null;
2364
- if (action.index === "first" || action.index === "last") out.push(lit(action.index));
2365
- else out.push(lit("nth"), lit(String(action.index)));
2366
- out.push(val(loc.value));
2367
- } else {
2368
- if (loc.by === "css") return null;
2369
- out.push(lit(loc.by), val(loc.value));
2370
- }
2371
- out.push(lit(abAction));
2372
- if (takesInput) out.push(val(action.value ?? ""));
2373
- if (loc.by === "role" && loc.name) out.push(lit("--name"), val(loc.name));
2374
- if (loc.by !== "css" && loc.exact) out.push(lit("--exact"));
2375
- return out;
2376
- }
2377
- //#endregion
2378
- //#region src/diagnose/prompt.ts
2379
- function buildDiagnosePrompt(input) {
2380
- const { script, specYaml, actions, failureLog, pageSnapshot, outputLanguage = "auto" } = input;
2381
- const numbered = numberLines(script);
2382
- const actionsSummary = actions.map((a, i) => {
2383
- const parts = [`${i + 1}. ${a.action}`];
2384
- if (a.assert) parts.push(`assert="${a.assert}"`);
2385
- if (a.locator) parts.push(`locator="${describeLocator(a.locator)}"`);
2386
- if (a.index !== void 0) parts.push(`index=${a.index}`);
2387
- if (a.value) parts.push(`value="${a.value}"`);
2388
- if (a.observation) parts.push(`→ ${a.observation}`);
2389
- return parts.join(" ");
2390
- }).join("\n");
2391
- return `You are diagnosing a failing E2E test. The test was generated from a recorded trace of the original interaction. Compare the failing run against the original spec and recorded actions to determine WHY the test failed and what the right fix is.
2392
-
2393
- ${outputLanguageBlock(outputLanguage, "`reasoning`, `reason`", "code, type names (TIMING_ISSUE, etc.)")}## You have read-only filesystem tools
2394
-
2395
- You can call \`Grep\`, \`Glob\`, and \`Read\` against the current repository before producing the JSON.
2396
-
2397
- For SELECTOR_DRIFT specifically the failure log is usually NOT enough on its own — the runner only reports "selector X not visible". To confirm a rename, search the application source for the *type* of selector that's failing:
2398
-
2399
- - For \`[aria-label='OLD']\` failures: \`Grep\` for \`aria-label=\` (or i18n key \`OLD\`) in the app source. If you find a near-miss like \`aria-label="NEW"\` whose text is a superset/rephrase of the failing label, that is your evidence.
2400
- - For \`[placeholder='OLD']\` failures: \`Grep\` for \`placeholder=\`.
2401
- - For \`[role='OLD']\` or \`[data-testid='OLD']\`: same pattern.
2402
- - For \`text=OLD\` failures: \`Grep\` the source / i18n bundles for \`OLD\`. Locale files (\`*.json\`, \`*.yml\`, \`messages.ts\`, etc.) often hold the canonical strings.
2403
-
2404
- You have **up to 10 tool turns**. Spend them on grep/read; do not loop. Only when you have concrete file:line evidence should you emit SELECTOR_DRIFT — otherwise prefer UNKNOWN with confidence < 0.4 and let the human decide.
2405
-
2406
- Do NOT attempt to write, edit, run shell commands, or hit the network. Only Grep/Glob/Read.
2407
-
2408
- ## Diagnosis categories
2409
-
2410
- Pick exactly ONE category. The output JSON must follow the shape for that category.
2411
-
2412
- 1. TIMING_ISSUE — element not yet present because the page hasn't loaded / navigated. Fix by inserting or extending sleeps.
2413
- {
2414
- "diagnosis": {
2415
- "type": "TIMING_ISSUE",
2416
- "fixes": [
2417
- { "kind": "insert", "line": <1-based>, "seconds": <int>, "reason": "<short>" },
2418
- { "kind": "increase", "line": <1-based of existing sleep>, "increase_to": <int>, "reason": "<short>" }
2419
- ]
2420
- },
2421
- "confidence": <0.0-1.0>,
2422
- "reasoning": "<why timing is the cause>"
2423
- }
2424
-
2425
- 2. OVER_ASSERTION — the test is asserting something the spec never required, OR a recorded assertion that is environment-dependent (e.g. a placeholder text that varies). The right fix is to remove those lines from the test.
2426
- {
2427
- "diagnosis": {
2428
- "type": "OVER_ASSERTION",
2429
- "lines": [<1-based line numbers to remove>],
2430
- "reason": "<short>"
2431
- },
2432
- "confidence": <0.0-1.0>,
2433
- "reasoning": "<why this assertion isn't required by the spec>"
2434
- }
2435
-
2436
- 3. SELECTOR_DRIFT — the page is healthy but a selector has been renamed/refined since the trace was recorded. The failure log will typically contain a snapshot showing the new selector. ONLY use this when you can name the exact replacement selector.
2437
- {
2438
- "diagnosis": {
2439
- "type": "SELECTOR_DRIFT",
2440
- "line": <1-based>,
2441
- "oldSelector": "<exact string in current line>",
2442
- "newSelector": "<exact replacement>",
2443
- "reason": "<short>"
2444
- },
2445
- "confidence": <0.0-1.0>,
2446
- "reasoning": "<evidence from failure log>"
2447
- }
2448
-
2449
- 4. DATA_MISSING — the test depends on data (a record, a setup, a logged-in state) that no longer exists. Not auto-fixable; the human must reseed or update the spec.
2450
- {
2451
- "diagnosis": { "type": "DATA_MISSING", "reason": "<what is missing>" },
2452
- "confidence": <0.0-1.0>,
2453
- "reasoning": "<evidence>"
2454
- }
2455
-
2456
- 5. UNKNOWN — none of the above fit, or evidence is too weak to choose.
2457
- {
2458
- "diagnosis": { "type": "UNKNOWN", "reason": "<short>" },
2459
- "confidence": <0.0-1.0>,
2460
- "reasoning": "<what you saw and why you can't classify>"
2461
- }
2462
-
2463
- ## Confidence guidance
2464
-
2465
- - 0.9-1.0: failure log directly shows the cause (e.g. "selector X not found, snapshot lists Y" → SELECTOR_DRIFT)
2466
- - 0.7-0.9: strong indirect evidence (e.g. timing pattern after navigation, or assertion text that doesn't appear in spec)
2467
- - 0.4-0.7: plausible classification but multiple categories could explain it
2468
- - < 0.4: prefer UNKNOWN over guessing
2469
-
2470
- ## Rules
2471
-
2472
- - Your **final** assistant message must start with \`{\` and end with \`}\` — a single JSON object, nothing before or after. No prose preamble like "Confirmed: ...", no markdown fences, no commentary, no tool calls in the same turn. If you have an analysis sentence, put it in the \`reasoning\` field.
2473
- - Line numbers refer to the numbered test script below (1-based).
2474
- - For SELECTOR_DRIFT, \`oldSelector\` must match a substring of the script at that line; \`newSelector\` must be backed by a concrete file:line you read with Grep/Read (do not invent). Cite the evidence in \`reasoning\`.
2475
- - For OVER_ASSERTION, only include lines that contain assert calls (\`abAssert*\`) or existence-checking waits (\`abWait\`); a recorded \`abWait("[selector]")\` is an implicit existence assertion and a valid removal candidate when the spec never required that element to be present.
2476
- - Cross-check assertions against the spec YAML. If the spec doesn't require the assertion, OVER_ASSERTION is the better diagnosis than SELECTOR_DRIFT.
2477
-
2478
- ## Test Spec (spec.yaml)
2479
- ${specYaml}
2480
-
2481
- ## Recorded Actions (ir.json summary)
2482
- ${actionsSummary}
2483
-
2484
- ## Test Script (with line numbers)
2485
- ${numbered}
2486
-
2487
- ## Failure Log
2488
- ${failureLog.slice(0, 4e3)}${pageSnapshot ? formatPageSnapshot(pageSnapshot) : ""}`;
461
+ //#region src/runtime/live-artifacts.ts
462
+ /**
463
+ * Build a sortable, unique run id. ISO8601 with `:` / `.` replaced so it's
464
+ * filename-safe, timestamp first so run directories still sort by time, and a
465
+ * random suffix because the timestamp alone does not separate two specs.
466
+ *
467
+ * The pool launches specs back-to-back, so at `--concurrency > 1` two of them
468
+ * land in the same millisecond. A spec that puts `${CCQA_RUN_ID}` in the name
469
+ * of something it creates would then share that name with its neighbour, and
470
+ * each would find — and delete — the other's row. Nothing fails; the
471
+ * assertions just read the wrong state.
472
+ *
473
+ * Caller is expected to mkdir the directory once and pass
474
+ * `runDir = <baseDir>/<runId>` to the path helpers below.
475
+ */
476
+ function buildRunId() {
477
+ return `${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`;
2489
478
  }
2490
479
  /**
2491
- * Page snapshot captured by ccqa right after the failure (agent-browser
2492
- * accessibility tree). When present, it usually decides SELECTOR_DRIFT vs
2493
- * TIMING_ISSUE: a near-miss aria-label / role / placeholder in the
2494
- * snapshot is direct evidence of a rename, while a tree that doesn't
2495
- * contain the failing locator at all (without a near-miss) points to a
2496
- * still-loading page or genuinely missing element.
480
+ * Per-step artifact paths under a run directory. `<runDir>/steps/<stepId>.*`.
481
+ * Three files per step:
482
+ * - <stepId>.before.png : screenshot taken BEFORE Claude executes the step.
483
+ * - <stepId>.after.png : screenshot taken AFTER Claude executes the step.
484
+ * - <stepId>.log.txt : full assistant transcript for the step (judgement
485
+ * reasoning, any STEP_RESULT lines, raw tool output
486
+ * summaries the model chose to keep).
2497
487
  */
2498
- function formatPageSnapshot(snapshot) {
2499
- return `
2500
-
2501
- ## Page Snapshot (accessibility tree captured right after the failure)
2502
-
2503
- This is the live state of the page when the test failed. Prefer this over your own assumptions:
2504
-
2505
- - If a near-miss of the failing selector appears here (e.g. failing \`[aria-label='A']\` and snapshot contains \`aria-label="A-prime"\`), that is direct evidence of SELECTOR_DRIFT — propose the snapshot's value as \`newSelector\`.
2506
- - If the failing locator is genuinely absent and no near-miss exists, the page may be still loading (TIMING_ISSUE) or the spec is asserting something not on this page (OVER_ASSERTION / DATA_MISSING).
2507
- - If the snapshot looks unrelated to the spec (e.g. error page, login wall), DATA_MISSING is likely.
2508
-
2509
- \`\`\`
2510
- ${snapshot}
2511
- \`\`\``;
488
+ function stepArtifactPaths(runDir, stepId) {
489
+ const dir = join(runDir, "steps");
490
+ return {
491
+ beforePng: join(dir, `${stepId}.before.png`),
492
+ afterPng: join(dir, `${stepId}.after.png`),
493
+ logTxt: join(dir, `${stepId}.log.txt`)
494
+ };
2512
495
  }
2513
496
  //#endregion
2514
- //#region src/diagnose/diagnose.ts
2515
- async function diagnose(input, options = {}) {
2516
- const { result: raw, isError } = await invokeClaudeStreaming({
2517
- prompt: buildDiagnosePrompt(input),
2518
- allowedTools: [
2519
- "Read",
2520
- "Grep",
2521
- "Glob"
2522
- ],
2523
- maxTurns: 20,
2524
- model: options.model
2525
- }, () => {});
2526
- if (isError) return {
2527
- result: null,
2528
- raw: raw ?? "",
2529
- sdkError: true
2530
- };
2531
- if (!raw) return {
2532
- result: null,
2533
- raw: "",
2534
- sdkError: false
497
+ //#region src/runtime/pool.ts
498
+ async function runPool(items, concurrency, fn, opts = {}) {
499
+ const results = new Array(items.length);
500
+ const needs = items.map((item) => opts.resources?.(item) ?? []);
501
+ const busy = /* @__PURE__ */ new Set();
502
+ const queued = new Set(items.map((_, i) => i));
503
+ const inFlight = /* @__PURE__ */ new Map();
504
+ const limit = Math.max(1, Math.min(concurrency, items.length));
505
+ const failures = [];
506
+ const start = (idx) => {
507
+ queued.delete(idx);
508
+ for (const name of needs[idx]) busy.add(name);
509
+ inFlight.set(idx, Promise.resolve().then(async () => {
510
+ try {
511
+ results[idx] = await fn(items[idx], idx);
512
+ } catch (err) {
513
+ failures.push(err);
514
+ } finally {
515
+ for (const name of needs[idx]) busy.delete(name);
516
+ inFlight.delete(idx);
517
+ }
518
+ }));
2535
519
  };
2536
- const candidates = extractJsonCandidates(raw);
2537
- for (const candidate of candidates) {
2538
- let parsed;
2539
- try {
2540
- parsed = JSON.parse(candidate);
2541
- } catch {
2542
- continue;
520
+ while (queued.size > 0 || inFlight.size > 0) {
521
+ if (failures.length === 0) {
522
+ for (const idx of queued) {
523
+ if (inFlight.size >= limit) break;
524
+ if (needs[idx].some((name) => busy.has(name))) continue;
525
+ start(idx);
526
+ }
527
+ if (inFlight.size === 0) throw new Error(`runPool: ${queued.size} item(s) unrunnable with nothing in flight`);
2543
528
  }
2544
- const normalised = normaliseResult(parsed);
2545
- if (normalised) return {
2546
- result: normalised,
2547
- raw,
2548
- sdkError: false
2549
- };
529
+ if (inFlight.size === 0) break;
530
+ await Promise.race(inFlight.values());
2550
531
  }
2551
- return {
2552
- result: {
2553
- diagnosis: {
2554
- type: "UNKNOWN",
2555
- reason: "diagnose returned no parseable diagnosis JSON"
2556
- },
2557
- confidence: 0,
2558
- reasoning: truncate$2(raw, 1e3)
2559
- },
2560
- raw,
2561
- sdkError: false
2562
- };
532
+ if (failures.length === 1) throw failures[0];
533
+ if (failures.length > 1) throw new AggregateError(failures, `${failures.length} items failed`);
534
+ return results;
535
+ }
536
+ //#endregion
537
+ //#region src/drift/auth.ts
538
+ /**
539
+ * Claude Code can also run against AWS Bedrock / Google Vertex AI, selected by
540
+ * these env toggles. Credentials then come from the cloud SDK's own chain
541
+ * (instance/task roles, gcloud auth, …), so none of the Anthropic-side probes
542
+ * below apply — a set toggle counts as auth being available. ccqa forwards the
543
+ * toggle verbatim; only "0"/"false" (any case) is treated as explicitly off.
544
+ */
545
+ const CLOUD_PROVIDER_ENV_KEYS = ["CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX"];
546
+ function cloudProviderEnabled() {
547
+ return CLOUD_PROVIDER_ENV_KEYS.some((key) => {
548
+ const value = process.env[key]?.trim().toLowerCase();
549
+ return value !== void 0 && value !== "" && value !== "0" && value !== "false";
550
+ });
2563
551
  }
2564
552
  /**
2565
- * Pull every plausible JSON object out of `raw`. We try, in order:
2566
- * 1. The whole string with code fences stripped (the prompt asks for
2567
- * JSON-only, so this is the happy path).
2568
- * 2. Each balanced `{...}` block found by scanning the text. The model
2569
- * sometimes prefixes the JSON with a "Confirmed: ..." sentence or
2570
- * mentions partial JSON in its tool-using reasoning; we want to
2571
- * try the *last* well-formed object first because it's most likely
2572
- * the final answer, then earlier ones as a fallback.
553
+ * Probe whether the host has any credential the Anthropic SDK can pick up:
554
+ * - one of CREDENTIAL_ENV_KEYS (API key, gateway bearer token, or the
555
+ * subscription token from `claude setup-token`)
556
+ * - CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
557
+ * endpoints authenticated by the cloud SDK's credential chain)
558
+ * - ~/.claude/.credentials.json (Claude Code login, file-based platforms)
559
+ * - macOS Keychain item "Claude Code-credentials" (Claude Code login on
560
+ * darwin stores the OAuth credentials in the Keychain, not on disk)
2573
561
  *
2574
- * The caller `JSON.parse`s each candidate and stops at the first match
2575
- * that normalises to a known DiagnosisResult.
562
+ * Claude-driven hooks are opt-in, so the caller only consults this after the
563
+ * user has asked for analysis. We never throw — auth absence is a normal flow
564
+ * that surfaces as "analysis skipped".
2576
565
  */
2577
- function extractJsonCandidates(raw) {
2578
- const out = [];
2579
- const stripped = stripFence(raw);
2580
- if (stripped) out.push(stripped);
2581
- const blocks = [];
2582
- let depth = 0;
2583
- let start = -1;
2584
- let inString = false;
2585
- let escaped = false;
2586
- for (let i = 0; i < raw.length; i++) {
2587
- const ch = raw[i];
2588
- if (inString) {
2589
- if (escaped) escaped = false;
2590
- else if (ch === "\\") escaped = true;
2591
- else if (ch === "\"") inString = false;
2592
- continue;
2593
- }
2594
- if (ch === "\"") {
2595
- inString = true;
2596
- continue;
2597
- }
2598
- if (ch === "{") {
2599
- if (depth === 0) start = i;
2600
- depth++;
2601
- } else if (ch === "}") {
2602
- depth--;
2603
- if (depth === 0 && start >= 0) {
2604
- blocks.push(raw.slice(start, i + 1));
2605
- start = -1;
2606
- }
2607
- }
2608
- }
2609
- for (let i = blocks.length - 1; i >= 0; i--) {
2610
- const block = blocks[i];
2611
- if (!out.includes(block)) out.push(block);
566
+ function driftAuthAvailable() {
567
+ for (const key of CREDENTIAL_ENV_KEYS) {
568
+ const value = process.env[key];
569
+ if (typeof value === "string" && value.length > 0) return { ok: true };
2612
570
  }
2613
- return out;
2614
- }
2615
- function truncate$2(s, max) {
2616
- return s.length <= max ? s : `${s.slice(0, max)}... [truncated, ${s.length - max} more chars]`;
2617
- }
2618
- function stripFence(raw) {
2619
- return raw.trim().replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
2620
- }
2621
- function normaliseResult(parsed) {
2622
- if (!isObject(parsed)) return null;
2623
- const diagnosis = normaliseDiagnosis(parsed["diagnosis"]);
2624
- if (!diagnosis) return null;
571
+ if (cloudProviderEnabled()) return { ok: true };
572
+ if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
573
+ if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
2625
574
  return {
2626
- diagnosis,
2627
- confidence: typeof parsed["confidence"] === "number" ? clamp(parsed["confidence"], 0, 1) : 0,
2628
- reasoning: typeof parsed["reasoning"] === "string" ? parsed["reasoning"] : ""
575
+ ok: false,
576
+ reason: `no ${CREDENTIAL_ENV_KEYS.join(" / ")} / Bedrock or Vertex env / claude login`
2629
577
  };
2630
578
  }
2631
- function normaliseDiagnosis(raw) {
2632
- if (!isObject(raw)) return null;
2633
- switch (raw["type"]) {
2634
- case "TIMING_ISSUE": {
2635
- const fixes = normaliseSleepFixes(raw["fixes"]);
2636
- if (fixes.length === 0) return null;
2637
- return {
2638
- type: "TIMING_ISSUE",
2639
- fixes
2640
- };
2641
- }
2642
- case "OVER_ASSERTION": {
2643
- const lines = Array.isArray(raw["lines"]) ? raw["lines"].filter((n) => typeof n === "number" && Number.isFinite(n)) : [];
2644
- if (lines.length === 0) return null;
2645
- return {
2646
- type: "OVER_ASSERTION",
2647
- lines,
2648
- reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
2649
- };
2650
- }
2651
- case "SELECTOR_DRIFT": {
2652
- const line = typeof raw["line"] === "number" ? raw["line"] : null;
2653
- const oldSelector = typeof raw["oldSelector"] === "string" ? raw["oldSelector"] : null;
2654
- const newSelector = typeof raw["newSelector"] === "string" ? raw["newSelector"] : null;
2655
- if (line === null || !oldSelector || !newSelector) return null;
2656
- return {
2657
- type: "SELECTOR_DRIFT",
2658
- line,
2659
- oldSelector,
2660
- newSelector,
2661
- reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
2662
- };
2663
- }
2664
- case "DATA_MISSING": return {
2665
- type: "DATA_MISSING",
2666
- reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
2667
- };
2668
- case "UNKNOWN": return {
2669
- type: "UNKNOWN",
2670
- reason: typeof raw["reason"] === "string" ? raw["reason"] : ""
2671
- };
2672
- default: return null;
2673
- }
2674
- }
2675
- function normaliseSleepFixes(raw) {
2676
- if (!Array.isArray(raw)) return [];
2677
- const out = [];
2678
- for (const item of raw) {
2679
- if (!isObject(item)) continue;
2680
- const line = typeof item["line"] === "number" ? item["line"] : null;
2681
- if (line === null) continue;
2682
- const reason = typeof item["reason"] === "string" ? item["reason"] : "";
2683
- if (item["kind"] === "insert") {
2684
- const seconds = typeof item["seconds"] === "number" ? item["seconds"] : null;
2685
- if (seconds === null) continue;
2686
- out.push({
2687
- kind: "insert",
2688
- line,
2689
- seconds,
2690
- reason
2691
- });
2692
- } else if (item["kind"] === "increase") {
2693
- const increaseTo = typeof item["increase_to"] === "number" ? item["increase_to"] : null;
2694
- if (increaseTo === null) continue;
2695
- out.push({
2696
- kind: "increase",
2697
- line,
2698
- increase_to: increaseTo,
2699
- reason
2700
- });
2701
- }
579
+ /**
580
+ * `security find-generic-password` without `-w` only checks the item's
581
+ * existence (exit 0) — it never reads the secret, so no Keychain unlock
582
+ * prompt is triggered. Resolved via PATH so tests can stub the binary.
583
+ */
584
+ function keychainHasClaudeCredentials() {
585
+ try {
586
+ return spawnSync("security", [
587
+ "find-generic-password",
588
+ "-s",
589
+ "Claude Code-credentials"
590
+ ], {
591
+ stdio: "ignore",
592
+ timeout: 3e3
593
+ }).status === 0;
594
+ } catch {
595
+ return false;
2702
596
  }
2703
- return out;
2704
- }
2705
- function isObject(v) {
2706
- return typeof v === "object" && v !== null && !Array.isArray(v);
2707
- }
2708
- function clamp(n, lo, hi) {
2709
- if (n < lo) return lo;
2710
- if (n > hi) return hi;
2711
- return n;
2712
597
  }
2713
598
  //#endregion
2714
599
  //#region src/prompts/custom-prompt.ts
@@ -4078,10 +1963,22 @@ When you are done exploring, reply with ONLY a JSON object (no explanation, no m
4078
1963
  function retryNote(error) {
4079
1964
  return `\n\n## Previous attempt rejected\n\nYour previous reply violated the output contract: ${error}\nReply again with ONLY the JSON object described in "Output format".`;
4080
1965
  }
1966
+ function formatStep(step) {
1967
+ if (isExpandedActionStep(step)) return `- ${step.id}: ${body(step.instruction)}\n expected: ${body(step.expected)}`;
1968
+ const source = step.from ? ` (read from \`${step.from}\`)` : "";
1969
+ return `- ${step.id}: judge by LLM${source}\n claim: ${body(step.judgeByLlm)}`;
1970
+ }
1971
+ /**
1972
+ * Step text is usually a block scalar, so it arrives with newlines. Left as-is
1973
+ * its later lines sit flush against the list and read as further steps.
1974
+ */
1975
+ function body(text) {
1976
+ return text.trim().split("\n").join("\n ");
1977
+ }
4081
1978
  function buildLlmGenPrompt(input) {
4082
1979
  const sections = [];
4083
1980
  sections.push(input.taskInstructions);
4084
- const steps = input.steps.map((s) => `- ${s.id}: ${s.instruction}\n expected: ${s.expected}`).join("\n");
1981
+ const steps = input.steps.map(formatStep).join("\n");
4085
1982
  sections.push(`## Test spec\n\nTitle: ${input.specTitle}\n\nSteps:\n${steps}`);
4086
1983
  if (input.draft) sections.push(`## Mechanical draft (recorded ground truth)\n\nPath: ${input.draft.path}\n\n\`\`\`
4087
1984
  ` + input.draft.contents + "\n```");
@@ -4941,13 +2838,14 @@ const PerspectiveStatusSchema = z.object({
4941
2838
  * One step of a spec, transcribed verbatim for display. Exactly one shape per
4942
2839
  * step: an include step carries the block name it invokes (its params stay in
4943
2840
  * the spec — they are wiring, not procedure), an action step carries the
4944
- * instruction and what it expects. Never authored or reworded here: like
2841
+ * instruction and what it expects, a judge step carries the claim it asserts. Never authored or reworded here: like
4945
2842
  * `title`, this is a mechanical copy the next `ccqa perspectives` rewrites.
4946
2843
  */
4947
2844
  const PerspectiveStepSchema = z.object({
4948
2845
  include: z.string().min(1).optional(),
4949
2846
  instruction: z.string().optional(),
4950
- expected: z.string().optional()
2847
+ expected: z.string().optional(),
2848
+ judgeByLlm: z.string().optional()
4951
2849
  }).strip();
4952
2850
  /**
4953
2851
  * One test case in the inventory.
@@ -8109,7 +6007,7 @@ function buildStepDescriptions(spec, blocks) {
8109
6007
  if (!spec) return /* @__PURE__ */ new Map();
8110
6008
  try {
8111
6009
  const expanded = expandSpec(spec, { blocks });
8112
- return new Map(expanded.map((s) => [s.id, s.expected.trim()]));
6010
+ return new Map(expanded.map((s) => [s.id, (isExpandedJudgeByLlmStep(s) ? s.judgeByLlm : s.expected).trim()]));
8113
6011
  } catch {
8114
6012
  return /* @__PURE__ */ new Map();
8115
6013
  }
@@ -8703,8 +6601,6 @@ async function generateWithLlmEngine(req) {
8703
6601
  meta("conventions", conventions.sections.length);
8704
6602
  const bundle = await loadPromptBundleFromHub(ctx.hub, req.target);
8705
6603
  if (bundle) meta("prompt-bundle", bundle.loaded.join(", "));
8706
- const blocks = await loadAllBlocks(ctx.cwd);
8707
- const steps = expandSpec(ctx.spec, { blocks });
8708
6604
  const policy = {
8709
6605
  cwd: ctx.cwd,
8710
6606
  outDirAbs: resolve(ctx.cwd, outDir),
@@ -8714,7 +6610,7 @@ async function generateWithLlmEngine(req) {
8714
6610
  const prompt = buildLlmGenPrompt({
8715
6611
  taskInstructions: req.taskInstructions,
8716
6612
  specTitle: ctx.spec.title,
8717
- steps,
6613
+ steps: req.steps,
8718
6614
  draft: req.draft,
8719
6615
  ...req.draftInvariant ? { draftInvariant: req.draftInvariant } : {},
8720
6616
  resources: resources.map(toPromptResource),
@@ -10621,6 +8517,17 @@ function resolveSpecsModes(specs, catalog) {
10621
8517
  }));
10622
8518
  }
10623
8519
  //#endregion
8520
+ //#region src/targets/agent-browser/judge-steps.ts
8521
+ /**
8522
+ * Lives apart from the plugin so the paths that refuse a claim can read the
8523
+ * same declaration the plugin publishes, without importing the plugin they
8524
+ * are part of.
8525
+ */
8526
+ const AGENT_BROWSER_JUDGE_STEPS = {
8527
+ supported: false,
8528
+ reason: "it drives a browser through a model, whose `expected` already decides each step"
8529
+ };
8530
+ //#endregion
10624
8531
  //#region src/cli/stale-blocks.ts
10625
8532
  /**
10626
8533
  * Hint when stale per-block artifacts (`test.spec.ts`, `actions.json`)
@@ -12199,7 +10106,7 @@ async function loadSpecInventory(cwd) {
12199
10106
  */
12200
10107
  function describeSteps(spec, blocks, specKey) {
12201
10108
  try {
12202
- return expandSpec(spec, { blocks }).map((s) => `${oneLine$1(s.instruction)} → ${oneLine$1(s.expected)}`);
10109
+ return expandSpec(spec, { blocks }).map(describeStepBody);
12203
10110
  } catch (e) {
12204
10111
  warn(`${specKey}: could not expand include steps (${e.message}) — showing block names instead`);
12205
10112
  return spec.steps.map(describeStep);
@@ -12207,6 +10114,10 @@ function describeSteps(spec, blocks, specKey) {
12207
10114
  }
12208
10115
  function describeStep(step) {
12209
10116
  if (isIncludeStep(step)) return `include block: ${step.include}`;
10117
+ return describeStepBody(step);
10118
+ }
10119
+ function describeStepBody(step) {
10120
+ if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
12210
10121
  return `${oneLine$1(step.instruction)} → ${oneLine$1(step.expected)}`;
12211
10122
  }
12212
10123
  function oneLine$1(text) {
@@ -14299,9 +12210,12 @@ async function runOneSpec(args) {
14299
12210
  };
14300
12211
  }
14301
12212
  const spec = parseTestSpec(specContent);
14302
- const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
12213
+ const steps = expandActionSteps(spec, { blocks: await loadAllBlocks(cwd) }, `${featureName}/${specName}`, {
12214
+ id: AGENT_BROWSER_TARGET,
12215
+ reason: AGENT_BROWSER_JUDGE_STEPS.reason
12216
+ });
14303
12217
  meta("spec", spec.title);
14304
- meta("steps", expanded.length);
12218
+ meta("steps", steps.length);
14305
12219
  const includes = collectIncludedBlockNames(spec);
14306
12220
  if (includes.length > 0) meta("blocks", includes.join(", "));
14307
12221
  const sessionName = generateLiveSessionName();
@@ -14346,13 +12260,13 @@ async function runOneSpec(args) {
14346
12260
  }
14347
12261
  try {
14348
12262
  const runId = buildRunId();
14349
- const envScrubMap = buildProseEnvScrubMap(spec, expanded, { CCQA_RUN_ID: runId });
12263
+ const envScrubMap = buildProseEnvScrubMap(spec, steps, { CCQA_RUN_ID: runId });
14350
12264
  const runDir = opts.out ?? join(specDir, "runs", runId);
14351
12265
  await mkdir(runDir, { recursive: true });
14352
12266
  meta("runDir", runDir);
14353
12267
  const result = await runLiveExecutor({
14354
12268
  spec: { title: spec.title },
14355
- steps: expanded,
12269
+ steps,
14356
12270
  runId,
14357
12271
  runDir,
14358
12272
  sessionName,
@@ -15386,15 +13300,18 @@ async function generateAgentBrowserTest(ctx) {
15386
13300
  const { spec, featureName, specName, cwd, fix } = ctx;
15387
13301
  const actions = ctx.recording;
15388
13302
  if (!actions) throw new Error(`the agent-browser target needs a recording — run \`ccqa record ${featureName}/${specName}\` first`);
15389
- const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
15390
- meta("steps", expanded.length);
13303
+ const steps = expandActionSteps(spec, { blocks: await loadAllBlocks(cwd) }, `${featureName}/${specName}`, {
13304
+ id: AGENT_BROWSER_TARGET,
13305
+ reason: AGENT_BROWSER_JUDGE_STEPS.reason
13306
+ });
13307
+ meta("steps", steps.length);
15391
13308
  meta("fix-mode", fix.mode);
15392
13309
  meta("language", ctx.language);
15393
13310
  blank();
15394
13311
  const cleanedActions = await cleanupActions(actions, ctx.model);
15395
13312
  if (cleanedActions.length !== actions.length) meta("cleaned", cleanedActions.length);
15396
- const markers = buildStepMarkers(expanded, cleanedActions);
15397
- const emptySteps = findEmptySteps(expanded, cleanedActions);
13313
+ const markers = buildStepMarkers(steps, cleanedActions);
13314
+ const emptySteps = findEmptySteps(steps, cleanedActions);
15398
13315
  const warnings = emptySteps.map((e) => `step ${e.stepId} has no kept actions — generated test will skip it (notice comment inserted).`);
15399
13316
  for (const w of warnings) warn(w);
15400
13317
  const scriptPath = await saveTestScript(featureName, specName, actionsToScript({
@@ -15467,6 +13384,15 @@ function buildStepMarkers(steps, actions) {
15467
13384
  }
15468
13385
  return markers;
15469
13386
  }
13387
+ /** The last action index each step owns — where anything that follows it belongs. */
13388
+ function lastActionIndexPerStep(actions) {
13389
+ const last = /* @__PURE__ */ new Map();
13390
+ for (let i = 0; i < actions.length; i++) {
13391
+ const id = actions[i].stepId;
13392
+ if (id) last.set(id, i);
13393
+ }
13394
+ return last;
13395
+ }
15470
13396
  /**
15471
13397
  * Spec steps that lost every action by the time the trace finished its
15472
13398
  * cleanup + validation passes. `actionsToScript` uses these to splice a
@@ -15482,11 +13408,7 @@ function buildStepMarkers(steps, actions) {
15482
13408
  function findEmptySteps(steps, cleanedActions) {
15483
13409
  const presentStepIds = /* @__PURE__ */ new Set();
15484
13410
  for (const a of cleanedActions) if (a.stepId) presentStepIds.add(a.stepId);
15485
- const lastActionIndexByStep = /* @__PURE__ */ new Map();
15486
- for (let i = 0; i < cleanedActions.length; i++) {
15487
- const id = cleanedActions[i].stepId;
15488
- if (id) lastActionIndexByStep.set(id, i);
15489
- }
13411
+ const lastActionIndexByStep = lastActionIndexPerStep(cleanedActions);
15490
13412
  const notices = [];
15491
13413
  let lastSeenSurvivorIndex = -1;
15492
13414
  for (const step of steps) {
@@ -15598,12 +13520,16 @@ const agentBrowserTarget = {
15598
13520
  browserCoverage: {
15599
13521
  browser: "cdp",
15600
13522
  cdpEndpoint: acquireAgentBrowserEndpoint
15601
- }
13523
+ },
13524
+ judgeSteps: AGENT_BROWSER_JUDGE_STEPS
15602
13525
  };
15603
13526
  //#endregion
15604
13527
  //#region src/targets/playwright/emit-mechanical.ts
15605
13528
  /** Module the emitted step-boundary capture calls import from. */
15606
13529
  const STEP_EVIDENCE_MODULE = "ccqa/step-evidence";
13530
+ /** Module the emitted judge calls import from, and the call they make. */
13531
+ const JUDGE_MODULE = "ccqa/judge";
13532
+ const JUDGE_CALL = "judgeByLlm";
15607
13533
  /** Capture call emitted when a step is entered / closed. Exported for the generation gate. */
15608
13534
  const STEP_EVIDENCE_BEFORE = "ccqaStepBefore";
15609
13535
  const STEP_EVIDENCE_AFTER = "ccqaStepAfter";
@@ -15621,12 +13547,32 @@ function stepEvidenceCall(fn, marker) {
15621
13547
  function stepEvidencePreserveRule() {
15622
13548
  return `**Keep the \`${STEP_EVIDENCE_MODULE}\` calls.** The draft's \`await ${STEP_EVIDENCE_BEFORE}(page, ...)\` / \`await ${STEP_EVIDENCE_AFTER}(page, ...)\` lines are load-bearing: ccqa run reads the per-step screenshots they capture. Keep both calls for every step, in place around that step's actions, with their exact \`(page, "<stepId>", "<source>")\` arguments — and keep the import. If you move a step's actions into a page-object method, leave these two calls in the test body around the call to that method; do NOT move them inside the page object. Never wrap a step in a closure to hold them.`;
15623
13549
  }
13550
+ /**
13551
+ * Told to the rewrite, because the alternative failure is silent: a claim
13552
+ * turned into a text match passes on the wording of one run, which is the
13553
+ * assertion the judge exists to replace.
13554
+ */
13555
+ function judgePreserveRule() {
13556
+ return `**Keep the \`${JUDGE_MODULE}\` calls.** The draft's \`await ${JUDGE_CALL}(page, "<claim>")\` lines assert a claim a model decides at run time, for output whose wording changes every run. Keep each call where it is, with its claim text unchanged, and keep the import. Do NOT replace one with \`toContainText\`, \`toHaveText\` or any other match on the answer's wording, and do not move it into a page object.`;
13557
+ }
15624
13558
  function emitPlaywrightDraft(input) {
15625
- const { actions, testName, stepMarkers = [] } = input;
13559
+ const { actions, testName, stepMarkers = [], judgements = [] } = input;
15626
13560
  const markerByIndex = new Map(stepMarkers.map((m) => [m.actionIndex, m]));
15627
13561
  const lines = [];
15628
13562
  let prevLine = null;
15629
13563
  let openMarker = null;
13564
+ const flushJudgements = (afterActionIndex) => {
13565
+ for (const { step } of judgements.filter((j) => j.afterActionIndex === afterActionIndex)) {
13566
+ if (openMarker) {
13567
+ lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker));
13568
+ openMarker = null;
13569
+ }
13570
+ if (lines.length > 0) lines.push("");
13571
+ lines.push(`// step: ${step.id} [${step.source}]`);
13572
+ lines.push(judgeCall(step));
13573
+ }
13574
+ };
13575
+ flushJudgements(-1);
15630
13576
  for (let i = 0; i < actions.length; i++) {
15631
13577
  const marker = markerByIndex.get(i);
15632
13578
  if (marker) {
@@ -15638,16 +13584,19 @@ function emitPlaywrightDraft(input) {
15638
13584
  }
15639
13585
  const action = actions[i];
15640
13586
  const line = actionToLine(action);
15641
- if (line === null) continue;
15642
- if (line === prevLine) continue;
15643
- if (action.replayUnstable) lines.push(`// [warn] replay-unstable: ${action.replayReason ?? "(no reason recorded)"}`);
15644
- lines.push(line);
15645
- prevLine = line;
13587
+ if (line !== null && line !== prevLine) {
13588
+ if (action.replayUnstable) lines.push(`// [warn] replay-unstable: ${action.replayReason ?? "(no reason recorded)"}`);
13589
+ lines.push(line);
13590
+ prevLine = line;
13591
+ }
13592
+ flushJudgements(i);
15646
13593
  }
15647
13594
  if (openMarker) lines.push(stepEvidenceCall(STEP_EVIDENCE_AFTER, openMarker));
13595
+ if (judgements.length > 0) lines.unshift("test.slow();", "");
15648
13596
  const body = lines.map((l) => l === "" ? "" : ` ${l}`).join("\n");
15649
13597
  return [
15650
13598
  `import { test, expect } from "@playwright/test";`,
13599
+ ...judgements.length > 0 ? [`import { ${JUDGE_CALL} } from ${j(JUDGE_MODULE)};`] : [],
15651
13600
  ...stepMarkers.length > 0 ? [`import { ${STEP_EVIDENCE_BEFORE}, ${STEP_EVIDENCE_AFTER} } from ${j(STEP_EVIDENCE_MODULE)};`] : [],
15652
13601
  "",
15653
13602
  `test(${j(testName)}, async ({ page }) => {`,
@@ -15828,6 +13777,11 @@ const j = (s) => JSON.stringify(s);
15828
13777
  * applies to user-supplied values).
15829
13778
  */
15830
13779
  const jExpr = (s) => envRefsToJsExpression(s);
13780
+ /** One claim, asserted through the judge. Exported so the generation gate can require it back. */
13781
+ function judgeCall(step) {
13782
+ const from = step.from === void 0 ? "" : `, ${jExpr(step.from)}`;
13783
+ return `await ${JUDGE_CALL}(page, ${bracedRefsToJsExpression(step.judgeByLlm.trim())}${from});`;
13784
+ }
15831
13785
  //#endregion
15832
13786
  //#region src/targets/playwright/browser-server.ts
15833
13787
  /**
@@ -16006,6 +13960,7 @@ const playwrightTarget = {
16006
13960
  existingOutput: existingPlaywrightOutput,
16007
13961
  runner: runCommandRunner,
16008
13962
  stepEvidence: { supported: true },
13963
+ judgeSteps: { supported: true },
16009
13964
  browserCoverage: {
16010
13965
  browser: "cdp",
16011
13966
  cdpEndpoint: acquirePlaywrightBrowser
@@ -16016,11 +13971,15 @@ async function generatePlaywrightTest(ctx) {
16016
13971
  const actions = ctx.recording;
16017
13972
  if (!actions) throw new Error(`the playwright target needs a recording — run \`ccqa record ${ctx.featureName}/${ctx.specName}\` first`);
16018
13973
  const blocks = await loadAllBlocks(ctx.cwd);
16019
- const stepMarkers = buildStepMarkers(expandSpec(ctx.spec, { blocks }), actions);
13974
+ const expanded = expandSpec(ctx.spec, { blocks });
13975
+ const stepMarkers = buildStepMarkers(expanded.filter(isExpandedActionStep), actions);
13976
+ const { judgements, warnings: judgeWarnings } = placeJudgements(expanded, actions, `${ctx.featureName}/${ctx.specName}`);
13977
+ for (const w of judgeWarnings) warn(w);
16020
13978
  const draft = emitPlaywrightDraft({
16021
13979
  actions,
16022
13980
  testName: ctx.spec.title,
16023
- stepMarkers
13981
+ stepMarkers,
13982
+ judgements
16024
13983
  });
16025
13984
  const outDir = ctx.targetConfig.outDir;
16026
13985
  const draftPath = outDir ? `${outDir}/${ctx.featureName}/${ctx.specName}.spec.ts` : `${specDirRel(ctx)}/test.spec.ts`;
@@ -16031,12 +13990,13 @@ async function generatePlaywrightTest(ctx) {
16031
13990
  const result = ctx.resources.length > 0 ? await generateWithLlmEngine({
16032
13991
  ctx,
16033
13992
  target: PLAYWRIGHT_TARGET,
13993
+ steps: expanded,
16034
13994
  taskInstructions: playwrightTaskInstructions(draftPath),
16035
13995
  draft: {
16036
13996
  path: draftPath,
16037
13997
  contents: draft
16038
13998
  },
16039
- draftInvariant: stepMarkers.length > 0 ? stepEvidencePreserveRule() : ""
13999
+ draftInvariant: [stepMarkers.length > 0 ? stepEvidencePreserveRule() : "", judgements.length > 0 ? judgePreserveRule() : ""].filter(Boolean).join("\n\n")
16040
14000
  }) : await finalizePreparedFiles({
16041
14001
  ctx,
16042
14002
  target: PLAYWRIGHT_TARGET,
@@ -16048,11 +14008,15 @@ async function generatePlaywrightTest(ctx) {
16048
14008
  summary: `Playwright spec compiled from ${actions.length} recorded action(s)`,
16049
14009
  warnings: []
16050
14010
  });
16051
- const missing = await missingInjectedCalls(result, stepMarkers);
14011
+ const missing = await missingInjectedCalls(result, stepMarkers, judgements);
16052
14012
  for (const w of missing) warn(w);
16053
14013
  return {
16054
14014
  ...result,
16055
- warnings: [...result.warnings, ...missing]
14015
+ warnings: [
14016
+ ...result.warnings,
14017
+ ...judgeWarnings,
14018
+ ...missing
14019
+ ]
16056
14020
  };
16057
14021
  }
16058
14022
  /**
@@ -16063,7 +14027,7 @@ async function generatePlaywrightTest(ctx) {
16063
14027
  * them); a file that can't be read is reported as missing everything rather
16064
14028
  * than passing silently.
16065
14029
  */
16066
- async function missingInjectedCalls(result, markers) {
14030
+ async function missingInjectedCalls(result, markers, judgements) {
16067
14031
  const corpus = (await Promise.all(result.files.filter((f) => f.kind === "test").map((f) => readFile(f.path, "utf8").catch(() => "")))).join("\n");
16068
14032
  const warnings = [];
16069
14033
  for (const m of markers) {
@@ -16071,6 +14035,10 @@ async function missingInjectedCalls(result, markers) {
16071
14035
  const hasAfter = corpus.includes(stepEvidenceCall(STEP_EVIDENCE_AFTER, m));
16072
14036
  if (!hasBefore || !hasAfter) warnings.push(`step ${m.stepId}: generated test is missing its ${STEP_EVIDENCE_BEFORE}/${STEP_EVIDENCE_AFTER} call(s) — that step will have no report screenshots. A rewrite pass must not drop them.`);
16073
14037
  }
14038
+ for (const { step } of judgements) {
14039
+ if (corpus.includes(judgeCall(step))) continue;
14040
+ warnings.push(`step ${step.id}: generated test is missing its ${JUDGE_CALL} call — that claim is never decided and the spec passes without testing it. A rewrite pass must not drop or reword it.`);
14041
+ }
16074
14042
  return warnings;
16075
14043
  }
16076
14044
  /**
@@ -16087,9 +14055,53 @@ async function existingPlaywrightOutput(ref, cwd) {
16087
14055
  const specTest = resolve(cwd, `${specDirRel(ref)}/test.spec.ts`);
16088
14056
  return stat(specTest).then(() => specTest, () => null);
16089
14057
  }
14058
+ /**
14059
+ * Each claim paired with the action index it is asserted after: the last
14060
+ * action of the nearest preceding step. A claim reads what the run has
14061
+ * produced so far, so emitting it at the end of the test would judge a page
14062
+ * later steps have already navigated away from.
14063
+ *
14064
+ * A claim whose preceding steps recorded nothing has no position the
14065
+ * recording can justify. It goes last and says so, because the alternative —
14066
+ * the previous claim's index, or the start — judges a page the spec never
14067
+ * meant, and a negative claim would pass there without being tested.
14068
+ */
14069
+ function placeJudgements(expanded, actions, specKey) {
14070
+ const lastIndex = lastActionIndexPerStep(actions);
14071
+ const judgements = [];
14072
+ const warnings = [];
14073
+ let afterActionIndex = null;
14074
+ for (const step of expanded) {
14075
+ if (!isExpandedJudgeByLlmStep(step)) {
14076
+ afterActionIndex = lastIndex.get(step.id) ?? afterActionIndex;
14077
+ continue;
14078
+ }
14079
+ if (afterActionIndex === null) {
14080
+ if (expanded.indexOf(step) === 0) throw new Error(`${specKey}: step ${step.id} uses \`judgeByLlm\` as the first step — there is nothing on the page to judge yet.`);
14081
+ warnings.push(`step ${step.id}: no recorded action belongs to the steps before this claim, so it is asserted at the end of the test instead of where the spec puts it. Re-record if the claim reads something a later step navigates away from.`);
14082
+ judgements.push({
14083
+ step,
14084
+ afterActionIndex: actions.length - 1
14085
+ });
14086
+ continue;
14087
+ }
14088
+ judgements.push({
14089
+ step,
14090
+ afterActionIndex
14091
+ });
14092
+ }
14093
+ return {
14094
+ judgements,
14095
+ warnings
14096
+ };
14097
+ }
16090
14098
  //#endregion
16091
14099
  //#region src/targets/runn/index.ts
16092
14100
  const RUNN_TARGET = "runn";
14101
+ const JUDGE_STEPS = {
14102
+ supported: false,
14103
+ reason: "runn runs API scenarios; there is no page to read a claim off"
14104
+ };
16093
14105
  /**
16094
14106
  * The runn target (input: "spec"): no record phase — `ccqa generate` compiles
16095
14107
  * the spec directly into a runn runbook (YAML) via the shared LLM engine. The
@@ -16111,14 +14123,20 @@ const runnTarget = {
16111
14123
  browser: "none",
16112
14124
  reason: "runn runs API scenarios; there is no browser to measure"
16113
14125
  },
14126
+ judgeSteps: JUDGE_STEPS,
16114
14127
  guidanceKind: RUNN_TARGET
16115
14128
  };
16116
14129
  /** Exported with the engine's invoke seam so unit tests can stub Claude. */
16117
14130
  async function generateRunnRunbook(ctx, invoke) {
16118
14131
  const outDir = ctx.targetConfig.outDir;
14132
+ const blocks = await loadAllBlocks(ctx.cwd);
16119
14133
  return generateWithLlmEngine({
16120
14134
  ctx,
16121
14135
  target: RUNN_TARGET,
14136
+ steps: expandActionSteps(ctx.spec, { blocks }, `${ctx.featureName}/${ctx.specName}`, {
14137
+ id: RUNN_TARGET,
14138
+ reason: JUDGE_STEPS.reason
14139
+ }),
16122
14140
  taskInstructions: runnTaskInstructions(outDir ? `${outDir}/${ctx.featureName}/${ctx.specName}.yaml` : `${specDirRel(ctx)}/runbook.yaml`),
16123
14141
  validateFile: validateRunnFile,
16124
14142
  invoke
@@ -18306,6 +16324,7 @@ function generateSessionName() {
18306
16324
  */
18307
16325
  function buildTraceSystemPrompt(input) {
18308
16326
  const sessionName = input.sessionName ?? generateSessionName();
16327
+ const firstStepId = input.steps[0]?.id ?? "step-01";
18309
16328
  const callerGuidance = input.instruction ? `## Caller Guidance
18310
16329
 
18311
16330
  The caller provided extra guidance for this recording — for example, a drift
@@ -18723,8 +16742,8 @@ Emit:
18723
16742
  AB_ACTION|cookies_clear
18724
16743
  \`\`\`
18725
16744
 
18726
- Then emit \`STEP_START|step-01|...\` and execute the first step, prefixing
18727
- every one of its agent-browser commands with \`CCQA_STEP=step-01\`. The first
16745
+ Then emit \`STEP_START|${firstStepId}|...\` and execute the first step, prefixing
16746
+ every one of its agent-browser commands with \`CCQA_STEP=${firstStepId}\`. The first
18728
16747
  step is responsible for opening the initial URL.
18729
16748
  `;
18730
16749
  }
@@ -19172,6 +17191,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
19172
17191
  await preflightAgentBrowserCommand();
19173
17192
  const spec = parseTestSpec(await readSpecFile(featureName, specName, opts.cwd));
19174
17193
  const expanded = expandSpec(spec, { blocks: await loadAllBlocks(opts.cwd) });
17194
+ const steps = expanded.filter(isExpandedActionStep);
19175
17195
  const sessionName = generateSessionName();
19176
17196
  const envScrub = buildSpecEnvScrub(spec, expanded, { CCQA_RUN_ID: sessionName });
19177
17197
  const envScrubMap = envScrub.map;
@@ -19181,14 +17201,14 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
19181
17201
  hint("load these vars before recording — pass --profile <name> (hub) or define them in .env — then re-record.");
19182
17202
  }
19183
17203
  meta("spec", spec.title);
19184
- meta("steps", expanded.length);
17204
+ meta("steps", steps.length);
19185
17205
  const includes = collectIncludedBlockNames(spec);
19186
17206
  if (includes.length > 0) meta("blocks", includes.join(", "));
19187
17207
  blank();
19188
17208
  opts.teardown?.trackSession(sessionName);
19189
17209
  const baseSystemPrompt = buildTraceSystemPrompt({
19190
17210
  title: spec.title,
19191
- steps: expanded,
17211
+ steps,
19192
17212
  sessionName,
19193
17213
  ...opts.instruction ? { instruction: opts.instruction } : {}
19194
17214
  });
@@ -19277,7 +17297,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
19277
17297
  meta("actions", validatedActions.length);
19278
17298
  meta("status", overallStatus.toUpperCase());
19279
17299
  if (overallStatus === "passed") {
19280
- for (const stepId of stepsWithoutAsserts(expanded.map((s) => s.id), validatedActions)) warn(`${stepId} recorded no assertion — nothing in the generated test verifies its 'expected'`);
17300
+ for (const stepId of stepsWithoutAsserts(steps.map((s) => s.id), validatedActions)) warn(`${stepId} recorded no assertion — nothing in the generated test verifies its 'expected'`);
19281
17301
  hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
19282
17302
  } else warn("trace FAILED — the recorded actions were saved beside the spec for diagnosis; the previous ir.json and generated code are left untouched");
19283
17303
  return {
@@ -19842,6 +17862,18 @@ async function runRecord(specPath, opts) {
19842
17862
  error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${featureName}/${specName}' instead`);
19843
17863
  process.exit(2);
19844
17864
  }
17865
+ if (!target.judgeSteps.supported) {
17866
+ const blocks = await loadAllBlocks(cwdForProfile);
17867
+ try {
17868
+ expandActionSteps(spec, { blocks }, `${featureName}/${specName}`, {
17869
+ id: target.id,
17870
+ reason: target.judgeSteps.reason
17871
+ });
17872
+ } catch (e) {
17873
+ error(e instanceof Error ? e.message : String(e));
17874
+ process.exit(2);
17875
+ }
17876
+ }
19845
17877
  if (spec.mode === "live") {
19846
17878
  error(`this spec is 'mode: live' — a live spec runs without a recording. Run 'ccqa run ${featureName}/${specName}' instead`);
19847
17879
  process.exit(2);
@@ -21572,7 +19604,8 @@ function readSpecMeta(specName, specYaml) {
21572
19604
  /**
21573
19605
  * The spec's procedure, copied verbatim for the inventory: an include step
21574
19606
  * keeps only the block name (its params are wiring, not procedure), an
21575
- * action step keeps its instruction/expected text. Anything malformed is
19607
+ * action step keeps its instruction/expected text, a judge step its claim.
19608
+ * Anything malformed is
21576
19609
  * skipped — the inventory never fails over one bad step, matching how the
21577
19610
  * rest of this sweep treats a broken spec.
21578
19611
  */
@@ -21583,6 +19616,7 @@ function transcribeSteps(raw) {
21583
19616
  if (typeof step !== "object" || step === null) continue;
21584
19617
  const s = step;
21585
19618
  if (typeof s.include === "string" && s.include.length > 0) steps.push({ include: s.include });
19619
+ else if (typeof s.judgeByLlm === "string" && s.judgeByLlm.length > 0) steps.push({ judgeByLlm: s.judgeByLlm });
21586
19620
  else if (typeof s.instruction === "string" && s.instruction.length > 0) steps.push({
21587
19621
  instruction: s.instruction,
21588
19622
  ...typeof s.expected === "string" && s.expected.length > 0 ? { expected: s.expected } : {}