ccqa 1.51.0 → 1.52.0

package/dist/runtime/judge.mjs

@@ -1,4 +1,1202 @@
-import { a as truncate, p as invokeClaudeStreaming, r as extractJsonCandidates } from "../diagnose-CZms9Cer.mjs";
+import { createRequire } from "node:module";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { query } from "@anthropic-ai/claude-agent-sdk";
+import { AsyncLocalStorage } from "node:async_hooks";
+import "yaml";
+import { z } from "zod";
+import { homedir } from "node:os";
+//#region src/cli/logger.ts
+/**
+* When a `withBuffer` scope is active, every log line (stdout and stderr) is
+* appended to its buffer instead of being written immediately. Parallel spec
+* runs use this so each spec's narration — including logs emitted deep inside
+* the live executor — flushes as one contiguous block, not interleaved.
+*/
+const bufferStore = new AsyncLocalStorage();
+const sinkStore = new AsyncLocalStorage();
+function emit(text, sink = process.stdout) {
+	const store = bufferStore.getStore();
+	if (store) {
+		store.out.push(text);
+		return;
+	}
+	const activeSink = sinkStore.getStore();
+	if (activeSink) {
+		activeSink.write(text);
+		return;
+	}
+	sink.write(text);
+}
+function write(scope, message, sink = process.stdout) {
+	emit(`[${scope}] ${message}\n`, sink);
+}
+function bash(command) {
+	emit(`  $ ${command.slice(0, 120)}\n`);
+}
+function warn(message) {
+	write("warn", message, process.stderr);
+}
+//#endregion
+//#region src/ir/from-agent-browser.ts
+/**
+* Normalization from the agent-browser side of the recorder into the IR.
+* The trace protocol emits one pipe-delimited `AB_ACTION|...` line per
+* browser action (see `src/prompts/trace.ts` and
+* `claude/invoke.ts:extractAbActionFromBashCommand`); `parseAbActionLine`
+* turns each line into a `RecordedAction`.
+*
+* The mapping is a deterministic re-encoding: `to-agent-browser.ts` is its
+* inverse, and the round-trip identity (ab argv → wire → IR → ab argv) is
+* pinned by `roundtrip.test.ts`.
+*/
+/**
+* Semantic locator strategies exposed by `agent-browser find`. Used by the
+* `find_*` wire commands when a target cannot be uniquely picked out by the
+* ALLOWED CSS forms (e.g. repeated `aria-label='1 reply'` rows where only
+* "the last one" is meaningful).
+*
+* `first` / `last` / `nth` are positional helpers whose value carries an
+* inner CSS selector (`nth` additionally needs an index); they normalize to
+* a `css` Locator plus `index`. The remaining strategies read the value as
+* the human-visible text/id and normalize to the matching `Locator.by`.
+*/
+const FIND_LOCATORS = [
+	"role",
+	"text",
+	"label",
+	"placeholder",
+	"alt",
+	"title",
+	"testid",
+	"first",
+	"last",
+	"nth"
+];
+/**
+* Actions reachable via `agent-browser find <locator> ... <action>`. Kept
+* here next to the locator list so all `find` wire knowledge lives in one
+* place — `claude/invoke.ts` imports these instead of redefining its own sets.
+*/
+const FIND_ACTIONS = [
+	"click",
+	"dblclick",
+	"fill",
+	"type",
+	"hover",
+	"focus",
+	"check",
+	"uncheck"
+];
+new Set(Object.keys({
+	element_enabled: "enabled",
+	element_disabled: "enabled",
+	element_checked: "checked",
+	element_unchecked: "checked"
+}));
+//#endregion
+//#region src/spec/yaml-schema.ts
+/**
+* An action step: one user-facing browser interaction. `instruction` and
+* `expected` are the natural-language description handed to Claude during
+* `ccqa trace`. URLs live inside `instruction`, either verbatim or via
+* `${ENV_VAR}` references (resolved at runtime).
+*/
+const ActionStepSchema = z.object({
+	instruction: z.string().min(1),
+	expected: z.string().min(1)
+}).strict();
+/**
+* An include step: invokes a reusable block (`.ccqa/blocks/<name>/spec.yaml`).
+* `params` values are plain strings; env refs (`${VAR}`) inside them are
+* resolved at expand time the same way step instructions are.
+*/
+const IncludeStepSchema = z.object({
+	include: z.string().min(1),
+	params: z.record(z.string(), z.string()).optional()
+}).strict();
+/**
+* A claim about the page decided by a model rather than by a selector match,
+* for output a run cannot predict. `from` narrows what it reads to one
+* element; omitted, the page's visible text. See docs/spec.md.
+*/
+const JudgeByLlmStepSchema = z.object({
+	judgeByLlm: z.string().min(1),
+	from: z.string().min(1).optional()
+}).strict();
+/**
+* A spec step is an action, an include, or a judge-by-LLM — discriminated by the
+* presence of the `include` / `judgeByLlm` key (see the predicates below).
+*/
+const StepSchema = z.union([
+	ActionStepSchema,
+	IncludeStepSchema,
+	JudgeByLlmStepSchema
+]);
+/**
+* Execution mode for `ccqa run`:
+*   - `deterministic` (default): vitest replays the recorded `test.spec.ts`.
+*   - `live`: Claude drives agent-browser per step (for fragile UIs where
+*     codegen is impractical). Cost ~$0.5 per spec.
+*/
+const SpecModeSchema = z.enum(["deterministic", "live"]);
+/**
+* A name a spec chooses that ccqa resolves to a path or looks up in a
+* registry. Restricted to a slug so it cannot escape a directory.
+*/
+function slug(what) {
+	return z.string().min(1).regex(/^[a-z0-9][a-z0-9._-]*$/i, `${what} must be a slug (letters, digits, '.', '_', '-'; no path separators)`);
+}
+/**
+* A saved browser session (cookies + localStorage) to restore before the spec
+* runs, resolved to `.ccqa/sessions/<profile>/<name>.json` at run time.
+*/
+const SessionNameSchema = slug("session name");
+/**
+* Sessions to restore before a `mode: live` spec runs: one name or a list,
+* always read back as a list. Multiple names are merged (their cookies +
+* localStorage are unioned) and restored together, so a spec can start
+* signed-in to several providers at once.
+*/
+const SessionFieldSchema = z.union([SessionNameSchema, z.array(SessionNameSchema).min(1)]).transform((v) => Array.isArray(v) ? v : [v]);
+/**
+* A generation-target id: which plugin turns this spec into runnable tests
+* (e.g. "agent-browser", "playwright", "runn"). Whether the id names a
+* registered target is the registry's responsibility, so new targets don't
+* require a schema change.
+*/
+const TargetIdSchema = slug("target");
+z.object({
+	title: z.string().min(1),
+	disabled: z.boolean().optional(),
+	target: TargetIdSchema.optional(),
+	mode: SpecModeSchema.optional(),
+	session: SessionFieldSchema.optional(),
+	steps: z.array(StepSchema).min(1)
+}).strict().superRefine((spec, ctx) => {
+	if (spec.target === void 0 || spec.target === "agent-browser") return;
+	for (const key of ["mode", "session"]) if (spec[key] !== void 0) ctx.addIssue({
+		code: "custom",
+		path: [key],
+		message: `\`${key}\` only applies to the agent-browser target — remove it or drop \`target: ${spec.target}\``
+	});
+});
+/**
+* A block param declaration. `required` defaults to true; only explicit
+* `required: false` makes it optional. `secret: true` flags the value as
+* sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
+* template literals so the secret never ends up baked into test.spec.ts.
+*/
+const BlockParamSchema = z.object({
+	name: z.string().min(1),
+	required: z.boolean().optional(),
+	secret: z.boolean().optional()
+}).strict();
+z.object({
+	title: z.string().min(1),
+	params: z.array(BlockParamSchema).optional(),
+	steps: z.array(z.union([ActionStepSchema, JudgeByLlmStepSchema])).min(1)
+}).strict();
+//#endregion
+//#region src/runtime/literal-scrub.ts
+/**
+* Opaque machine-generated id shapes. Only the unambiguous forms are listed:
+* long digit or hex runs also appear in addresses a step legitimately names
+* (a date path, a numeric tenant id, a content hash), and a false hit here
+* dead-ends that step — the guard in `claude/invoke.ts` blocks the `open`
+* and the navigate check below drops the action. A missed id still fails
+* verification at record time, so the trade is deliberate.
+*
+* Boundaries are alphanumeric lookarounds, not `\b`, so `item_01H8…`
+* (word-char `_`) is still caught; `i` accepts lowercase ULIDs (a 26-letter
+* lowercase run inside a URL is rare enough to risk). Ids that come from
+* `${ENV_VAR}` values are symbolised before either check runs, never match.
+*/
+const OPAQUE_ID_PATTERNS = [{
+	id: "ulid",
+	pattern: /(?<![0-9A-Za-z])[0-9A-HJKMNP-TV-Z]{26}(?![0-9A-Za-z])/i,
+	label: "ULID"
+}, {
+	id: "uuid",
+	pattern: /(?<![0-9A-Za-z])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9A-Za-z])/i,
+	label: "UUID"
+}];
+/** First opaque-id hit in `text`, or null. Also used by the `open` guard in `claude/invoke.ts`. */
+function findOpaqueIdSegment(text) {
+	for (const p of OPAQUE_ID_PATTERNS) {
+		const m = text.match(p.pattern);
+		if (m) return {
+			patternId: p.id,
+			match: m[0]
+		};
+	}
+	return null;
+}
+//#endregion
+//#region src/runtime/env-scrub.ts
+/**
+* Replace every occurrence of an env value with its `${VAR}` placeholder in
+* `text`. **Caller invariant**: the map must be sorted longest-value-first
+* so a shorter value doesn't shadow a longer one that contains it as a
+* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
+*/
+function scrubEnvValues(text, scrubMap) {
+	if (scrubMap.length === 0) return text;
+	let out = text;
+	for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
+	return out;
+}
+//#endregion
+//#region src/claude/native-binary.ts
+const require = createRequire(import.meta.url);
+/**
+* The agent SDK launches Claude through a native `claude` binary that ships in
+* a per-platform package (`@anthropic-ai/claude-agent-sdk-<platform>-<cpu>`),
+* declared as an *optional* dependency of the SDK. Optional means a consumer's
+* lockfile can omit it without any install-time error — and then every Claude
+* call fails at runtime with a message that never reaches our logs. Resolving
+* the package up front lets us say so once, in a line that names the fix.
+*
+* ccqa's own package.json repeats these packages in `optionalDependencies` for
+* the same reason: a second declaration gives the resolver another chance to
+* record them. Keep that list's version range in step with the SDK's.
+*/
+function nativeBinaryPackage(platform = process.platform, arch = process.arch, musl = isMusl(platform)) {
+	return `@anthropic-ai/claude-agent-sdk-${platform}-${arch === "arm64" ? "arm64" : "x64"}${platform === "linux" && musl ? "-musl" : ""}`;
+}
+/**
+* musl builds (Alpine and friends) need their own binary. Node doesn't expose
+* the libc flavour directly; the absence of `glibcVersionRuntime` in the
+* process report is the usual proxy.
+*/
+function isMusl(platform) {
+	if (platform !== "linux") return false;
+	return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
+}
+/**
+* Name of the platform package this host needs, or `null` when it resolves.
+* The per-platform packages have no `exports`, so the manifest is reachable.
+*/
+function missingNativeBinaryPackage(resolve = require.resolve) {
+	const pkg = nativeBinaryPackage();
+	try {
+		resolve(`${pkg}/package.json`);
+		return null;
+	} catch {
+		return pkg;
+	}
+}
+/** Advice shown when the binary is absent — the package name plus how to fix it. */
+function missingNativeBinaryMessage(pkg) {
+	return `${pkg} is not installed. The Claude Agent SDK needs it to start Claude on this platform, so every Claude-backed command (run in live mode, drift, diagnose) will fail. It ships as an optional dependency of the SDK, which a lockfile can drop silently: reinstall without omitting optional dependencies, or add it to your project as a direct dependency pinned to the same version as @anthropic-ai/claude-agent-sdk.`;
+}
+//#endregion
+//#region src/claude/cost-tally.ts
+/**
+* Sum every Claude invocation made inside a scope.
+*
+* A command like `record` calls Claude several times — the browser trace, the
+* codegen cleanup, one diagnosis per auto-fix retry — and the caller wants one
+* number for the whole command. Threading a cost out of each of those return
+* types would touch every layer in between, so the tally is scoped instead:
+* `invokeClaudeStreaming` adds to whichever scope is active, and nothing
+* between the two has to know.
+*
+* Scoped rather than module-global because commands run specs concurrently
+* (`drift` uses a pool). Two scopes must not fold into each other.
+*/
+const tallyStore = new AsyncLocalStorage();
+/** Record one invocation against the active scope. No-op outside one. */
+function tallyInvocation(cost) {
+	tallyStore.getStore()?.push(cost);
+}
+//#endregion
+//#region src/claude/env-keys.ts
+/**
+* Variables that carry a credential the Claude Code process can use on its
+* own, with no login on the host: an API key, a gateway bearer token, or a
+* subscription token from `claude setup-token`.
+*/
+const CREDENTIAL_ENV_KEYS = [
+	"ANTHROPIC_API_KEY",
+	"ANTHROPIC_AUTH_TOKEN",
+	"CLAUDE_CODE_OAUTH_TOKEN"
+];
+/**
+* Standard Claude Code environment variables that select the API endpoint and
+* credentials. ccqa forwards whichever of these are set to the underlying
+* Claude Code process; it does not read or interpret their values.
+*
+* - `ANTHROPIC_BASE_URL`      — the API endpoint to send requests to.
+* - `ANTHROPIC_AUTH_TOKEN`    — sent as `Authorization: Bearer <token>`.
+* - `ANTHROPIC_API_KEY`       — API key, when used instead of a token.
+* - `ANTHROPIC_CUSTOM_HEADERS` — extra request headers.
+* - `CLAUDE_CODE_OAUTH_TOKEN` — long-lived subscription token from
+*   `claude setup-token`, the headless-CI counterpart of a login.
+*/
+const ENDPOINT_ENV_KEYS = [
+	"ANTHROPIC_BASE_URL",
+	"ANTHROPIC_CUSTOM_HEADERS",
+	...CREDENTIAL_ENV_KEYS
+];
+//#endregion
+//#region src/claude/invoke.ts
+/** The built-in tools an allow-list names: `Bash(*)` is `Bash`; `mcp__*` are not built-ins. */
+function builtinToolNames(allowedTools) {
+	const names = allowedTools.map((entry) => entry.replace(/\(.*\)$/, "")).filter((name) => !name.startsWith("mcp__"));
+	return [...new Set(names)];
+}
+function resolveModel(explicit) {
+	if (explicit) return explicit;
+	const envModel = process.env["CCQA_MODEL"];
+	return envModel && envModel.length > 0 ? envModel : void 0;
+}
+/**
+* When both credentials are present the OAuth token wins and the API key is
+* dropped. Left to the CLI the API key would win, which makes "switch a CI
+* job to the subscription token" require unwiring the key everywhere; with
+* this rule, adding the one variable is the whole switch, and removing it is
+* the whole rollback. The one place the rule lives — both the resolved view
+* and the env the SDK receives apply it through here.
+*/
+function preferOauthToken(env) {
+	if (env["CLAUDE_CODE_OAUTH_TOKEN"]) delete env["ANTHROPIC_API_KEY"];
+}
+/**
+* Drop endpoint variables that are present but empty, so an empty value never
+* reaches the Claude Code process as an override. "Set to nothing" is how a
+* caller that cannot omit the key says "use the default" — a CI job wiring
+* `ANTHROPIC_BASE_URL` from an unset repository variable, most of all.
+*/
+function withoutEmptyEndpointVars(env) {
+	const out = { ...env };
+	for (const key of ENDPOINT_ENV_KEYS) if (out[key] === "") delete out[key];
+	return out;
+}
+/**
+* The environment actually handed to the Claude Code process: the full process
+* environment with the caller's overrides on top, empty endpoint variables
+* dropped, and — when both credentials survive the merge — the API key removed
+* so the OAuth token wins.
+*
+* That removal MUST happen on the env the SDK receives, not only on the
+* resolved view: left to the CLI the API key would win, silently moving every
+* call from the subscription to metered billing when a CI job wires both
+* (which is exactly what happened before this function existed).
+*/
+function buildInvocationEnv(env) {
+	const merged = withoutEmptyEndpointVars({
+		...process.env,
+		...env
+	});
+	preferOauthToken(merged);
+	merged["CLAUDE_CODE_DISABLE_AUTO_MEMORY"] = "1";
+	return merged;
+}
+let nativeBinaryWarned = false;
+/**
+* Warn once per process when the SDK's per-platform native binary is missing:
+* every Claude call is about to fail, and the opaque per-step errors alone are
+* expensive to trace back to a lockfile that dropped an optional dependency.
+*/
+function warnOnceIfNativeBinaryMissing() {
+	if (nativeBinaryWarned) return;
+	nativeBinaryWarned = true;
+	const missing = missingNativeBinaryPackage();
+	if (missing) warn(missingNativeBinaryMessage(missing));
+}
+/** Whole minutes read best, but the ceiling is set in ms and may be seconds. */
+function formatDuration(ms) {
+	if (ms < 6e4 || ms % 6e4 !== 0) return `${Math.round(ms / 1e3)}s`;
+	const minutes = ms / 6e4;
+	return `${minutes} minute${minutes === 1 ? "" : "s"}`;
+}
+async function invokeClaudeStreaming(options, onEvent) {
+	const { prompt, systemPrompt, allowedTools, disableThinking = false, mcpServers, maxTurns, timeoutMs, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false, additionalDirectories } = options;
+	const resolvedModel = resolveModel(model);
+	const mergedEnv = buildInvocationEnv(env);
+	const abortController = new AbortController();
+	let lastAbToolUseId = null;
+	let lastAbAnswerHolds = null;
+	const claimAbToolUse = (toolUseId) => {
+		if (toolUseId !== lastAbToolUseId) return false;
+		lastAbToolUseId = null;
+		lastAbAnswerHolds = null;
+		return true;
+	};
+	const sdkOptions = {
+		systemPrompt,
+		maxTurns,
+		allowedTools,
+		tools: builtinToolNames(allowedTools),
+		strictMcpConfig: true,
+		settingSources: [],
+		permissionMode: "bypassPermissions",
+		allowDangerouslySkipPermissions: true,
+		abortController,
+		...resolvedModel ? { model: resolvedModel } : {},
+		...cwd ? { cwd } : {},
+		...additionalDirectories?.length ? { additionalDirectories } : {},
+		env: mergedEnv,
+		...mcpServers ? { mcpServers } : {},
+		...disableThinking ? { thinking: { type: "disabled" } } : {},
+		hooks: onAbAction || onAbActionFailed ? {
+			PreToolUse: [{ hooks: [async (input) => {
+				if (input.hook_event_name !== "PreToolUse") return {};
+				if (input.tool_name !== "Bash") return {};
+				const cmd = input.tool_input?.["command"];
+				if (typeof cmd !== "string") return {};
+				if (!relaxAbConstraints) {
+					if (isBlockedAbSubcommand(cmd)) return {
+						decision: "block",
+						reason: "This agent-browser subcommand is not allowed because it cannot be recorded as a structured test action. Use only the standard commands: click, check, fill, select, hover, press, wait, find (with role/text/label/placeholder/alt/title/testid/first/last/nth). Take a fresh snapshot to find the correct selector."
+					};
+					if (hasRefSelector(cmd)) return {
+						decision: "block",
+						reason: "@ref selectors (like @e14) are session-specific and change every run. They cannot be used in generated tests. Use one of the allowed selector formats instead: [aria-label='...'], text=..., [placeholder='...'], or [type='password']. Take a fresh snapshot and find the element's aria-label or visible text. If an allowed selector already clicks the element but nothing happens, the element is clipped by an inner scroll container: `scrollintoview` it (addressed by a CSS selector, not `text=`) and click again — a @ref would not have fixed that either."
+					};
+					const bareTag = findPositionalBareTag(cmd);
+					if (bareTag !== null) return {
+						decision: "block",
+						reason: `\`find ${bareTag.locator}\` with a bare tag selector (\`${bareTag.selector}\`) is rejected: it matches every <${bareTag.selector}> on the page and is non-deterministic on replay. Pass a specific attribute selector instead, e.g. \`find ${bareTag.locator} "[aria-label='...']" ${bareTag.action}\` or \`find ${bareTag.locator} "[data-qa='...']" ${bareTag.action}\`. Take a fresh snapshot to find the right attribute.`
+					};
+					if (hasMultipleAbInvocations(cmd)) return {
+						decision: "block",
+						reason: "Run each `agent-browser` call as its own Bash command. Chaining multiple invocations with &&, ;, |, or || prevents ccqa from recording them as discrete steps and lets failed attempts leak into the trace. Issue one Bash tool call per agent-browser command."
+					};
+					if (hasErrorSuppression(cmd)) return {
+						decision: "block",
+						reason: "Do not suppress errors on `agent-browser` commands. Remove `|| true`, `|| :`, `2>/dev/null`, `; true`, and similar redirects so ccqa can detect failures and roll back unsuccessful attempts. Run the command standalone and let it surface its exit code."
+					};
+					const runProducedUrl = findRunProducedOpenUrl(cmd, envScrubMap);
+					if (runProducedUrl !== null) return {
+						decision: "block",
+						reason: `Do not open ${runProducedUrl} — that address was produced by this run (the id in it belongs to a record this run created), so the generated test would open a record later runs do not have. Reach the page the way a person does: click through from where the run already is. If the spec's instruction really names this exact address, put the id in a profile variable so it survives replay.`
+					};
+				}
+				const assertMarker = relaxAbConstraints ? null : extractCcqaAssertFromBashCommand(cmd);
+				const ab = relaxAbConstraints ? null : extractAbActionFromBashCommand(cmd) ?? (assertMarker !== null ? extractObservationAbAction(cmd) : null);
+				if ((ab !== null || assertMarker !== null) && onAbAction) {
+					lastAbToolUseId = input.tool_use_id;
+					lastAbAnswerHolds = assertMarker === null ? null : markerHolds(assertMarker, ab);
+					const stepId = extractCcqaStepFromBashCommand(cmd);
+					onAbAction({
+						...ab !== null ? { abAction: ab } : {},
+						...stepId ? { stepId } : {},
+						...assertMarker !== null ? { assertMarker } : {},
+						...hasCcqaSecretPrefix(cmd) ? { secret: true } : {}
+					});
+				} else lastAbToolUseId = null;
+				return {};
+			}] }],
+			PostToolUse: [{ hooks: [async (input) => {
+				if (input.hook_event_name !== "PostToolUse") return {};
+				if (input.tool_name !== "Bash") return {};
+				const holds = lastAbAnswerHolds;
+				const output = bashToolOutput(input.tool_response);
+				const contradicted = holds !== null && output !== null && !holds(output);
+				if (!isBashToolResponseError(input.tool_response) && !contradicted) return {};
+				if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
+				return {};
+			}] }],
+			PostToolUseFailure: [{ hooks: [async (input) => {
+				if (input.hook_event_name !== "PostToolUseFailure") return {};
+				if (input.tool_name !== "Bash") return {};
+				if (claimAbToolUse(input.tool_use_id) && onAbActionFailed) onAbActionFailed();
+				return {};
+			}] }]
+		} : void 0
+	};
+	warnOnceIfNativeBinaryMissing();
+	const capTimer = timeoutMs === void 0 ? null : setTimeout(() => abortController.abort(), timeoutMs);
+	capTimer?.unref?.();
+	let result = "";
+	let answered = false;
+	let isError = false;
+	let errorDetail = null;
+	let cost = {
+		totalCostUsd: null,
+		durationMs: null,
+		durationApiMs: null,
+		numTurns: null,
+		inputTokens: null,
+		cacheCreationInputTokens: null,
+		cacheReadInputTokens: null,
+		outputTokens: null,
+		models: []
+	};
+	const q = await buildMessageStream(prompt, sdkOptions);
+	try {
+		for await (const msg of q) {
+			onEvent(msg);
+			if (msg.type === "assistant" && !silenceBashLog) {
+				for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
+					const cmd = block.input?.["command"];
+					if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
+				}
+			}
+			if (msg.type === "result") {
+				answered = true;
+				isError = msg.is_error ?? false;
+				if (msg.subtype === "success") result = msg.result;
+				else {
+					result = "";
+					errorDetail = `SDK reported ${msg.subtype}`;
+				}
+				cost = extractInvocationCost(msg);
+			}
+		}
+	} catch (err) {
+		isError = true;
+		errorDetail = err instanceof Error ? err.message : String(err);
+		if (!result) result = errorDetail;
+	} finally {
+		if (capTimer) clearTimeout(capTimer);
+	}
+	if (abortController.signal.aborted && timeoutMs !== void 0 && !answered) {
+		isError = true;
+		errorDetail = `stopped after ${formatDuration(timeoutMs)} (host time limit)`;
+		result = errorDetail;
+	}
+	tallyInvocation(cost);
+	return {
+		result,
+		isError,
+		errorDetail,
+		cost
+	};
+}
+/**
+* Pull the cost / usage / turn / duration fields off the SDK `result` message.
+* The SDK's success and error result shapes share these fields, so we read
+* them defensively as `unknown` and coerce — newer SDK versions may rename a
+* field without breaking our extraction.
+*/
+function extractInvocationCost(msg) {
+	const m = msg;
+	const usage = m["usage"];
+	const modelUsage = m["modelUsage"];
+	const num = (v) => typeof v === "number" && Number.isFinite(v) ? v : null;
+	const models = modelUsage && typeof modelUsage === "object" ? Object.keys(modelUsage) : [];
+	return {
+		totalCostUsd: pricedForClaude(models) ? num(m["total_cost_usd"]) : null,
+		durationMs: num(m["duration_ms"]),
+		durationApiMs: num(m["duration_api_ms"]),
+		numTurns: num(m["num_turns"]),
+		inputTokens: num(usage?.["input_tokens"]),
+		cacheCreationInputTokens: num(usage?.["cache_creation_input_tokens"]),
+		cacheReadInputTokens: num(usage?.["cache_read_input_tokens"]),
+		outputTokens: num(usage?.["output_tokens"]),
+		models
+	};
+}
+/**
+* The SDK prices an unknown model id at a default Claude rate rather than
+* returning null, so a self-hosted model would report dollars nobody is billed.
+*/
+function pricedForClaude(models) {
+	return models.every((id) => /claude/i.test(id));
+}
+const BLOCKED_AB_SUBCOMMANDS = new Set([
+	"eval",
+	"js",
+	"label",
+	"textbox"
+]);
+/**
+* Shell-aware tokenizer: splits a command string into tokens respecting single/double quotes.
+* e.g. `click "[role='dialog'] button:last-child"` → ["click", "[role='dialog'] button:last-child"]
+*/
+function shellTokenize(s) {
+	const tokens = [];
+	let cur = "";
+	let quote = null;
+	for (let i = 0; i < s.length; i++) {
+		const ch = s[i];
+		if (quote) if (ch === quote) quote = null;
+		else cur += ch;
+		else if (ch === "\"" || ch === "'") quote = ch;
+		else if (ch === " " || ch === "	") {
+			if (cur) {
+				tokens.push(cur);
+				cur = "";
+			}
+		} else cur += ch;
+	}
+	if (cur) tokens.push(cur);
+	return tokens;
+}
+/**
+* Positional tokens of an `agent-browser [flags] <subcommand> [args...]`
+* command string, value-taking flags dropped. Empty when `cmd` is not an
+* agent-browser call.
+*/
+function abPositionalTokens(cmd) {
+	const abIdx = cmd.indexOf("agent-browser");
+	if (abIdx === -1) return [];
+	const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
+	let i = 0;
+	while (i < parts.length && parts[i].startsWith("-")) i += 2;
+	return parts.slice(i);
+}
+/** Extracts the subcommand from an `agent-browser [flags] <subcommand> [args...]` command string. */
+function extractAbSubcommand(cmd) {
+	return abPositionalTokens(cmd)[0] ?? null;
+}
+/** Returns true if the agent-browser subcommand is blocked (eval/js/find/etc). */
+function isBlockedAbSubcommand(cmd) {
+	const sub = extractAbSubcommand(cmd);
+	return sub !== null && BLOCKED_AB_SUBCOMMANDS.has(sub);
+}
+/**
+* Detects "the Bash tool returned an error" from a SDK PostToolUse hook's
+* `tool_response`. The SDK can shape this two ways depending on how Claude
+* Code reports Bash failures:
+*
+*   - `{ is_error: true, ... }`              — the canonical Bash failure shape
+*   - `{ output, exitCode, killed?, ... }`   — the BashOutput shape; treat
+*                                              non-zero exit / kill as error
+*
+* We accept either. Anything else (including missing fields) is treated as a
+* successful response so we never roll back over an unrelated tool call.
+*/
+function isBashToolResponseError(tool_response) {
+	if (tool_response === null || typeof tool_response !== "object") return false;
+	const r = tool_response;
+	if (r["is_error"] === true) return true;
+	if (typeof r["exitCode"] === "number" && r["exitCode"] !== 0) return true;
+	if (r["killed"] === true) return true;
+	return false;
+}
+/** The Bash tool's own output, when the response carries one. */
+function bashToolOutput(tool_response) {
+	if (tool_response === null || typeof tool_response !== "object") return null;
+	const output = tool_response["output"];
+	return typeof output === "string" ? output : null;
+}
+/**
+* Whether a marked probe's printed answer bears its marker out.
+*
+* `get count`, `get url` and `is` all exit 0 whatever they answer, so the
+* exit code says only that the probe ran. The marker says what the step
+* expected; this reads the answer and says whether it agreed. An unknown
+* marker, or a probe with nothing to compare, answers null — nothing to
+* check is not the same as a check that failed.
+*/
+function markerHolds(marker, abAction) {
+	if (marker.startsWith("url_contains:")) {
+		const substring = marker.slice(13);
+		return substring ? (out) => out.includes(substring) : null;
+	}
+	const parts = abAction === null ? [] : abAction.split("|");
+	if (parts[1] === "get_count") {
+		const present = (out) => Number.parseInt(out.trim(), 10);
+		if (marker === "element_visible") return (out) => present(out) > 0;
+		if (marker === "element_not_visible") return (out) => present(out) === 0;
+		return null;
+	}
+	if (parts[1] === "is") {
+		const yes = marker === "element_enabled" || marker === "element_checked";
+		if (!yes && !(marker === "element_disabled" || marker === "element_unchecked")) return null;
+		return (out) => out.trim() === String(yes);
+	}
+	return null;
+}
+/**
+* Detect `agent-browser ... find first|last|nth <bare-tag> <action>`. A bare
+* tag inside a *positional* finder matches every element of that tag on the
+* page, so "the last button" picks a different element whenever the page
+* shape shifts — recorded tests built on top are flaky by construction. The
+* check is narrow on purpose: `find role button --name X` is fine because
+* role + accessible name stays stable.
+*/
+function findPositionalBareTag(cmd) {
+	if (extractAbSubcommand(cmd) !== "find") return null;
+	const abIdx = cmd.indexOf("agent-browser");
+	const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
+	let i = 0;
+	while (i < parts.length && parts[i].startsWith("-")) i += 2;
+	const locator = parts[i + 1];
+	if (locator !== "first" && locator !== "last" && locator !== "nth") return null;
+	const innerIdx = locator === "nth" ? i + 3 : i + 2;
+	const inner = parts[innerIdx];
+	const action = parts[innerIdx + 1] ?? "";
+	if (!inner) return null;
+	if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(inner)) return null;
+	return {
+		locator,
+		selector: inner,
+		action
+	};
+}
+/** Returns true if any argument to an agent-browser command uses a @ref selector (e.g. @e14). */
+function hasRefSelector(cmd) {
+	const abIdx = cmd.indexOf("agent-browser");
+	if (abIdx === -1) return false;
+	const parts = shellTokenize(cmd.slice(abIdx + 13).trim());
+	let i = 0;
+	while (i < parts.length && parts[i].startsWith("-")) i += 2;
+	i++;
+	for (; i < parts.length; i++) if (/^@/.test(parts[i])) return true;
+	return false;
+}
+/**
+* Detect `agent-browser open <url>` whose address carries an opaque
+* machine-generated id (ULID / UUID) that no `${ENV_VAR}` value accounts
+* for. Such an address was produced by the run itself — this is the
+* mechanical form of the trace prompt's "Open only a URL the step names"
+* rule, which alone does not stop every model. Returns the env-scrubbed URL
+* for the block message, or null when the command is fine.
+*/
+function findRunProducedOpenUrl(cmd, envScrubMap) {
+	const [sub, url] = abPositionalTokens(cmd);
+	if (sub !== "open" || !url) return null;
+	const scrubbed = scrubEnvValues(url, envScrubMap);
+	return findOpaqueIdSegment(scrubbed) !== null ? scrubbed : null;
+}
+/**
+* Split `cmd` into shell statements at unquoted separators (`;`, `|`, `&`,
+* newline; consecutive separator chars like `&&` count once). String
+* literals are honoured so `fill "a;b"` stays a single statement. This is a
+* heuristic split (no subshell grammar), shared by the compound-invocation
+* guard and the CCQA_STEP prefix extraction so both agree on what "one
+* command" means.
+*/
+function splitShellStatements(cmd) {
+	const statements = [];
+	let start = 0;
+	let quote = null;
+	for (let i = 0; i < cmd.length; i++) {
+		const ch = cmd[i];
+		if (quote) {
+			if (ch === quote) quote = null;
+			continue;
+		}
+		if (ch === "\"" || ch === "'" || ch === "`") {
+			quote = ch;
+			continue;
+		}
+		if (ch === ";" || ch === "|" || ch === "&" || ch === "\n") {
+			statements.push(cmd.slice(start, i));
+			while (i + 1 < cmd.length && (cmd[i + 1] === "|" || cmd[i + 1] === "&" || cmd[i + 1] === ";" || cmd[i + 1] === "\n")) i++;
+			start = i + 1;
+		}
+	}
+	statements.push(cmd.slice(start));
+	return statements;
+}
+/** One leading `KEY=value` env assignment; value may be single/double-quoted. */
+const ENV_ASSIGN_HEAD_RE = /^([A-Za-z_][A-Za-z0-9_]*)=(?:"([^"]*)"|'([^']*)'|(\S*))(?:\s+|$)/;
+/**
+* Split a statement into its leading `KEY=value` env assignments and the
+* command they prefix. An assignment whose value is a command substitution
+* (`$(...)` / backticks) is NOT treated as a prefix — `result=$(agent-browser
+* ... snapshot)` is an assignment statement, not an agent-browser invocation,
+* and must stay invisible to the guards below.
+*/
+function splitLeadingEnvAssignments(statement) {
+	const env = /* @__PURE__ */ new Map();
+	let command = statement.trimStart();
+	for (;;) {
+		const m = ENV_ASSIGN_HEAD_RE.exec(command);
+		if (!m) break;
+		const value = m[2] ?? m[3] ?? m[4] ?? "";
+		if (value.startsWith("$(") || value.startsWith("`")) return {
+			env: /* @__PURE__ */ new Map(),
+			command: statement.trimStart()
+		};
+		env.set(m[1], value);
+		command = command.slice(m[0].length);
+	}
+	return {
+		env,
+		command
+	};
+}
+/** True when `command` starts with `agent-browser` as the command word. */
+function isAgentBrowserHead(command) {
+	if (!command.startsWith("agent-browser")) return false;
+	const after = command[13];
+	return after === void 0 || !/[A-Za-z0-9_\-]/.test(after);
+}
+/** Step ids passed via `CCQA_STEP=<step-id>` must be a plain slug. */
+const STEP_SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
+/**
+* Extract the step id from the `CCQA_STEP=<step-id>` env prefix on the
+* agent-browser invocation in `cmd` (e.g. `CCQA_STEP=step-03 agent-browser
+* --session s click "text=Submit"`). The prefix may sit anywhere in the
+* leading env-assignment run (`FOO=x CCQA_STEP=step-02 agent-browser ...`),
+* and the invocation may be a later statement of a compound command
+* (`cd app && CCQA_STEP=step-01 agent-browser ...`). Returns null when the
+* prefix is absent or its value is not a valid slug — callers then fall back
+* to the STEP_START text protocol.
+*/
+function extractCcqaStepFromBashCommand(cmd) {
+	const value = ccqaEnvPrefix(cmd, "CCQA_STEP");
+	return value !== null && STEP_SLUG_RE.test(value) ? value : null;
+}
+/**
+* The value of a `CCQA_*` env prefix on the agent-browser invocation in `cmd`.
+*
+* The three markers the trace protocol carries this way — the step, the assert,
+* the secret — differ only in what they do with the value, so how a prefix is
+* found is written once: changing what counts as an invocation (a compound
+* command, another leading assignment) must not reach two of them and miss one.
+*/
+function ccqaEnvPrefix(cmd, name) {
+	for (const statement of splitShellStatements(cmd)) {
+		const { env, command } = splitLeadingEnvAssignments(statement);
+		if (!isAgentBrowserHead(command)) continue;
+		return env.get(name) ?? null;
+	}
+	return null;
+}
+/**
+* Extract the assert marker from the `CCQA_ASSERT=<marker>` env prefix on
+* the agent-browser invocation in `cmd`, e.g. `CCQA_STEP=step-03
+* CCQA_ASSERT=1 agent-browser --session s wait --text "Submitted" --timeout
+* 3000`. The marker declares that the command verifies a step signal;
+* `promoteMarkedAssert` maps it onto recorded assert action(s). Returns the
+* raw value — semantic validation (which markers combine with which
+* commands) happens at promotion time so mismatches surface as warnings
+* instead of being silently dropped here. Returns null when the prefix is
+* absent or empty.
+*/
+function extractCcqaAssertFromBashCommand(cmd) {
+	const value = ccqaEnvPrefix(cmd, "CCQA_ASSERT");
+	return value !== null && value.length > 0 ? value : null;
+}
+/**
+* Whether the agent-browser invocation in `cmd` carries `CCQA_SECRET=1`,
+* which the trace prompt asks for on a command that types into a password
+* field. The same channel as `CCQA_STEP` and `CCQA_ASSERT`, for the same
+* reason: the command line is the one place the fact is observable.
+*/
+function hasCcqaSecretPrefix(cmd) {
+	const value = ccqaEnvPrefix(cmd, "CCQA_SECRET");
+	return value !== null && value !== "" && value !== "0";
+}
+/**
+* Returns true when `cmd` contains more than one `agent-browser` invocation
+* chained together via shell operators (`&&`, `||`, `;`, `|`, newline). The
+* PreToolUse hook only records ONE AB_ACTION per Bash call, so chained
+* invocations would silently drop every intermediate failure — turning
+* "I tried four selectors before one worked" into a clean-looking trace
+* with five orphaned actions that later fail at replay.
+*
+* Counts statements whose command word is `agent-browser`, skipping any
+* leading env assignments (the trace protocol prefixes every invocation
+* with `CCQA_STEP=<step-id>`). String literals are honoured so
+* `agent-browser fill 'agent-browser'` doesn't false-fire.
+*/
+function hasMultipleAbInvocations(cmd) {
+	let count = 0;
+	for (const statement of splitShellStatements(cmd)) {
+		if (!isAgentBrowserHead(splitLeadingEnvAssignments(statement).command)) continue;
+		count++;
+		if (count > 1) return true;
+	}
+	return false;
+}
+/**
+* Returns true when an `agent-browser` command in `cmd` has its exit
+* status hidden by a shell decorator that would prevent ccqa from rolling
+* back a failed attempt:
+*
+*   - trailing `|| true` / `|| :` / `; true` (force exit 0)
+*   - `2>/dev/null` and friends (drop stderr, sometimes paired with `|| true`)
+*
+* The agent-browser command itself returns exit 1 on selector miss, so
+* once one of these is present the PostToolUse hook sees `is_error=false`
+* and the bad attempt sneaks into ir.json.
+*/
+function hasErrorSuppression(cmd) {
+	if (cmd.indexOf("agent-browser") === -1) return false;
+	if (/\|\|\s*(true|:|\s*$|#)/.test(cmd)) return true;
+	if (/;\s*(true|:)\b/.test(cmd)) return true;
+	if (/2\s*>\s*\/dev\/null/.test(cmd)) return true;
+	if (/&\s*>\s*\/dev\/null/.test(cmd)) return true;
+	return false;
+}
+/**
+* Parse an `agent-browser --session <name> <cmd> [args...]` bash command
+* and return the corresponding AB_ACTION line, or null if not an agent-browser call.
+*/
+function extractAbActionFromBashCommand(cmd) {
+	const subCmd = extractAbSubcommand(cmd);
+	if (!subCmd) return null;
+	const abIdx = cmd.indexOf("agent-browser");
+	const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
+	let i = 0;
+	while (i < parts.length && parts[i].startsWith("-")) i += 2;
+	const args = parts.slice(i + 1);
+	switch (subCmd) {
+		case "cookies":
+			if (args[0] === "clear") return "AB_ACTION|cookies_clear";
+			return null;
+		case "open": return `AB_ACTION|open|${args[0] ?? ""}`;
+		case "press": return `AB_ACTION|press|${args[0] ?? ""}`;
+		case "scroll": return `AB_ACTION|scroll|${args.join("|")}`;
+		case "click":
+		case "dblclick":
+		case "check":
+		case "uncheck":
+		case "hover":
+		case "wait": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}`;
+		case "fill":
+		case "type":
+		case "select": return `AB_ACTION|${subCmd}|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
+		case "drag": return `AB_ACTION|drag|${args[0] ?? ""}|${args[1] ?? ""}|${args[2] ?? ""}`;
+		case "upload": {
+			const sel = args[0] ?? "";
+			const files = args.slice(1);
+			if (!sel || files.length === 0) return null;
+			return `AB_ACTION|upload|${sel}|${files.join("|")}`;
+		}
+		case "snapshot": return null;
+		case "find": return extractFindAbAction(args);
+		default: return null;
+	}
+}
+/**
+* Wire lines for the observation-only probes `get count <sel>`, `get url` and
+* `is <state> <sel>`.
+* These commands read state without mutating it, so they have no place in
+* the replay sequence and `extractAbActionFromBashCommand` ignores them.
+* They matter only when a `CCQA_ASSERT=<marker>` env prefix declares the
+* probe verifies a step signal — the hook layer then surfaces them via this
+* function so `promoteMarkedAssert` can turn them into recorded asserts.
+* Only consulted when a marker is present; unmarked `get` commands stay
+* unobserved as before.
+*/
+function extractObservationAbAction(cmd) {
+	const sub = extractAbSubcommand(cmd);
+	if (sub !== "get" && sub !== "is") return null;
+	const abIdx = cmd.indexOf("agent-browser");
+	const parts = shellTokenize(cmd.slice(abIdx + 13).trim()).filter((t) => !/^(2?>|[|&>])/.test(t));
+	let i = 0;
+	while (i < parts.length && parts[i].startsWith("-")) i += 2;
+	const args = parts.slice(i + 1);
+	if (sub === "is") return args[0] && args[1] ? `AB_ACTION|is|${args[0]}|${args[1]}` : null;
+	if (args[0] === "count" && args[1]) return `AB_ACTION|get_count|${args[1]}`;
+	if (args[0] === "url") return "AB_ACTION|get_url";
+	return null;
+}
+const FIND_ACTION_SET = new Set(FIND_ACTIONS);
+const FIND_LOCATOR_SET = new Set(FIND_LOCATORS);
+/**
+* Parse the positional tokens of `agent-browser find <locator> <value> [...]
+* <action> [fillValue]` and produce a canonical
+*   `AB_ACTION|find_<action>|<locator>|<value>|<extra>|<exact>|...|<label>`
+* line. The wire format keeps a fixed positional layout across locators so
+* downstream `parseAbActionLine` in `ir/from-agent-browser.ts` can split on
+* `|` alone:
+*
+*   <extra> is `--name` value for role, integer index for nth, "" otherwise.
+*   <exact> is the literal "exact" if --exact was passed, "" otherwise.
+*
+* Returns null for malformed invocations — the caller treats null as "not a
+* structured action" and the Bash command still runs unobserved.
+*/
+function extractFindAbAction(args) {
+	const locator = args[0];
+	if (!locator || !FIND_LOCATOR_SET.has(locator)) return null;
+	let i = 1;
+	let value = args[i] ?? "";
+	i++;
+	let extra = "";
+	if (locator === "nth") {
+		extra = value;
+		value = args[i] ?? "";
+		i++;
+	}
+	let action = "";
+	let name = "";
+	let exact = "";
+	let fillValue = "";
+	for (; i < args.length; i++) {
+		const tok = args[i];
+		if (tok === "--name") {
+			name = args[i + 1] ?? "";
+			i++;
+		} else if (tok === "--exact") exact = "exact";
+		else if (FIND_ACTION_SET.has(tok)) action = tok;
+		else if (tok === "text" && action === "" && locator === "role") action = tok;
+		else if (action) fillValue = tok;
+	}
+	if (!action) return null;
+	if (locator === "role") extra = name;
+	const command = `find_${action}`;
+	if (action === "fill" || action === "type") return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|${fillValue}|`;
+	return `AB_ACTION|${command}|${locator}|${value}|${extra}|${exact}|`;
+}
+async function buildMessageStream(prompt, options) {
+	const mockFile = process.env["CCQA_CLAUDE_MOCK_FILE"];
+	if (mockFile) return replayMockMessages(mockFile, options);
+	return query({
+		prompt,
+		options
+	});
+}
+async function* replayMockMessages(path, options) {
+	const raw = await readFile(path, "utf8");
+	for (const line of raw.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		const msg = JSON.parse(trimmed);
+		await fireMockPreToolUseHooks(msg, options);
+		yield msg;
+	}
+}
+/**
+* The real SDK fires PreToolUse hooks as it executes tool calls; the JSONL
+* replay approximates that by invoking the configured PreToolUse hooks for
+* every Bash tool_use block before yielding its message, so e2e stubs
+* exercise the AB_ACTION recording path (including CCQA_STEP step
+* attribution). Hook decisions are ignored and post-tool hooks are not
+* simulated — the replay runs no tools, so there is nothing to block or fail.
+*/
+async function fireMockPreToolUseHooks(msg, options) {
+	const matchers = options.hooks?.PreToolUse;
+	if (!matchers || msg.type !== "assistant") return;
+	for (const block of msg.message.content ?? []) {
+		if (block.type !== "tool_use" || block.name !== "Bash") continue;
+		const input = {
+			hook_event_name: "PreToolUse",
+			tool_name: "Bash",
+			tool_input: block.input,
+			tool_use_id: block.id,
+			session_id: "mock",
+			transcript_path: "",
+			cwd: process.cwd()
+		};
+		for (const matcher of matchers) for (const hook of matcher.hooks) await hook(input, block.id, { signal: new AbortController().signal });
+	}
+}
+//#endregion
+//#region src/drift/auth.ts
+/**
+* Claude Code can also run against AWS Bedrock / Google Vertex AI, selected by
+* these env toggles. Credentials then come from the cloud SDK's own chain
+* (instance/task roles, gcloud auth, …), so none of the Anthropic-side probes
+* below apply — a set toggle counts as auth being available. ccqa forwards the
+* toggle verbatim; only "0"/"false" (any case) is treated as explicitly off.
+*/
+const CLOUD_PROVIDER_ENV_KEYS = ["CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX"];
+function cloudProviderEnabled() {
+	return CLOUD_PROVIDER_ENV_KEYS.some((key) => {
+		const value = process.env[key]?.trim().toLowerCase();
+		return value !== void 0 && value !== "" && value !== "0" && value !== "false";
+	});
+}
+/**
+* Probe whether the host has any credential the Anthropic SDK can pick up:
+*   - one of CREDENTIAL_ENV_KEYS (API key, gateway bearer token, or the
+*     subscription token from `claude setup-token`)
+*   - CLAUDE_CODE_USE_BEDROCK / CLAUDE_CODE_USE_VERTEX (cloud-provider
+*     endpoints authenticated by the cloud SDK's credential chain)
+*   - ~/.claude/.credentials.json (Claude Code login, file-based platforms)
+*   - macOS Keychain item "Claude Code-credentials" (Claude Code login on
+*     darwin stores the OAuth credentials in the Keychain, not on disk)
+*
+* Claude-driven hooks are opt-in, so the caller only consults this after the
+* user has asked for analysis. We never throw — auth absence is a normal flow
+* that surfaces as "analysis skipped".
+*/
+function driftAuthAvailable() {
+	for (const key of CREDENTIAL_ENV_KEYS) {
+		const value = process.env[key];
+		if (typeof value === "string" && value.length > 0) return { ok: true };
+	}
+	if (cloudProviderEnabled()) return { ok: true };
+	if (existsSync(join(homedir(), ".claude", ".credentials.json"))) return { ok: true };
+	if (process.platform === "darwin" && keychainHasClaudeCredentials()) return { ok: true };
+	return {
+		ok: false,
+		reason: `no ${CREDENTIAL_ENV_KEYS.join(" / ")} / Bedrock or Vertex env / claude login`
+	};
+}
+/**
+* `security find-generic-password` without `-w` only checks the item's
+* existence (exit 0) — it never reads the secret, so no Keychain unlock
+* prompt is triggered. Resolved via PATH so tests can stub the binary.
+*/
+function keychainHasClaudeCredentials() {
+	try {
+		return spawnSync("security", [
+			"find-generic-password",
+			"-s",
+			"Claude Code-credentials"
+		], {
+			stdio: "ignore",
+			timeout: 3e3
+		}).status === 0;
+	} catch {
+		return false;
+	}
+}
+//#endregion
+//#region src/diagnose/diagnose.ts
+/**
+* Pull every plausible JSON object out of `raw`. We try, in order:
+*   1. The whole string with code fences stripped (the prompt asks for
+*      JSON-only, so this is the happy path).
+*   2. Each balanced `{...}` block found by scanning the text. The model
+*      sometimes prefixes the JSON with a "Confirmed: ..." sentence or
+*      mentions partial JSON in its tool-using reasoning; we want to
+*      try the *last* well-formed object first because it's most likely
+*      the final answer, then earlier ones as a fallback.
+*
+* The caller `JSON.parse`s each candidate and stops at the first match
+* that normalises to a known DiagnosisResult.
+*/
+function extractJsonCandidates(raw) {
+	const out = [];
+	const stripped = stripFence(raw);
+	if (stripped) out.push(stripped);
+	const blocks = [];
+	let depth = 0;
+	let start = -1;
+	let inString = false;
+	let escaped = false;
+	for (let i = 0; i < raw.length; i++) {
+		const ch = raw[i];
+		if (inString) {
+			if (escaped) escaped = false;
+			else if (ch === "\\") escaped = true;
+			else if (ch === "\"") inString = false;
+			continue;
+		}
+		if (ch === "\"") {
+			inString = true;
+			continue;
+		}
+		if (ch === "{") {
+			if (depth === 0) start = i;
+			depth++;
+		} else if (ch === "}") {
+			depth--;
+			if (depth === 0 && start >= 0) {
+				blocks.push(raw.slice(start, i + 1));
+				start = -1;
+			}
+		}
+	}
+	for (let i = blocks.length - 1; i >= 0; i--) {
+		const block = blocks[i];
+		if (!out.includes(block)) out.push(block);
+	}
+	return out;
+}
+function truncate(s, max) {
+	return s.length <= max ? s : `${s.slice(0, max)}... [truncated, ${s.length - max} more chars]`;
+}
+function stripFence(raw) {
+	return raw.trim().replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "").trim();
+}
+//#endregion
 //#region src/runtime/judge.ts
 const SYSTEM_PROMPT = [
 	"You decide whether a claim holds for a piece of text taken from a web page under test.",
@@ -13,6 +1211,8 @@ const SYSTEM_PROMPT = [
 const MAX_TEXT_CHARS = 2e4;
 /** A judge runs inside a test, so a turn that will not finish has to fail rather than hold the run. */
 const JUDGE_TIMEOUT_MS = 6e4;
+/** Name the verdict is attached under, so a report can find it by name. */
+const ATTACHMENT_NAME = "ccqa-judge";
 /**
 * Fails the test unless a model agrees the claim holds for the text read from
 * `from` (a selector; omitted, the page's body). The reason the model gave
@@ -20,12 +1220,37 @@ const JUDGE_TIMEOUT_MS = 6e4;
 *
 * A selector matching several elements judges the first, as Playwright's
 * page-level `innerText` does — narrow it if that is not what you mean.
+*
+* A string third argument is shorthand for `{ from: <string> }` — the form
+* generated tests already carry. Pass `testInfo` (Playwright's per-test
+* handle) to attach the verdict to the test's report, pass or fail.
 */
-async function judgeByLlm(page, claim, from = "body") {
+async function judgeByLlm(page, claim, options) {
+	const opts = typeof options === "string" ? { from: options } : options ?? {};
+	const from = opts.from ?? "body";
+	const model = opts.model ?? process.env["CCQA_JUDGE_MODEL"];
 	const verdict = await decideClaim({
 		claim,
-		text: await page.innerText(from)
+		text: await page.innerText(from),
+		...model ? { model } : {}
 	});
+	if (opts.testInfo) {
+		const attachment = {
+			body: JSON.stringify({
+				claim,
+				from,
+				ok: verdict.ok,
+				reason: verdict.reason
+			}, null, 2),
+			contentType: "application/json"
+		};
+		try {
+			await opts.testInfo.attach(ATTACHMENT_NAME, attachment);
+		} catch (err) {
+			const reason = err instanceof Error ? err.message : String(err);
+			process.stderr.write(`judgeByLlm: could not attach "${ATTACHMENT_NAME}" (${reason})\n`);
+		}
+	}
 	if (!verdict.ok) throw new Error(`judgeByLlm: the claim did not hold (${verdict.reason || "no reason given"})\n  claim: ${claim}\n  read from: ${from}`);
 }
 /**
@@ -35,6 +1260,7 @@ async function judgeByLlm(page, claim, from = "body") {
 * claim never goes silently unjudged.
 */
 async function decideClaim(input) {
+	if (!driftAuthAvailable().ok) throw new Error("judgeByLlm needs Claude credentials: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN (a Claude subscription token) in the environment that runs the test — a Bedrock or Vertex environment, or a local `claude login`, also count. Optionally set CCQA_JUDGE_MODEL to pick the model for judgements (otherwise CCQA_MODEL, then the Claude Code default).");
 	const { result, isError, errorDetail } = await invokeClaudeStreaming({
 		prompt: [
 			"## Claim",