ccqa 1.28.0 → 1.29.0

package/dist/bin/ccqa.mjs

@@ -237,17 +237,11 @@ const DEFAULT_SPEC_MODE = "deterministic";
 * `required: false` makes it optional. `secret: true` flags the value as
 * sensitive — codegen renders such values as `process.env.<NAME> ?? ""`
 * template literals so the secret never ends up baked into test.spec.ts.
-* `dummy` is a placeholder value surfaced by the draft / drift prompts
-* (which see the block in isolation, before any include site exists);
-* `description` is the param's semantic role, also consumed by those
-* prompts and by spec authors browsing the block.
 */
 const BlockParamSchema = z.object({
 	name: z.string().min(1),
 	required: z.boolean().optional(),
-	secret: z.boolean().optional(),
-	dummy: z.string().optional(),
-	description: z.string().optional()
+	secret: z.boolean().optional()
 }).strict();
 /**
 * Block schema. Block steps are restricted to ActionStep — nested blocks are
@@ -270,15 +264,25 @@ function isParamRequired(param) {
 }
 //#endregion
 //#region src/spec/parser.ts
-/**
-* Fields the schema used to accept, and what to do now. The schema is
-* `.strict()`, so a spec still carrying one fails on an "unrecognized key"
-* that says nothing about why it stopped being recognised. Folded into
-* `humanizeIssue`'s `unrecognized_keys` branch (below) rather than checked
-* ahead of validation, so the migration note reaches both `parseTestSpec` and
-* `parseBlockSpec` through the one place that already rewrites that error.
-*/
-const REMOVED_FIELDS = { relatedPaths: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field." };
+/** The spec/block root (an `unrecognized_keys` issue there has an empty path). */
+const atRoot = (path) => path.length === 0;
+/** A block param entry — the issue path is `params.<index>`. */
+const atBlockParam = (path) => path.length === 2 && path[0] === "params" && typeof path[1] === "number";
+const UNREAD_PARAM_FIELD = "nothing reads it (a block param reaches the prompts as its name, required and secret only). Delete the line.";
+const REMOVED_FIELDS = {
+	relatedPaths: {
+		at: atRoot,
+		message: "which specs a change affects is now decided by `ccqa select-specs`, which reads the diff instead of a declared path list. Delete the field."
+	},
+	dummy: {
+		at: atBlockParam,
+		message: UNREAD_PARAM_FIELD
+	},
+	description: {
+		at: atBlockParam,
+		message: UNREAD_PARAM_FIELD
+	}
+};
 /** Parse a spec.yaml. Schema rejections are rewritten with actionable messages. */
 function parseTestSpec(content, source = "spec.yaml") {
 	const raw = parseYamlOrThrow(content, source);
@@ -336,9 +340,9 @@ function humanizeIssue(issue, isBlock) {
 	if (issue.code === "unrecognized_keys") {
 		const keys = Array.isArray(issue.keys) ? issue.keys : [];
 		if (isBlock && keys.includes("include")) return `Nested blocks are not supported — flatten by inlining the included block's steps into this block.`;
-		const removed = keys.filter((k) => k in REMOVED_FIELDS);
-		const stillUnknown = keys.filter((k) => !(k in REMOVED_FIELDS));
-		const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k]}`);
+		const removed = keys.filter((k) => REMOVED_FIELDS[k]?.at(issue.path));
+		const stillUnknown = keys.filter((k) => !REMOVED_FIELDS[k]?.at(issue.path));
+		const parts = removed.map((k) => `\`${k}\` is no longer part of the spec schema — ${REMOVED_FIELDS[k].message}`);
 		if (stillUnknown.length > 0) parts.push(`Unknown keys: ${stillUnknown.join(", ")}`);
 		return parts.join(" ");
 	}
@@ -512,8 +516,8 @@ async function loadAllBlocks(cwd) {
 * Co-located with `loadAllBlocks` so callers don't have to remember the
 * isParamRequired / secret-default mapping.
 */
-async function loadAvailableBlocks(cwd) {
-	return [...(await loadAllBlocks(cwd)).entries()].map(([name, block]) => ({
+function projectAvailableBlocks(blocks) {
+	return [...blocks.entries()].map(([name, block]) => ({
 		name,
 		title: block.title,
 		params: (block.params ?? []).map((p) => ({
@@ -523,6 +527,10 @@ async function loadAvailableBlocks(cwd) {
 		}))
 	}));
 }
+/** `loadAllBlocks` + `projectAvailableBlocks`, for callers that need only the projection. */
+async function loadAvailableBlocks(cwd) {
+	return projectAvailableBlocks(await loadAllBlocks(cwd));
+}
 const USER_PROMPT_MAX_BYTES = 32768;
 /**
 * Load the prompt bundle from the hub for one guidance kind ("record" /
@@ -1298,6 +1306,95 @@ function opt(key, value) {
 	return value ? { [key]: value } : {};
 }
 //#endregion
+//#region src/runtime/env-scrub.ts
+/**
+* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
+* mentioned in the spec OR in any of its expanded (block-inlined) steps.
+* Used at trace time to scrub recorded Claude-text outputs so a value the
+* spec author intentionally threaded through `process.env` is preserved as
+* `${VAR}` in `ir.json` rather than baked in as the concrete
+* trace-time value.
+*
+* Why we walk `spec.steps` AND `expanded`:
+*   - `spec.steps` carries the spec's own `instruction` / `expected` + each
+*     include's raw `params` (which may themselves be `${ENV}` refs).
+*   - `expanded` carries the inlined block-internal steps, whose
+*     `instruction` / `expected` may *also* contain `${ENV}` refs that
+*     don't go through include params.
+*
+* Only refs whose env value is currently non-empty land in the map —
+* scrubbing against an empty string would corrupt unrelated empty strings
+* in the action stream. Names whose env is unset are returned via
+* `unresolved` so the caller can warn the user.
+*
+* Longer values sort first so a `${SHORT}` whose value is a substring of a
+* `${LONG}` value doesn't clobber the longer one.
+*
+* `title` is deliberately NOT scanned — it never reaches the recorded action
+* stream.
+*/
+function buildSpecEnvScrub(spec, expanded) {
+	const refNames = /* @__PURE__ */ new Set();
+	for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
+	else {
+		collect(step.instruction, refNames);
+		collect(step.expected, refNames);
+	}
+	for (const step of expanded) {
+		collect(step.instruction, refNames);
+		collect(step.expected, refNames);
+	}
+	const map = [];
+	const unresolved = [];
+	for (const name of refNames) {
+		const value = process.env[name];
+		if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
+		else unresolved.push(name);
+	}
+	map.sort((a, b) => b[0].length - a[0].length);
+	return {
+		map,
+		unresolved
+	};
+}
+function collect(value, into) {
+	for (const name of iterEnvRefNames(value)) into.add(name);
+}
+/** Shorter than this, a value is no secret and matches inside ordinary words. */
+const MIN_PROSE_SCRUB_LENGTH = 4;
+/** Long enough to clear the length bar, still ordinary prose / JSON. */
+const COMMON_PROSE_VALUES = new Set([
+	"true",
+	"false",
+	"null",
+	"none",
+	"undefined"
+]);
+/**
+* Scrub map for model output, built like {@link buildSpecEnvScrub} but
+* without the values that read as ordinary text (`"1"`, `"true"`): prose
+* runs to paragraphs, where replacing every occurrence of such a value
+* costs more meaning than it protects. Record's own scrub keeps them for
+* its single command lines; the live path reuses this one map for its Bash
+* command log too, trading that short-value coverage for not building a
+* second map.
+*/
+function buildProseEnvScrubMap(spec, expanded) {
+	return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
+}
+/**
+* Replace every occurrence of an env value with its `${VAR}` placeholder in
+* `text`. **Caller invariant**: the map must be sorted longest-value-first
+* so a shorter value doesn't shadow a longer one that contains it as a
+* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
+*/
+function scrubEnvValues(text, scrubMap) {
+	if (scrubMap.length === 0) return text;
+	let out = text;
+	for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
+	return out;
+}
+//#endregion
 //#region src/claude/native-binary.ts
 const require$1 = createRequire(import.meta.url);
 /**
@@ -1457,7 +1554,7 @@ function warnOnceIfNativeBinaryMissing() {
 	if (missing) warn(missingNativeBinaryMessage(missing));
 }
 async function invokeClaudeStreaming(options, onEvent) {
-	const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, relaxAbConstraints = false } = options;
+	const { prompt, systemPrompt, allowedTools, disableBuiltinTools = false, disableThinking = false, mcpServers, maxTurns, env, model, cwd, onAbAction, onAbActionFailed, silenceBashLog = false, envScrubMap = [], relaxAbConstraints = false } = options;
 	const resolvedModel = resolveModel(model);
 	const hasEndpointEnv = Object.keys(resolveEndpointEnv()).length > 0;
 	const mergedEnv = env || hasEndpointEnv ? withoutEmptyEndpointVars({
@@ -1561,7 +1658,7 @@ async function invokeClaudeStreaming(options, onEvent) {
 			if (msg.type === "assistant" && !silenceBashLog) {
 				for (const block of msg.message.content ?? []) if (block.type === "tool_use" && block.name === "Bash") {
 					const cmd = block.input?.["command"];
-					if (typeof cmd === "string") bash(cmd);
+					if (typeof cmd === "string") bash(scrubEnvValues(cmd, envScrubMap));
 				}
 			}
 			if (msg.type === "result") {
@@ -3486,6 +3583,36 @@ function buildDiffMcpServer(getFileDiff) {
 * with confidence 0 rather than throwing — the report must always render.
 */
 async function analyzeFailure(input, options) {
+	return scrubOutcome(await classifyFailure(input, options), options.envScrubMap ?? []);
+}
+/**
+* Mask the profile values the classifier may have quoted: its Read/Grep reach
+* the repository, so a local `.env` is in reach of its prose even when the
+* evidence it was handed is clean. A literal match — a value the model
+* paraphrases still gets through. Only what the classifier authored is
+* covered: the row's other evidence (`failureLogExcerpt`, `diffExcerpt`)
+* passes to the report and the hub as its producer wrote it.
+*/
+function scrubOutcome(outcome, scrubMap) {
+	if (scrubMap.length === 0) return outcome;
+	const scrub = (text) => scrubEnvValues(text, scrubMap);
+	const { analysis } = outcome;
+	return {
+		...outcome,
+		raw: scrub(outcome.raw),
+		analysis: {
+			...analysis,
+			headline: scrub(analysis.headline),
+			recommendation: scrub(analysis.recommendation),
+			reasoning: scrub(analysis.reasoning),
+			evidence: analysis.evidence.map((item) => ({
+				...item,
+				detail: scrub(item.detail)
+			}))
+		}
+	};
+}
+async function classifyFailure(input, options) {
 	const { result: raw, isError } = await invokeClaudeStreaming({
 		prompt: buildFailureAnalysisPrompt(input),
 		allowedTools: [
@@ -6264,6 +6391,7 @@ function createFailureAnalysisPass(deps) {
 		};
 		const baselineMissing = specDiffResult.ok ? null : specDiffResult.skip;
 		info(`failure analysis: ${featureName}/${specName}${baselineMissing ? " (no baseline — classifying from current source)" : ""}`);
+		const envScrubMap = specEnvScrubMap(input.parsedSpec, deps.parsedBlocks);
 		const outcome = await analyzeFailure({
 			script: await input.readScript(),
 			hasGeneratedSurface: true,
@@ -6283,7 +6411,8 @@ function createFailureAnalysisPass(deps) {
 		}, {
 			...deps.model ? { model: deps.model } : {},
 			cwd: deps.cwd,
-			getFileDiff: specDiff?.fileDiff ?? (() => null)
+			getFileDiff: specDiff?.fileDiff ?? (() => null),
+			envScrubMap
 		});
 		if (!printedHeader) {
 			printedHeader = true;
@@ -6297,6 +6426,20 @@ function createFailureAnalysisPass(deps) {
 		};
 	} };
 }
+/**
+* The `${VAR}` values this spec resolved, for masking them out of the
+* classifier's prose. Read from `process.env` here rather than at run start
+* (where the live path builds its own): a profile is applied once per
+* invocation, so this is still what the spec ran against.
+*/
+function specEnvScrubMap(spec, blocks) {
+	if (spec === null) return [];
+	try {
+		return buildProseEnvScrubMap(spec, expandSpec(spec, { blocks }));
+	} catch {
+		return buildProseEnvScrubMap(spec, []);
+	}
+}
 /** One classified spec's line in the failure-analysis block. */
 function printAnalysis(featureName, specName, analysis) {
 	const pct = Math.round(analysis.confidence * 100);
@@ -6349,6 +6492,7 @@ async function analyzeExternalRows(rows, run) {
 			readScript: () => readGeneratedTestSources(ref, deps.cwd),
 			failureLog: row.failureLogExcerpt ?? "",
 			specYaml: row.specYaml,
+			parsedSpec: tryParseTestSpec(row.specYaml),
 			target: row.target ?? "agent-browser",
 			artifactsDir: readableArtifactsDir(ref, deps)
 		});
@@ -9365,93 +9509,6 @@ function isStorageStateShape(state) {
 	return typeof state === "object" && state !== null && Array.isArray(state.cookies) && Array.isArray(state.origins);
 }
 //#endregion
-//#region src/runtime/env-scrub.ts
-/**
-* Build a list of `[envValue, "${VAR}"]` pairs for every `${VAR}` reference
-* mentioned in the spec OR in any of its expanded (block-inlined) steps.
-* Used at trace time to scrub recorded Claude-text outputs so a value the
-* spec author intentionally threaded through `process.env` is preserved as
-* `${VAR}` in `ir.json` rather than baked in as the concrete
-* trace-time value.
-*
-* Why we walk `spec.steps` AND `expanded`:
-*   - `spec.steps` carries the spec's own `instruction` / `expected` + each
-*     include's raw `params` (which may themselves be `${ENV}` refs).
-*   - `expanded` carries the inlined block-internal steps, whose
-*     `instruction` / `expected` may *also* contain `${ENV}` refs that
-*     don't go through include params.
-*
-* Only refs whose env value is currently non-empty land in the map —
-* scrubbing against an empty string would corrupt unrelated empty strings
-* in the action stream. Names whose env is unset are returned via
-* `unresolved` so the caller can warn the user.
-*
-* Longer values sort first so a `${SHORT}` whose value is a substring of a
-* `${LONG}` value doesn't clobber the longer one.
-*
-* `title` is deliberately NOT scanned — it never reaches the recorded action
-* stream.
-*/
-function buildSpecEnvScrub(spec, expanded) {
-	const refNames = /* @__PURE__ */ new Set();
-	for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
-	else {
-		collect(step.instruction, refNames);
-		collect(step.expected, refNames);
-	}
-	for (const step of expanded) {
-		collect(step.instruction, refNames);
-		collect(step.expected, refNames);
-	}
-	const map = [];
-	const unresolved = [];
-	for (const name of refNames) {
-		const value = process.env[name];
-		if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
-		else unresolved.push(name);
-	}
-	map.sort((a, b) => b[0].length - a[0].length);
-	return {
-		map,
-		unresolved
-	};
-}
-function collect(value, into) {
-	for (const name of iterEnvRefNames(value)) into.add(name);
-}
-/** Shorter than this, a value is no secret and matches inside ordinary words. */
-const MIN_PROSE_SCRUB_LENGTH = 4;
-/** Long enough to clear the length bar, still ordinary prose / JSON. */
-const COMMON_PROSE_VALUES = new Set([
-	"true",
-	"false",
-	"null",
-	"none",
-	"undefined"
-]);
-/**
-* Scrub map for a live run, built like {@link buildSpecEnvScrub} but without
-* the values that read as ordinary text (`"1"`, `"true"`). A live step records
-* paragraphs of model prose, so replacing every occurrence of such a value
-* would cost more meaning than it protects; record scrubs single command
-* lines, where the same trade favours keeping them.
-*/
-function buildLiveEnvScrubMap(spec, expanded) {
-	return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
-}
-/**
-* Replace every occurrence of an env value with its `${VAR}` placeholder in
-* `text`. **Caller invariant**: the map must be sorted longest-value-first
-* so a shorter value doesn't shadow a longer one that contains it as a
-* substring. `buildSpecEnvScrub` upholds this; hand-built maps should too.
-*/
-function scrubEnvValues(text, scrubMap) {
-	if (scrubMap.length === 0) return text;
-	let out = text;
-	for (const [value, placeholder] of scrubMap) if (out.includes(value)) out = out.replaceAll(value, placeholder);
-	return out;
-}
-//#endregion
 //#region src/claude/agent-browser-invoke.ts
 function agentBrowserInvokeBase(input) {
 	return {
@@ -9749,6 +9806,7 @@ async function runLiveExecutor(input) {
 				prompt: userPrompt,
 				systemPrompt,
 				model: input.model,
+				envScrubMap: input.envScrubMap,
 				relaxAbConstraints: true
 			}, (msg) => {
 				if (msg.type !== "assistant") return;
@@ -10297,7 +10355,7 @@ async function runOneSpec(args) {
 	}
 	const spec = parseTestSpec(specContent);
 	const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
-	const envScrubMap = buildLiveEnvScrubMap(spec, expanded);
+	const envScrubMap = buildProseEnvScrubMap(spec, expanded);
 	meta("spec", spec.title);
 	meta("steps", expanded.length);
 	const includes = collectIncludedBlockNames(spec);
@@ -10359,6 +10417,7 @@ async function runOneSpec(args) {
 			specName,
 			runDir,
 			specYaml: specContent,
+			envScrubMap,
 			result
 		};
 	} finally {
@@ -10417,7 +10476,8 @@ async function analyzeOneLiveFailure(r, diffProvider, auth, blocks, opts, cwd) {
 	}, {
 		...opts.model ? { model: opts.model } : {},
 		cwd,
-		getFileDiff: specDiff?.fileDiff ?? (() => null)
+		getFileDiff: specDiff?.fileDiff ?? (() => null),
+		envScrubMap: r.envScrubMap
 	});
 	const pct = Math.round(outcome.analysis.confidence * 100);
 	const headline = outcome.analysis.headline.trim() || (outcome.analysis.reasoning.split("\n")[0] ?? "").trim();
@@ -12916,6 +12976,7 @@ async function executeRun(targets, opts) {
 	});
 	const triageUserPromptHash = triageUserPrompt ? hashTriageUserPrompt(triageUserPrompt) : null;
 	const reportDir = resolveReportDir(opts.reportDir, cwd);
+	const parsedBlocks = await loadAllBlocks(cwd);
 	const analysisDeps = {
 		diffProvider,
 		auth: diffProvider ? driftAuthAvailable() : {
@@ -12924,7 +12985,8 @@ async function executeRun(targets, opts) {
 		},
 		cwd,
 		reportDir,
-		blocks: await loadAvailableBlocks(cwd),
+		blocks: projectAvailableBlocks(parsedBlocks),
+		parsedBlocks,
 		...opts.model ? { model: opts.model } : {},
 		...opts.language ? { language: opts.language } : {},
 		customPrompt,
@@ -13373,14 +13435,13 @@ function failedSpec(s) {
 * specs share). Degrades — never throws — when Claude auth or the git diff
 * aren't available. Caller writes report.json.
 */
-async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }) {
-	const allBlocks = await loadAllBlocks(cwd);
+async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass, deps }) {
 	const results = [];
 	for (const s of summaries) {
 		const assertions = collectAssertions(s);
 		const specYaml = await tryReadSpecFile(s.featureName, s.specName, cwd);
 		const parsedSpec = tryParseTestSpec(specYaml);
-		const stepDescriptions = buildStepDescriptions(parsedSpec, allBlocks);
+		const stepDescriptions = buildStepDescriptions(parsedSpec, deps.parsedBlocks);
 		const evidence = await loadEvidenceForSpec(s.evidenceDir, reportDir, stepDescriptions);
 		const base = {
 			feature: s.featureName,
@@ -13416,6 +13477,7 @@ async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass }
 			readScript: () => readScriptSafe(s.scriptFile),
 			failureLog,
 			specYaml,
+			parsedSpec,
 			target: AGENT_BROWSER_TARGET
 		});
 		results.push({
@@ -14833,6 +14895,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
 			runId: sessionName
 		}),
 		model,
+		envScrubMap,
 		onAbAction: ({ abAction, stepId, assertMarker }) => {
 			const stepForCommand = stepTracker.fromCommand(stepId);
 			const line = abAction === void 0 ? null : scrubEnvValues(abAction, envScrubMap);

package/dist/hub-client/index.d.mts

@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
     running: "running";
   }>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
     specs: z.ZodNumber;
@@ -621,9 +621,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
 declare const RunReportDataSchema: z.ZodObject<{
   schemaVersion: z.ZodLiteral<1>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   createdAt: z.ZodString;
   runId: z.ZodNullable<z.ZodString>;

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.28.0",
+  "version": "1.29.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.28.0",
+  "version": "1.29.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {