ccqa 1.34.0 → 1.35.1

package/README.md

@@ -197,6 +197,26 @@ tenant, an account, a role — not an environment: ccqa tracks one
 verification environment
 ([ADR-0013](./docs/adr/0013-one-verification-environment.md)).
 
+## Agent skills
+
+`skills/` ships guides that let a coding agent (e.g. Claude Code) drive ccqa
+end-to-end on its own:
+
+- **ccqa-record** — create a new test case: pin down the behavior, write the
+  spec, choose deterministic or live mode, record, run to green.
+- **ccqa-rerecord** — bring a flagged or failing test case back to green:
+  read the hub's finding, decide what went stale, repair that, re-record.
+- **ccqa-resolve** — clear everything the hub is holding for a person: read
+  the verdict, order the rows, route each to a product fix, a re-recording or
+  an environment repair.
+
+Install them with the [skills CLI](https://github.com/vercel-labs/skills)
+into a consuming project (or `-g` for all projects):
+
+```bash
+npx skills add <this-repo> --skill ccqa-record --skill ccqa-rerecord --skill ccqa-resolve
+```
+
 ## Documentation
 
 | I want to… | Read |

package/dist/bin/ccqa.mjs

@@ -89,17 +89,19 @@ function* iterEnvRefNames(value) {
 	}
 }
 /**
-* Resolve every `$VAR` / `${VAR}` reference against the current process env.
+* Resolve every `$VAR` / `${VAR}` reference against `overrides`, then the
+* current process env. `overrides` carries values an invoker injects into a
+* child process (e.g. CCQA_RUN_ID), which beat the parent env there.
 *
 * Missing variables expand to the empty string, mirroring `sh` behaviour.
 * Throwing would force ccqa to be invoked with every var set even for
 * unused blocks, which is more user-hostile than letting the test fail
 * downstream with a clearer message ("login form rejected: empty password").
 */
-function resolveEnvRefs(value) {
+function resolveEnvRefs(value, overrides = {}) {
 	return value.replace(ENV_VAR_RE, (_, braced, plain) => {
 		const name = braced ?? plain ?? "";
-		return process.env[name] ?? "";
+		return overrides[name] ?? process.env[name] ?? "";
 	});
 }
 /**
@@ -408,6 +410,7 @@ function collectIncludedBlockNames(spec) {
 const CCQA_DIR = ".ccqa";
 const SPEC_FILE = "spec.yaml";
 const RECORDING_FILE = "ir.json";
+const FAILED_RECORDING_FILE = "ir.failed.json";
 const PERSPECTIVES_FILE = "perspectives.yaml";
 const PERSPECTIVES_MD_FILE = "perspectives.md";
 function getCcqaDir(cwd = process.cwd()) {
@@ -476,9 +479,22 @@ async function saveRecording(featureName, specName, actions, cwd) {
 	await mkdir(specDir, { recursive: true });
 	const recordingPath = join(specDir, RECORDING_FILE);
 	await writeFile(recordingPath, JSON.stringify(actions, null, 2), "utf-8");
-	await Promise.all(LEGACY_RECORDING_FILES.map((f) => unlink(join(specDir, f)).catch(() => {})));
+	await Promise.all([...LEGACY_RECORDING_FILES, FAILED_RECORDING_FILE].map((f) => unlink(join(specDir, f)).catch(() => {})));
 	return recordingPath;
 }
+/**
+* Persist the actions of a trace that FAILED. They go to a side file so a
+* recording that did not demonstrate the spec can never replace one that
+* did — `ir.json` and the generated code stay whatever they were. The next
+* successful {@link saveRecording} deletes the file.
+*/
+async function saveFailedRecording(featureName, specName, actions, cwd) {
+	const specDir = getSpecDir(featureName, specName, cwd);
+	await mkdir(specDir, { recursive: true });
+	const path = join(specDir, FAILED_RECORDING_FILE);
+	await writeFile(path, JSON.stringify(actions, null, 2), "utf-8");
+	return path;
+}
 function getBlocksDir(cwd) {
 	return join(getCcqaDir(cwd), "blocks");
 }
@@ -1326,10 +1342,12 @@ function opt(key, value) {
 *     `instruction` / `expected` may *also* contain `${ENV}` refs that
 *     don't go through include params.
 *
-* Only refs whose env value is currently non-empty land in the map —
-* scrubbing against an empty string would corrupt unrelated empty strings
-* in the action stream. Names whose env is unset are returned via
-* `unresolved` so the caller can warn the user.
+* Each ref resolves against `overrides` first, then `process.env` —
+* `overrides` carries values the invoker injects into the child process,
+* which beat the parent env there. Only refs that resolve non-empty land in
+* the map — scrubbing against an empty string would corrupt unrelated empty
+* strings in the action stream; the rest are returned via `unresolved` so
+* the caller can warn the user.
 *
 * Longer values sort first so a `${SHORT}` whose value is a substring of a
 * `${LONG}` value doesn't clobber the longer one.
@@ -1337,7 +1355,7 @@ function opt(key, value) {
 * `title` is deliberately NOT scanned — it never reaches the recorded action
 * stream.
 */
-function buildSpecEnvScrub(spec, expanded) {
+function buildSpecEnvScrub(spec, expanded, overrides = {}) {
 	const refNames = /* @__PURE__ */ new Set();
 	for (const step of spec.steps) if (isIncludeStep(step)) for (const v of Object.values(step.params ?? {})) collect(v, refNames);
 	else {
@@ -1351,7 +1369,7 @@ function buildSpecEnvScrub(spec, expanded) {
 	const map = [];
 	const unresolved = [];
 	for (const name of refNames) {
-		const value = process.env[name];
+		const value = overrides[name] ?? process.env[name];
 		if (typeof value === "string" && value.length > 0) map.push([value, "${" + name + "}"]);
 		else unresolved.push(name);
 	}
@@ -1383,8 +1401,8 @@ const COMMON_PROSE_VALUES = new Set([
 * command log too, trading that short-value coverage for not building a
 * second map.
 */
-function buildProseEnvScrubMap(spec, expanded) {
-	return buildSpecEnvScrub(spec, expanded).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
+function buildProseEnvScrubMap(spec, expanded, overrides = {}) {
+	return buildSpecEnvScrub(spec, expanded, overrides).map.filter(([value]) => value.length >= MIN_PROSE_SCRUB_LENGTH && !COMMON_PROSE_VALUES.has(value.toLowerCase()));
 }
 /**
 * Replace every occurrence of an env value with its `${VAR}` placeholder in
@@ -10570,7 +10588,6 @@ async function runOneSpec(args) {
 	}
 	const spec = parseTestSpec(specContent);
 	const expanded = expandSpec(spec, { blocks: await loadAllBlocks(cwd) });
-	const envScrubMap = buildProseEnvScrubMap(spec, expanded);
 	meta("spec", spec.title);
 	meta("steps", expanded.length);
 	const includes = collectIncludedBlockNames(spec);
@@ -10600,6 +10617,7 @@ async function runOneSpec(args) {
 	}
 	try {
 		const runId = buildRunId();
+		const envScrubMap = buildProseEnvScrubMap(spec, expanded, { CCQA_RUN_ID: runId });
 		const runDir = opts.out ?? join(specDir, "runs", runId);
 		await mkdir(runDir, { recursive: true });
 		meta("runDir", runDir);
@@ -11835,10 +11853,11 @@ async function runVitest(scriptPath, agentBrowserSession) {
 		"--config",
 		bundledVitestConfigPath(),
 		scriptPath
-	], agentBrowserSession ? { env: {
+	], { env: {
 		...process.env,
-		AGENT_BROWSER_SESSION: agentBrowserSession
-	} } : {});
+		CCQA_RUN_ID: buildRunId(),
+		...agentBrowserSession ? { AGENT_BROWSER_SESSION: agentBrowserSession } : {}
+	} });
 	const currentScript = await readFile(scriptPath, "utf8");
 	return {
 		exitCode,
@@ -14030,6 +14049,10 @@ function createRunTeardown() {
 		untrackSession(name) {
 			sessions.delete(name);
 		},
+		async closeTracked(name) {
+			await closeSession(name);
+			sessions.delete(name);
+		},
 		onFinalize(fn) {
 			finalizers.push(fn);
 		},
@@ -14304,6 +14327,7 @@ CCQA_STEP=<step-id> agent-browser --session SESSION upload "<input[type=file] se
 - \`@ref\` / \`@e1\` / \`e14\` — reference IDs are session-specific and change every run.
 - **Bare tag selectors**: \`button\`, \`a\`, \`div\`, \`td\`, \`tr\`, \`main a\`, \`table tbody tr:nth-child(N)\`. These match every element of that tag and are non-deterministic on replay. **This includes the inner selector inside \`find first/last/nth\`** — see the \`find\` rules below.
 - \`[role='button']\` or \`[type='checkbox']\` alone — matches too many elements.
+- **Playwright-only pseudo-classes**: \`:has-text()\`, \`:text-is()\`, \`:text-matches()\`, \`:visible\`. agent-browser's CSS engine does not implement them — they match nothing and every command using them fails. Use \`text=...\` or plain CSS instead.
 - JavaScript execution (\`eval\`, \`js\`) — blocked by the hook layer.
 
 ### \`find\` subset (fallback when no ALLOWED CSS uniquely targets the element)
@@ -14685,9 +14709,9 @@ const ASSERT_TIMEOUT_MS = 1e4;
 * has no side effect; assert types whose codegen forms aren't directly
 * verifiable here fall through to the caller's `unverifiable` fallback).
 */
-function actionToAbArgs(action, sessionName) {
+function actionToAbArgs(action, sessionName, envOverrides = {}) {
 	const base = ["--session", sessionName];
-	const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s);
+	const sub = (s) => s === void 0 ? "" : resolveEnvRefs(s, envOverrides);
 	switch (action.action) {
 		case "snapshot": return null;
 		case "assert": return assertToAbArgs(action, sub, sessionName);
@@ -14772,8 +14796,8 @@ const NO_STEP_ID = "__no_step__";
 * `wait <selector>`); everything else spawns the agent-browser argv. A single
 * hard-timeout (SIGTERM) retry covers the daemon's occasional under-load drop.
 */
-function runValidationAction(action, sessionName) {
-	const built = actionToAbArgs(action, sessionName);
+function runValidationAction(action, sessionName, envOverrides = {}) {
+	const built = actionToAbArgs(action, sessionName, envOverrides);
 	if (built === null) return {
 		skipped: true,
 		ok: false,
@@ -14817,7 +14841,7 @@ function validateActions(actions, opts) {
 			});
 			continue;
 		}
-		const outcome = runValidationAction(action, opts.sessionName);
+		const outcome = runValidationAction(action, opts.sessionName, opts.envOverrides);
 		if (outcome.skipped) {
 			kept.push(action);
 			continue;
@@ -14894,7 +14918,7 @@ function rescueLostSteps(actions, kept, dropped, opts) {
 	for (const [stepId, drops] of lostStepDrops.entries()) {
 		let anyForThisStep = false;
 		for (const d of drops) {
-			const outcome = runValidationAction(d.action, opts.sessionName);
+			const outcome = runValidationAction(d.action, opts.sessionName, opts.envOverrides);
 			if (outcome.skipped) continue;
 			if (outcome.ok) {
 				rescuedIndices.add(d.index);
@@ -15076,12 +15100,22 @@ function formatUnstableDrop(drop) {
 }
 //#endregion
 //#region src/cli/trace.ts
+/**
+* Step ids (in spec order) whose kept actions include no assertion. A step
+* with none produced a test that performs the step but verifies nothing
+* about its `expected` — surfaced as a per-step warning after validation.
+*/
+function stepsWithoutAsserts(stepIds, actions) {
+	const withAssert = new Set(actions.filter((a) => a.action === "assert" && a.stepId !== void 0).map((a) => a.stepId));
+	return stepIds.filter((id) => !withAssert.has(id));
+}
 async function runTrace(featureName, specName, model, validationMode = "lenient", language, opts = {}) {
 	header("trace", `${featureName}/${specName}`);
 	await preflightAgentBrowserCommand();
 	const spec = parseTestSpec(await readSpecFile(featureName, specName, opts.cwd));
 	const expanded = expandSpec(spec, { blocks: await loadAllBlocks(opts.cwd) });
-	const envScrub = buildSpecEnvScrub(spec, expanded);
+	const sessionName = generateSessionName();
+	const envScrub = buildSpecEnvScrub(spec, expanded, { CCQA_RUN_ID: sessionName });
 	const envScrubMap = envScrub.map;
 	if (envScrub.unresolved.length > 0) {
 		warn(`spec references env var(s) that are unset at record time: ${envScrub.unresolved.join(", ")}`);
@@ -15093,7 +15127,7 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
 	const includes = collectIncludedBlockNames(spec);
 	if (includes.length > 0) meta("blocks", includes.join(", "));
 	blank();
-	const sessionName = generateSessionName();
+	opts.teardown?.trackSession(sessionName);
 	const baseSystemPrompt = buildTraceSystemPrompt({
 		title: spec.title,
 		steps: expanded,
@@ -15176,13 +15210,18 @@ async function runTrace(featureName, specName, model, validationMode = "lenient"
 		}
 	});
 	if (isError) overallStatus = "failed";
-	const validatedActions = validateAndReport(dedupAndReport(scrubAndReport(traceActions)), validationMode);
-	const recordingPath = await saveRecording(featureName, specName, validatedActions, opts.cwd);
+	const dedupedActions = dedupAndReport(scrubAndReport(traceActions));
+	const validatedActions = overallStatus === "passed" || opts.validateFailedTrace === true ? validateAndReport(dedupedActions, validationMode, { CCQA_RUN_ID: sessionName }, opts.teardown) : dedupedActions;
+	opts.teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
+	const recordingPath = overallStatus === "passed" ? await saveRecording(featureName, specName, validatedActions, opts.cwd) : await saveFailedRecording(featureName, specName, validatedActions, opts.cwd);
 	blank();
 	meta("saved", recordingPath);
 	meta("actions", validatedActions.length);
 	meta("status", overallStatus.toUpperCase());
-	hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
+	if (overallStatus === "passed") {
+		for (const stepId of stepsWithoutAsserts(expanded.map((s) => s.id), validatedActions)) warn(`${stepId} recorded no assertion — nothing in the generated test verifies its 'expected'`);
+		hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
+	} else warn("trace FAILED — the recorded actions were saved beside the spec for diagnosis; the previous ir.json and generated code are left untouched");
 	return {
 		status: overallStatus,
 		statusLines,
@@ -15323,26 +15362,30 @@ function isAdjacentDuplicate(a, b) {
 }
 /**
 * Run the post-trace replay validation and emit user-visible drop reports.
-* Splitting this out keeps `runTrace` readable; the function is pure aside
-* from `log.*` and the agent-browser invocations inside `validateActions`.
+* Splitting this out keeps `runTrace` readable; side effects are `log.*`,
+* the agent-browser invocations inside `validateActions`, and the replay
+* session's registration/close on `teardown`.
 *
 * In lenient mode (the default) failing actions are NOT removed — they're
 * tagged with `replayUnstable: true` and merged back into the output stream
 * in their original order so codegen can still emit them (with a `// [warn]`
 * comment) and let the auto-fix loop decide what to do.
 */
-function validateAndReport(actions, mode) {
+function validateAndReport(actions, mode, envOverrides, teardown) {
 	if (actions.length === 0) return actions;
 	const sessionName = `${generateSessionName()}-validate`;
+	teardown?.trackSession(sessionName);
 	blank();
 	info(`post-trace validation in ${mode} mode (replaying ${actions.length} recorded action(s))...`);
 	const { kept, unstable, dropped, rescuedSteps = [] } = validateActions(actions, {
 		sessionName,
 		mode,
+		envOverrides,
 		onProgress: (i, total, action) => {
 			progress(i, total, validationProgressLabel(action));
 		}
 	});
+	teardown?.closeTracked(sessionName) ?? closeSession(sessionName);
 	progressEnd();
 	if (rescuedSteps.length > 0) info(`rescued ${rescuedSteps.length} step(s) that had lost every action: ${rescuedSteps.join(", ")}`);
 	if (mode === "lenient") {
@@ -15565,6 +15608,10 @@ function resolveTargetOrExit(resolve) {
 async function runGenerateLocked(featureName, specName, opts, cwd) {
 	const specYaml = await readSpecFile(featureName, specName, cwd);
 	const spec = parseTestSpec(specYaml);
+	if (spec.mode === "live") {
+		error(`this spec is 'mode: live' — a live spec runs without generated code. Run 'ccqa run ${featureName}/${specName}' instead`);
+		process.exit(2);
+	}
 	const config = await loadProjectConfig(cwd);
 	const target = resolveTargetOrExit(() => opts.targetOverride !== void 0 ? resolveTargetOverride(spec, opts.targetOverride) : resolveTarget(spec, config));
 	meta("target", target.id + (opts.targetOverride !== void 0 ? " (--target override)" : ""));
@@ -15737,6 +15784,10 @@ async function runRecord(specPath, opts) {
 		error(`target "${target.id}" does not use a browser recording — run 'ccqa generate ${featureName}/${specName}' instead`);
 		process.exit(2);
 	}
+	if (spec.mode === "live") {
+		error(`this spec is 'mode: live' — a live spec runs without a recording. Run 'ccqa run ${featureName}/${specName}' instead`);
+		process.exit(2);
+	}
 	const project = opts.hubProfile !== void 0 ? resolveProject(opts) : void 0;
 	if (opts.hubProfile !== void 0) await applyProfileFromOption({
 		profile: opts.hubProfile,
@@ -15804,6 +15855,8 @@ async function runRecord(specPath, opts) {
 			const traceResult = await runTrace(featureName, specName, opts.model, opts.traceValidation ?? "lenient", language, {
 				cwd: cwdForProfile,
 				hubContext,
+				teardown,
+				validateFailedTrace: opts.learnHubTracePrompt === true,
 				...opts.instruction ? { instruction: opts.instruction } : {},
 				onStep: (stepId) => {
 					tracingStep = stepId;
@@ -15821,7 +15874,7 @@ async function runRecord(specPath, opts) {
 				...opts.model ? { model: opts.model } : {},
 				...language ? { language } : {}
 			});
-			if (!opts.traceOnly) generated = (await runGenerate(featureName, specName, {
+			if (!opts.traceOnly && traceResult.status === "passed") generated = (await runGenerate(featureName, specName, {
 				maxRetries: parseInt(opts.autoFixMaxRetries ?? "3", 10),
 				fixMode: toFixMode(opts.autoFix ?? "interactive"),
 				force: opts.overwrite ?? false,
@@ -16089,7 +16142,9 @@ Ask whether the **intent** the step describes still exists in the product:
 - The intent exists, but the string or selector the spec names is gone or renamed → **TEST_DRIFT**. Cite where the replacement lives.
 - The intent itself is gone, or deliberately different → **SPEC_CHANGE**. Cite the source that shows the new shape.
 
-A renamed button is TEST_DRIFT. A button that no longer exists because the flow was replaced is SPEC_CHANGE. If the source shows a rename you can point at, prefer TEST_DRIFT; if you cannot find where the intent went, SPEC_CHANGE is the most you may claim, and UNKNOWN when even that is a guess.
+A renamed button is TEST_DRIFT. A button that no longer exists because the flow was replaced is SPEC_CHANGE. If the source shows a rename you can point at, prefer TEST_DRIFT.
+
+SPEC_CHANGE is the more expensive answer — it sends a human to rewrite or retire the spec — so it takes the *stronger* evidence, not the weaker. Failing to find where the intent went is not a finding; that is UNKNOWN. Claim SPEC_CHANGE only when you can point at the source that shows the new shape, or at where the implementation would sit if it still existed.
 
 ## Which surface drifted
 
@@ -16122,6 +16177,8 @@ sit nearby.
 
 - **No drift is a claim, not a default.** Make it after picking the concrete strings from *every* surface you were given — the spec's \`expected\` and the generated code's selectors alike — and finding each of them in the source. Clearing the test case because one surface checked out is the most common way to miss a real finding. If you never looked, the honest answer is UNKNOWN.
 - **A finding needs a citation.** Every TEST_DRIFT and SPEC_CHANGE must carry at least one \`evidence\` entry with a real \`file\`, and a line where you can give one. A label with no citation is a guess wearing a verdict's clothes — answer UNKNOWN instead.
+- **A citation must apply to the case at hand.** Finding the string is not the end of it — read what encloses the line before you cite it. A line inside a guard, behind an early \`return\`, or in a branch this spec's steps never enter says nothing about this spec. Name the conditions that must hold for that line to run, and check the spec puts the product in them. A citation that only proves the line exists is not evidence.
+- **A comment is not the code.** Comments in the product's source say what someone intended, and they rarely restate the conditions they sit under. A line reading "this is not supported", sitting inside a guarded branch, is true only inside that branch. Cite the control flow you traced, not the sentence you found.
 - **Do not report style.** Wording you would have phrased differently is not drift. Report only what would make a replay fail, or what asks about something the product no longer does.
 - \`confidence\` is about the label: how sure you are it is the right one, not how bad the finding is.
 
@@ -16131,6 +16188,7 @@ sit nearby.
 2. \`Grep\` the source for them, at the page, component or handler the step is about.
 3. For \`include\` steps, confirm the block exists under \`.ccqa/blocks/<name>/spec.yaml\` and that every \`params\` key is declared on it.
 4. When a string is missing, look for what replaced it before concluding. Where it went is what decides the label.
+5. Before citing any line, read the block that encloses it. Which conditions must hold for it to run, and does the spec put the product in those conditions? A line that only runs in a case the spec never enters proves nothing about the spec.
 
 ${guidance.userPromptBlock ?? ""}${guidance.customPromptBlock ?? ""}## Output (STRICT)
 
@@ -16488,7 +16546,7 @@ function determineExitCode(results, threshold) {
 //#endregion
 //#region src/drift/to-report.ts
 /** Tracks the drift prompt's own version — the two must never drift apart. */
-const DRIFT_REPORT_PROMPT_VERSION = "6";
+const DRIFT_REPORT_PROMPT_VERSION = "7";
 /**
 * Spec-level status under the given threshold, mirroring determineExitCode's
 * per-spec logic (exit-code.ts) but scoped to a single SpecResult.

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.34.0",
+  "version": "1.35.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.34.0",
+  "version": "1.35.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {