ccqa 1.60.1 → 1.60.3

package/dist/bin/ccqa.mjs

@@ -1650,10 +1650,10 @@ async function saveRecording(ref, actions, cleanup = [], stepDigests) {
 *
 * One directory, not one per attempt: a project whose tests belong to its own
 * runner never calls `ccqa run`, so this stands in for the run report the
-* evidence table reads — and what stands in for it is the attempt that passed,
-* which there is only ever one of. `ccqa generate` clears it before each
-* attempt and removes it when none passed, because screenshots of a failing
-* run are a debugging aid and this table presents them as the case working.
+* evidence table reads — and what stands in for it is the attempt that passed
+* last, the version generation keeps. `ccqa generate` replaces it only on a
+* pass and removes it when none passed, because screenshots of a failing run
+* are a debugging aid and this table presents them as the case working.
 */
 function caseRunDir(ref) {
 	return join(ref.dir, EVIDENCE_DIR);
@@ -6535,7 +6535,7 @@ function parseStepComment(line) {
 * every step then reads as deciding nothing and every rewrite is rejected.
 */
 function stepLines(source) {
-	return source.replace(STEP_CALL, (call) => call.replace(/\s*\n\s*/g, " ")).split("\n");
+	return source.replace(STEP_CALL, (call) => "\n".repeat(call.split("\n").length - 1) + call.replace(/\s*\n\s*/g, " ")).split("\n");
 }
 /**
 * The step id a `test.step` title names — the block form only.
@@ -8338,6 +8338,31 @@ function exportedNames(source) {
 }
 //#endregion
 //#region src/targets/emitted-review.ts
+/**
+* A mechanical read of the code a generation just wrote, before it is called
+* done.
+*
+* The fix loop already runs the project's own test command and checks, so
+* what reaches review is code that compiles, lints and passes. None of that
+* answers whether it is code a reviewer would take: an assertion scoped to
+* nothing passes, a locator named for a row that matches page-wide text
+* passes, a test that checks five things the case never asked about passes.
+* Those are what this reads for, and its findings go back through the same
+* fix loop rather than to a human.
+*
+* Every rule here was calibrated against a large hand-written suite: a rule
+* that fires on code people wrote and reviewed is a rule that is wrong,
+* however reasonable it sounds. Two candidates died that way —
+* "a locator named `…Row` must use `getByRole("row")`" fired across a
+* sizable share of the hand-written page objects, because that suite names
+* elements for what they look like rather than for their ARIA role, and
+* "never `.first()` in an assertion" fired on deliberate uses that each
+* carried the comment their own guidelines ask for. A third died against
+* ccqa's own output: the step comment a generated test opens each step with
+* looked like narration, and is what the evidence table reads back to say
+* which assertion belongs to which step. Keep that bar for anything added
+* here.
+*/
 function reviewEmittedFiles(input) {
 	const locators = collectLocators(input.files);
 	const said = input.caseText.join("\n");
@@ -8346,7 +8371,7 @@ function reviewEmittedFiles(input) {
 		...containerOfPageText(file, source),
 		...unjustifiedFirst(file, source),
 		...describeEcho(file, source),
-		...unaskedAssertions(file, source, locators, said),
+		...unaskedAssertions(file, source, locators, said, input.blockSteps),
 		...weakerTwin(file, source),
 		...unreached(file, source, input)
 	])];
@@ -8601,9 +8626,26 @@ function assertedStrings(text, locators) {
 * precondition broke rather than four steps later, is worth doing and is
 * written down when it is.
 */
-function unaskedAssertions(file, source, locators, said) {
+function unaskedAssertions(file, source, locators, said, blockSteps = /* @__PURE__ */ new Set()) {
 	const all = lines(source);
+	const boundaries = stepLines(source);
+	let step = null;
+	let depth = null;
 	return all.flatMap((text, i) => {
+		const opened = parseStepComment(boundaries[i] ?? "");
+		if (opened !== null) {
+			step = opened;
+			depth = boundaries[i].includes("test.step(") ? 0 : null;
+		}
+		if (depth !== null) {
+			const before = depth;
+			depth += (text.match(/\{/g)?.length ?? 0) - (text.match(/\}/g)?.length ?? 0);
+			if (before > 0 && depth <= 0) {
+				step = null;
+				depth = null;
+			}
+		}
+		if (step !== null && blockSteps.has(step)) return [];
 		if (!/\bexpect(?:\.soft)?\(/.test(text)) return [];
 		const strings = assertedStrings(text, locators);
 		if (strings.length === 0 || strings.some((s) => said.includes(s))) return [];
@@ -9302,8 +9344,12 @@ const ViolationsSchema = z.object({ ruleViolations: z.array(z.unknown()) });
 * treats it as one it could not obtain.
 */
 const MAX_REVIEW_TURNS = 80;
-/** A wedged call, not a slow one: a review still silent after this is stuck. */
-const REVIEW_TIMEOUT_MS = 10 * 6e4;
+/**
+* A wedged call, not a slow one. A review of a real suite reads the test, its
+* page objects and many of the suite's files to count conventions, which runs
+* past ten minutes; one that is still going after twenty is stuck.
+*/
+const REVIEW_TIMEOUT_MS = 20 * 6e4;
 /** Read-only: a reviewer reads the repository and changes nothing in it. */
 const REVIEW_TOOLS = [
 	"Read",
@@ -14746,7 +14792,7 @@ const LLM_GEN_ALLOWED_TOOLS = [
 const LlmFileSchema = z.object({
 	path: z.string().min(1),
 	contents: z.string(),
-	kind: z.string().default("test")
+	kind: z.string().optional()
 });
 const LlmOutputSchema = z.object({
 	files: z.array(LlmFileSchema),
@@ -14778,21 +14824,29 @@ function parseLlmGenOutput(raw) {
 		const issues = res.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`);
 		throw new Error(`reply JSON does not match {files:[{path,contents,kind}],summary}: ${issues.join("; ")}`);
 	}
-	const kindWarnings = [];
 	return {
-		files: res.data.files.map((f) => {
-			if (f.kind === "test" || f.kind === "support") return {
-				...f,
-				kind: f.kind
-			};
-			kindWarnings.push(`file "${f.path}": unknown kind "${f.kind}" coerced to "test"`);
-			return {
-				...f,
-				kind: "test"
-			};
-		}),
-		summary: res.data.summary,
-		kindWarnings
+		files: res.data.files.map((f) => ({
+			path: f.path,
+			contents: f.contents,
+			kind: f.kind === "support" ? "support" : "test"
+		})),
+		summary: res.data.summary
+	};
+}
+/**
+* The file at the configured test path is the test and every other file is
+* support — whatever the reply labelled them. The path already says which one
+* a file is, and a model that labels a page object "test" would otherwise fail
+* the whole generation over a word.
+*/
+function kindByPath(output, policy) {
+	const testAbs = resolve(policy.cwd, policy.testPath);
+	return {
+		...output,
+		files: output.files.map((f) => ({
+			...f,
+			kind: resolve(policy.cwd, f.path) === testAbs ? "test" : "support"
+		}))
 	};
 }
 /**
@@ -14952,7 +15006,6 @@ async function generateWithLlmEngine(req) {
 		requireTestFile: true,
 		allowEmpty: false
 	}), "run");
-	warnings.push(...output.kindWarnings);
 	return finalizeAndVerify({
 		ctx,
 		target: req.target,
@@ -15013,12 +15066,14 @@ async function finalizeAndVerify(p) {
 		review
 	};
 }
+const FINDINGS_LEFT_OPEN = "generated with review findings still open — the files are kept and the findings are recorded against the case, but nothing acted on them";
 /**
 * The runCommand verification loop. One round asks the project's own commands,
-* then the review, then the run — and a round is over when all three are
-* clean. On anything else the output goes to Claude for a corrected set and
+* then the review and the run together — and a round is over when all three
+* are clean. On anything else the output goes to Claude for a corrected set and
 * the next round asks all three again: a fix written for a failing run can
-* break a rule the round before it cleared.
+* break a rule the round before it cleared, and one written for a finding can
+* break the run.
 *
 * `fix.mode` mirrors the agent-browser target's fix UX (which only ever
 * prompts inside its own fix loop, never for the first write):
@@ -15030,8 +15085,9 @@ async function finalizeAndVerify(p) {
 *     verify decides pass/fail.
 *
 * Exhaustion (or a declined / skipped fix) keeps the files on disk and reports
-* `passed: false`. Targets without a runCommand are generate-only here and
-* pass trivially.
+* `passed: false` — unless an earlier round's files passed their run: those
+* are put back and the generate passes with the findings open. Targets without
+* a runCommand are generate-only here and pass trivially.
 */
 async function runVerificationLoop(p, state) {
 	const runCommand = p.ctx.targetConfig.runCommand;
@@ -15061,33 +15117,61 @@ async function runVerificationLoop(p, state) {
 	const askReviewer = maxRetries > 0;
 	let reviewerGone = false;
 	if (!askReviewer && p.reading) info("no fix round could act on a review — the reviewer is not asked; the mechanical read still runs");
+	let passing = null;
+	const original = /* @__PURE__ */ new Map();
+	const settle = async (attempt) => {
+		if (passing === null) {
+			await clearCaseRun(p.ctx.ref);
+			return finish(false);
+		}
+		if (passing.attempt !== attempt) {
+			await restoreFiles(state, passing.files, original);
+			review = passing.review;
+			warn("the final round did not pass — restored the files of the last round that did");
+		}
+		warn(FINDINGS_LEFT_OPEN);
+		return finish(true);
+	};
+	if (captures) await clearCaseRun(p.ctx.ref);
 	for (let attempt = 0;; attempt++) {
 		let failing = await runCheckCommands(p.ctx);
 		let asked = [];
 		if (failing === null) {
 			await refreshFromDisk(state);
-			const [reading, mechanical] = await Promise.all([readingOfEmitted(p, state, askReviewer && !reviewerGone), reviewOfEmitted(p.ctx, state, p.writeRoots)]);
+			const questions = [
+				readingOfEmitted(p, state, askReviewer && !reviewerGone),
+				reviewOfEmitted(p.ctx, state, p.writeRoots),
+				verifyRun(p, state, runCommand, attempt, captures)
+			];
+			const [reading, mechanical, run] = await Promise.all(questions).catch(async (err) => {
+				await Promise.allSettled(questions);
+				await clearCaseRun(p.ctx.ref);
+				throw err;
+			});
 			review = reading;
-			if (review?.reviewerFailed) reviewerGone = true;
-			const unchecked = uncheckedSteps(review, spent);
-			const violations = guideViolations(review, spent);
+			if (run.exitCode === 0) passing = {
+				files: new Map(state),
+				review,
+				attempt
+			};
+			const heard = review?.reviewerFailed ? void 0 : review;
+			if (review && !heard) reviewerGone = true;
+			const unchecked = uncheckedSteps(heard, spent);
+			const violations = guideViolations(heard, spent);
 			const reads = allReadings(mechanical, unchecked, violations);
-			if (reads !== null && attempt < maxRetries) {
-				asked = [...unchecked?.asked ?? [], ...violations?.asked ?? []];
-				failing = reads;
-			} else {
-				if (reads !== null) warn("generated with review findings still open — the files are kept and the findings are recorded against the case, but nothing acted on them");
-				const run = await verifyRun(p, state, runCommand, attempt, captures);
-				if (run.exitCode === 0) return finish(true);
-				failing = run;
+			asked = [...unchecked?.asked ?? [], ...violations?.asked ?? []];
+			if (run.exitCode !== 0) failing = reads === null ? run : allReadings(run, reads) ?? run;
+			else if (reads !== null && attempt < maxRetries) failing = reads;
+			else {
+				if (reads !== null) warn(FINDINGS_LEFT_OPEN);
+				return finish(true);
 			}
 		}
 		const outputTail = scrubEnvValues(tail(failing.output), outputScrub);
 		if (attempt >= maxRetries) {
-			warn(`verification still failing after ${maxRetries} fix attempt(s) — generated files kept`);
+			warn(`verification still failing after ${maxRetries} fix attempt(s)`);
 			logOutputTail(outputTail, warn);
-			await clearCaseRun(p.ctx.ref);
-			return finish(false);
+			return settle(attempt);
 		}
 		fix(`verification failed (exit ${failing.exitCode}) — requesting a fix (${attempt + 1}/${maxRetries})`);
 		logOutputTail(outputTail, fix);
@@ -15125,9 +15209,12 @@ async function runVerificationLoop(p, state) {
 			continue;
 		}
 		if (p.ctx.fix.mode === "interactive" && !await confirmFixWrite(output.files, p.ctx.cwd)) {
-			info("fix not applied (declined) — keeping current files");
-			await clearCaseRun(p.ctx.ref);
-			return finish(false);
+			info("fix not applied (declined)");
+			return settle(attempt);
+		}
+		for (const f of output.files) {
+			const rel = relative(p.ctx.cwd, resolve(p.ctx.cwd, f.path));
+			if (!state.has(rel) && !original.has(rel)) original.set(rel, await readFile(resolve(p.ctx.cwd, f.path), "utf8").catch(() => null));
 		}
 		await writeGeneratedFiles(p.ctx.cwd, output.files, state);
 		review = void 0;
@@ -15135,17 +15222,12 @@ async function runVerificationLoop(p, state) {
 }
 /**
 * One verification run of the files as they are now, in the shape the fix loop
-* consumes. The round's last question and its most expensive: it is asked only
-* of files the project's own commands and the review have already cleared.
+* consumes. Asked only of files the project's own commands have cleared.
 */
 async function verifyRun(p, state, runCommand, attempt, captures) {
 	const testFiles = [...state.entries()].filter(([, f]) => f.kind === "test").map(([rel]) => rel);
 	const artifactsDir = await mkdtemp(join(tmpdir(), "ccqa-verify-artifacts-"));
-	const evidenceDir = captures ? caseRunDir(p.ctx.ref) : null;
-	if (evidenceDir) {
-		await clearCaseRun(p.ctx.ref);
-		await mkdir(evidenceDir, { recursive: true });
-	}
+	const evidenceDir = captures ? await mkdtemp(join(tmpdir(), "ccqa-verify-evidence-")) : null;
 	let command = substituteArtifactsDir(substituteRunCommandFiles(runCommand, testFiles), artifactsDir);
 	let traceUnavailable;
 	if (evidenceDir) {
@@ -15169,6 +15251,8 @@ async function verifyRun(p, state, runCommand, attempt, captures) {
 				warn: (message) => warn(`${p.ctx.ref.id}: ${message}`)
 			});
 			if (unavailable) warn(unavailable);
+			await clearCaseRun(p.ctx.ref);
+			await cp(evidenceDir, caseRunDir(p.ctx.ref), { recursive: true });
 		}
 		return {
 			...result,
@@ -15179,6 +15263,10 @@ async function verifyRun(p, state, runCommand, attempt, captures) {
 			recursive: true,
 			force: true
 		});
+		if (evidenceDir) await rm(evidenceDir, {
+			recursive: true,
+			force: true
+		});
 	}
 }
 /**
@@ -15199,6 +15287,10 @@ async function runCheckCommands(ctx) {
 	}
 	return null;
 }
+function blockStepIds(ctx) {
+	const included = new Set(collectIncludedBlockNames(ctx.spec));
+	return new Set(ctx.steps.filter((s) => included.has(s.source)).map((s) => s.id));
+}
 /**
 * The mechanical read of what this attempt wrote, shaped like a failed check
 * so the fix loop carries it the same way.
@@ -15216,6 +15308,7 @@ async function reviewOfEmitted(ctx, state, writeRoots) {
 			...ctx.expectations,
 			...ctx.cleanupExpectations
 		],
+		blockSteps: blockStepIds(ctx),
 		testPath: ctx.testPath
 	});
 	if (findings.length === 0) return null;
@@ -15261,6 +15354,23 @@ async function refreshFromDisk(state) {
 	}
 }
 /**
+* Put the files back as `to` holds them. A path written after `to` was taken
+* goes back to what it held before the loop wrote it, or away if it was new.
+*/
+async function restoreFiles(state, to, original) {
+	for (const [rel, f] of state) {
+		if (to.has(rel)) continue;
+		const was = original.get(rel) ?? null;
+		if (was === null) await rm(f.abs, { force: true });
+		else await writeFile(f.abs, was, "utf8");
+	}
+	state.clear();
+	for (const [rel, f] of to) {
+		await writeFile(f.abs, f.contents, "utf8");
+		state.set(rel, f);
+	}
+}
+/**
 * The reading's findings, shaped like a failed check so the fix loop carries
 * them the same way — or null when there is nothing to act on.
 *
@@ -15273,9 +15383,8 @@ async function refreshFromDisk(state) {
 *
 * What the round asks for is verified like any other rewrite, and that can
 * end red: an assertion strengthened to check what the case claims may simply
-* not hold. The generate then reports failed — on the test run, not on this —
-* and the files are kept. That is the honest outcome, and the alternative
-* (restoring what passed weakly) would hide a case the product does not meet.
+* not hold. The loop then goes back to the last version that passed, and the
+* finding stays open on the record rather than hidden.
 *
 * A review that could not be obtained is not acted on either: spending a round
 * answering a question nobody asked is worse than leaving it unanswered. Nor
@@ -15421,7 +15530,7 @@ function allReadings(...reports) {
 	const found = reports.filter((r) => r !== null);
 	if (found.length === 0) return null;
 	return {
-		exitCode: 1,
+		exitCode: found[0].exitCode,
 		command: found.map((r) => r.command).join(" + "),
 		output: found.map((r) => `${r.command}\n\n${r.output}`).join("\n\n")
 	};
@@ -15461,8 +15570,7 @@ async function invokeForFiles(p) {
 			model: p.ctx.model
 		}, () => {});
 		if (isError) throw new Error(`Claude invocation failed: ${tail(result)}`);
-		const output = parseLlmGenOutput(result);
-		for (const w of output.kindWarnings) warn(w);
+		const output = kindByPath(parseLlmGenOutput(result), p.policy);
 		const errors = validateOutput(output, p);
 		if (errors.length > 0) throw new Error(errors.join("; "));
 		return output;
@@ -15505,8 +15613,7 @@ function validateOutput(output, p) {
 		if (fileError) errors.push(fileError);
 	}
 	const tests = output.files.filter((f) => f.kind === "test");
-	if (p.requireTestFile && tests.length === 0) errors.push("output contains no \"kind\": \"test\" file");
-	if (tests.length > 1) errors.push(`output contains ${tests.length} "kind": "test" files; a spec has exactly one`);
+	if (p.requireTestFile && tests.length === 0) errors.push(`the test file must be written to ${p.policy.testPath}; the reply has no file there`);
 	return errors;
 }
 function toPromptResource(r) {

package/dist/hub-client/index.d.cts

@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
     running: "running";
   }>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
     specs: z.ZodNumber;
@@ -831,9 +831,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
 declare const RunReportDataSchema: z.ZodObject<{
   schemaVersion: z.ZodLiteral<1>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   createdAt: z.ZodString;
   runId: z.ZodNullable<z.ZodString>;

package/dist/hub-client/index.d.mts

@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
     running: "running";
   }>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
     specs: z.ZodNumber;
@@ -831,9 +831,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
 declare const RunReportDataSchema: z.ZodObject<{
   schemaVersion: z.ZodLiteral<1>;
   kind: z.ZodDefault<z.ZodEnum<{
+    record: "record";
     run: "run";
     drift: "drift";
-    record: "record";
   }>>;
   createdAt: z.ZodString;
   runId: z.ZodNullable<z.ZodString>;

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.60.1",
+  "version": "1.60.3",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.60.1",
+  "version": "1.60.3",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {