ccqa 1.50.0 → 1.50.1

package/dist/bin/ccqa.mjs

@@ -8603,11 +8603,11 @@ async function buildLiveTranscriptExcerpt(result, options = {}) {
 	return combined.length > maxBytes ? `${combined.slice(0, maxBytes)}\n…[transcript excerpt truncated at ${maxBytes} bytes]` : combined;
 }
 function formatPreviousStep(step) {
-	const reason = oneLine$2(step.reasoning) || "(no reason given)";
+	const reason = oneLine$3(step.reasoning) || "(no reason given)";
 	return `[${step.stepId} ${step.status}: ${reason}]`;
 }
 async function formatFailingStep(step, headBytes, tailBytes) {
-	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$2(step.instruction)}\nExpected: ${oneLine$2(step.expected)}\nReasoning (Claude's verdict): ${oneLine$2(step.reasoning) || "(none)"}`;
+	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$3(step.instruction)}\nExpected: ${oneLine$3(step.expected)}\nReasoning (Claude's verdict): ${oneLine$3(step.reasoning) || "(none)"}`;
 	if (!step.logTxt) return `${header}\n(No assistant log file recorded for this step.)`;
 	const raw = await readFile(step.logTxt, "utf-8").catch((err) => {
 		return `[log file unreadable: ${err instanceof Error ? err.message : String(err)}]`;
@@ -8617,7 +8617,7 @@ async function formatFailingStep(step, headBytes, tailBytes) {
 	const tail = raw.slice(raw.length - tailBytes);
 	return `${header}\n--- assistant log (head ${headBytes}B) ---\n${head}\n…[${raw.length - headBytes - tailBytes} bytes omitted]…\n--- assistant log (tail ${tailBytes}B) ---\n${tail}`;
 }
-function oneLine$2(s) {
+function oneLine$3(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -10129,10 +10129,10 @@ function describeStep(step) {
 	return describeStepBody(step);
 }
 function describeStepBody(step) {
-	if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
-	return `${oneLine$1(step.instruction)} → ${oneLine$1(step.expected)}`;
+	if (isJudgeBody(step)) return `judge: ${oneLine$2(step.judgeByLlm)}`;
+	return `${oneLine$2(step.instruction)} → ${oneLine$2(step.expected)}`;
 }
-function oneLine$1(text) {
+function oneLine$2(text) {
 	return text.trim().replace(/\s+/g, " ");
 }
 function emptyDeployLog() {
@@ -12432,15 +12432,15 @@ function renderRunMarkdown(featureName, specName, result) {
 	].join("\n") + result.steps.map((s) => [
 		`## ${s.stepId} — ${s.status}`,
 		`- duration: ${(s.durationMs / 1e3).toFixed(1)}s`,
-		`- instruction: ${oneLine(s.instruction)}`,
-		`- expected: ${oneLine(s.expected)}`,
-		`- reasoning: ${oneLine(s.reasoning)}`,
+		`- instruction: ${oneLine$1(s.instruction)}`,
+		`- expected: ${oneLine$1(s.expected)}`,
+		`- reasoning: ${oneLine$1(s.reasoning)}`,
 		...s.beforePng ? [`- before: ${s.beforePng}`] : [],
 		...s.afterPng ? [`- after: ${s.afterPng}`] : [],
 		""
 	].join("\n")).join("\n");
 }
-function oneLine(s) {
+function oneLine$1(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -14004,6 +14004,136 @@ async function waitForCdp(port) {
 	}
 }
 //#endregion
+//#region src/prompts/verifies-spec.ts
+/**
+* Asks whether a generated test actually decides what its spec claims.
+*
+* The generation loop's only bar is "the test goes green", and a rewrite that
+* weakens an assertion clears that bar as easily as one that keeps it. Green
+* therefore does not mean checked, and nothing else looks. Observed cases: a
+* step whose expectation was "the linked page opens" asserting instead that
+* the *link* is still visible on the page it clicked from; another asserting
+* on a navigation element unrelated to the step.
+*
+* Deliberately narrow. It reads only what the step says and what the code
+* does, and reports the step as unchecked when the two do not line up. It
+* does not review style, coverage, or whether the expectation is a good one.
+*/
+function verifiesSpecPrompt(input) {
+	return [
+		"You are reviewing whether a generated end-to-end test decides what its spec says.",
+		"",
+		"For each step below, the test must contain assertions that could FAIL if the",
+		"step's `expected` stopped holding. Report a step when:",
+		"",
+		"- its assertions cannot fail while the product is broken in the way the",
+		"  expectation describes (e.g. the expectation says a page opens, and the",
+		"  code only re-checks the element it clicked);",
+		"- what it asserts on is unrelated to what the step did (e.g. a navigation",
+		"  element that is present on every page);",
+		"- it depends on something that varies between runs and is not part of the",
+		"  expectation (a count, an index, a position, wording that changes);",
+		"- it has no assertion at all.",
+		"",
+		"Do NOT report: style, naming, structure, missing coverage the spec never",
+		"asked for, or an expectation you merely disagree with. A step that checks",
+		"less than you would have written, but still fails when the expectation",
+		"breaks, is fine.",
+		"",
+		"## Steps",
+		"",
+		...input.steps.map(stepLine),
+		"",
+		"## Generated test",
+		"",
+		"```",
+		input.source,
+		"```",
+		"",
+		"Answer with one json block and nothing else:",
+		"",
+		"```json",
+		"{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ] }",
+		"```",
+		"",
+		"`problem` is one sentence naming what the step claims and what the code",
+		"checks instead. Empty `findings` means every step is decided.",
+		languageDirective(input.language)
+	].join("\n");
+}
+function stepLine(step) {
+	if (!isExpandedActionStep(step)) return `- ${step.id}: judged by a model at run time — its claim is asserted by the injected call, so it needs no other assertion.`;
+	return [
+		`- ${step.id}`,
+		`  does: ${oneLine(step.instruction)}`,
+		`  expected: ${oneLine(step.expected)}`
+	].join("\n");
+}
+function oneLine(text) {
+	return text.trim().split("\n").map((l) => l.trim()).join(" ");
+}
+//#endregion
+//#region src/targets/verifies-spec.ts
+const FindingsSchema = z.object({ findings: z.array(z.object({
+	stepId: z.string(),
+	problem: z.string()
+})) });
+/**
+* Findings in the model's answer, or null when it did not answer in the
+* agreed shape. Null is not "no findings": the caller says so rather than
+* reporting a clean review it never got.
+*/
+function parseVerifiesSpecFindings(answer) {
+	const json = extractJsonBlock(answer);
+	if (!json) return null;
+	try {
+		return FindingsSchema.parse(JSON.parse(json)).findings;
+	} catch {
+		return null;
+	}
+}
+/** The warning a finding becomes, phrased so the reader knows the test is green for nothing. */
+function formatFinding(finding) {
+	return `step ${finding.stepId}: the generated test passes without deciding what this step claims — ${finding.problem}`;
+}
+/**
+* Read the generated test back and ask whether each step is actually decided
+* (see `verifiesSpecPrompt`). Returns warnings; an empty list means either a
+* clean review or one that could not be obtained, and the difference is
+* logged rather than encoded — a review that failed must not read as a pass,
+* but it must also not fail the generate that produced working files.
+*/
+async function reviewGeneratedTest(input) {
+	const source = (await Promise.all(input.result.files.filter((f) => f.kind === "test").map((f) => readFile(f.path, "utf8").catch(() => "")))).filter((s) => s.length > 0).join("\n\n");
+	if (source.length === 0) {
+		warn("could not check whether the generated test decides its spec (no test file to read)");
+		return [];
+	}
+	const { result: answer, isError } = await (input.invoke ?? invokeClaudeStreaming)({
+		prompt: verifiesSpecPrompt({
+			steps: input.steps,
+			source,
+			language: input.language
+		}),
+		allowedTools: [],
+		disableThinking: true,
+		maxTurns: 1,
+		silenceBashLog: true,
+		...input.model ? { model: input.model } : {},
+		cwd: input.cwd
+	}, () => {});
+	if (isError) {
+		warn("could not check whether the generated test decides its spec (Claude returned an error)");
+		return [];
+	}
+	const findings = parseVerifiesSpecFindings(answer);
+	if (findings === null) {
+		warn("could not check whether the generated test decides its spec (no usable answer)");
+		return [];
+	}
+	return findings.map(formatFinding);
+}
+//#endregion
 //#region src/targets/playwright/index.ts
 const PLAYWRIGHT_TARGET = "playwright";
 /**
@@ -14075,12 +14205,21 @@ async function generatePlaywrightTest(ctx) {
 	});
 	const missing = await missingInjectedCalls(result, stepMarkers, judgements);
 	for (const w of missing) warn(w);
+	const unchecked = await reviewGeneratedTest({
+		result,
+		steps: expanded,
+		language: ctx.language,
+		...ctx.model ? { model: ctx.model } : {},
+		cwd: ctx.cwd
+	});
+	for (const w of unchecked) warn(w);
 	return {
 		...result,
 		warnings: [
 			...result.warnings,
 			...judgeWarnings,
-			...missing
+			...missing,
+			...unchecked
 		]
 	};
 }
@@ -15238,7 +15377,7 @@ async function executeRun(targets, opts) {
 	const declared = [...new Set(specs.flatMap(resources))];
 	if (declared.length > 0) meta("serial groups", declared.join(", "));
 	let waitingOnGroup = [];
-	if (hubCtx && rerunProfile !== null) {
+	if (hubCtx && rerunProfile !== null && forExecution) {
 		const held = await holdSpecs(hubCtx, rerunProfile, specs, resources, opts.teardown);
 		waitingOnGroup = specs.flatMap((spec) => {
 			const groups = resources(spec).filter((n) => held.deniedResources.includes(n));

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.50.0",
+  "version": "1.50.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.50.0",
+  "version": "1.50.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {