ccqa 1.49.0 → 1.50.1

package/dist/bin/ccqa.mjs

@@ -1502,6 +1502,8 @@ const ReportSpecResultSchema = z.object({
 		failed: z.number()
 	}).nullable(),
 	durationMs: z.number().nullable(),
+	startedAt: z.string().optional(),
+	finishedAt: z.string().optional(),
 	assertions: z.array(ReportAssertionSchema).nullable(),
 	analysis: FailureAnalysisSchema.nullable(),
 	analysisSkipped: z.string().nullable(),
@@ -6198,6 +6200,7 @@ async function runOneSpec$1(ref, opts, blocks) {
 	meta("command", command);
 	blank();
 	const started = Date.now();
+	const startedAt = new Date(started).toISOString();
 	let outcome;
 	let spawnFailure;
 	let measured;
@@ -6218,6 +6221,7 @@ async function runOneSpec$1(ref, opts, blocks) {
 	const coverageFields = coverageRowFields(opts, measured, attachError);
 	if (spawnFailure !== void 0 || outcome === void 0) return {
 		...didNotExecute(`could not spawn runCommand: ${spawnFailure ?? "unknown error"}`, "the runCommand could not be spawned"),
+		startedAt,
 		...coverageFields
 	};
 	const durationMs = Date.now() - started;
@@ -6244,6 +6248,8 @@ async function runOneSpec$1(ref, opts, blocks) {
 			status: "passed"
 		}),
 		target: opts.targetId,
+		startedAt,
+		finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
 		durationMs,
 		...artifactFields,
 		...evidenceFields,
@@ -6251,6 +6257,8 @@ async function runOneSpec$1(ref, opts, blocks) {
 	};
 	return {
 		...failedRow([`command failed (exit ${outcome.exitCode}): ${command}`, outcome.tail.length > 0 ? `--- output (tail) ---\n${outcome.tail}` : null].filter((p) => p !== null).join("\n")),
+		startedAt,
+		finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
 		durationMs,
 		...artifactFields,
 		...evidenceFields,
@@ -8595,11 +8603,11 @@ async function buildLiveTranscriptExcerpt(result, options = {}) {
 	return combined.length > maxBytes ? `${combined.slice(0, maxBytes)}\n…[transcript excerpt truncated at ${maxBytes} bytes]` : combined;
 }
 function formatPreviousStep(step) {
-	const reason = oneLine$2(step.reasoning) || "(no reason given)";
+	const reason = oneLine$3(step.reasoning) || "(no reason given)";
 	return `[${step.stepId} ${step.status}: ${reason}]`;
 }
 async function formatFailingStep(step, headBytes, tailBytes) {
-	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$2(step.instruction)}\nExpected: ${oneLine$2(step.expected)}\nReasoning (Claude's verdict): ${oneLine$2(step.reasoning) || "(none)"}`;
+	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$3(step.instruction)}\nExpected: ${oneLine$3(step.expected)}\nReasoning (Claude's verdict): ${oneLine$3(step.reasoning) || "(none)"}`;
 	if (!step.logTxt) return `${header}\n(No assistant log file recorded for this step.)`;
 	const raw = await readFile(step.logTxt, "utf-8").catch((err) => {
 		return `[log file unreadable: ${err instanceof Error ? err.message : String(err)}]`;
@@ -8609,7 +8617,7 @@ async function formatFailingStep(step, headBytes, tailBytes) {
 	const tail = raw.slice(raw.length - tailBytes);
 	return `${header}\n--- assistant log (head ${headBytes}B) ---\n${head}\n…[${raw.length - headBytes - tailBytes} bytes omitted]…\n--- assistant log (tail ${tailBytes}B) ---\n${tail}`;
 }
-function oneLine$2(s) {
+function oneLine$3(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -10121,10 +10129,10 @@ function describeStep(step) {
 	return describeStepBody(step);
 }
 function describeStepBody(step) {
-	if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
-	return `${oneLine$1(step.instruction)} → ${oneLine$1(step.expected)}`;
+	if (isJudgeBody(step)) return `judge: ${oneLine$2(step.judgeByLlm)}`;
+	return `${oneLine$2(step.instruction)} → ${oneLine$2(step.expected)}`;
 }
-function oneLine$1(text) {
+function oneLine$2(text) {
 	return text.trim().replace(/\s+/g, " ");
 }
 function emptyDeployLog() {
@@ -10189,6 +10197,36 @@ function foldTouchIndex(current, entry, selection) {
 	}
 	return out;
 }
+/**
+* How much clock skew between the runner (which stamps a row's window) and
+* the hub (which stamps a deploy) the placement tolerates. The window is
+* widened by this on both ends, so a skewed clock costs a row its credit
+* rather than crediting a row that straddled.
+*/
+const PLACEMENT_SKEW_MS = 1e4;
+/**
+* Place one row's execution window against the deploy log (ADR-0027).
+*
+* The window is widened by {@link PLACEMENT_SKEW_MS} and its start is
+* exclusive, so a deploy landing exactly as the spec began reads as a
+* straddle. Both choices err the same way: toward `ambiguous`, never toward
+* crediting a spec with a commit it may not have exercised.
+*
+* `entries` must be in append order, which the log guarantees and which is
+* time order.
+*/
+function placeRowInDeployLog(entries, window) {
+	const before = (ms, inclusive) => entries.findLast((e) => {
+		const at = Date.parse(e.at);
+		return inclusive ? at <= ms : at < ms;
+	});
+	const opened = before(window.startMs - PLACEMENT_SKEW_MS, false);
+	const closed = before(Math.max(window.startMs, window.endMs) + PLACEMENT_SKEW_MS, true);
+	return {
+		deployedSha: opened?.sha ?? null,
+		deployedShaAmbiguous: opened?.sha !== closed?.sha
+	};
+}
 //#endregion
 //#region src/coverage/resolve-stream.ts
 /**
@@ -11774,6 +11812,8 @@ async function liveRunToReportResult(args) {
 		target: AGENT_BROWSER_TARGET,
 		status: result.status,
 		testCounts: null,
+		startedAt: result.startedAt,
+		finishedAt: new Date(Date.parse(result.startedAt) + result.durationMs).toISOString(),
 		durationMs: result.durationMs,
 		assertions: null,
 		analysis: null,
@@ -12392,15 +12432,15 @@ function renderRunMarkdown(featureName, specName, result) {
 	].join("\n") + result.steps.map((s) => [
 		`## ${s.stepId} — ${s.status}`,
 		`- duration: ${(s.durationMs / 1e3).toFixed(1)}s`,
-		`- instruction: ${oneLine(s.instruction)}`,
-		`- expected: ${oneLine(s.expected)}`,
-		`- reasoning: ${oneLine(s.reasoning)}`,
+		`- instruction: ${oneLine$1(s.instruction)}`,
+		`- expected: ${oneLine$1(s.expected)}`,
+		`- reasoning: ${oneLine$1(s.reasoning)}`,
 		...s.beforePng ? [`- before: ${s.beforePng}`] : [],
 		...s.afterPng ? [`- after: ${s.afterPng}`] : [],
 		""
 	].join("\n")).join("\n");
 }
-function oneLine(s) {
+function oneLine$1(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -13964,6 +14004,136 @@ async function waitForCdp(port) {
 	}
 }
 //#endregion
+//#region src/prompts/verifies-spec.ts
+/**
+* Asks whether a generated test actually decides what its spec claims.
+*
+* The generation loop's only bar is "the test goes green", and a rewrite that
+* weakens an assertion clears that bar as easily as one that keeps it. Green
+* therefore does not mean checked, and nothing else looks. Observed cases: a
+* step whose expectation was "the linked page opens" asserting instead that
+* the *link* is still visible on the page it clicked from; another asserting
+* on a navigation element unrelated to the step.
+*
+* Deliberately narrow. It reads only what the step says and what the code
+* does, and reports the step as unchecked when the two do not line up. It
+* does not review style, coverage, or whether the expectation is a good one.
+*/
+function verifiesSpecPrompt(input) {
+	return [
+		"You are reviewing whether a generated end-to-end test decides what its spec says.",
+		"",
+		"For each step below, the test must contain assertions that could FAIL if the",
+		"step's `expected` stopped holding. Report a step when:",
+		"",
+		"- its assertions cannot fail while the product is broken in the way the",
+		"  expectation describes (e.g. the expectation says a page opens, and the",
+		"  code only re-checks the element it clicked);",
+		"- what it asserts on is unrelated to what the step did (e.g. a navigation",
+		"  element that is present on every page);",
+		"- it depends on something that varies between runs and is not part of the",
+		"  expectation (a count, an index, a position, wording that changes);",
+		"- it has no assertion at all.",
+		"",
+		"Do NOT report: style, naming, structure, missing coverage the spec never",
+		"asked for, or an expectation you merely disagree with. A step that checks",
+		"less than you would have written, but still fails when the expectation",
+		"breaks, is fine.",
+		"",
+		"## Steps",
+		"",
+		...input.steps.map(stepLine),
+		"",
+		"## Generated test",
+		"",
+		"```",
+		input.source,
+		"```",
+		"",
+		"Answer with one json block and nothing else:",
+		"",
+		"```json",
+		"{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ] }",
+		"```",
+		"",
+		"`problem` is one sentence naming what the step claims and what the code",
+		"checks instead. Empty `findings` means every step is decided.",
+		languageDirective(input.language)
+	].join("\n");
+}
+function stepLine(step) {
+	if (!isExpandedActionStep(step)) return `- ${step.id}: judged by a model at run time — its claim is asserted by the injected call, so it needs no other assertion.`;
+	return [
+		`- ${step.id}`,
+		`  does: ${oneLine(step.instruction)}`,
+		`  expected: ${oneLine(step.expected)}`
+	].join("\n");
+}
+function oneLine(text) {
+	return text.trim().split("\n").map((l) => l.trim()).join(" ");
+}
+//#endregion
+//#region src/targets/verifies-spec.ts
+const FindingsSchema = z.object({ findings: z.array(z.object({
+	stepId: z.string(),
+	problem: z.string()
+})) });
+/**
+* Findings in the model's answer, or null when it did not answer in the
+* agreed shape. Null is not "no findings": the caller says so rather than
+* reporting a clean review it never got.
+*/
+function parseVerifiesSpecFindings(answer) {
+	const json = extractJsonBlock(answer);
+	if (!json) return null;
+	try {
+		return FindingsSchema.parse(JSON.parse(json)).findings;
+	} catch {
+		return null;
+	}
+}
+/** The warning a finding becomes, phrased so the reader knows the test is green for nothing. */
+function formatFinding(finding) {
+	return `step ${finding.stepId}: the generated test passes without deciding what this step claims — ${finding.problem}`;
+}
+/**
+* Read the generated test back and ask whether each step is actually decided
+* (see `verifiesSpecPrompt`). Returns warnings; an empty list means either a
+* clean review or one that could not be obtained, and the difference is
+* logged rather than encoded — a review that failed must not read as a pass,
+* but it must also not fail the generate that produced working files.
+*/
+async function reviewGeneratedTest(input) {
+	const source = (await Promise.all(input.result.files.filter((f) => f.kind === "test").map((f) => readFile(f.path, "utf8").catch(() => "")))).filter((s) => s.length > 0).join("\n\n");
+	if (source.length === 0) {
+		warn("could not check whether the generated test decides its spec (no test file to read)");
+		return [];
+	}
+	const { result: answer, isError } = await (input.invoke ?? invokeClaudeStreaming)({
+		prompt: verifiesSpecPrompt({
+			steps: input.steps,
+			source,
+			language: input.language
+		}),
+		allowedTools: [],
+		disableThinking: true,
+		maxTurns: 1,
+		silenceBashLog: true,
+		...input.model ? { model: input.model } : {},
+		cwd: input.cwd
+	}, () => {});
+	if (isError) {
+		warn("could not check whether the generated test decides its spec (Claude returned an error)");
+		return [];
+	}
+	const findings = parseVerifiesSpecFindings(answer);
+	if (findings === null) {
+		warn("could not check whether the generated test decides its spec (no usable answer)");
+		return [];
+	}
+	return findings.map(formatFinding);
+}
+//#endregion
 //#region src/targets/playwright/index.ts
 const PLAYWRIGHT_TARGET = "playwright";
 /**
@@ -14035,12 +14205,21 @@ async function generatePlaywrightTest(ctx) {
 	});
 	const missing = await missingInjectedCalls(result, stepMarkers, judgements);
 	for (const w of missing) warn(w);
+	const unchecked = await reviewGeneratedTest({
+		result,
+		steps: expanded,
+		language: ctx.language,
+		...ctx.model ? { model: ctx.model } : {},
+		cwd: ctx.cwd
+	});
+	for (const w of unchecked) warn(w);
 	return {
 		...result,
 		warnings: [
 			...result.warnings,
 			...judgeWarnings,
-			...missing
+			...missing,
+			...unchecked
 		]
 	};
 }
@@ -15198,7 +15377,7 @@ async function executeRun(targets, opts) {
 	const declared = [...new Set(specs.flatMap(resources))];
 	if (declared.length > 0) meta("serial groups", declared.join(", "));
 	let waitingOnGroup = [];
-	if (hubCtx && rerunProfile !== null) {
+	if (hubCtx && rerunProfile !== null && forExecution) {
 		const held = await holdSpecs(hubCtx, rerunProfile, specs, resources, opts.teardown);
 		waitingOnGroup = specs.flatMap((spec) => {
 			const groups = resources(spec).filter((n) => held.deniedResources.includes(n));
@@ -15729,6 +15908,7 @@ async function runOneDeterministicSpec(spec, index, ctx) {
 		CCQA_RUN_ID: runId
 	};
 	if (evidenceDir) specEnv[EVIDENCE_DIR_ENV] = evidenceDir;
+	const startedAt = (/* @__PURE__ */ new Date()).toISOString();
 	const proc = spawnVitestStreaming([
 		"run",
 		"--config",
@@ -15745,11 +15925,14 @@ async function runOneDeterministicSpec(spec, index, ctx) {
 	await Promise.all([streamFiltered(proc.stdout, sink, tail), streamFiltered(proc.stderr, sink, tail)]);
 	const specExitCode = await proc.exited;
 	blank();
+	const report = await readReport$1(reportFile);
 	return {
 		featureName,
 		specName,
 		scriptFile,
-		report: await readReport$1(reportFile),
+		startedAt,
+		finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		report,
 		exitCode: specExitCode,
 		outputTail: tail ? tail.toString() : null,
 		evidenceDir
@@ -15778,6 +15961,8 @@ async function analyzeDeterministicSummaries(summaries, cwd, reportDir, { pass,
 			spec: s.specName,
 			title: parsedSpec?.title ?? null,
 			target: AGENT_BROWSER_TARGET,
+			startedAt: s.startedAt,
+			finishedAt: s.finishedAt,
 			testCounts: s.report ? {
 				total: s.report.numTotalTests,
 				passed: s.report.numPassedTests,
@@ -20186,17 +20371,20 @@ function countSpecs(results) {
 async function updateSpecLedger(storage, run, results) {
 	const { gitHead, branch } = run;
 	if (run.kind !== "run" || !gitHead || !branch) return;
-	const entry = {
+	const placeRow = await rowDeployPlacer(storage, run);
+	const base = {
 		gitHead,
 		runId: run.id,
-		at: run.reportCreatedAt,
-		deployedSha: run.deployedSha ?? null,
-		deployedShaAmbiguous: run.deployedShaAmbiguous ?? false
+		at: run.reportCreatedAt
 	};
 	const ledger = emptyLedger();
 	for (const row of results) {
 		if (row.status === "skipped") continue;
 		const key = `${row.feature}/${row.spec}`;
+		const entry = {
+			...base,
+			...placeRow(row)
+		};
 		ledger.run[key] = entry;
 		if (row.status === "passed") ledger.green[key] = entry;
 		else ledger.red[key] = redEntry(entry, row);
@@ -20299,6 +20487,38 @@ async function resolveDeployedSha(storage, kind, project, profile, explicit) {
 	};
 }
 /**
+* Place each row against the deploy log by its own execution window rather
+* than the run's (ADR-0027), so a run that outlives a deploy loses only the
+* specs that straddled it.
+*
+* Falls back to the run's placement where a finer answer isn't available: a
+* client-asserted sha is the caller's claim about its whole run, and a row
+* with no `startedAt` (an older client) has nothing finer to say. A row with
+* no end is measured to now, which only widens its window.
+*/
+async function rowDeployPlacer(storage, run) {
+	const runPlacement = {
+		deployedSha: run.deployedSha ?? null,
+		deployedShaAmbiguous: run.deployedShaAmbiguous ?? false
+	};
+	if (run.deployedShaSource !== "hub-deploy-log") return () => runPlacement;
+	const log = await storage.deploys.getLog(run.project, run.profile ?? "default").catch((err) => {
+		console.error(`hub: could not read the deploy log to place rows of run "${run.id}": ${errMsg(err)}`);
+		return null;
+	});
+	if (!log) return () => runPlacement;
+	const sealedAt = Date.now();
+	return (row) => {
+		const startMs = row.startedAt ? Date.parse(row.startedAt) : NaN;
+		if (Number.isNaN(startMs)) return runPlacement;
+		const endMs = row.finishedAt ? Date.parse(row.finishedAt) : sealedAt;
+		return placeRowInDeployLog(log.entries, {
+			startMs,
+			endMs: Number.isNaN(endMs) ? sealedAt : endMs
+		});
+	};
+}
+/**
 * True when the deploy-log head moved while the run was open: the run
 * straddled a deploy, so which commit it exercised is not knowable and re-run
 * selection must report `unknown` instead of picking one. Only meaningful for

package/dist/hub-client/index.d.mts

@@ -35,9 +35,9 @@ declare const RunSchema: z.ZodObject<{
     running: "running";
   }>;
   kind: z.ZodDefault<z.ZodEnum<{
-    record: "record";
     run: "run";
     drift: "drift";
+    record: "record";
   }>>;
   drift: z.ZodDefault<z.ZodNullable<z.ZodObject<{
     specs: z.ZodNumber;
@@ -640,8 +640,8 @@ declare const ReportSpecResultSchema: z.ZodObject<{
   title: z.ZodNullable<z.ZodString>;
   target: z.ZodOptional<z.ZodString>;
   mode: z.ZodOptional<z.ZodEnum<{
-    deterministic: "deterministic";
     live: "live";
+    deterministic: "deterministic";
   }>>;
   status: z.ZodEnum<{
     passed: "passed";
@@ -655,6 +655,8 @@ declare const ReportSpecResultSchema: z.ZodObject<{
     failed: z.ZodNumber;
   }, z.core.$strip>>;
   durationMs: z.ZodNullable<z.ZodNumber>;
+  startedAt: z.ZodOptional<z.ZodString>;
+  finishedAt: z.ZodOptional<z.ZodString>;
   assertions: z.ZodNullable<z.ZodArray<z.ZodObject<{
     name: z.ZodString;
     status: z.ZodEnum<{
@@ -811,9 +813,9 @@ type ReportSpecResult = z.infer<typeof ReportSpecResultSchema>;
 declare const RunReportDataSchema: z.ZodObject<{
   schemaVersion: z.ZodLiteral<1>;
   kind: z.ZodDefault<z.ZodEnum<{
-    record: "record";
     run: "run";
     drift: "drift";
+    record: "record";
   }>>;
   createdAt: z.ZodString;
   runId: z.ZodNullable<z.ZodString>;
@@ -854,8 +856,8 @@ declare const RunReportDataSchema: z.ZodObject<{
     title: z.ZodNullable<z.ZodString>;
     target: z.ZodOptional<z.ZodString>;
     mode: z.ZodOptional<z.ZodEnum<{
-      deterministic: "deterministic";
       live: "live";
+      deterministic: "deterministic";
     }>>;
     status: z.ZodEnum<{
       passed: "passed";
@@ -869,6 +871,8 @@ declare const RunReportDataSchema: z.ZodObject<{
       failed: z.ZodNumber;
     }, z.core.$strip>>;
     durationMs: z.ZodNullable<z.ZodNumber>;
+    startedAt: z.ZodOptional<z.ZodString>;
+    finishedAt: z.ZodOptional<z.ZodString>;
     assertions: z.ZodNullable<z.ZodArray<z.ZodObject<{
       name: z.ZodString;
       status: z.ZodEnum<{

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.49.0",
+  "version": "1.50.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.49.0",
+  "version": "1.50.1",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {