ccqa 1.50.0 → 1.51.0

package/dist/bin/ccqa.mjs

@@ -8603,11 +8603,11 @@ async function buildLiveTranscriptExcerpt(result, options = {}) {
 	return combined.length > maxBytes ? `${combined.slice(0, maxBytes)}\n…[transcript excerpt truncated at ${maxBytes} bytes]` : combined;
 }
 function formatPreviousStep(step) {
-	const reason = oneLine$2(step.reasoning) || "(no reason given)";
+	const reason = oneLine$3(step.reasoning) || "(no reason given)";
 	return `[${step.stepId} ${step.status}: ${reason}]`;
 }
 async function formatFailingStep(step, headBytes, tailBytes) {
-	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$2(step.instruction)}\nExpected: ${oneLine$2(step.expected)}\nReasoning (Claude's verdict): ${oneLine$2(step.reasoning) || "(none)"}`;
+	const header = `\n>>> Failed step ${step.stepId}\nInstruction: ${oneLine$3(step.instruction)}\nExpected: ${oneLine$3(step.expected)}\nReasoning (Claude's verdict): ${oneLine$3(step.reasoning) || "(none)"}`;
 	if (!step.logTxt) return `${header}\n(No assistant log file recorded for this step.)`;
 	const raw = await readFile(step.logTxt, "utf-8").catch((err) => {
 		return `[log file unreadable: ${err instanceof Error ? err.message : String(err)}]`;
@@ -8617,7 +8617,7 @@ async function formatFailingStep(step, headBytes, tailBytes) {
 	const tail = raw.slice(raw.length - tailBytes);
 	return `${header}\n--- assistant log (head ${headBytes}B) ---\n${head}\n…[${raw.length - headBytes - tailBytes} bytes omitted]…\n--- assistant log (tail ${tailBytes}B) ---\n${tail}`;
 }
-function oneLine$2(s) {
+function oneLine$3(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -10129,10 +10129,10 @@ function describeStep(step) {
 	return describeStepBody(step);
 }
 function describeStepBody(step) {
-	if (isJudgeBody(step)) return `judge: ${oneLine$1(step.judgeByLlm)}`;
-	return `${oneLine$1(step.instruction)} → ${oneLine$1(step.expected)}`;
+	if (isJudgeBody(step)) return `judge: ${oneLine$2(step.judgeByLlm)}`;
+	return `${oneLine$2(step.instruction)} → ${oneLine$2(step.expected)}`;
 }
-function oneLine$1(text) {
+function oneLine$2(text) {
 	return text.trim().replace(/\s+/g, " ");
 }
 function emptyDeployLog() {
@@ -12432,15 +12432,15 @@ function renderRunMarkdown(featureName, specName, result) {
 	].join("\n") + result.steps.map((s) => [
 		`## ${s.stepId} — ${s.status}`,
 		`- duration: ${(s.durationMs / 1e3).toFixed(1)}s`,
-		`- instruction: ${oneLine(s.instruction)}`,
-		`- expected: ${oneLine(s.expected)}`,
-		`- reasoning: ${oneLine(s.reasoning)}`,
+		`- instruction: ${oneLine$1(s.instruction)}`,
+		`- expected: ${oneLine$1(s.expected)}`,
+		`- reasoning: ${oneLine$1(s.reasoning)}`,
 		...s.beforePng ? [`- before: ${s.beforePng}`] : [],
 		...s.afterPng ? [`- after: ${s.afterPng}`] : [],
 		""
 	].join("\n")).join("\n");
 }
-function oneLine(s) {
+function oneLine$1(s) {
 	return s.replace(/\s+/g, " ").trim();
 }
 //#endregion
@@ -14004,6 +14004,136 @@ async function waitForCdp(port) {
 	}
 }
 //#endregion
+//#region src/prompts/verifies-spec.ts
+/**
+* Asks whether a generated test actually decides what its spec claims.
+*
+* The generation loop's only bar is "the test goes green", and a rewrite that
+* weakens an assertion clears that bar as easily as one that keeps it. Green
+* therefore does not mean checked, and nothing else looks. Observed cases: a
+* step whose expectation was "the linked page opens" asserting instead that
+* the *link* is still visible on the page it clicked from; another asserting
+* on a navigation element unrelated to the step.
+*
+* Deliberately narrow. It reads only what the step says and what the code
+* does, and reports the step as unchecked when the two do not line up. It
+* does not review style, coverage, or whether the expectation is a good one.
+*/
+function verifiesSpecPrompt(input) {
+	return [
+		"You are reviewing whether a generated end-to-end test decides what its spec says.",
+		"",
+		"For each step below, the test must contain assertions that could FAIL if the",
+		"step's `expected` stopped holding. Report a step when:",
+		"",
+		"- its assertions cannot fail while the product is broken in the way the",
+		"  expectation describes (e.g. the expectation says a page opens, and the",
+		"  code only re-checks the element it clicked);",
+		"- what it asserts on is unrelated to what the step did (e.g. a navigation",
+		"  element that is present on every page);",
+		"- it depends on something that varies between runs and is not part of the",
+		"  expectation (a count, an index, a position, wording that changes);",
+		"- it has no assertion at all.",
+		"",
+		"Do NOT report: style, naming, structure, missing coverage the spec never",
+		"asked for, or an expectation you merely disagree with. A step that checks",
+		"less than you would have written, but still fails when the expectation",
+		"breaks, is fine.",
+		"",
+		"## Steps",
+		"",
+		...input.steps.map(stepLine),
+		"",
+		"## Generated test",
+		"",
+		"```",
+		input.source,
+		"```",
+		"",
+		"Answer with one json block and nothing else:",
+		"",
+		"```json",
+		"{ \"findings\": [ { \"stepId\": \"step-05\", \"problem\": \"…\" } ] }",
+		"```",
+		"",
+		"`problem` is one sentence naming what the step claims and what the code",
+		"checks instead. Empty `findings` means every step is decided.",
+		languageDirective(input.language)
+	].join("\n");
+}
+function stepLine(step) {
+	if (!isExpandedActionStep(step)) return `- ${step.id}: judged by a model at run time — its claim is asserted by the injected call, so it needs no other assertion.`;
+	return [
+		`- ${step.id}`,
+		`  does: ${oneLine(step.instruction)}`,
+		`  expected: ${oneLine(step.expected)}`
+	].join("\n");
+}
+function oneLine(text) {
+	return text.trim().split("\n").map((l) => l.trim()).join(" ");
+}
+//#endregion
+//#region src/targets/verifies-spec.ts
+const FindingsSchema = z.object({ findings: z.array(z.object({
+	stepId: z.string(),
+	problem: z.string()
+})) });
+/**
+* Findings in the model's answer, or null when it did not answer in the
+* agreed shape. Null is not "no findings": the caller says so rather than
+* reporting a clean review it never got.
+*/
+function parseVerifiesSpecFindings(answer) {
+	const json = extractJsonBlock(answer);
+	if (!json) return null;
+	try {
+		return FindingsSchema.parse(JSON.parse(json)).findings;
+	} catch {
+		return null;
+	}
+}
+/** The warning a finding becomes, phrased so the reader knows the test is green for nothing. */
+function formatFinding(finding) {
+	return `step ${finding.stepId}: the generated test passes without deciding what this step claims — ${finding.problem}`;
+}
+/**
+* Read the generated test back and ask whether each step is actually decided
+* (see `verifiesSpecPrompt`). Returns warnings; an empty list means either a
+* clean review or one that could not be obtained, and the difference is
+* logged rather than encoded — a review that failed must not read as a pass,
+* but it must also not fail the generate that produced working files.
+*/
+async function reviewGeneratedTest(input) {
+	const source = (await Promise.all(input.result.files.filter((f) => f.kind === "test").map((f) => readFile(f.path, "utf8").catch(() => "")))).filter((s) => s.length > 0).join("\n\n");
+	if (source.length === 0) {
+		warn("could not check whether the generated test decides its spec (no test file to read)");
+		return [];
+	}
+	const { result: answer, isError } = await (input.invoke ?? invokeClaudeStreaming)({
+		prompt: verifiesSpecPrompt({
+			steps: input.steps,
+			source,
+			language: input.language
+		}),
+		allowedTools: [],
+		disableThinking: true,
+		maxTurns: 1,
+		silenceBashLog: true,
+		...input.model ? { model: input.model } : {},
+		cwd: input.cwd
+	}, () => {});
+	if (isError) {
+		warn("could not check whether the generated test decides its spec (Claude returned an error)");
+		return [];
+	}
+	const findings = parseVerifiesSpecFindings(answer);
+	if (findings === null) {
+		warn("could not check whether the generated test decides its spec (no usable answer)");
+		return [];
+	}
+	return findings.map(formatFinding);
+}
+//#endregion
 //#region src/targets/playwright/index.ts
 const PLAYWRIGHT_TARGET = "playwright";
 /**
@@ -14075,12 +14205,21 @@ async function generatePlaywrightTest(ctx) {
 	});
 	const missing = await missingInjectedCalls(result, stepMarkers, judgements);
 	for (const w of missing) warn(w);
+	const unchecked = await reviewGeneratedTest({
+		result,
+		steps: expanded,
+		language: ctx.language,
+		...ctx.model ? { model: ctx.model } : {},
+		cwd: ctx.cwd
+	});
+	for (const w of unchecked) warn(w);
 	return {
 		...result,
 		warnings: [
 			...result.warnings,
 			...judgeWarnings,
-			...missing
+			...missing,
+			...unchecked
 		]
 	};
 }
@@ -14525,32 +14664,42 @@ function requireReportToHubConnection(conn) {
 	process.exit(2);
 }
 /**
-* Open the run a `--report-to-hub` command patches into. Failure is fatal: a
-* job that asked to publish and cannot reach the hub has not done what it was
-* told. Thrown rather than exited, so a caller's `finally` still runs (the
-* audit releases its spec claims there). Not retried: a dropped response after
-* the hub committed would leave a second orphan running run.
+* Open the run a `--report-to-hub` command patches into, and name it on stderr
+* — stdout belongs to the report (`--report-format json`), and a caller that
+* links to the run needs its id before the command ends (docs/hub.md).
+*
+* Failure is fatal: a job that asked to publish and cannot reach the hub has
+* not done what it was told. Thrown rather than exited, so a caller's
+* `finally` still runs (the audit releases its spec claims there). Not
+* retried: a dropped response after the hub committed would leave a second
+* orphan running run.
 */
-async function openHubRun(kind, conn, cwd, profile) {
-	const [branch, gitHead] = await Promise.all([detectBranch(cwd), getGitHead(cwd)]);
+async function openHubRun(kind, conn, cwd, opts = {}) {
+	const [branch, gitHead] = await Promise.all([detectBranch(cwd), opts.gitHead !== void 0 ? opts.gitHead : getGitHead(cwd)]);
+	let opened;
 	try {
-		const run = await conn.hub.openRun({
+		opened = await conn.hub.openRun({
 			project: conn.project,
 			kind,
 			...branch ? { branch } : {},
-			...profile ? { profile } : {},
+			...opts.profile ? { profile: opts.profile } : {},
 			...gitHead ? { gitHead } : {},
+			...opts.deployedSha ? { deployedSha: opts.deployedSha } : {},
 			...ciProvenance()
 		});
-		return {
-			hub: conn.hub,
-			kind,
-			runId: run.id,
-			gitHead
-		};
 	} catch (err) {
 		throw new RunUsageError(`--report-to-hub: could not open a run on the hub (${errMessage(err)})`);
 	}
+	process.stderr.write(`[hub-run] ${JSON.stringify({
+		id: opened.id,
+		kind
+	})}\n`);
+	return {
+		hub: conn.hub,
+		kind,
+		runId: opened.id,
+		gitHead
+	};
 }
 /**
 * Close an open run with its final rows and envelope, answering whether it
@@ -15238,7 +15387,7 @@ async function executeRun(targets, opts) {
 	const declared = [...new Set(specs.flatMap(resources))];
 	if (declared.length > 0) meta("serial groups", declared.join(", "));
 	let waitingOnGroup = [];
-	if (hubCtx && rerunProfile !== null) {
+	if (hubCtx && rerunProfile !== null && forExecution) {
 		const held = await holdSpecs(hubCtx, rerunProfile, specs, resources, opts.teardown);
 		waitingOnGroup = specs.flatMap((spec) => {
 			const groups = resources(spec).filter((n) => held.deniedResources.includes(n));
@@ -15298,25 +15447,17 @@ async function executeRun(targets, opts) {
 			reportDir: null
 		};
 	}
-	const det = await runDeterministicSpecs(detSpecs, opts, cwd, reportDir, resources);
 	let hubRunId = null;
 	let hubSink;
 	let hubPublishBroken = false;
-	if (hubCtx != null && opts.reportToHub) try {
-		const branch = await detectBranch(cwd);
-		const opened = await hubCtx.hub.openRun({
-			project: hubCtx.project,
-			...branch ? { branch } : {},
+	if (hubCtx != null && opts.reportToHub) {
+		const { runId } = await openHubRun("run", hubCtx, cwd, {
 			...opts.hubProfile ? { profile: opts.hubProfile } : {},
-			...git.head ? { gitHead: git.head } : {},
-			...deployedSha ? { deployedSha } : {},
-			...ciProvenance(),
-			kind: "run"
+			gitHead: git.head,
+			deployedSha
 		});
-		hubRunId = opened.id;
-		info(`hub: incremental run opened (${opened.id})`);
-		await coverage?.linkHubRun(opened.id);
-		const runId = opened.id;
+		hubRunId = runId;
+		await coverage?.linkHubRun(runId);
 		let hubPatchEverSucceeded = false;
 		hubSink = { onUpsert: async (row) => {
 			try {
@@ -15334,8 +15475,6 @@ async function executeRun(targets, opts) {
 				} else warn(`hub: incremental push failed for ${row.feature}/${row.spec}: ${errMessage(err)}`);
 			}
 		} };
-	} catch (err) {
-		throw new RunUsageError(`--report-to-hub: could not open a run on the hub (${errMessage(err)})`);
 	}
 	const incrementalReport = createIncrementalReport(reportDir, buildReportEnvelope({
 		git,
@@ -15360,6 +15499,7 @@ async function executeRun(targets, opts) {
 			warn(`hub: could not finalize interrupted run ${hubRunId}: ${errMessage(err)}`);
 		}
 	});
+	const det = await runDeterministicSpecs(detSpecs, opts, cwd, reportDir, resources);
 	const externalRows = await runExternalSpecs(dispatch, {
 		cwd,
 		reportDir,
@@ -17847,8 +17987,7 @@ async function runRecord(specPath, opts) {
 		}
 		throw e;
 	});
-	const push = pushConn ? await openHubRun("record", pushConn, cwdForProfile, opts.hubProfile) : null;
-	if (push) info(`hub: record run opened (${push.runId})`);
+	const push = pushConn ? await openHubRun("record", pushConn, cwdForProfile, { ...opts.hubProfile ? { profile: opts.hubProfile } : {} }) : null;
 	let recorded = false;
 	let sealed = true;
 	let tracingStep;
@@ -18926,10 +19065,7 @@ async function runAudit(specPath, opts) {
 	let push = null;
 	try {
 		promptCtx = await resolveAuditPromptContext(opts, cwd);
-		if (pushConn) {
-			push = await openHubRun("drift", pushConn, cwd, opts.hubProfile);
-			if (format === "text") info(`hub: incremental drift run opened (${push.runId})`);
-		}
+		if (pushConn) push = await openHubRun("drift", pushConn, cwd, { ...opts.hubProfile ? { profile: opts.hubProfile } : {} });
 		results = await analyzeDrift({
 			targets,
 			cwd,
@@ -20474,7 +20610,7 @@ function createPatchRunHandler(config) {
 		sendJson(ctx.res, 200, updated);
 	};
 }
-/** GET /api/v1/runs?project&branch&status&kind&since&until&limit */
+/** GET /api/v1/runs?project&branch&status&kind&ciRunId&since&until&limit */
 function createListRunsHandler(storage) {
 	return async (ctx) => {
 		const project = ctx.url.searchParams.get("project");
@@ -20482,11 +20618,13 @@ function createListRunsHandler(storage) {
 		const status = ctx.url.searchParams.get("status");
 		const limitRaw = ctx.url.searchParams.get("limit");
 		const kindsRaw = ctx.url.searchParams.get("kind");
+		const ciRunId = boundedParam(ctx.url.searchParams.get("ciRunId"), "ciRunId", 128);
 		const runs = await storage.runs.list({
 			...project ? { project } : {},
 			...branch ? { branch } : {},
 			...status ? { status } : {},
 			...kindsRaw ? { kinds: kindsRaw.split(",").map(requireKind) } : {},
+			...ciRunId ? { ciRunId } : {},
 			...requireWindowParams(ctx.url),
 			...limitRaw ? { limit: Number(limitRaw) } : {}
 		});
@@ -24678,12 +24816,15 @@ const CLIENT_JS = `
     // Disconnected: the full-screen login gate is the only thing to show.
     if (!state.token) { showAuthGate(false); return; }
     showAuthGate(true);
+    // A run is addressed by its own id, not through a project, so a link from
+    // outside (a chat message, a CI job) opens it even in a browser that has
+    // never picked one — ahead of the gate below, which would drop the link.
+    var m = parseRunHash(location.hash);
+    if (m) { openRunDetail(m.runId, m.caseKey); return; }
     // With no project chosen yet, the Projects picker is the only useful view —
     // land there (e.g. right after login) instead of an empty Runs list, and
-    // gate any deep-linked #/runs or #/secrets to it too.
+    // gate any deep-linked project-scoped view to it too.
     if (location.hash === "#/projects" || !state.project) { openProjects(); return; }
-    var m = parseRunHash(location.hash);
-    if (m) { openRunDetail(m.runId, m.caseKey); return; }
     if (location.hash === "#/perspectives") { openPerspectives(); return; }
     if (location.hash === "#/coverage") { openCoverage(); return; }
     if (location.hash === "#/secrets") { openSecrets(); return; }
@@ -26267,6 +26408,11 @@ const CLIENT_JS = `
     var runGone = false;
 
     apiFetch("/api/v1/runs/" + encodeURIComponent(runId)).then(function (run) {
+      // Arrived by an outside link: the selected project is whatever the last
+      // visit or the boot auto-select left, which need not be this run's.
+      // Follow the run, so the nav around it answers for the project the page
+      // is actually showing.
+      if (run.project && run.project !== state.project) adoptProject(run.project);
       renderRunHead(run);
     }).catch(function (err) {
       runGone = err.status === 404;
@@ -28344,14 +28490,20 @@ const CLIENT_JS = `
     updateNavGate();
   }
 
-  // Scope to a project and land on its Runs view. Shared by the top menu, the
-  // Projects grid, and the "new project" flow. Switching project restores the
-  // profile last chosen for that project (or "default" if none was saved);
-  // the Secrets tab reloads the profile list when opened.
-  function chooseProject(p) {
+  // Scope the UI to a project, restoring the profile last chosen for it (or
+  // "default" if none was saved); the Secrets tab reloads the profile list when
+  // opened. Separate from the navigation below because arriving on a run by
+  // link adopts a project without going anywhere.
+  function adoptProject(p) {
     setProject(p);
     storeProject(p);
     setProfile(storedProfileForProject(p) || "default");
+  }
+
+  // Scope to a project and land on its Runs view. Shared by the top menu, the
+  // Projects grid, and the "new project" flow.
+  function chooseProject(p) {
+    adoptProject(p);
     location.hash = "#/runs";
     route();
   }
@@ -30049,7 +30201,7 @@ function createFileRunStore(root) {
 		async delete(id) {
 			await serialize(runMetaPath(root, id), () => removePath(runDir(root, id)));
 		},
-		async list({ project, branch, status, kinds, since, until, limit }) {
+		async list({ project, branch, status, kinds, ciRunId, since, until, limit }) {
 			const ids = await listSubdirsOrEmpty(runsDir(root));
 			const inWindow = windowFilter({
 				since,
@@ -30063,6 +30215,7 @@ function createFileRunStore(root) {
 				if (branch !== void 0 && run.branch !== branch) continue;
 				if (status !== void 0 && run.status !== status) continue;
 				if (kinds !== void 0 && !kinds.includes(run.kind)) continue;
+				if (ciRunId !== void 0 && run.ciRunId !== ciRunId) continue;
 				if (!inWindow(run.createdAt)) continue;
 				runs.push(run);
 			}

package/dist/hub-client/index.d.mts

@@ -1147,9 +1147,10 @@ interface HubClient {
    * `gitHead` stamps the run's commit at open time, so even a run that dies
    * before its final reconcile patch is attributable to a commit.
    *
-   * `deployedSha` does the same for the environment's commit. The hub would
-   * otherwise read its own deploy-log head here, and the open happens after
-   * the deterministic phase, not before the first spec.
+   * `deployedSha` does the same for the environment's commit: the baseline the
+   * caller selected against, which a `--rerun` inherits from an earlier run.
+   * Left unset the hub reads its own deploy-log head, which by the time this
+   * call lands can already name a later deploy.
    */
   openRun(meta: {
     project: string;
@@ -1165,11 +1166,13 @@ interface HubClient {
   }): Promise<Run>;
   /** Add finished spec rows (+ evidence) to a running run; `done` closes it. */
   patchRun(id: string, body: PatchRunRequest): Promise<Run>;
+  /** `ciRunId` answers "which runs did this CI job create", whatever the window. */
   listRuns(q?: {
     project?: string;
     branch?: string;
     status?: RunStatus;
     kind?: Run["kind"];
+    ciRunId?: string;
     limit?: number;
   }): Promise<Run[]>;
   getRun(id: string): Promise<Run>;

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.50.0",
+  "version": "1.51.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.50.0",
+  "version": "1.51.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {