ccqa 1.46.1 → 1.47.0

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.46.1",
+  "version": "1.47.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {
@@ -26,6 +26,10 @@
     "./hub-client": {
       "types": "./dist/hub-client/index.d.mts",
       "import": "./dist/hub-client/index.mjs"
+    },
+    "./judge": {
+      "types": "./dist/runtime/judge.d.mts",
+      "import": "./dist/runtime/judge.mjs"
     }
   },
   "files": [

package/dist/runtime/judge.d.mts

@@ -0,0 +1,46 @@
+//#region src/runtime/judge.d.ts
+/** One decided claim. */
+interface Verdict {
+  ok: boolean;
+  /** Why, in one sentence. Carried into the failure message so a red test says what was wrong. */
+  reason: string;
+}
+/**
+ * The slice of Playwright's `Page` this needs. Structural because ccqa does
+ * not depend on Playwright, which also lets a caller pass anything else that
+ * can hand over text.
+ */
+interface TextSource {
+  innerText(selector: string): Promise<string>;
+}
+/**
+ * Fails the test unless a model agrees the claim holds for the text read from
+ * `from` (a selector; omitted, the page's body). The reason the model gave
+ * rides in the failure, so a red test says what was wrong.
+ *
+ * A selector matching several elements judges the first, as Playwright's
+ * page-level `innerText` does — narrow it if that is not what you mean.
+ */
+declare function judgeByLlm(page: TextSource, claim: string, from?: string): Promise<void>;
+interface ClaimInput {
+  /** The claim to decide, as written in the spec's `judgeByLlm`. */
+  claim: string;
+  /** The text it is decided against. */
+  text: string;
+  model?: string;
+  cwd?: string;
+}
+/**
+ * Decides a claim about text a run cannot predict — a generated answer, a
+ * summary. A model that will not answer, or answers something this cannot
+ * read, is an unmade decision rather than a passing one: it throws, so a
+ * claim never goes silently unjudged.
+ */
+declare function decideClaim(input: ClaimInput): Promise<Verdict>;
+/**
+ * The model is asked for bare JSON but sometimes wraps it in prose or a fence.
+ * An answer with no readable verdict is a failure to decide, not a false one.
+ */
+declare function parseVerdict(answer: string): Verdict;
+//#endregion
+export { ClaimInput, TextSource, Verdict, decideClaim, judgeByLlm, parseVerdict };

package/dist/runtime/judge.mjs

@@ -0,0 +1,78 @@
+import { a as truncate, p as invokeClaudeStreaming, r as extractJsonCandidates } from "../diagnose-CSQzwS5f.mjs";
+//#region src/runtime/judge.ts
+const SYSTEM_PROMPT = [
+	"You decide whether a claim holds for a piece of text taken from a web page under test.",
+	"",
+	"Judge only what the claim says. Do not reward text that is merely well-formed,",
+	"and do not fail text for wording, length, or formatting the claim does not mention.",
+	"An empty or error-like text fails any claim about content.",
+	"",
+	"Answer with one line of JSON and nothing else: {\"ok\": true|false, \"reason\": \"<one sentence>\"}"
+].join("\n");
+/** Beyond this the tail is dropped, so one runaway page cannot fill the model's context. */
+const MAX_TEXT_CHARS = 2e4;
+/** A judge runs inside a test, so a turn that will not finish has to fail rather than hold the run. */
+const JUDGE_TIMEOUT_MS = 6e4;
+/**
+* Fails the test unless a model agrees the claim holds for the text read from
+* `from` (a selector; omitted, the page's body). The reason the model gave
+* rides in the failure, so a red test says what was wrong.
+*
+* A selector matching several elements judges the first, as Playwright's
+* page-level `innerText` does — narrow it if that is not what you mean.
+*/
+async function judgeByLlm(page, claim, from = "body") {
+	const verdict = await decideClaim({
+		claim,
+		text: await page.innerText(from)
+	});
+	if (!verdict.ok) throw new Error(`judgeByLlm: the claim did not hold (${verdict.reason || "no reason given"})\n  claim: ${claim}\n  read from: ${from}`);
+}
+/**
+* Decides a claim about text a run cannot predict — a generated answer, a
+* summary. A model that will not answer, or answers something this cannot
+* read, is an unmade decision rather than a passing one: it throws, so a
+* claim never goes silently unjudged.
+*/
+async function decideClaim(input) {
+	const { result, isError, errorDetail } = await invokeClaudeStreaming({
+		prompt: [
+			"## Claim",
+			input.claim.trim(),
+			"",
+			"## Text",
+			truncate(input.text.trim(), MAX_TEXT_CHARS) || "(the page yielded no text)"
+		].join("\n"),
+		systemPrompt: SYSTEM_PROMPT,
+		allowedTools: [],
+		maxTurns: 1,
+		disableThinking: true,
+		timeoutMs: JUDGE_TIMEOUT_MS,
+		...input.model ? { model: input.model } : {},
+		...input.cwd ? { cwd: input.cwd } : {}
+	}, () => {});
+	if (isError) throw new Error(`judgeByLlm: the model did not answer (${errorDetail || result})`);
+	return parseVerdict(result);
+}
+/**
+* The model is asked for bare JSON but sometimes wraps it in prose or a fence.
+* An answer with no readable verdict is a failure to decide, not a false one.
+*/
+function parseVerdict(answer) {
+	for (const candidate of extractJsonCandidates(answer)) {
+		let parsed;
+		try {
+			parsed = JSON.parse(candidate);
+		} catch {
+			continue;
+		}
+		const verdict = parsed;
+		if (typeof verdict?.ok === "boolean") return {
+			ok: verdict.ok,
+			reason: typeof verdict.reason === "string" ? verdict.reason : ""
+		};
+	}
+	throw new Error(`judgeByLlm: no verdict in the model's answer (${truncate(answer, 200)})`);
+}
+//#endregion
+export { decideClaim, judgeByLlm, parseVerdict };

package/dist/runtime/step-evidence.mjs

@@ -1,4 +1,4 @@
-import { i as sanitizeStepId, t as EVIDENCE_DIR_ENV } from "../evidence-constants-C425F7ZG.mjs";
+import { i as sanitizeStepId, t as EVIDENCE_DIR_ENV } from "../evidence-constants-Cm_S_5od.mjs";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 //#region src/runtime/step-evidence.ts

package/dist/runtime/test-helpers.mjs

@@ -1,5 +1,5 @@
-import { i as sanitizeStepId, n as FAILURE_SOURCE, r as FAILURE_STEP_ID, t as EVIDENCE_DIR_ENV } from "../evidence-constants-C425F7ZG.mjs";
-import { n as spawnAB, t as sleepSync } from "../spawn-ab-Bm34WBui.mjs";
+import { i as sanitizeStepId, n as FAILURE_SOURCE, r as FAILURE_STEP_ID, t as EVIDENCE_DIR_ENV } from "../evidence-constants-Cm_S_5od.mjs";
+import { n as spawnAB, t as sleepSync } from "../spawn-ab-CR_Sr7wh.mjs";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { dirname, isAbsolute, join, resolve } from "node:path";
 //#region src/runtime/test-helpers.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ccqa",
-  "version": "1.46.1",
+  "version": "1.47.0",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "repository": {
@@ -26,6 +26,10 @@
     "./hub-client": {
       "types": "./dist/hub-client/index.d.mts",
       "import": "./dist/hub-client/index.mjs"
+    },
+    "./judge": {
+      "types": "./dist/runtime/judge.d.mts",
+      "import": "./dist/runtime/judge.mjs"
     }
   },
   "files": [