jeopi-cli 16.2.30 → 16.4.0

package/src/goals/runtime.ts

@@ -6,6 +6,13 @@ import { assertSubstantiveCompletionEvidence } from "./completion-evidence";
 import { GoalEvidenceTracker, isVerificationSignal, MUTATION_TOOL_NAMES } from "./evidence-gate";
 import type { Goal, GoalBudgetSteering, GoalModeState, GoalRuntimeEvent, GoalTokenUsage } from "./state";
 
+export interface GoalCompletionVerdict {
+	verdict: "okay" | "iterate" | "reject";
+	justification: string;
+	summary?: string;
+	requiredFixes: string[];
+}
+
 export interface GoalRuntimeHost {
 	getState(): GoalModeState | undefined;
 	setState(state: GoalModeState | undefined): void;
@@ -17,6 +24,16 @@ export interface GoalRuntimeHost {
 		content: string;
 		deliverAs?: "steer" | "followUp" | "nextTurn";
 	}): Promise<void>;
+	/**
+	 * Spawn an independent grader sub-agent to verdict a completion claim against
+	 * the objective and the actual repo diff (article gap: "the agent that wrote
+	 * the code is not the agent that grades it"). The sub-agent sees only the
+	 * objective + evidence text passed here plus its own repo inspection — never
+	 * this session's transcript/reasoning. Optional: hosts that don't wire this
+	 * (e.g. test harnesses, `goal.verifyCompletion` disabled) skip independent
+	 * grading and fall back to the deterministic evidence gate alone.
+	 */
+	verifyCompletion?(objective: string, evidence: string): Promise<GoalCompletionVerdict>;
 	now?(): number;
 }
 
@@ -122,6 +139,8 @@ export class GoalRuntime {
 	#turnSnapshot: GoalTurnSnapshot | undefined;
 	#wallClock: GoalWallClockSnapshot;
 	#budgetReportedFor: string | undefined;
+	/** Set when an independent-verifier non-okay verdict was bounced once; cleared by any new mutation/verification or a fresh goal. Mirrors GoalEvidenceTracker's bypass latch, kept separate because it gates a different (probabilistic) signal. */
+	#verifierBypassArmed = false;
 	#accountingTail: Promise<void> = Promise.resolve();
 
 	constructor(host: GoalRuntimeHost) {
@@ -219,10 +238,12 @@ export class GoalRuntime {
 		if (!this.#hasAccountingState()) return;
 		if (MUTATION_TOOL_NAMES[toolName]) {
 			this.#evidence.recordMutation();
+			this.#verifierBypassArmed = false;
 			return;
 		}
 		if (detail && isVerificationSignal(detail.command ?? "", detail.outputHead ?? "")) {
 			this.#evidence.recordVerification();
+			this.#verifierBypassArmed = false;
 		}
 	}
 
@@ -418,6 +439,7 @@ export class GoalRuntime {
 			const state = this.#createGoalState(objective, input.tokenBudget);
 			this.#budgetReportedFor = undefined;
 			this.#evidence.reset();
+			this.#verifierBypassArmed = false;
 			this.#markActiveAccounting(state.goal);
 			await this.#commitState(state, { persist: "goal" });
 			return state;
@@ -437,6 +459,7 @@ export class GoalRuntime {
 			const state = this.#createGoalState(objective, input.tokenBudget);
 			this.#budgetReportedFor = undefined;
 			this.#evidence.reset();
+			this.#verifierBypassArmed = false;
 			this.#markActiveAccounting(state.goal);
 			await this.#commitState(state, { persist: "goal" });
 			return state;
@@ -522,6 +545,34 @@ export class GoalRuntime {
 			if (gate.block) {
 				throw new Error(gate.message);
 			}
+			// Independent grader (goal-verifier sub-agent, see GoalRuntimeHost.verifyCompletion):
+			// maker != grader, per the compounding-loop primitive — the same agent that did
+			// the work does not get to be the sole judge of whether it's done. Same bounce-once
+			// shape as the deterministic gate above, kept as a separate latch since this signal
+			// is probabilistic (an LLM verdict), not mechanical.
+			if (this.#host.verifyCompletion) {
+				const verdict = await this.#host.verifyCompletion(state.goal.objective, verifiedEvidence);
+				if (verdict.verdict !== "okay") {
+					if (this.#verifierBypassArmed) {
+						// Second attempt with no new mutation/verification since the bounce:
+						// escape hatch for a verifier false-positive-reject/iterate.
+						this.#verifierBypassArmed = false;
+					} else {
+						this.#verifierBypassArmed = true;
+						const fixes =
+							verdict.requiredFixes.length > 0
+								? `\nRequired fixes:\n${verdict.requiredFixes.map(fix => `- ${fix}`).join("\n")}`
+								: "";
+						throw new Error(
+							`Completion blocked: independent verifier verdict was "${verdict.verdict}", not "okay". ` +
+								`${verdict.justification}${fixes}\nAddress the gaps above, then call complete again. ` +
+								"If the verifier is wrong, call complete again unchanged to override.",
+						);
+					}
+				} else {
+					this.#verifierBypassArmed = false;
+				}
+			}
 			state.enabled = false;
 			state.goal.status = "complete";
 			state.goal.completionEvidence = verifiedEvidence;

package/src/goals/verifier.ts

@@ -0,0 +1,100 @@
+/**
+ * Independent-verifier bridge for goal-mode completion (see GoalRuntimeHost.verifyCompletion
+ * in runtime.ts). Spawns the bundled `goal-verifier` sub-agent — a fresh session with its own
+ * kernel, its own read-only tool set, and no exposure to the maker's transcript/reasoning —
+ * to grade a completion claim against the objective and the actual repo diff. Reuses the eval
+ * tool's `agent()` bridge as a plain async call (no eval kernel required outside eval cells;
+ * `withBridgeTimeoutPause` in agent-bridge.ts is a no-op when no `emitStatus` sink is wired).
+ */
+import { runEvalAgent } from "../eval/agent-bridge";
+import { getBundledAgent } from "../task/agents";
+import type { ToolSession } from "../tools";
+import type { GoalCompletionVerdict } from "./runtime";
+
+const VERIFIER_AGENT_NAME = "goal-verifier";
+
+/** Cap the evidence claim embedded in the verifier prompt; the verifier re-derives ground truth from the repo, not from a long claim. */
+const MAX_EVIDENCE_CHARS = 4000;
+
+function parseVerdict(output: string): GoalCompletionVerdict {
+	let data: unknown;
+	try {
+		data = JSON.parse(output);
+	} catch {
+		// Verifier didn't return a structured payload (refused, crashed, plain-text
+		// reply). Fail open to "iterate" rather than silently trusting completion —
+		// a broken verifier bounces once instead of being bypassed unnoticed.
+		return {
+			verdict: "iterate",
+			justification: `goal-verifier did not return a structured verdict. Raw output: ${output.slice(0, 500)}`,
+			requiredFixes: [],
+		};
+	}
+	if (data === null || typeof data !== "object") {
+		return {
+			verdict: "iterate",
+			justification: "goal-verifier returned a non-object payload.",
+			requiredFixes: [],
+		};
+	}
+	const record = data as Record<string, unknown>;
+	const verdict = record.verdict;
+	if (verdict !== "okay" && verdict !== "iterate" && verdict !== "reject") {
+		return {
+			verdict: "iterate",
+			justification: "goal-verifier returned an unrecognized verdict value.",
+			requiredFixes: [],
+		};
+	}
+	const rawFixes = record.required_fixes;
+	const requiredFixes = Array.isArray(rawFixes)
+		? rawFixes.filter((fix): fix is string => typeof fix === "string")
+		: [];
+	const justification = typeof record.justification === "string" ? record.justification : "";
+	const summary = typeof record.summary === "string" ? record.summary : undefined;
+	return { verdict, justification, summary, requiredFixes };
+}
+
+/**
+ * Grade a goal-completion claim with the independent `goal-verifier` sub-agent.
+ * Never throws: a spawn failure (depth limit, disabled agent, no model) fails
+ * open to `iterate` so it costs one corrective bounce, the same as a genuine
+ * gap — never a silent skip of independent grading, never a permanent block.
+ */
+export async function verifyGoalCompletion(
+	session: ToolSession,
+	objective: string,
+	evidence: string,
+): Promise<GoalCompletionVerdict> {
+	const truncatedEvidence =
+		evidence.length > MAX_EVIDENCE_CHARS ? `${evidence.slice(0, MAX_EVIDENCE_CHARS)}\n[truncated]` : evidence;
+	const verifierPrompt =
+		`<objective>\n${objective}\n</objective>\n\n` +
+		`<evidence>\n${truncatedEvidence}\n</evidence>\n\n` +
+		"Verdict whether the objective above is genuinely satisfied by the repo's current state. " +
+		"The evidence is the maker's claim, not a fact — verify it independently.";
+	try {
+		// Explicit schema, not the agent's own frontmatter default: runEvalAgent (the
+		// eval `agent()` bridge) only applies `outputSchema` when the caller supplies
+		// one — unlike the `task` tool, it never falls back to `effectiveAgent.output`
+		// (task/index.ts). Read the bundled definition's own `output` so this can't
+		// drift from goal-verifier.md's declared shape. Omit the key entirely when
+		// undefined: `Object.hasOwn` (agent-bridge.ts's `structured` check) is true
+		// for an explicit `schema: undefined`, which would wrongly disable the
+		// agent's frontmatter-schema fallback in the bridge.
+		const schema = getBundledAgent(VERIFIER_AGENT_NAME)?.output;
+		const args =
+			schema === undefined
+				? { prompt: verifierPrompt, agent: VERIFIER_AGENT_NAME }
+				: { prompt: verifierPrompt, agent: VERIFIER_AGENT_NAME, schema };
+		const result = await runEvalAgent(args, { session });
+		return parseVerdict(result.text);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		return {
+			verdict: "iterate",
+			justification: `Independent verifier could not run: ${message}`,
+			requiredFixes: [],
+		};
+	}
+}