@underactive/pi-topping-moa-fusion 0.1.0

package/src/moa/verificationCriteria.ts

@@ -0,0 +1,127 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+
+import type { AgentConfig } from "../agents/discovery.ts";
+import { CancelRun, type CancelSession } from "../runtime/cancelRun.ts";
+import { getFinalOutput, isFailedResult } from "../runtime/results.ts";
+import { runSingleAgent } from "../runtime/runner.ts";
+import { modelRefLabel, type ModelRef, type ThinkingLevel } from "../shared/modelRefs.ts";
+import { activityLoopCount } from "../ui/agentStatus.ts";
+import type { ObserveSession } from "../ui/observeOverlay.ts";
+import type { MoaProgressWidget } from "../ui/moaProgressWidget.ts";
+import { buildRetryCorrection } from "./planlessRetry.ts";
+import { modelExtensionOptions, resolveContextWindow, resolveModelCost } from "./modelRuntime.ts";
+
+export const VERIFICATION_CRITERIA_HEADING = "## Verification Criteria";
+export const MAX_VERIFICATION_CRITERIA = 30;
+
+export const CRITERIA_TASK_PREAMBLE =
+	"You are the read-only SYNTHESIZER in a Mixture-of-Agents planning run. Your only deliverable is a pass/fail verification checklist for the already approved plan, emitted as markdown text in your reply. Do NOT implement, edit, or run anything. Having only read-only tools is expected and is never a blocker.";
+
+export interface VerificationCriterion {
+	id: string;
+	text: string;
+}
+
+export function buildCriteriaTask(plan: string): string {
+	return [
+		CRITERIA_TASK_PREAMBLE,
+		"---",
+		`## Approved plan (frozen)\n${plan}`,
+		"---",
+		`Your output must contain exactly one ${VERIFICATION_CRITERIA_HEADING} section and no prose outside it. Emit 3–${MAX_VERIFICATION_CRITERIA} bullets in this exact form:\n- **C<n>:** <observable binary condition> — <how to check: file, symbol, grep, expected state>\n\nEvery criterion must be checkable by reading the repository only; do not require execution. Cover at least one criterion per plan step in plan order, every named file or call site, and implied regression/scope constraints (such as preserving an existing export or avoiding unrelated files). Never reference proposer labels or model names.`,
+	].join("\n\n");
+}
+
+/** Collect `- …`/`* …` bullets under a heading, stopping at the next heading. */
+export function sectionBullets(output: string, heading: RegExp): string[] {
+	const lines = output.split("\n");
+	const start = lines.findIndex((line) => heading.test(line));
+	if (start < 0) return [];
+	const bullets: string[] = [];
+	for (let i = start + 1; i < lines.length; i++) {
+		const line = lines[i];
+		if (/^\s*#{1,6}\s/.test(line)) break;
+		const match = line.match(/^\s*[-*]\s+(.*\S)\s*$/);
+		if (match) bullets.push(match[1].trim());
+	}
+	return bullets;
+}
+
+export function parseVerificationCriteria(output: string): VerificationCriterion[] {
+	const seen = new Set<string>();
+	const criteria: VerificationCriterion[] = [];
+	for (const bullet of sectionBullets(output, /^\s*##\s+Verification Criteria\b/i)) {
+		const match = bullet.match(/^\*\*(C\d+):?\*\*:?[\s]*(.+)$/i);
+		if (!match) continue;
+		const id = match[1].toUpperCase();
+		if (seen.has(id)) continue;
+		seen.add(id);
+		criteria.push({ id, text: match[2].trim() });
+		if (criteria.length >= MAX_VERIFICATION_CRITERIA) break;
+	}
+	return criteria;
+}
+
+export function formatCriteriaMarkdown(criteria: VerificationCriterion[]): string {
+	return [VERIFICATION_CRITERIA_HEADING, ...criteria.map((criterion) => `- **${criterion.id}:** ${criterion.text}`)].join("\n");
+}
+
+export const CRITERIA_RETRY_HEADER =
+	"IMPORTANT: Your previous output did not contain any parseable verification criteria. Emit only the required ## Verification Criteria section now, with 3–30 bullets in the exact **C<n>:** format. Do not ask questions, implement anything, or add prose outside the section.";
+
+export function buildCriteriaRetryTask(originalTask: string, previousOutput: string): string {
+	return [originalTask, "", "---", "", buildRetryCorrection(CRITERIA_RETRY_HEADER, previousOutput)].join("\n");
+}
+
+export async function runCriteriaGeneration(input: {
+	ctx: ExtensionContext;
+	agents: AgentConfig[];
+	synthesizer: ModelRef;
+	thinking: ThinkingLevel | undefined;
+	plan: string;
+	session?: CancelSession;
+	widget?: MoaProgressWidget;
+	observe?: ObserveSession;
+}): Promise<{ criteria: VerificationCriterion[]; markdown: string } | undefined> {
+	const { ctx, agents, synthesizer, thinking, plan, session, widget } = input;
+	widget?.switchToSynthesizing(synthesizer, "writing verification criteria…", thinking);
+	const run = new CancelRun();
+	const slot = run.add(modelRefLabel(synthesizer));
+	if (session) {
+		session.title = "MoA verification criteria";
+		session.run = run;
+		session.getExtras = () => {
+			const status = widget?.getRoleStatus("Synthesize");
+			return { contextTokens: status?.contextTokens, contextWindow: status?.ref ? resolveContextWindow(ctx, status.ref) : undefined, activity: status?.activity, loopCount: activityLoopCount(status?.activity, status?.activityHistory) };
+		};
+	}
+	const runTask = async (task: string): Promise<string | undefined> => {
+		widget?.updateRoleTranscript("Synthesize", []);
+		try {
+			const result = await runSingleAgent(ctx.cwd, agents, "moa-synthesizer", task, undefined, slot.signal, undefined, modelRefLabel(synthesizer), thinking, {
+				...modelExtensionOptions(ctx, synthesizer),
+				resolveOnAbort: true,
+				onProgress: (progress) => {
+					widget?.updateRoleUsage("Synthesize", progress.usage.contextTokens, progress.usage.turns, progress.usage.toolCalls, resolveModelCost(ctx, synthesizer, progress.usage));
+					if (progress.activity) widget?.updateRoleActivity("Synthesize", progress.activity);
+					if (progress.outputActivity) widget?.updateRoleOutput("Synthesize", progress.outputActivity.tokens, progress.outputActivity.revision);
+					widget?.updateRoleTranscript("Synthesize", progress.messages, progress.partialAssistant);
+				},
+			});
+			widget?.updateRoleTranscript("Synthesize", result.messages);
+			if (result.cancelled || run.cancelAllRequested || isFailedResult(result)) return undefined;
+			return getFinalOutput(result.messages);
+		} catch { return undefined; }
+	};
+	try {
+		let output = await runTask(buildCriteriaTask(plan));
+		let criteria = output ? parseVerificationCriteria(output) : [];
+		if (criteria.length === 0 && output !== undefined) {
+			output = await runTask(buildCriteriaRetryTask(buildCriteriaTask(plan), output));
+			criteria = output ? parseVerificationCriteria(output) : [];
+		}
+		return criteria.length > 0 ? { criteria, markdown: formatCriteriaMarkdown(criteria) } : undefined;
+	} finally {
+		if (session) { session.run = undefined; session.getExtras = undefined; }
+	}
+}

package/src/moa/verifyGate.ts

@@ -0,0 +1,137 @@
+/**
+ * Deterministic script gate + working-tree diff for the verification phase.
+ *
+ * The read-only verifier subprocess cannot run tests itself (it is spawned with
+ * the same `--tools read,grep,find,ls` + read-only env handshake as every other
+ * planning agent), so the orchestrator runs the project's own check/lint/test
+ * scripts in the parent process and hands the results to the verifier as
+ * evidence. `captureImplementationDiff` likewise runs in the parent — never in
+ * the read-only subprocess — so a git diff of what the implementer changed can
+ * be attached to the verifier task.
+ *
+ * The gate (`discoverVerifyScripts`/`runVerifyScript`) is ported nearly verbatim
+ * from the sibling pi-topping-persona-audit's `src/verify.ts`, with a local
+ * `tail` and a local result type so it carries no persona-audit dependencies.
+ * It runs AFTER the read-only verifier's mutation-tripwire check, because
+ * `npm test` can legitimately write snapshots/coverage and must not trip a
+ * false "verifier modified the working tree" alarm.
+ */
+
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+
+/** Preferred script discovery order. */
+const SCRIPT_ORDER = ["check", "lint", "test"] as const;
+
+const SCRIPT_TIMEOUT_MS = 5 * 60_000;
+const OUTPUT_TAIL_CHARS = 4_000;
+
+/** Keep the trailing `maxChars` of long output, prefixing an ellipsis when cut. */
+function tail(text: string, maxChars: number): string {
+	const trimmed = text.trim();
+	return trimmed.length > maxChars ? `…${trimmed.slice(-maxChars)}` : trimmed;
+}
+
+export interface VerifyResult {
+	script: string;
+	command: string;
+	status: "pass" | "fail";
+	exitCode: number;
+	relevantOutput: string;
+}
+
+/** Discover which of check/lint/test exist in package.json#scripts, in that order. */
+export async function discoverVerifyScripts(cwd: string): Promise<string[]> {
+	let raw: string;
+	try {
+		raw = await readFile(`${cwd}/package.json`, "utf-8");
+	} catch {
+		return [];
+	}
+
+	let scripts: Record<string, unknown>;
+	try {
+		const pkg = JSON.parse(raw) as { scripts?: Record<string, unknown> };
+		scripts = pkg.scripts && typeof pkg.scripts === "object" ? pkg.scripts : {};
+	} catch {
+		return [];
+	}
+
+	return SCRIPT_ORDER.filter((name) => typeof scripts[name] === "string");
+}
+
+/** Run one discovered verification script via `npm run <script>`. Never throws. */
+export async function runVerifyScript(cwd: string, script: string, signal?: AbortSignal): Promise<VerifyResult> {
+	const command = `npm run ${script}`;
+	try {
+		const { stdout, stderr } = await execFileAsync("npm", ["run", script], {
+			cwd,
+			encoding: "utf-8",
+			timeout: SCRIPT_TIMEOUT_MS,
+			maxBuffer: 16 * 1024 * 1024,
+			signal,
+		});
+		return {
+			script,
+			command,
+			status: "pass",
+			exitCode: 0,
+			relevantOutput: tail(`${stdout}\n${stderr}`, OUTPUT_TAIL_CHARS),
+		};
+	} catch (error) {
+		const err = error as { code?: number; stdout?: string; stderr?: string; message?: string };
+		return {
+			script,
+			command,
+			status: "fail",
+			exitCode: typeof err.code === "number" ? err.code : 1,
+			relevantOutput: tail(`${err.stdout ?? ""}\n${err.stderr ?? ""}`, OUTPUT_TAIL_CHARS) || (err.message ?? "unknown error"),
+		};
+	}
+}
+
+const DIFF_MAX_BYTES = 64 * 1024;
+
+/**
+ * Best-effort git diff of the current working tree against HEAD, for the
+ * verifier task. Returns null outside a git repo. The diff includes any
+ * pre-existing uncommitted edits, so the verifier task caveats it as a pointer
+ * to what moved rather than proof of what this implementation changed.
+ */
+export async function captureImplementationDiff(cwd: string): Promise<string | null> {
+	const run = async (args: string[]): Promise<string | null> => {
+		try {
+			const { stdout } = await execFileAsync("git", args, {
+				cwd,
+				encoding: "utf-8",
+				maxBuffer: 16 * 1024 * 1024,
+			});
+			return stdout;
+		} catch {
+			return null;
+		}
+	};
+
+	const stat = await run(["diff", "HEAD", "--stat"]);
+	if (stat === null) return null; // not a git repo (or git unavailable)
+
+	let full = (await run(["diff", "HEAD"])) ?? "";
+	let truncated = false;
+	if (Buffer.byteLength(full, "utf-8") > DIFF_MAX_BYTES) {
+		full = full.slice(0, DIFF_MAX_BYTES);
+		truncated = true;
+	}
+
+	const untracked = await run(["ls-files", "--others", "--exclude-standard"]);
+
+	const sections: string[] = [];
+	if (stat.trim()) sections.push(`Diffstat (git diff HEAD --stat):\n${stat.trim()}`);
+	if (untracked?.trim()) sections.push(`Untracked files:\n${untracked.trim()}`);
+	if (full.trim()) {
+		sections.push(`Diff (git diff HEAD)${truncated ? " — TRUNCATED to 64 KB" : ""}:\n${full.trim()}`);
+	}
+	return sections.length > 0 ? sections.join("\n\n") : "(no changes detected against HEAD)";
+}

package/src/opinion/opinionContract.ts

@@ -0,0 +1,21 @@
+import { buildRetryCorrection } from "../moa/planlessRetry.ts";
+
+export const OPINION_TASK_PREAMBLE =
+	"You are a read-only analyst in a multi-model opinion run. Your only deliverable is one independent, repo-grounded opinion emitted as markdown in your reply. Do not implement anything, edit files, or run commands, builds, or tests. Read-only tools are expected and never a blocker. No later agent will merge or judge your answer, so commit to a clear recommendation. You are running headless: resolve open questions with stated assumptions rather than asking the user or waiting.";
+
+export const OPINION_HEADING = /^#{1,6}\s.*\b(opinion|answer|recommendation|assessment|verdict)\b/im;
+
+export function looksLikeOpinion(output: string): boolean {
+	return OPINION_HEADING.test(output);
+}
+
+export const OPINION_RETRY_HEADER =
+	"IMPORTANT: You already attempted this opinion task once and stopped without emitting a complete opinion. You are running headless: do not ask questions, wait for clarification, or refuse because only read-only tools are available. Re-emit the COMPLETE independent opinion now, using the required headings beginning with ## Opinion.";
+
+export function buildOpinionTask(question: string): string {
+	return [OPINION_TASK_PREAMBLE, "", "Question:", question].join("\n");
+}
+
+export function buildOpinionRetryTask(originalTask: string, previousOutput: string): string {
+	return [originalTask, "", "---", "", buildRetryCorrection(OPINION_RETRY_HEADER, previousOutput)].join("\n");
+}

package/src/opinion/opinionFanout.ts

@@ -0,0 +1,135 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+
+import type { AgentConfig } from "../agents/discovery.ts";
+import { runWidgetFanout } from "../moa/fanoutWiring.ts";
+import { modelExtensionOptions, resolveContextWindow } from "../moa/modelRuntime.ts";
+import type { CancelSession } from "../runtime/cancelRun.ts";
+import { CancelRun } from "../runtime/cancelRun.ts";
+import { getFinalOutput, isFailedResult, type SingleResult } from "../runtime/results.ts";
+import type { ModelParallelAgentTask } from "../runtime/runner.ts";
+import { modelRefLabel, type ModelRef, type ThinkingLevel } from "../shared/modelRefs.ts";
+import { activityLoopCount } from "../ui/agentStatus.ts";
+import type { MoaProgressWidget } from "../ui/moaProgressWidget.ts";
+import type { ObserveSession } from "../ui/observeOverlay.ts";
+import { buildOpinionRetryTask, buildOpinionTask, looksLikeOpinion } from "./opinionContract.ts";
+
+export interface OpinionFanoutHost {
+	getActiveObserveSession(): ObserveSession | undefined;
+	setActiveObserveSession(session: ObserveSession | undefined): void;
+}
+
+export interface OpinionFanoutOptions {
+	host: OpinionFanoutHost;
+	ctx: ExtensionContext;
+	question: string;
+	models: ModelRef[];
+	thinking: (ThinkingLevel | undefined)[];
+	session: CancelSession;
+	widget: MoaProgressWidget;
+	agents: AgentConfig[];
+}
+
+export type OpinionFanoutResult =
+	| { status: "done"; results: SingleResult[] }
+	| { status: "cancelled" };
+
+export async function runOpinionFanout(options: OpinionFanoutOptions): Promise<OpinionFanoutResult> {
+	const { host, ctx, question, models, thinking, session, widget, agents } = options;
+	const originalTask = buildOpinionTask(question);
+	widget.startFanout(models, thinking);
+	const observe: ObserveSession = {
+		title: "MoA opinions",
+		phase: "fanout",
+		agents: models.map((ref) => ({
+			label: ref.id,
+			model: ref.id,
+			task: originalTask,
+			messages: [],
+			state: "working",
+		})),
+		overlayOpen: false,
+	};
+	const previousObserveSession = host.getActiveObserveSession();
+	host.setActiveObserveSession(observe);
+
+	try {
+		const runBatch = (
+			tasks: ModelParallelAgentTask[],
+			indexMap: (index: number) => number,
+			currentRun: CancelRun,
+		): Promise<SingleResult[]> => runWidgetFanout({ ctx, agents, tasks, run: currentRun, session, widget, observe, models, indexMap });
+
+		const run = new CancelRun();
+		session.title = "Opinion agents";
+		session.getExtras = (index) => {
+			const status = widget.getStatus(index);
+			if (!status) return undefined;
+			return {
+				contextTokens: status.contextTokens,
+				contextWindow: resolveContextWindow(ctx, status.ref),
+				activity: status.activity,
+				loopCount: activityLoopCount(status.activity, status.activityHistory),
+			};
+		};
+		const tasks = models.map((ref, index) => {
+			const label = modelRefLabel(ref);
+			return {
+				agent: "moa-opinion",
+				task: originalTask,
+				model: label,
+				thinking: thinking[index],
+				...modelExtensionOptions(ctx, ref),
+				signal: run.add(label).signal,
+			};
+		});
+		const results = await runBatch(tasks, (index) => index, run);
+		if (run.cancelAllRequested) return { status: "cancelled" };
+
+		const retryIndices: number[] = [];
+		for (let index = 0; index < results.length; index++) {
+			const result = results[index];
+			if (!result.cancelled && !isFailedResult(result) && !looksLikeOpinion(getFinalOutput(result.messages))) retryIndices.push(index);
+		}
+		if (retryIndices.length > 0) {
+			const retryRun = new CancelRun();
+			session.title = "Opinion agents (retry)";
+			session.getExtras = (index) => {
+				const status = widget.getStatus(retryIndices[index]);
+				if (!status) return undefined;
+				return {
+					contextTokens: status.contextTokens,
+					contextWindow: resolveContextWindow(ctx, status.ref),
+					activity: status.activity,
+					loopCount: activityLoopCount(status.activity, status.activityHistory),
+				};
+			};
+			const retryTasks = retryIndices.map((opinionIndex) => {
+				const ref = models[opinionIndex];
+				const label = modelRefLabel(ref);
+				const retryTask = buildOpinionRetryTask(originalTask, getFinalOutput(results[opinionIndex].messages));
+				widget.update(opinionIndex, "working", "retrying: no opinion produced");
+				const observed = observe.agents[opinionIndex];
+				if (observed) {
+					observed.state = "working";
+					observed.task = retryTask;
+				}
+				return {
+					agent: "moa-opinion",
+					task: retryTask,
+					model: label,
+					thinking: thinking[opinionIndex],
+					...modelExtensionOptions(ctx, ref),
+					signal: retryRun.add(label).signal,
+				};
+			});
+			const retryResults = await runBatch(retryTasks, (index) => retryIndices[index], retryRun);
+			for (let index = 0; index < retryResults.length; index++) results[retryIndices[index]] = retryResults[index];
+			if (retryRun.cancelAllRequested) return { status: "cancelled" };
+		}
+
+		return { status: "done", results };
+	} finally {
+		observe.closeOverlay?.();
+		host.setActiveObserveSession(previousObserveSession);
+	}
+}

package/src/opinion/opinionFile.ts

@@ -0,0 +1,38 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { CONFIG_DIR_NAME } from "@earendil-works/pi-coding-agent";
+import { isValidPlanSlug } from "../planning/planFile.ts";
+
+export type RepoOpinionFileKind = "opinion-prompt" | "opinions";
+
+function repoOpinionSuffix(kind: RepoOpinionFileKind): string {
+	return kind === "opinion-prompt" ? "__opinion-prompt" : "__opinions";
+}
+
+export function getRepoOpinionDirectory(repoCwd: string): string {
+	const opinionDir = path.join(repoCwd, CONFIG_DIR_NAME, "mf-opinion");
+	try {
+		fs.mkdirSync(opinionDir, { recursive: true });
+	} catch {
+		// Best effort; the write will report a useful error if this failed.
+	}
+	return opinionDir;
+}
+
+export function saveRepoOpinionFile(
+	content: string,
+	repoCwd: string,
+	baseSlug: string,
+	kind: RepoOpinionFileKind,
+): string {
+	if (!isValidPlanSlug(baseSlug)) throw new Error("Invalid repository opinion slug");
+	const opinionDir = getRepoOpinionDirectory(repoCwd);
+	const filePath = path.join(opinionDir, `${baseSlug}${repoOpinionSuffix(kind)}.md`);
+	fs.writeFileSync(filePath, content, { encoding: "utf-8" });
+	return filePath;
+}
+
+export function repoOpinionDisplayPath(baseSlug: string, kind: RepoOpinionFileKind): string {
+	if (!isValidPlanSlug(baseSlug)) throw new Error("Invalid repository opinion slug");
+	return `${CONFIG_DIR_NAME}/mf-opinion/${baseSlug}${repoOpinionSuffix(kind)}.md`;
+}

package/src/opinion/opinionResults.ts

@@ -0,0 +1,73 @@
+import type { ModelRef, ThinkingLevel } from "../shared/modelRefs.ts";
+import { modelRefLabel } from "../shared/modelRefs.ts";
+import type { SingleResult } from "../runtime/results.ts";
+import { getFinalOutput, getResultOutput, isFailedResult } from "../runtime/results.ts";
+
+export interface OpinionOutcome {
+	ref: ModelRef;
+	thinking: ThinkingLevel | undefined;
+	status: "done" | "error" | "cancelled";
+	text: string;
+}
+
+export function collectOpinionOutcomes(
+	models: ModelRef[],
+	thinking: (ThinkingLevel | undefined)[],
+	results: SingleResult[],
+): OpinionOutcome[] {
+	return models.map((ref, index) => {
+		const result = results[index];
+		if (!result) {
+			return { ref, thinking: thinking[index], status: "error", text: "No result was returned." };
+		}
+		if (result.cancelled) {
+			return { ref, thinking: thinking[index], status: "cancelled", text: getResultOutput(result) };
+		}
+		if (isFailedResult(result)) {
+			return { ref, thinking: thinking[index], status: "error", text: getResultOutput(result) };
+		}
+		return {
+			ref,
+			thinking: thinking[index],
+			status: "done",
+			text: getFinalOutput(result.messages).trim() || "(no output)",
+		};
+	});
+}
+
+export function demoteHeadings(markdown: string): string {
+	let fence: "```" | "~~~" | undefined;
+	return markdown.split("\n").map((line) => {
+		const fenceMatch = line.match(/^\s*(```|~~~)/);
+		if (fenceMatch) {
+			const marker = fenceMatch[1] as "```" | "~~~";
+			if (!fence) fence = marker;
+			else if (fence === marker) fence = undefined;
+			return line;
+		}
+		if (fence) return line;
+		return line.replace(/^(\s*)(#{1,6})(\s+)/, (_match, indent: string, hashes: string, space: string) =>
+			`${indent}${"#".repeat(Math.min(6, hashes.length + 1))}${space}`,
+		);
+	}).join("\n");
+}
+
+function firstLine(text: string): string {
+	return text.trim().split(/\r?\n/, 1)[0] || "Unknown error";
+}
+
+export function formatOpinionsMarkdown(
+	question: string,
+	outcomes: OpinionOutcome[],
+	name?: string,
+): string {
+	const title = name ? `# MoA Opinions — ${name}` : "# MoA Opinions";
+	const sections = outcomes.map((outcome, index) => {
+		const thinking = outcome.thinking ?? "default";
+		const heading = `## Opinion ${index + 1} — ${modelRefLabel(outcome.ref)} (thinking: ${thinking})`;
+		if (outcome.status === "cancelled") return `${heading}\n\n_Cancelled by user._`;
+		if (outcome.status === "error") return `${heading}\n\n_Failed: ${firstLine(outcome.text)}_`;
+		return `${heading}\n\n${demoteHeadings(outcome.text)}`;
+	});
+	return [title, "", `**Question:** ${question}`, "", sections.join("\n\n---\n\n")].join("\n").trimEnd();
+}