@bastani/atomic 0.9.14-alpha.3 → 0.9.14-alpha.5

package/dist/builtin/workflows/builtin/adversarial-verification-runner.ts

@@ -1,104 +1,391 @@
-import { writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import { Type, type Static } from "typebox";
-import type { WorkflowRunContext, WorkflowSerializableValue } from "../src/shared/types.js";
-import { renderReducerPrompt, renderRepairPrompt, renderVerifierPrompt, renderWorkerPrompt } from "./adversarial-verification-prompts.js";
+import { Type } from "typebox";
+import type { WorkflowRunContext, WorkflowSerializableValue, WorkflowTaskResult } from "../src/shared/types.js";
+import {
+	renderConsolidatorPrompt,
+	renderRepairPrompt,
+	renderWorkerPrompt,
+} from "./adversarial-verification-prompts.js";
+import {
+	build_scoring_prompt,
+	scoring_prompt_reads,
+	warm_first_fan_out,
+	type ScoringCandidate,
+	type SharedHead,
+} from "./verification-prompts.js";
 import { stableArtifactRoot } from "./pattern-artifact-root.js";
+import { fold_usage } from "./verification-usage.js";
+import {
+	decide_verification,
+	normalize_criteria,
+	parse_rubric,
+	VERIFICATION_SCALE,
+	type Criterion,
+	type Criteria,
+	type CriterionScore,
+	type VerificationDecision,
+} from "./verification-criteria.js";
 
-const verifierSchema = Type.Object({
-  verdict: Type.Union([Type.Literal("pass"), Type.Literal("fail")]),
-  evidence: Type.Array(Type.String()),
-  blocking_findings: Type.Array(Type.String()),
+export const DEFAULT_CRITERIA = {
+	task_fit: "The candidate satisfies the literal task.",
+	evidence: "Important claims cite observable evidence, and file findings cite file:line where applicable.",
+	completeness: "Relevant validation is executed and reported with commands run and observed output, and no blocking correctness, safety, or completeness gap remains.",
+};
+
+// V2 deliberately requires every fanned-out cell to return a schema-valid score
+// after its bounded re-ask wave. A missing score is loudly Indeterminate rather
+// than silently narrowing the mean, and quorum is not caller-configurable.
+const QUORUM_FRACTION = 1;
+
+const criterionScoreSchema = Type.Object({
+	criterion_id: Type.String(),
+	score: VERIFICATION_SCALE.schema,
+	evidence: Type.Array(Type.String()),
+	findings: Type.Array(Type.Object({
+		finding: Type.String(),
+		severity: Type.Union([Type.Literal("veto"), Type.Literal("blocking"), Type.Literal("note")]),
+	}, { additionalProperties: false })),
 }, { additionalProperties: false });
-const reducerSchema = Type.Object({
-  decision: Type.Union([Type.Literal("accept"), Type.Literal("reject"), Type.Literal("repair")]),
-  rationale: Type.String(),
-  remaining_work: Type.Array(Type.String()),
+const consolidatorSchema = Type.Object({
+	repair_guidance: Type.String(),
+	remaining_work: Type.Array(Type.String()),
 }, { additionalProperties: false });
 
-type VerifierDecision = Static<typeof verifierSchema>;
-type ReducerDecision = Static<typeof reducerSchema>;
-type Inputs = { readonly task: string; readonly verifier_count: number; readonly max_repairs: number } & Record<string, WorkflowSerializableValue>;
+type CriterionScoreReport = {
+	readonly criterion_id: string;
+	readonly score: number;
+	readonly evidence: readonly string[];
+	readonly findings: readonly {
+		readonly finding: string;
+		readonly severity: "veto" | "blocking" | "note";
+	}[];
+};
+type ConsolidatedReport = {
+	readonly repair_guidance: string;
+	readonly remaining_work: readonly string[];
+};
+type Inputs = {
+	readonly task: string;
+	readonly verifier_count: number;
+	readonly max_repairs: number;
+	readonly criteria?: string | Record<string, string>;
+	readonly accept_mean?: number;
+	readonly reask_limit?: number;
+};
 export type AdversarialVerificationResult = {
-  readonly result: string;
-  readonly approved: boolean;
-  readonly repairs_completed: number;
-  readonly candidate_path: string;
-  readonly review_report_path: string;
-  readonly verifier_artifact_paths: string[];
-  readonly artifact_dir: string;
-  readonly remaining_work: string[];
+	readonly approved: boolean;
+	readonly mean_score: number;
+	readonly score_table_path: string;
+	readonly repairs_completed: number;
+	readonly candidate_path: string;
+	readonly review_report_path: string;
+	readonly remaining_work: string[];
+};
+
+type VerificationCell = {
+	readonly criterion: Criterion;
+	readonly verifierIndex: number;
+	readonly name: string;
+	readonly artifactPath: string;
+};
+type ValidResult = {
+	readonly cell: VerificationCell;
+	readonly report: CriterionScoreReport;
+	readonly artifactPath: string;
+};
+type InvalidArtifact = {
+	readonly invalid: true;
+	readonly stage: string;
 };
 
-function structured<T extends WorkflowSerializableValue>(value: WorkflowSerializableValue | undefined, guard: (candidate: WorkflowSerializableValue) => candidate is T): T | undefined {
-  return value !== undefined && guard(value) ? value : undefined;
-}
 function isRecord(value: WorkflowSerializableValue): value is Record<string, WorkflowSerializableValue> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
+	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function isVerifier(value: WorkflowSerializableValue): value is VerifierDecision {
-  return isRecord(value) && (value.verdict === "pass" || value.verdict === "fail") && Array.isArray(value.evidence) && value.evidence.every((item) => typeof item === "string") && Array.isArray(value.blocking_findings) && value.blocking_findings.every((item) => typeof item === "string");
+
+function hasOnlyKeys(value: Record<string, WorkflowSerializableValue>, keys: readonly string[]): boolean {
+	const actual = Object.keys(value);
+	return actual.length === keys.length && keys.every((key) => key in value);
 }
-function isReducer(value: WorkflowSerializableValue): value is ReducerDecision {
-  return isRecord(value) && (value.decision === "accept" || value.decision === "reject" || value.decision === "repair") && typeof value.rationale === "string" && Array.isArray(value.remaining_work) && value.remaining_work.every((item) => typeof item === "string");
+
+function isStringArray(value: WorkflowSerializableValue | undefined): value is string[] {
+	return Array.isArray(value) && value.every((entry) => typeof entry === "string");
 }
-const INVALID_VERIFIER_REPORT: VerifierDecision = {
-  verdict: "fail",
-  evidence: [],
-  blocking_findings: ["Verifier stage produced no valid structured report."],
-};
+
+function isStringRecord(value: WorkflowSerializableValue): value is Record<string, string> {
+	return isRecord(value) && Object.values(value).every((entry) => typeof entry === "string");
+}
+
+function isCriterionFinding(value: WorkflowSerializableValue): boolean {
+	return isRecord(value)
+		&& hasOnlyKeys(value, ["finding", "severity"])
+		&& typeof value.finding === "string"
+		&& (value.severity === "veto" || value.severity === "blocking" || value.severity === "note");
+}
+
+function isCriterionScore(value: WorkflowSerializableValue | undefined, criterionId: string): value is CriterionScoreReport {
+	return value !== undefined
+		&& isRecord(value)
+		&& hasOnlyKeys(value, ["criterion_id", "score", "evidence", "findings"])
+		&& value.criterion_id === criterionId
+		&& typeof value.score === "number"
+		&& Number.isInteger(value.score)
+		&& value.score >= VERIFICATION_SCALE.min
+		&& value.score <= VERIFICATION_SCALE.max
+		&& isStringArray(value.evidence)
+		&& Array.isArray(value.findings)
+		&& value.findings.every(isCriterionFinding);
+}
+
+function isConsolidatedReport(value: WorkflowSerializableValue | undefined): value is ConsolidatedReport {
+	return value !== undefined
+		&& isRecord(value)
+		&& hasOnlyKeys(value, ["repair_guidance", "remaining_work"])
+		&& typeof value.repair_guidance === "string"
+		&& isStringArray(value.remaining_work);
+}
+
+function structured<T extends WorkflowSerializableValue>(
+	value: WorkflowSerializableValue | undefined,
+	guard: (candidate: WorkflowSerializableValue | undefined) => candidate is T,
+): T | undefined {
+	return guard(value) ? value : undefined;
+}
+
+function resolveCriteria(value: WorkflowSerializableValue | undefined): Criteria {
+	if (value === undefined) {
+		return { groundTruthNote: "", criteria: normalize_criteria(DEFAULT_CRITERIA) };
+	}
+	if (typeof value === "string") return parse_rubric(value);
+	if (isStringRecord(value)) return { groundTruthNote: "", criteria: normalize_criteria(value) };
+	throw new TypeError("criteria must be a criteria.md string or a record of criterion descriptions");
+}
+
+function renderCriteriaMarkdown(criteria: Criteria): string {
+	const lines = ["# Verification criteria"];
+	if (criteria.groundTruthNote.length > 0) lines.push("## Ground Truth Note", criteria.groundTruthNote);
+	lines.push("## Criteria");
+	for (const item of criteria.criteria) {
+		lines.push(`### ${item.name} {#${item.id}}`, item.description);
+	}
+	return `${lines.join("\n\n")}\n`;
+}
+
+function meanScore(scores: readonly CriterionScoreReport[]): number {
+	if (scores.length === 0) return 0;
+	return scores.reduce((total, score) => total + score.score, 0) / scores.length;
+}
+
+function toCriterionScore(report: CriterionScoreReport): CriterionScore {
+	return {
+		criterionId: report.criterion_id,
+		score: report.score,
+		evidence: report.evidence,
+		findings: report.findings,
+	};
+}
+
+function findingText(decision: Extract<VerificationDecision, { kind: "repair" }>): string[] {
+	return decision.findings.map((finding) => finding.finding);
+}
+
+function quorumEvidence(missing: number, invalidCount: number, expectedCount: number, reaskLimit: number): string {
+	return `Quorum failure: ${missing} of ${expectedCount} criterion scores remain missing after ${reaskLimit} re-ask wave(s); ${invalidCount} report attempts were invalid or missing.`;
+}
+
+function stageName(cell: VerificationCell, reaskWave: number): string {
+	return reaskWave === 0 ? cell.name : `${cell.name}-reask-${reaskWave}`;
+}
+
+function verifierOutputFormat(criterionId: string): string {
+	return `Call structured_output with criterion_id (set to ${criterionId}), score (1–20), evidence (string array), and findings containing finding and severity (veto, blocking, or note).`;
+}
+
+function scoreStep(
+	cell: VerificationCell,
+	reaskWave: number,
+	head: SharedHead,
+	criteriaPath: string,
+) {
+	const promptHead: SharedHead = {
+		...head,
+		outputFormat: verifierOutputFormat(cell.criterion.id),
+	};
+	return {
+		name: stageName(cell, reaskWave),
+		prompt: build_scoring_prompt(promptHead, cell.criterion),
+		context: "fresh" as const,
+		reads: [criteriaPath, ...scoring_prompt_reads(promptHead)],
+		schema: criterionScoreSchema,
+	};
+}
+
+async function writeInvalid(path: string, stage: string): Promise<void> {
+	const marker: InvalidArtifact = { invalid: true, stage };
+	await writeFile(path, `${JSON.stringify(marker, null, 2)}\n`);
+}
+
 export async function runAdversarialVerification(ctx: WorkflowRunContext<Inputs>): Promise<AdversarialVerificationResult> {
-  const root = await stableArtifactRoot(ctx, "adversarial-verification");
-  const rubricPath = join(root, "rubric.md");
-  const candidatePath = join(root, "candidate.md");
-  await writeFile(rubricPath, ["# Verification rubric", "- The candidate satisfies the literal task.", "- Important claims cite observable evidence.", "- Relevant validation is executed and reported with commands run and observed output.", "- File findings cite file:line evidence where applicable.", "- No blocking correctness, safety, or completeness gap remains."].join("\n"));
-  await ctx.task("worker", { prompt: renderWorkerPrompt(ctx.inputs.task), context: "fresh", output: candidatePath, outputMode: "file-only" });
-
-  let repairsCompleted = 0;
-  let reviewReportPath!: string;
-  let verifierArtifactPaths: string[] = [];
-  let decision: ReducerDecision = { decision: "reject", rationale: "No valid reducer decision was produced.", remaining_work: ["Reducer did not return a valid structured decision."] };
-  for (;;) {
-    verifierArtifactPaths = Array.from({ length: ctx.inputs.verifier_count }, (_, index) => join(root, `verification-${repairsCompleted}-${index + 1}.json`));
-    const reports = await ctx.parallel(verifierArtifactPaths.map((_, index) => ({
-      name: `verifier-${repairsCompleted}-${index + 1}`,
-      prompt: renderVerifierPrompt(ctx.inputs.task, candidatePath, rubricPath),
-      context: "fresh" as const,
-      reads: [candidatePath, rubricPath],
-      schema: verifierSchema,
-    })), { concurrency: Math.min(ctx.inputs.verifier_count, 4), failFast: false });
-    // Verifier reports are inter-stage data: the reducer reads schema-shaped
-    // JSON from these paths, so the runner persists the structured decisions
-    // itself rather than routing them through the stage artifact channel.
-    await Promise.all(verifierArtifactPaths.map(async (path, index) => {
-      const report = structured(reports[index]?.structured, isVerifier) ?? INVALID_VERIFIER_REPORT;
-      await writeFile(path, `${JSON.stringify(report, null, 2)}\n`);
-    }));
-    const validReports = reports.map((report) => structured(report.structured, isVerifier)).filter((report): report is VerifierDecision => report !== undefined);
-    const allVerifiersPassed = validReports.length === ctx.inputs.verifier_count && validReports.every((report) => report.verdict === "pass");
-    await writeFile(join(root, `verification-summary-${repairsCompleted}.json`), JSON.stringify(validReports, null, 2));
-    reviewReportPath = join(root, `review-${repairsCompleted}.json`);
-    const reduced = await ctx.task(`reducer-${repairsCompleted}`, {
-      prompt: renderReducerPrompt(ctx.inputs.task, candidatePath, verifierArtifactPaths, repairsCompleted, ctx.inputs.max_repairs),
-      context: "fresh", reads: [candidatePath, rubricPath, ...verifierArtifactPaths], schema: reducerSchema,
-    });
-    decision = structured(reduced.structured, isReducer) ?? decision;
-    // Same inter-stage contract as the verifier reports above: the repair
-    // stage reads this path as the reducer's structured decision.
-    await writeFile(reviewReportPath, `${JSON.stringify(decision, null, 2)}\n`);
-    if (decision.decision === "accept" && !allVerifiersPassed) {
-      const remaining = validReports.flatMap((report) => report.blocking_findings);
-      decision = repairsCompleted < ctx.inputs.max_repairs
-        ? { decision: "repair", rationale: "Independent verification did not unanimously pass.", remaining_work: remaining }
-        : { decision: "reject", rationale: "Independent verification did not pass before the repair bound was exhausted.", remaining_work: remaining };
-    }
-    if (decision.decision === "repair" && repairsCompleted >= ctx.inputs.max_repairs) {
-      decision = { ...decision, decision: "reject", rationale: `${decision.rationale} Repair bound exhausted.` };
-    }
-    if (decision.decision !== "repair") break;
-    repairsCompleted += 1;
-    await ctx.task(`repair-${repairsCompleted}`, { prompt: renderRepairPrompt(ctx.inputs.task, candidatePath, reviewReportPath), context: "fresh", reads: [candidatePath, reviewReportPath], output: candidatePath, outputMode: "file-only" });
-  }
-  const approved = decision.decision === "accept";
-  return { result: decision.rationale, approved, repairs_completed: repairsCompleted, candidate_path: candidatePath, review_report_path: reviewReportPath, verifier_artifact_paths: verifierArtifactPaths, artifact_dir: root, remaining_work: approved ? [] : decision.remaining_work };
+	const root = await stableArtifactRoot(ctx, "adversarial-verification");
+	const criteriaPath = join(root, "criteria.md");
+	const candidatePath = join(root, "candidate.md");
+	const resolvedCriteria = resolveCriteria(ctx.inputs.criteria);
+	await writeFile(criteriaPath, renderCriteriaMarkdown(resolvedCriteria));
+	await ctx.task("worker", { prompt: renderWorkerPrompt(ctx.inputs.task), context: "fresh", output: candidatePath, outputMode: "file-only" });
+
+	const verifierCount = ctx.inputs.verifier_count ?? 3;
+	const maxRepairs = ctx.inputs.max_repairs ?? 2;
+	const acceptMean = ctx.inputs.accept_mean ?? 14;
+	const reaskLimit = Math.max(0, Math.floor(ctx.inputs.reask_limit ?? 1));
+	const expectedCount = resolvedCriteria.criteria.length * verifierCount;
+	let repairsCompleted = 0;
+	let consecutiveIndeterminate = 0;
+	let finalDecision: VerificationDecision;
+	let finalMean = 0;
+	let scoreTablePath: string;
+	let reviewReportPath: string;
+	let remainingWork: string[] = [];
+
+	for (let round = 0; ; round += 1) {
+		const candidateBody = await readFile(candidatePath, "utf8");
+		const scoringHead: SharedHead = {
+			task: ctx.inputs.task,
+			groundTruthNote: resolvedCriteria.groundTruthNote,
+			candidates: [{ path: candidatePath, body: candidateBody } satisfies ScoringCandidate],
+			scaleAnchors: VERIFICATION_SCALE.anchors,
+		};
+		const cells: VerificationCell[] = [];
+		for (const criterion of resolvedCriteria.criteria) {
+			for (let index = 0; index < verifierCount; index += 1) {
+				const name = `verifier-${round}-${criterion.id}-${index + 1}`;
+				cells.push({
+					criterion,
+					verifierIndex: index + 1,
+					name,
+					artifactPath: join(root, `verification-${round}-${criterion.id}-${index + 1}.json`),
+				});
+			}
+		}
+
+		const runWave = async (
+			pending: readonly VerificationCell[],
+			reaskWave: number,
+		): Promise<{ readonly valid: readonly ValidResult[]; readonly invalid: readonly VerificationCell[]; readonly results: readonly WorkflowTaskResult[] }> => {
+			if (pending.length === 0) return { valid: [], invalid: [], results: [] };
+			const reports = await warm_first_fan_out(
+				ctx,
+				pending.map((cell) => scoreStep(cell, reaskWave, scoringHead, criteriaPath)),
+				() => scoringHead,
+				{ concurrency: Math.min(pending.length, 4), failFast: false },
+			);
+			const byName = new Map<string, (typeof reports)[number]>();
+			for (const report of reports) {
+				const name = report.name ?? report.stageName;
+				if (name !== undefined) byName.set(name, report);
+			}
+			const valid: ValidResult[] = [];
+			const invalid: VerificationCell[] = [];
+			for (const cell of pending) {
+				const report = byName.get(stageName(cell, reaskWave));
+				const artifactPath = reaskWave === 0
+					? cell.artifactPath
+					: cell.artifactPath.replace(/\.json$/, `-reask-${reaskWave}.json`);
+				if (isCriterionScore(report?.structured, cell.criterion.id)) {
+					await writeFile(artifactPath, `${JSON.stringify(report.structured, null, 2)}\n`);
+					valid.push({ cell, report: report.structured, artifactPath });
+				} else {
+					await writeInvalid(artifactPath, stageName(cell, reaskWave));
+					invalid.push(cell);
+				}
+			}
+			return { valid, invalid, results: reports };
+		};
+
+		const roundResults: WorkflowTaskResult[] = [];
+		let pending = [...cells];
+		const validResults: ValidResult[] = [];
+		let invalidCount = 0;
+		for (let reaskWave = 0; reaskWave <= reaskLimit; reaskWave += 1) {
+			const wave = await runWave(pending, reaskWave);
+			roundResults.push(...wave.results);
+			validResults.push(...wave.valid);
+			invalidCount += wave.invalid.length;
+			pending = [...wave.invalid];
+			if (pending.length === 0) break;
+		}
+
+		const validReportsByCell = new Map(validResults.map((item) => [item.cell.name, item.report]));
+		const scoreReports = cells.flatMap((cell) => {
+			const report = validReportsByCell.get(cell.name);
+			return report === undefined ? [] : [report];
+		});
+		const scores = scoreReports.map(toCriterionScore);
+		const roundResult = { scores, invalidCount, expectedCount };
+		const decision = decide_verification(roundResult, { acceptMean, quorumFraction: QUORUM_FRACTION });
+		finalDecision = decision;
+		finalMean = meanScore(scoreReports);
+		scoreTablePath = join(root, `verification-summary-${round}.json`);
+		reviewReportPath = join(root, `review-${round}.json`);
+		const summary = {
+			scores: scoreReports,
+			mean: finalMean,
+			invalidCount,
+			decision,
+			usage: fold_usage(roundResults),
+		};
+
+		if (decision.kind === "accept") {
+			remainingWork = [];
+			await writeFile(scoreTablePath, `${JSON.stringify(summary, null, 2)}\n`);
+			await writeFile(reviewReportPath, `${JSON.stringify({ decision, remaining_work: [] }, null, 2)}\n`);
+			break;
+		}
+
+		if (decision.kind === "indeterminate") {
+			consecutiveIndeterminate += 1;
+			remainingWork = [quorumEvidence(decision.missing, invalidCount, expectedCount, reaskLimit)];
+			await writeFile(scoreTablePath, `${JSON.stringify(summary, null, 2)}\n`);
+			await writeFile(reviewReportPath, `${JSON.stringify({ decision, evidence: remainingWork, remaining_work: remainingWork }, null, 2)}\n`);
+			if (consecutiveIndeterminate >= 2) break;
+			continue;
+		}
+
+		consecutiveIndeterminate = 0;
+		const confirmedFindings = findingText(decision);
+		const scorePaths = validResults.map((item) => item.artifactPath);
+		const consolidated = await ctx.task(`consolidate-findings-${round}`, {
+			prompt: renderConsolidatorPrompt(ctx.inputs.task, candidatePath, scorePaths, repairsCompleted, maxRepairs),
+			context: "fresh",
+			reads: [candidatePath, criteriaPath, ...scorePaths],
+			schema: consolidatorSchema,
+		});
+		const fallbackRemaining = confirmedFindings.length > 0
+			? confirmedFindings
+			: [`Mean score ${decision.mean} is below the acceptance threshold ${acceptMean}.`];
+		const consolidatedReport: ConsolidatedReport = structured(consolidated.structured, isConsolidatedReport)
+			?? { repair_guidance: "Confirmed verifier findings require repair.", remaining_work: fallbackRemaining };
+		remainingWork = fallbackRemaining;
+		await writeFile(scoreTablePath, `${JSON.stringify(summary, null, 2)}\n`);
+		await writeFile(reviewReportPath, `${JSON.stringify({ ...consolidatedReport, remaining_work: remainingWork }, null, 2)}\n`);
+
+		if (repairsCompleted >= maxRepairs) break;
+		repairsCompleted += 1;
+		await ctx.task(`repair-${repairsCompleted}`, {
+			prompt: renderRepairPrompt(ctx.inputs.task, candidatePath, reviewReportPath),
+			context: "fresh",
+			reads: [candidatePath, reviewReportPath],
+			output: candidatePath,
+			outputMode: "file-only",
+		});
+	}
+
+	return {
+		approved: finalDecision.kind === "accept",
+		mean_score: finalMean,
+		score_table_path: scoreTablePath,
+		repairs_completed: repairsCompleted,
+		candidate_path: candidatePath,
+		review_report_path: reviewReportPath,
+		remaining_work: finalDecision.kind === "accept" ? [] : remainingWork,
+	};
 }

package/dist/builtin/workflows/builtin/adversarial-verification.d.ts

@@ -1,11 +1,35 @@
 import type { WorkflowDefinition, WorkflowInputValues, WorkflowOutputValues } from "../src/authoring.js";
-export type AdversarialVerificationInputs = WorkflowInputValues & { readonly task: string; readonly verifier_count: number; readonly max_repairs: number };
-export type AdversarialVerificationRunInputs = WorkflowInputValues & { readonly task: string; readonly verifier_count?: number; readonly max_repairs?: number };
+
+export type AdversarialVerificationCriteria = string | Record<string, string>;
+export type AdversarialVerificationInputs = WorkflowInputValues & {
+  readonly task: string;
+  readonly verifier_count: number;
+  readonly max_repairs: number;
+  readonly criteria: AdversarialVerificationCriteria;
+  readonly accept_mean: number;
+  readonly reask_limit: number;
+};
+export type AdversarialVerificationRunInputs = WorkflowInputValues & {
+  readonly task: string;
+  readonly verifier_count?: number;
+  readonly max_repairs?: number;
+  readonly criteria?: AdversarialVerificationCriteria;
+  readonly accept_mean?: number;
+  readonly reask_limit?: number;
+};
 export type AdversarialVerificationOutputs = WorkflowOutputValues & {
-  readonly result: string; readonly approved: boolean; readonly repairs_completed: number;
-  readonly candidate_path: string; readonly review_report_path: string;
-  readonly verifier_artifact_paths: string[]; readonly artifact_dir: string; readonly remaining_work: string[];
+  readonly approved: boolean;
+  readonly mean_score: number;
+  readonly score_table_path: string;
+  readonly repairs_completed: number;
+  readonly candidate_path: string;
+  readonly review_report_path: string;
+  readonly remaining_work: string[];
 };
-export type AdversarialVerificationDefinition = WorkflowDefinition<AdversarialVerificationInputs, AdversarialVerificationOutputs, AdversarialVerificationRunInputs>;
+export type AdversarialVerificationDefinition = WorkflowDefinition<
+  AdversarialVerificationInputs,
+  AdversarialVerificationOutputs,
+  AdversarialVerificationRunInputs
+>;
 declare const workflow: AdversarialVerificationDefinition;
 export default workflow;

package/dist/builtin/workflows/builtin/adversarial-verification.ts

@@ -1,29 +1,34 @@
 import { Type } from "typebox";
 import { workflow } from "../src/authoring/workflow.js";
 import { withSteeringPropagationContext } from "./steering-context.js";
-import { runAdversarialVerification } from "./adversarial-verification-runner.js";
+import { DEFAULT_CRITERIA, runAdversarialVerification } from "./adversarial-verification-runner.js";
 
 export default workflow({
   name: "adversarial-verification",
-  description: "Produce a candidate, challenge it with fresh-context rubric-based verifiers, and reduce their evidence through a bounded repair loop.",
+  description: "Produce a candidate, score independent per-criterion verifier reports, and apply a deterministic mean-and-veto gate with bounded repairs.",
   // The 15-minute default, stated rather than inherited: this is a per-workflow
   // product decision, so a future change to the global default must not silently
   // re-cadence a long autonomous run.
   heartbeatIntervalMinutes: 15,
   inputs: {
     task: Type.String({ description: "Task whose candidate result must be independently verified." }),
-    verifier_count: Type.Integer({ minimum: 1, maximum: 5, default: 3, description: "Number of independent verifiers per review round." }),
+    verifier_count: Type.Integer({ minimum: 1, maximum: 5, default: 3, description: "Number of independent verifiers for each criterion per round." }),
     max_repairs: Type.Integer({ minimum: 0, maximum: 5, default: 2, description: "Maximum candidate repair rounds before rejection." }),
+    criteria: Type.Union([
+      Type.String(),
+      Type.Record(Type.String(), Type.String()),
+    ], { default: DEFAULT_CRITERIA, description: "Criteria record of name-to-description entries, or criteria.md markdown." }),
+    accept_mean: Type.Number({ default: 14, description: "Mean score required for acceptance on the 1–20 verification scale." }),
+    reask_limit: Type.Integer({ minimum: 0, default: 1, description: "Maximum bounded re-ask waves for invalid criterion reports." }),
   },
   outputs: {
-    result: Type.String({ description: "Final reducer rationale." }),
-    approved: Type.Boolean({ description: "Whether verification accepted the candidate." }),
+    approved: Type.Boolean({ description: "Whether the deterministic mean-and-veto gate accepted the candidate." }),
+    mean_score: Type.Number({ description: "Mean score of the final round's schema-valid criterion reports." }),
+    score_table_path: Type.String({ description: "Path to the final round per-criterion score summary." }),
     repairs_completed: Type.Integer({ description: "Number of repair rounds performed." }),
     candidate_path: Type.String({ description: "Path to the final candidate artifact." }),
-    review_report_path: Type.String({ description: "Path to the final reducer report." }),
-    verifier_artifact_paths: Type.Array(Type.String(), { description: "Paths to final-round verifier reports." }),
-    artifact_dir: Type.String({ description: "Directory containing run artifacts." }),
-    remaining_work: Type.Array(Type.String(), { description: "Unresolved blocking findings when not approved." }),
+    review_report_path: Type.String({ description: "Path to the final consolidated findings or quorum report." }),
+    remaining_work: Type.Array(Type.String(), { description: "Unresolved findings or quorum evidence when not approved." }),
   },
   run: async (ctx) => await runAdversarialVerification(withSteeringPropagationContext(ctx)),
 });

package/dist/builtin/workflows/builtin/generate-and-filter-prompts.ts

@@ -1,6 +1,14 @@
+import type { ScoringCandidate, SharedHead } from "./verification-prompts.js";
+import { build_scoring_prompt, scoring_prompt_reads } from "./verification-prompts.js";
+
 const GROUNDED_REPORTING = "Before reporting progress, audit each claim against a tool result from this session. Report only work you can point to evidence for; say so explicitly when something is unverified.";
 const READABLE_REPORT = "Lead with the outcome. Keep facts, decisions, caveats, and next steps; drop background and repetition. Use complete, readable sentences rather than compressed fragments.";
 
+export interface RenderedJudgePrompt {
+	readonly prompt: string;
+	readonly reads: readonly string[];
+}
+
 export function renderGeneratorPrompt(task: string, ordinal: number): string {
   return `<role>\nYou independently generate candidate ${ordinal}; do not imitate or assume other candidates.\n</role>\n\n<success_criteria>\nOne distinct, concrete candidate states its value, constraints, risks, and how it can be evaluated.\n</success_criteria>\n\n<stop_rules>\nStop after one self-contained, evaluable candidate; do not add alternatives.\n</stop_rules>\n\n<output_format>\nA candidate artifact with title, proposal, criteria-based rationale, risks, and evaluation evidence. ${READABLE_REPORT}\n${GROUNDED_REPORTING}\n</output_format>\n\n<objective>\n${task}\n</objective>`;
 }
@@ -9,10 +17,25 @@ export function renderFilterPrompt(task: string, candidatePaths: readonly string
   return `<artifacts>\nRead every candidate: ${candidatePaths.join(", ")}\n</artifacts>\n\n<role>\nYou deduplicate and filter independently generated candidates.\n</role>\n\n<rubric>\nFirst collapse substantively equivalent candidates. Then score fit to the task, feasibility, evidence, distinctiveness, and risk. Near-duplicates must not gain weight by repetition. Record every discarded candidate and a concrete reason.\n</rubric>\n\n<success_criteria>\nAt most ${shortlistSize} strongest distinct candidates remain, ranked by the rubric, and every discarded candidate has a concrete reason.\n</success_criteria>\n\n<stop_rules>\nStop after every candidate is shortlisted once or recorded as discarded.\n</stop_rules>\n\n<output_format>\nCall structured_output with shortlist (candidate artifact paths in ranked order) and discarded entries containing path and a concise, criteria-based reason.\n</output_format>\n\n<objective>\nSelect at most ${shortlistSize} strongest candidates for: ${task}\n</objective>`;
 }
 
-export function renderJudgePrompt(task: string, filterPath: string, shortlistSize: number): string {
-  return `<artifacts>\nRead the filter report at ${filterPath} and every candidate path it references.\n</artifacts>\n\n<role>\nYou independently judge the filtered shortlist against the explicit rubric.\n</role>\n\n<rubric>\nCheck task fit, feasibility, evidence, distinctiveness, and material risk. Do not restore a duplicate merely because it is phrased differently.\n</rubric>\n\n<success_criteria>\nAt most ${shortlistSize} distinct candidate paths are ranked by rubric-grounded evidence.\n</success_criteria>\n\n<stop_rules>\nStop after evaluating every filtered candidate and ranking the qualifying paths.\n</stop_rules>\n\n<output_format>\nCall structured_output with shortlist and a concise, criteria-based rationale in complete sentences.\n</output_format>\n\n<objective>\nRank the candidates that best satisfy: ${task}\n</objective>`;
+export function renderJudgePrompt(
+	task: string,
+	filterPath: string,
+	shortlistSize: number,
+	candidates: readonly ScoringCandidate[] = [],
+): RenderedJudgePrompt {
+	const head: SharedHead = {
+		task: `Rank at most ${shortlistSize} distinct candidate paths for: ${task}.`,
+		groundTruthNote: `Read the filter report at ${filterPath} and every candidate path it references before judging.`,
+		candidates,
+		outputFormat: "Call structured_output with shortlist (candidate paths in ranked order) and rationale (a concise, criteria-based explanation).",
+	};
+	const prompt = build_scoring_prompt(head, {
+		id: "filtered_shortlist",
+		name: "Filtered shortlist",
+		description: "Check task fit, feasibility, evidence, distinctiveness, and material risk; do not restore a duplicate merely because it is phrased differently.",
+	});
+	return { prompt, reads: [filterPath, ...scoring_prompt_reads(head)] };
 }
-
 export function renderFinalShortlistPrompt(task: string, decisionPath: string): string {
   return `<artifact>\nRead the authoritative selection at ${decisionPath}; follow its order and do not add candidates.\n</artifact>\n\n<role>\nYou present a concise, actionable final shortlist so the reader can choose the next evaluation without reading the selection session.\n</role>\n\n<success_criteria>\nEvery selected candidate appears once in authoritative order with its differentiator, evidence, tradeoffs, and recommended next evaluation.\n</success_criteria>\n\n<stop_rules>\nStop after presenting every selected candidate once; do not add or reorder candidates.\n</stop_rules>\n\n<output_format>\nRanked markdown shortlist with candidate path, differentiator, evidence, tradeoffs, and recommended next evaluation. ${READABLE_REPORT}\n${GROUNDED_REPORTING}\n</output_format>\n\n<objective>\nSummarize the selected candidates for: ${task}\n</objective>`;
 }