lazycodex-ai 5.0.0-beta.34 → 5.0.0-beta.36

package/packages/omo-codex/plugin/components/ulw-loop/test/checkpoint-final.test.ts

@@ -114,6 +114,32 @@ describe("checkpointUlwLoop final story surface resolution", () => {
 });
 
 describe("checkpointUlwLoop final story", () => {
+	it("#given a mismatched codex objective and multiple gate defects #when completing the final checkpoint #then reports both validation domains", async () => {
+		const repo = await repoWith(
+			plan([passGoal("G001", { status: "complete" }), passGoal("G002")], { activeGoalId: "G002" }),
+		);
+		const gate = JSON.parse(await qualityGateJson(repo)) as Record<string, Record<string, unknown>>;
+		const manualQa = requiredSection(gate, "manualQa");
+		manualQa["surfaceEvidence"] = [];
+		requiredSection(gate, "gateReview")["recommendation"] = "REJECT";
+
+		try {
+			await checkpointUlwLoop(repo, {
+				goalId: "G002",
+				status: "complete",
+				evidence: "final work complete and validation passed",
+				codexGoalJson: snapshot("complete", "wrong objective"),
+				qualityGateJson: JSON.stringify(gate),
+			});
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(Error);
+			if (!(error instanceof Error)) throw error;
+			expect(error.message).toContain("Codex goal objective mismatch");
+			expect(error.message).toContain("manualQa.surfaceEvidence");
+			expect(error.message).toContain("gateReview.recommendation");
+		}
+	});
 	it("requires quality-gate-json for the final goal complete", async () => {
 		const repo = await repoWith(
 			plan([passGoal("G001", { status: "complete" }), passGoal("G002")], { activeGoalId: "G002" }),

package/packages/omo-codex/plugin/components/ulw-loop/test/checkpoint-template.test.ts

@@ -1,10 +1,12 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { existsSync, statSync } from "node:fs";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import { ulwLoopCommand } from "../src/cli-commands.js";
 import { writePlan } from "../src/plan-io.js";
+import { validateQualityGate } from "../src/quality-gate.js";
 import { goal, plan } from "./fixtures/checkpoint-builders.js";
 
 let repo: string;
@@ -32,7 +34,50 @@ async function run(args: string[]): Promise<Record<string, unknown>> {
 	return JSON.parse(output.join(""));
 }
 
+function fillTemplate(value: unknown): unknown {
+	if (typeof value === "string") {
+		return value.replace(/<replace:[^>]+>/g, "plausible evidence");
+	}
+	if (Array.isArray(value)) return value.map(fillTemplate);
+	if (value !== null && typeof value === "object") {
+		return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, fillTemplate(item)]));
+	}
+	return value;
+}
+
 describe("checkpoint --print-template", () => {
+	it("#given a printed template #when every placeholder is filled #then the quality gate validates", async () => {
+		await writePlan(
+			repo,
+			plan([goal({ id: "G001", attempt: 2 })], {
+				evidenceLayoutVersion: 2,
+				activeGoalId: "G001",
+				codexObjective: "Exact objective from goals.json",
+			}),
+		);
+		process.env["OMO_AGENT_TOOLKIT_SURFACE"] = "omo-senpi";
+		const result = await run(["--print-template", "--goal-id", "G001", "--json"]);
+		const template = fillTemplate(result["qualityGateTemplate"]) as Record<string, unknown>;
+		const paths = [
+			...((template["manualQa"] as Record<string, unknown>)["artifactRefs"] as Array<Record<string, unknown>>),
+		];
+		await Promise.all(
+			[
+				...paths.map((artifact) => String(artifact["path"])),
+				String((template["gateReview"] as Record<string, unknown>)["reportPath"]),
+			].map(async (path) => {
+				const absolute = resolve(repo, path);
+				await mkdir(resolve(absolute, ".."), { recursive: true });
+				await writeFile(absolute, "evidence");
+			}),
+		);
+		const validated = validateQualityGate(template, {
+			repoRoot: repo,
+			fs: { existsSync, statSync },
+			reviewerSurface: "omo-senpi",
+		});
+		expect(validated.gateReview.recommendation).toBe("APPROVE");
+	});
 	it("#given an active senpi v2 plan #when printed with only goal-id #then emits the senpi gate skeleton", async () => {
 		await writePlan(
 			repo,

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-aggregate-basics.test.ts

@@ -0,0 +1,258 @@
+import { existsSync, statSync } from "node:fs";
+import { describe, expect, it } from "vitest";
+import { validateQualityGate } from "../src/quality-gate.js";
+import { UlwLoopError } from "../src/types.js";
+
+const opts = { repoRoot: process.cwd(), fs: { existsSync, statSync }, reviewerSurface: "omo-senpi" as const };
+describe("aggregated quality gate basics", () => {
+	it("#given rejected recommendation, empty surface evidence, and invalid artifact kind #when validating #then reports all three defects", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa evidence",
+				surfaceEvidence: [],
+				adversarialCases: [
+					{
+						id: "adv",
+						criterionRef: "C001",
+						scenario: "scenario",
+						expectedBehavior: "reject",
+						verdict: "passed",
+						artifactRefs: [],
+					},
+				],
+				artifactRefs: [
+					{
+						id: "artifact",
+						kind: "invalid-kind",
+						description: "artifact",
+						path: "test/fixtures/artifacts/cli-pass.txt",
+					},
+				],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "REJECT",
+				evidence: "gate evidence",
+				reportPath: "test/fixtures/artifacts/gate-review.md",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			expect(Array.isArray(fields) ? fields.map((item) => item["field"]) : []).toEqual(
+				expect.arrayContaining([
+					"gateReview.recommendation",
+					"manualQa.surfaceEvidence",
+					"manualQa.artifactRefs[0].kind",
+				]),
+			);
+		}
+	});
+
+	it("#given an empty existing artifact and another defect #when validating #then reports both defects", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa evidence",
+				surfaceEvidence: [
+					{
+						id: "surface",
+						criterionRef: "C001",
+						surface: "cli",
+						invocation: "run",
+						verdict: "passed",
+						artifactRefs: ["empty"],
+					},
+				],
+				adversarialCases: [
+					{
+						id: "adv",
+						criterionRef: "C001",
+						scenario: "scenario",
+						expectedBehavior: "reject",
+						verdict: "passed",
+						artifactRefs: [],
+					},
+				],
+				artifactRefs: [{ id: "empty", kind: "cli-transcript", description: "empty", path: "empty.txt" }],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "REJECT",
+				evidence: "gate evidence",
+				reportPath: "test/fixtures/artifacts/gate-review.md",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+
+		try {
+			validateQualityGate(gate, {
+				repoRoot: process.cwd(),
+				reviewerSurface: "omo-senpi",
+				fs: {
+					existsSync: (path) => path.endsWith("empty.txt") || path.endsWith("gate-review.md"),
+					statSync: () => ({ size: 0 }),
+				},
+			});
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			expect(Array.isArray(fields) ? fields.map((item) => item["field"]) : []).toEqual(
+				expect.arrayContaining(["manualQa.artifactRefs[0].path", "gateReview.recommendation"]),
+			);
+		}
+	});
+	it("#given four independent defects #when validating #then reports all defects in one error", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "placeholder",
+				surfaceEvidence: [],
+				adversarialCases: [],
+				artifactRefs: [
+					{ id: "missing", kind: "cli-transcript", description: "artifact", path: "does-not-exist.txt" },
+				],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "REJECT",
+				reportPath: "test/fixtures/artifacts/gate-review.md",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, opts);
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID");
+			expect(error.details?.["field"]).toBe("manualQa.evidence");
+			expect(error.details?.["fields"]).toEqual(
+				expect.arrayContaining([
+					{ field: "manualQa.evidence", message: expect.stringContaining("placeholder") },
+					{ field: "gateReview.evidence", message: expect.stringContaining("non-empty") },
+					{ field: "gateReview.recommendation", message: expect.stringContaining("APPROVE") },
+					{ field: "manualQa.artifactRefs[0].path", message: expect.stringContaining("existing") },
+				]),
+			);
+		}
+	});
+
+	it("#given missing required fields #when validating #then reports each field in one error", () => {
+		const gate = {
+			manualQa: { evidence: "qa", artifactRefs: [] },
+			gateReview: { by: "invalid", recommendation: "REJECT", evidence: "", reportPath: "", blockers: ["blocker"] },
+			iteration: { fullRerun: false, status: "failed", rerunCommands: [], evidence: "" },
+			criteriaCoverage: {
+				totalCriteria: "bad",
+				passCount: 0,
+				originalIntent: "",
+				desiredOutcome: "",
+				userOutcomeReview: "",
+				adversarialClassesCovered: [],
+			},
+		};
+		expect(() => validateQualityGate(gate, { reviewerSurface: "omo-senpi" })).toThrow(/gateReview\.by/);
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			expect(Array.isArray(fields) ? fields.map((field) => field["field"]) : []).toEqual(
+				expect.arrayContaining([
+					"gateReview.by",
+					"iteration.status",
+					"criteriaCoverage.totalCriteria",
+					"manualQa.artifactRefs",
+				]),
+			);
+		}
+	});
+
+	it("#given more than twenty-five defects #when validating #then caps the list and flags truncation", () => {
+		const refs = Array.from({ length: 30 }, (_, index) => ({
+			id: `ref-${index}`,
+			kind: "cli-transcript",
+			description: "artifact",
+			path: `does-not-exist-${index}.txt`,
+		}));
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "real evidence",
+				surfaceEvidence: [],
+				adversarialCases: [],
+				artifactRefs: refs,
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "APPROVE",
+				reportPath: "test/fixtures/artifacts/gate-review.md",
+				evidence: "gate evidence",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, opts);
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			expect(Array.isArray(fields) && fields.length).toBe(25);
+			expect(error.details?.["truncated"]).toBe(true);
+			expect(error.message).toContain("25+");
+		}
+	});
+});

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-cap-and-dedupe.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+import { throwQualityGateDefects } from "../src/quality-gate-fields.js";
+import { UlwLoopError } from "../src/types.js";
+
+describe("quality gate cap and dedupe", () => {
+	it("#given duplicate defects beyond the cap #when formatting #then deduplicates before capping", () => {
+		const defects = Array.from({ length: 30 }, (_, index) => ({
+			field: index < 10 ? "duplicate" : `field-${index - 10}`,
+			message: index < 10 ? "same defect" : `defect-${index - 10}`,
+		}));
+		try {
+			throwQualityGateDefects(defects);
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			expect(Array.isArray(fields) ? fields.length : 0).toBe(21);
+			expect(error.details?.["truncated"]).toBeUndefined();
+		}
+	});
+});

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-poisoning-cascades.test.ts

@@ -0,0 +1,97 @@
+import { existsSync } from "node:fs";
+import { describe, expect, it } from "vitest";
+import { validateQualityGate } from "../src/quality-gate.js";
+import { UlwLoopError } from "../src/types.js";
+
+const shared = {
+	gateReview: {
+		by: "category:deep",
+		recommendation: "APPROVE",
+		evidence: "review",
+		reportPath: "report.txt",
+		blockers: [],
+	},
+	iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+	criteriaCoverage: {
+		totalCriteria: 1,
+		passCount: 1,
+		originalIntent: "intent",
+		desiredOutcome: "outcome",
+		userOutcomeReview: "review",
+		adversarialClassesCovered: ["none"],
+	},
+};
+function fields(gate: Record<string, unknown>, withFs = false): readonly string[] {
+	try {
+		validateQualityGate(gate, {
+			reviewerSurface: "omo-senpi",
+			...(withFs ? { repoRoot: process.cwd(), fs: { existsSync, statSync: () => ({ size: 1 }) } } : {}),
+		});
+	} catch (error) {
+		if (error instanceof UlwLoopError) {
+			const list = error.details?.["fields"];
+			return Array.isArray(list) ? list.map((item) => `${String(item["field"])}: ${String(item["message"])}`) : [];
+		}
+		throw error;
+	}
+	throw new Error("expected validation failure");
+}
+function manual(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+	return {
+		by: "main-session",
+		status: "passed",
+		evidence: "qa",
+		surfaceEvidence: [],
+		adversarialCases: [],
+		artifactRefs: [{ id: "a", kind: "log", description: "x", path: "x.txt" }],
+		...overrides,
+	};
+}
+describe("quality gate poisoning contract", () => {
+	it.each([
+		[
+			"invalid artifact kind",
+			[{ id: "a", kind: "invalid-kind", description: "x", path: "x.txt" }],
+			(list: readonly string[]) => {
+				expect(list.some((x) => x.startsWith("manualQa.artifactRefs[0].kind:"))).toBe(true);
+				expect(list.some((x) => x.includes("references unknown artifact a"))).toBe(false);
+			},
+		],
+		[
+			"duplicate id with invalid kind",
+			[
+				{ id: "a", kind: "cli-transcript", description: "x", path: "x.txt" },
+				{ id: "a", kind: "invalid-kind", description: "x", path: "x.txt" },
+			],
+			(list: readonly string[]) => {
+				expect(list.some((x) => x.startsWith("manualQa.artifactRefs[1].kind:"))).toBe(true);
+				expect(list.some((x) => x.includes("duplicate a"))).toBe(true);
+			},
+		],
+	] as const)("#given %s #when validating #then preserves independent defects", (_name, artifactRefs, assertResult) =>
+		assertResult(fields({ ...shared, manualQa: manual({ artifactRefs }) })),
+	);
+	it("#given a poisoned numeric input #when validating #then skips its cross-field comparison", () => {
+		const list = fields({
+			...shared,
+			criteriaCoverage: { ...shared.criteriaCoverage, passCount: "bad" },
+			manualQa: manual(),
+		});
+		expect(list.some((x) => x.includes("passCount") && x.includes("numeric"))).toBe(true);
+		expect(list.some((x) => x.includes("cover totalCriteria"))).toBe(false);
+	});
+	it("#given poisoned parent rows #when validating #then suppresses all child defects", () => {
+		const list = fields({ ...shared, manualQa: manual({ surfaceEvidence: [null], adversarialCases: [null] }) });
+		expect(list.some((x) => x.startsWith("manualQa.surfaceEvidence[0]:"))).toBe(true);
+		expect(list.some((x) => x.startsWith("manualQa.surfaceEvidence[0]."))).toBe(false);
+		expect(list.some((x) => x.startsWith("manualQa.adversarialCases[0]:"))).toBe(true);
+	});
+	it("#given a poisoned report path #when validating with fs options #then skips file checks", () => {
+		const list = fields(
+			{ ...shared, gateReview: { ...shared.gateReview, reportPath: "tbd" }, manualQa: manual() },
+			true,
+		);
+		expect(list.some((x) => x.includes("reportPath") && x.includes("placeholder"))).toBe(true);
+		expect(list.some((x) => x.includes("gateReview.reportPath") && x.includes("existing artifact"))).toBe(false);
+	});
+});