lazycodex-ai 5.0.0-beta.4 → 5.0.0-beta.40

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-senpi-surface.test.ts

@@ -0,0 +1,131 @@
+import { readFile } from "node:fs/promises";
+
+import { describe, expect, it } from "vitest";
+
+import { validateQualityGate } from "../src/quality-gate.js";
+import { UlwLoopError } from "../src/types.js";
+
+const COMMON = {
+	manualQa: {
+		status: "passed",
+		evidence: "Ran senpi manual QA with artifact-backed evidence.",
+		surfaceEvidence: [
+			{
+				id: "surface-cli-pass",
+				criterionRef: "C1",
+				surface: "cli",
+				invocation: "omo-agent-toolkit ulw-loop checkpoint --status complete",
+				verdict: "passed",
+				artifactRefs: ["artifact-cli-pass"],
+			},
+		],
+		adversarialCases: [
+			{
+				id: "adv-role",
+				criterionRef: "C2",
+				scenario: "a reviewer role is invalid",
+				expectedBehavior: "the gate rejects the invalid role",
+				verdict: "passed",
+				artifactRefs: ["artifact-cli-pass"],
+			},
+		],
+		artifactRefs: [
+			{
+				id: "artifact-cli-pass",
+				kind: "cli-transcript",
+				description: "CLI transcript for valid checkpoint.",
+				path: "cli-pass.txt",
+			},
+		],
+	},
+	iteration: {
+		fullRerun: true,
+		status: "passed",
+		rerunCommands: ["bunx vitest run"],
+		evidence: "Focused tests passed.",
+	},
+	criteriaCoverage: {
+		totalCriteria: 2,
+		passCount: 2,
+		originalIntent: "User wanted a valid senpi quality gate.",
+		desiredOutcome: "The senpi gate accepts its four required lanes.",
+		userOutcomeReview: "The requested senpi outcome is covered.",
+		adversarialClassesCovered: ["role_mismatch"],
+	},
+} as const;
+
+function senpiGate(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+	return {
+		...COMMON,
+		manualQa: { ...COMMON.manualQa, by: "main-session" },
+		gateReview: {
+			by: "category:deep",
+			recommendation: "APPROVE",
+			reportPath: "gate-review.md",
+			evidence: "Verified the senpi gate.",
+			blockers: [],
+		},
+		...overrides,
+	};
+}
+
+function section(input: Record<string, unknown>, key: string): Record<string, unknown> {
+	const value = input[key];
+	if (typeof value !== "object" || value === null || Array.isArray(value))
+		throw new Error(`missing object section: ${key}`);
+	return value as Record<string, unknown>;
+}
+
+function errorFor(input: unknown): UlwLoopError {
+	try {
+		validateQualityGate(input, { reviewerSurface: "omo-senpi" });
+	} catch (error) {
+		if (error instanceof UlwLoopError) return error;
+		throw error;
+	}
+	throw new Error("Expected quality gate validation to fail");
+}
+
+describe("validateQualityGate omo-senpi surface", () => {
+	it("#given a four-section senpi gate #when validated #then it accepts main-session manual QA and category:deep review", () => {
+		expect(() => validateQualityGate(senpiGate(), { reviewerSurface: "omo-senpi" })).not.toThrow();
+	});
+
+	it("#given a senpi gate with codeReview #when validated #then it rejects the forbidden codeReview lane", () => {
+		const error = errorFor(senpiGate({ codeReview: { by: "anything" } }));
+		expect(error.details?.["field"]).toContain("codeReview");
+		expect(error.message).toContain("omo-senpi gate has no codeReview lane");
+	});
+
+	it.each(["category:deep", "category:unspecified-high", "category:unspecified-low"])(
+		"#given gateReview.by %s #when validated on senpi #then it accepts the category role",
+		(by) => {
+			expect(() =>
+				validateQualityGate(senpiGate({ gateReview: { ...section(senpiGate(), "gateReview"), by } }), {
+					reviewerSurface: "omo-senpi",
+				}),
+			).not.toThrow();
+		},
+	);
+
+	it.each(["omo-senpi-gate-reviewer", "lazycodex-gate-reviewer", "arbitrary"])(
+		"#given gateReview.by %s #when validated on senpi #then it rejects the non-category role",
+		(by) => {
+			const error = errorFor(senpiGate({ gateReview: { ...section(senpiGate(), "gateReview"), by } }));
+			expect(error.message).toContain("gateReview.by");
+		},
+	);
+
+	it.each(["omo-senpi-qa-executor", "lazycodex-qa-executor", "arbitrary"])(
+		"#given manualQa.by %s #when validated on senpi #then it rejects the non-main-session role",
+		(by) => {
+			const error = errorFor(senpiGate({ manualQa: { ...COMMON.manualQa, by } }));
+			expect(error.message).toContain("manualQa.by");
+		},
+	);
+
+	it("#given the existing lazycodex fixture #when validated on the default surface #then it remains valid", async () => {
+		const raw = await readFile(new URL("./fixtures/sample-quality-gate.json", import.meta.url), "utf8");
+		expect(() => validateQualityGate(JSON.parse(raw))).not.toThrow();
+	});
+});

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate-single-report.test.ts

@@ -0,0 +1,241 @@
+import { describe, expect, it } from "vitest";
+import { validateQualityGate } from "../src/quality-gate.js";
+import { UlwLoopError } from "../src/types.js";
+
+describe("quality gate single report per invalid value", () => {
+	it("#given an empty artifact reference id #when validating #then reports only the array item defect", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa",
+				surfaceEvidence: [
+					{
+						id: "surface",
+						criterionRef: "C001",
+						surface: "cli",
+						invocation: "run",
+						verdict: "passed",
+						artifactRefs: [""],
+					},
+				],
+				adversarialCases: [
+					{
+						id: "adv",
+						criterionRef: "C001",
+						scenario: "scenario",
+						expectedBehavior: "reject",
+						verdict: "passed",
+						artifactRefs: [],
+					},
+				],
+				artifactRefs: [{ id: "artifact", kind: "cli-transcript", description: "artifact", path: "artifact.txt" }],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "APPROVE",
+				evidence: "review",
+				reportPath: "report.txt",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			const artifactFields = Array.isArray(fields)
+				? fields.filter((item) => item["field"] === "manualQa.surfaceEvidence[0].artifactRefs")
+				: [];
+			expect(artifactFields).toHaveLength(1);
+		}
+	});
+
+	it("#given a malformed artifact entry #when validating #then does not derive downstream artifact defects", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa",
+				surfaceEvidence: [],
+				adversarialCases: [],
+				artifactRefs: [null],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "APPROVE",
+				evidence: "review",
+				reportPath: "report.txt",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			const artifactFields = Array.isArray(fields)
+				? fields.filter((item) => String(item["field"]).startsWith("manualQa.artifactRefs[0]"))
+				: [];
+			expect(artifactFields).toHaveLength(1);
+			expect(artifactFields[0]?.["field"]).toBe("manualQa.artifactRefs[0]");
+		}
+	});
+
+	it("#given an invalid artifact kind #when validating #then does not derive compatibility defects", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa",
+				surfaceEvidence: [
+					{
+						id: "surface",
+						criterionRef: "C001",
+						// A data surface accepts only data-diff, so the validator's "log" fallback
+						// for the invalid kind WOULD trip the compatibility check if poisoning
+						// were removed; a cli surface would let the fallback pass silently.
+						surface: "data",
+						invocation: "run",
+						verdict: "passed",
+						artifactRefs: ["artifact"],
+					},
+				],
+				adversarialCases: [
+					{
+						id: "adv",
+						criterionRef: "C001",
+						scenario: "scenario",
+						expectedBehavior: "reject",
+						verdict: "passed",
+						artifactRefs: [],
+					},
+				],
+				artifactRefs: [{ id: "artifact", kind: "invalid", description: "artifact", path: "artifact.txt" }],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "APPROVE",
+				evidence: "review",
+				reportPath: "report.txt",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			const labels = Array.isArray(fields) ? fields.map((item) => item["field"]) : [];
+			expect(labels).toContain("manualQa.artifactRefs[0].kind");
+			expect(labels).not.toContain("manualQa.surfaceEvidence");
+			expect(labels).not.toContain("manualQa.surfaceEvidence[0].artifactRefs");
+			expect(labels.filter((label) => label === "manualQa.artifactRefs[0].kind")).toHaveLength(1);
+		}
+	});
+	it("#given an empty gate #when validating #then reports each missing section once", () => {
+		try {
+			validateQualityGate({}, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			const labels = Array.isArray(fields) ? fields.map((item) => item["field"]) : [];
+			expect(labels).toEqual(expect.arrayContaining(["manualQa", "gateReview", "iteration", "criteriaCoverage"]));
+			expect(new Set(labels).size).toBe(labels.length);
+		}
+	});
+
+	it("#given a not-applicable surface verdict #when validating #then reports that verdict defect once", () => {
+		const gate = {
+			manualQa: {
+				by: "main-session",
+				status: "passed",
+				evidence: "qa",
+				surfaceEvidence: [
+					{
+						id: "surface",
+						criterionRef: "C001",
+						surface: "cli",
+						invocation: "run",
+						verdict: "not_applicable",
+						artifactRefs: [],
+					},
+				],
+				adversarialCases: [
+					{
+						id: "adv",
+						criterionRef: "C001",
+						scenario: "scenario",
+						expectedBehavior: "reject",
+						verdict: "passed",
+						artifactRefs: [],
+					},
+				],
+				artifactRefs: [{ id: "artifact", kind: "cli-transcript", description: "artifact", path: "artifact.txt" }],
+			},
+			gateReview: {
+				by: "category:deep",
+				recommendation: "APPROVE",
+				evidence: "review",
+				reportPath: "report.txt",
+				blockers: [],
+			},
+			iteration: { fullRerun: true, status: "passed", rerunCommands: ["test"], evidence: "rerun" },
+			criteriaCoverage: {
+				totalCriteria: 1,
+				passCount: 1,
+				originalIntent: "intent",
+				desiredOutcome: "outcome",
+				userOutcomeReview: "review",
+				adversarialClassesCovered: ["none"],
+			},
+		};
+		try {
+			validateQualityGate(gate, { reviewerSurface: "omo-senpi" });
+			throw new Error("expected validation failure");
+		} catch (error) {
+			expect(error).toBeInstanceOf(UlwLoopError);
+			if (!(error instanceof UlwLoopError)) throw error;
+			const fields = error.details?.["fields"];
+			const verdictFields = Array.isArray(fields)
+				? fields.filter((item) => item["field"] === "manualQa.surfaceEvidence[0].verdict")
+				: [];
+			expect(verdictFields).toHaveLength(1);
+		}
+	});
+});

package/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate.test.ts

@@ -111,7 +111,10 @@ describe("validateQualityGate", () => {
 			"gateReview",
 			"iteration",
 			"manualQa",
+			"surface",
 		]);
+		expect("codeReview" in gate).toBe(true);
+		if (!("codeReview" in gate)) throw new Error("expected lazycodex codeReview section");
 		expect(gate.codeReview.codeQualityStatus).toBe("CLEAR");
 		expect(gate).toMatchObject({
 			criteriaCoverage: { totalCriteria: 9, passCount: 9, userOutcomeReview: expect.stringContaining("user") },
@@ -127,6 +130,8 @@ describe("validateQualityGate", () => {
 		const gate = validateQualityGate(parsed, FS_OPTS);
 
 		// then
+		expect("codeReview" in gate).toBe(true);
+		if (!("codeReview" in gate)) throw new Error("expected lazycodex codeReview section");
 		expect(gate.codeReview.recommendation).toBe("APPROVE");
 		expect(gate.manualQa.artifactRefs).toHaveLength(5);
 	});
@@ -161,20 +166,60 @@ describe("validateQualityGate", () => {
 		expect(error.message).toContain("missing-artifact");
 	});
 
-	it("#given incompatible surface artifact kind #when validated #then it rejects the gate", () => {
+	it("#given incompatible surface artifact kind #when validated #then it rejects the gate with compatible kinds", () => {
 		// when
 		const error = getQualityGateError(
 			makeGate({
 				manualQa: {
 					...VALID_GATE.manualQa,
-					artifactRefs: [{ ...VALID_GATE.manualQa.artifactRefs[0], kind: "http-dump" }],
+					artifactRefs: [
+						{ ...VALID_GATE.manualQa.artifactRefs[0], kind: "http-dump" },
+						VALID_GATE.manualQa.artifactRefs[1],
+					],
 				},
 			}),
 		);
 
 		// then
 		expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID");
-		expect(error.message).toContain("cli");
+		expect(error.message).toContain(
+			'manualQa.surfaceEvidence cli artifact http-dump is incompatible; surface "cli" accepts artifact kinds: cli-transcript, log.',
+		);
+	});
+
+	it("#given an unsupported artifact kind #when validated #then it lists allowed kinds and report path guidance", () => {
+		const error = getQualityGateError(
+			makeGate({
+				manualQa: {
+					...VALID_GATE.manualQa,
+					artifactRefs: [
+						{ ...VALID_GATE.manualQa.artifactRefs[0], kind: "review-report" },
+						VALID_GATE.manualQa.artifactRefs[1],
+					],
+				},
+			}),
+		);
+
+		expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID");
+		expect(error.message).toContain(
+			"manualQa.artifactRefs[0].kind must be a supported artifact kind (cli-transcript, log, screenshot, image, http-dump, data-diff); review/QA reports belong in codeReview.reportPath or gateReview.reportPath, not artifactRefs.",
+		);
+	});
+
+	it("#given an unsupported manual QA surface #when validated #then it lists allowed surfaces", () => {
+		const error = getQualityGateError(
+			makeGate({
+				manualQa: {
+					...VALID_GATE.manualQa,
+					surfaceEvidence: [{ ...VALID_GATE.manualQa.surfaceEvidence[0], surface: "terminal" }],
+				},
+			}),
+		);
+
+		expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID");
+		expect(error.message).toContain(
+			"manualQa.surfaceEvidence[0].surface must be a supported manual QA surface (cli, http, tmux, browser, gui, data).",
+		);
 	});
 
 	it("#given placeholder evidence and artifact path #when validated #then it rejects placeholders", () => {