@mainahq/core 0.3.0 → 0.5.0

package/src/verify/__tests__/typecheck.test.ts

@@ -0,0 +1,160 @@
+/**
+ * Tests for built-in type checking in the verify pipeline.
+ *
+ * Verifies that runTypecheck() spawns the correct language-specific
+ * type checker and parses its output into Finding[].
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+describe("runTypecheck", () => {
+	let testDir: string;
+
+	beforeEach(() => {
+		testDir = join(tmpdir(), `maina-typecheck-test-${Date.now()}`);
+		mkdirSync(testDir, { recursive: true });
+	});
+
+	afterEach(() => {
+		rmSync(testDir, { recursive: true, force: true });
+	});
+
+	it("should parse tsc --noEmit output into Finding[]", async () => {
+		// Create a tsconfig.json and install typescript locally
+		writeFileSync(
+			join(testDir, "tsconfig.json"),
+			JSON.stringify({
+				compilerOptions: { strict: true, noEmit: true },
+				include: ["*.ts"],
+			}),
+		);
+		// Install typescript locally so tsc is in node_modules/.bin
+		const install = Bun.spawnSync(["bun", "add", "typescript"], {
+			cwd: testDir,
+		});
+		if (install.exitCode !== 0) {
+			// Skip if we can't install (CI without network, etc.)
+			return;
+		}
+
+		// Create a file with a type error
+		writeFileSync(
+			join(testDir, "bad.ts"),
+			'const x: number = "not a number";\n',
+		);
+
+		const { runTypecheck } = await import("../typecheck");
+		const result = await runTypecheck(["bad.ts"], testDir);
+
+		expect(result.findings.length).toBeGreaterThan(0);
+		const first = result.findings[0];
+		if (!first) throw new Error("Expected at least one finding");
+		expect(first.tool).toBe("tsc");
+		expect(first.severity).toBe("error");
+		expect(first.file).toContain("bad.ts");
+		expect(first.line).toBeGreaterThan(0);
+		expect(result.duration).toBeGreaterThanOrEqual(0);
+	});
+
+	it("should return empty findings for clean file", async () => {
+		writeFileSync(
+			join(testDir, "tsconfig.json"),
+			JSON.stringify({
+				compilerOptions: { strict: true, noEmit: true },
+				include: ["*.ts"],
+			}),
+		);
+		Bun.spawnSync(["bun", "add", "typescript"], { cwd: testDir });
+
+		writeFileSync(join(testDir, "good.ts"), "const x: number = 42;\n");
+
+		const { runTypecheck } = await import("../typecheck");
+		const result = await runTypecheck(["good.ts"], testDir);
+
+		expect(result.findings).toEqual([]);
+		expect(result.tool).toBe("tsc");
+	});
+
+	it("should skip with info when tsc is not found", async () => {
+		// Use a non-existent tool path
+		const { runTypecheck } = await import("../typecheck");
+		const result = await runTypecheck(["file.ts"], testDir, {
+			command: "nonexistent-tsc-binary",
+		});
+
+		expect(result.findings).toEqual([]);
+		expect(result.skipped).toBe(true);
+	});
+
+	it("should detect language and use appropriate checker", async () => {
+		const { getTypecheckCommand } = await import("../typecheck");
+
+		expect(getTypecheckCommand("typescript")).toEqual(
+			expect.objectContaining({ tool: "tsc" }),
+		);
+		expect(getTypecheckCommand("python")).toEqual(
+			expect.objectContaining({ tool: "mypy" }),
+		);
+		expect(getTypecheckCommand("go")).toEqual(
+			expect.objectContaining({ tool: "go-vet" }),
+		);
+		expect(getTypecheckCommand("rust")).toEqual(
+			expect.objectContaining({ tool: "cargo-check" }),
+		);
+		expect(getTypecheckCommand("csharp")).toEqual(
+			expect.objectContaining({ tool: "dotnet-build" }),
+		);
+		expect(getTypecheckCommand("java")).toEqual(
+			expect.objectContaining({ tool: "javac" }),
+		);
+	});
+
+	it("should parse multiple errors from tsc output", async () => {
+		const { parseTscOutput } = await import("../typecheck");
+
+		const tscOutput = [
+			"src/foo.ts(10,5): error TS2322: Type 'string' is not assignable to type 'number'.",
+			"src/bar.ts(3,1): error TS2304: Cannot find name 'x'.",
+		].join("\n");
+
+		const findings = parseTscOutput(tscOutput);
+
+		expect(findings).toHaveLength(2);
+		expect(findings[0]).toEqual({
+			tool: "tsc",
+			file: "src/foo.ts",
+			line: 10,
+			column: 5,
+			message: "TS2322: Type 'string' is not assignable to type 'number'.",
+			severity: "error",
+			ruleId: "TS2322",
+		});
+		expect(findings[1]).toEqual({
+			tool: "tsc",
+			file: "src/bar.ts",
+			line: 3,
+			column: 1,
+			message: "TS2304: Cannot find name 'x'.",
+			severity: "error",
+			ruleId: "TS2304",
+		});
+	});
+
+	it("should return empty for empty tsc output", async () => {
+		const { parseTscOutput } = await import("../typecheck");
+		const findings = parseTscOutput("");
+		expect(findings).toEqual([]);
+	});
+
+	it("should handle no tsconfig.json gracefully", async () => {
+		// No tsconfig.json in testDir
+		const { runTypecheck } = await import("../typecheck");
+		const result = await runTypecheck(["file.ts"], testDir);
+
+		// Should not crash, should skip or return empty
+		expect(result.skipped).toBe(true);
+	});
+});

package/src/verify/__tests__/zap.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, it } from "bun:test";
+import { parseZapJson, runZap } from "../zap";
+
+// ─── parseZapJson ─────────────────────────────────────────────────────────
+
+describe("parseZapJson", () => {
+	it("should return empty array for empty alerts", () => {
+		const json = JSON.stringify({ site: [{ alerts: [] }] });
+		const findings = parseZapJson(json);
+		expect(findings).toEqual([]);
+	});
+
+	it("should parse a single alert from ZAP JSON output", () => {
+		const json = JSON.stringify({
+			site: [
+				{
+					alerts: [
+						{
+							pluginid: "10021",
+							alert: "X-Content-Type-Options Header Missing",
+							riskdesc: "Low (Medium)",
+							desc: "The Anti-MIME-Sniffing header is not set.",
+							instances: [
+								{
+									uri: "https://example.com/api/health",
+									method: "GET",
+								},
+							],
+						},
+					],
+				},
+			],
+		});
+
+		const findings = parseZapJson(json);
+		expect(findings.length).toBe(1);
+		expect(findings[0]?.tool).toBe("zap");
+		expect(findings[0]?.file).toBe("https://example.com/api/health");
+		expect(findings[0]?.line).toBe(0);
+		expect(findings[0]?.message).toContain(
+			"X-Content-Type-Options Header Missing",
+		);
+		expect(findings[0]?.severity).toBe("info");
+		expect(findings[0]?.ruleId).toBe("10021");
+	});
+
+	it("should map ZAP risk levels to severity correctly", () => {
+		const makeZap = (riskdesc: string) =>
+			JSON.stringify({
+				site: [
+					{
+						alerts: [
+							{
+								pluginid: "10001",
+								alert: "Test Alert",
+								riskdesc,
+								desc: "Test description",
+								instances: [{ uri: "https://example.com", method: "GET" }],
+							},
+						],
+					},
+				],
+			});
+
+		expect(parseZapJson(makeZap("High (Medium)"))[0]?.severity).toBe("error");
+		expect(parseZapJson(makeZap("Medium (Low)"))[0]?.severity).toBe("warning");
+		expect(parseZapJson(makeZap("Low (Medium)"))[0]?.severity).toBe("info");
+		expect(parseZapJson(makeZap("Informational (Low)"))[0]?.severity).toBe(
+			"info",
+		);
+	});
+
+	it("should handle multiple alerts with multiple instances", () => {
+		const json = JSON.stringify({
+			site: [
+				{
+					alerts: [
+						{
+							pluginid: "10021",
+							alert: "Alert A",
+							riskdesc: "High (Medium)",
+							desc: "Description A",
+							instances: [
+								{ uri: "https://example.com/a", method: "GET" },
+								{ uri: "https://example.com/b", method: "POST" },
+							],
+						},
+						{
+							pluginid: "10022",
+							alert: "Alert B",
+							riskdesc: "Low (Low)",
+							desc: "Description B",
+							instances: [{ uri: "https://example.com/c", method: "GET" }],
+						},
+					],
+				},
+			],
+		});
+
+		const findings = parseZapJson(json);
+		expect(findings.length).toBe(3);
+		expect(findings[0]?.file).toBe("https://example.com/a");
+		expect(findings[1]?.file).toBe("https://example.com/b");
+		expect(findings[2]?.file).toBe("https://example.com/c");
+	});
+
+	it("should handle alerts with no instances", () => {
+		const json = JSON.stringify({
+			site: [
+				{
+					alerts: [
+						{
+							pluginid: "10021",
+							alert: "No Instances Alert",
+							riskdesc: "Medium (Medium)",
+							desc: "No instances",
+							instances: [],
+						},
+					],
+				},
+			],
+		});
+
+		const findings = parseZapJson(json);
+		expect(findings.length).toBe(1);
+		expect(findings[0]?.file).toBe("");
+		expect(findings[0]?.message).toContain("No Instances Alert");
+	});
+
+	it("should return empty array for invalid JSON", () => {
+		const findings = parseZapJson("not valid json {{{");
+		expect(findings).toEqual([]);
+	});
+
+	it("should return empty array for malformed structure", () => {
+		const findings = parseZapJson(JSON.stringify({ unexpected: true }));
+		expect(findings).toEqual([]);
+	});
+
+	it("should handle missing site array gracefully", () => {
+		const findings = parseZapJson(JSON.stringify({ site: "not-an-array" }));
+		expect(findings).toEqual([]);
+	});
+
+	it("should handle site entries without alerts", () => {
+		const json = JSON.stringify({
+			site: [{ name: "example.com" }],
+		});
+		const findings = parseZapJson(json);
+		expect(findings).toEqual([]);
+	});
+});
+
+// ─── runZap ───────────────────────────────────────────────────────────────
+
+describe("runZap", () => {
+	it("should skip when docker is not available", async () => {
+		const result = await runZap({
+			targetUrl: "https://example.com",
+			cwd: "/tmp",
+			available: false,
+		});
+		expect(result.findings).toEqual([]);
+		expect(result.skipped).toBe(true);
+	});
+
+	it("should skip when no targetUrl is provided", async () => {
+		const result = await runZap({
+			targetUrl: "",
+			cwd: "/tmp",
+			available: true,
+		});
+		expect(result.findings).toEqual([]);
+		expect(result.skipped).toBe(true);
+	});
+
+	it("should return correct result shape", async () => {
+		const result = await runZap({
+			targetUrl: "https://example.com",
+			cwd: "/tmp",
+			available: false,
+		});
+		expect(result).toHaveProperty("findings");
+		expect(result).toHaveProperty("skipped");
+		expect(Array.isArray(result.findings)).toBe(true);
+		expect(typeof result.skipped).toBe("boolean");
+	});
+});

package/src/verify/consistency.ts

@@ -0,0 +1,199 @@
+/**
+ * Cross-function Consistency Check — deterministic AST-based analysis.
+ *
+ * Catches the class of bug that lost Maina 2 points in the Tier 3 benchmark:
+ * functions that call a validator on one code path but skip it on another.
+ *
+ * Two modes:
+ * 1. Spec-based: reads spec.md / constitution.md for stated constraints,
+ *    builds a rule set, checks compliance
+ * 2. Heuristic: if no spec exists, looks for inconsistent validator usage
+ *    patterns across related functions
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { Finding } from "./diff-filter";
+
+// ─── Types ────────────────────────────────────────────────────────────────
+
+export interface ConsistencyRule {
+	pattern: string;
+	source: "spec" | "heuristic";
+}
+
+export interface ConsistencyResult {
+	findings: Finding[];
+	rulesChecked: number;
+}
+
+// ─── Rule Extraction ─────────────────────────────────────────────────────
+
+/**
+ * Extract consistency rules from spec/constitution content.
+ * Looks for patterns like "use X when Y", "always call X", "validate with X".
+ */
+function extractRulesFromSpec(content: string): ConsistencyRule[] {
+	const rules: ConsistencyRule[] = [];
+	const patterns = [
+		/always (?:use|call|check|validate with) (\w+)/gi,
+		/must (?:use|call|check) (\w+)/gi,
+		/validate.*(?:with|using) (\w+)/gi,
+	];
+
+	for (const pattern of patterns) {
+		for (const match of content.matchAll(pattern)) {
+			rules.push({
+				pattern: match[1] ?? "",
+				source: "spec",
+			});
+		}
+	}
+
+	return rules;
+}
+
+/**
+ * Load spec/constitution content from the maina directory.
+ */
+function loadSpecContent(mainaDir: string): string {
+	const paths = [join(mainaDir, "constitution.md"), join(mainaDir, "spec.md")];
+
+	const parts: string[] = [];
+	for (const p of paths) {
+		if (existsSync(p)) {
+			parts.push(readFileSync(p, "utf-8"));
+		}
+	}
+
+	return parts.join("\n");
+}
+
+// ─── Heuristic Analysis ──────────────────────────────────────────────────
+
+/**
+ * Find functions that call validators inconsistently.
+ * If functionA calls isValid(x) and functionB doesn't but both take similar
+ * params, that's a potential inconsistency.
+ */
+function findHeuristicIssues(source: string, file: string): Finding[] {
+	const findings: Finding[] = [];
+
+	// Extract function calls per function body
+	const functionPattern =
+		/function\s+(\w+)\s*\([^)]*\)\s*\{([^}]*(?:\{[^}]*\}[^}]*)*)\}/g;
+	const functions: Array<{ name: string; body: string; line: number }> = [];
+
+	for (const match of source.matchAll(functionPattern)) {
+		const lineNumber = source.substring(0, match.index ?? 0).split("\n").length;
+		functions.push({
+			name: match[1] ?? "",
+			body: match[2] ?? "",
+			line: lineNumber,
+		});
+	}
+
+	// Find validator-like calls (isX, validateX, checkX)
+	const validatorPattern = /\b(is[A-Z]\w+|validate\w+|check\w+)\s*\(/g;
+	const validatorsByFunction = new Map<string, Set<string>>();
+
+	for (const fn of functions) {
+		const validators = new Set<string>();
+		for (const validatorMatch of fn.body.matchAll(validatorPattern)) {
+			validators.add(validatorMatch[1] ?? "");
+		}
+		validatorsByFunction.set(fn.name, validators);
+	}
+
+	// Compare: if most functions use a validator but one doesn't, flag it
+	const allValidators = new Set<string>();
+	for (const validators of validatorsByFunction.values()) {
+		for (const v of validators) allValidators.add(v);
+	}
+
+	if (functions.length >= 2) {
+		for (const validator of allValidators) {
+			const usersCount = Array.from(validatorsByFunction.values()).filter((v) =>
+				v.has(validator),
+			).length;
+
+			// If majority uses it but some don't, flag the ones that don't
+			if (usersCount > 0 && usersCount < functions.length) {
+				for (const fn of functions) {
+					const fnValidators = validatorsByFunction.get(fn.name);
+					if (fnValidators && !fnValidators.has(validator)) {
+						findings.push({
+							tool: "consistency",
+							file,
+							line: fn.line,
+							message: `Function '${fn.name}' does not call '${validator}' — other functions in this file do. Possible inconsistency.`,
+							severity: "warning",
+							ruleId: `consistency/${validator}`,
+						});
+					}
+				}
+			}
+		}
+	}
+
+	return findings;
+}
+
+// ─── Main ────────────────────────────────────────────────────────────────
+
+export async function checkConsistency(
+	files: string[],
+	cwd: string,
+	mainaDir: string,
+): Promise<ConsistencyResult> {
+	const specContent = existsSync(mainaDir) ? loadSpecContent(mainaDir) : "";
+	const specRules = extractRulesFromSpec(specContent);
+	const allFindings: Finding[] = [];
+
+	for (const file of files) {
+		const filePath = join(cwd, file);
+		if (!existsSync(filePath)) continue;
+
+		const source = readFileSync(filePath, "utf-8");
+
+		// Check spec-based rules
+		for (const rule of specRules) {
+			const callPattern = new RegExp(`\\b${rule.pattern}\\s*\\(`, "g");
+			const fnPattern = /function\s+(\w+)/g;
+
+			// Find functions that should use this pattern but don't
+			for (const fnMatch of source.matchAll(fnPattern)) {
+				const fnStart = fnMatch.index ?? 0;
+				const fnEnd = source.indexOf("}", fnStart + 1);
+				if (fnEnd === -1) continue;
+
+				const fnBody = source.substring(fnStart, fnEnd);
+				if (!callPattern.test(fnBody)) {
+					const relatedTerms = rule.pattern
+						.toLowerCase()
+						.replace(/^is|^validate|^check/, "");
+					if (fnBody.toLowerCase().includes(relatedTerms)) {
+						const line = source.substring(0, fnStart).split("\n").length;
+						allFindings.push({
+							tool: "consistency",
+							file,
+							line,
+							message: `Spec requires '${rule.pattern}' — function '${fnMatch[1]}' may need it.`,
+							severity: "warning",
+							ruleId: `consistency/spec-${rule.pattern}`,
+						});
+					}
+				}
+			}
+		}
+
+		// Heuristic checks (always run)
+		allFindings.push(...findHeuristicIssues(source, file));
+	}
+
+	return {
+		findings: allFindings,
+		rulesChecked: specRules.length + 1, // +1 for heuristic check
+	};
+}

package/src/verify/detect.ts

@@ -26,7 +26,9 @@ export type ToolName =
 	| "dotnet-format"
 	| "checkstyle"
 	| "spotbugs"
-	| "pmd";
+	| "pmd"
+	| "zap"
+	| "lighthouse";
 
 export interface DetectedTool {
 	name: string;
@@ -55,6 +57,8 @@ export const TOOL_REGISTRY: Record<
 	checkstyle: { command: "checkstyle", versionFlag: "--version" },
 	spotbugs: { command: "spotbugs", versionFlag: "-version" },
 	pmd: { command: "pmd", versionFlag: "--version" },
+	zap: { command: "docker", versionFlag: "--version" },
+	lighthouse: { command: "lighthouse", versionFlag: "--version" },
 };
 
 /**