agent-gauntlet 0.2.2 → 0.4.0

package/src/utils/log-parser.ts

@@ -1,15 +1,14 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import type { ReviewFullJsonOutput } from "../gates/result.js";
 
-export interface PreviousViolation {
-	file: string;
-	line: number | string;
-	issue: string;
-	fix?: string;
-}
+export type { PreviousViolation } from "../gates/result.js";
+
+import type { PreviousViolation } from "../gates/result.js";
 
 export interface AdapterFailure {
 	adapterName: string; // e.g., 'claude', 'gemini'
+	reviewIndex?: number; // 1-based review index from @N in filename
 	violations: PreviousViolation[];
 }
 
@@ -22,189 +21,388 @@ export interface GateFailures {
 }
 
 /**
- * Parses a single log file to extract failures per adapter.
- * Only processes review gates (ignores check gates).
+ * Parse a review filename to extract the job ID, adapter, review index, and run number.
+ * Pattern: <jobId>_<adapter>@<reviewIndex>.<runNumber>.(log|json)
+ * Returns null if the filename doesn't match the review pattern.
  */
-export async function parseLogFile(
-	logPath: string,
+export function parseReviewFilename(filename: string): {
+	jobId: string;
+	adapter: string;
+	reviewIndex: number;
+	runNumber: number;
+	ext: string;
+} | null {
+	// Match: <prefix>_<adapter>@<index>.<runNum>.(log|json)
+	const m = filename.match(/^(.+)_([^@]+)@(\d+)\.(\d+)\.(log|json)$/);
+	if (!m) return null;
+	return {
+		jobId: m[1]!,
+		adapter: m[2]!,
+		reviewIndex: parseInt(m[3]!, 10),
+		runNumber: parseInt(m[4]!, 10),
+		ext: m[5]!,
+	};
+}
+
+/**
+ * Parses a JSON review file.
+ */
+export async function parseJsonReviewFile(
+	jsonPath: string,
 ): Promise<GateFailures | null> {
 	try {
-		const content = await fs.readFile(logPath, "utf-8");
-		const filename = path.basename(logPath);
+		const content = await fs.readFile(jsonPath, "utf-8");
+		const data: ReviewFullJsonOutput = JSON.parse(content);
+		const filename = path.basename(jsonPath);
+
+		// Extract jobId: strip the _adapter@index.runNum.json suffix
+		const parsed = parseReviewFilename(filename);
+		const jobId = parsed ? parsed.jobId : filename.replace(/\.\d+\.json$/, "");
 
-		// Check if it's a review log by content marker
-		if (!content.includes("--- Review Output")) {
+		if (data.status === "pass") {
 			return null;
 		}
 
-		// Use the sanitized filename as the Job ID key
-		const jobId = filename.replace(/\.log$/, "");
-
-		// We can't reliably parse entryPoint/gateName from sanitized filename
-		// leaving them empty for now as they aren't critical for the map lookup
-		const gateName = "";
-		const entryPoint = "";
-
-		const adapterFailures: AdapterFailure[] = [];
-
-		// Split by sections using `--- Review Output (adapterName) ---` markers
-		const sectionRegex = /--- Review Output \(([^)]+)\) ---/g;
-
-		let match: RegExpExecArray | null;
-		const sections: { adapter: string; startIndex: number }[] = [];
-
-		for (;;) {
-			match = sectionRegex.exec(content);
-			if (!match) break;
-			sections.push({
-				adapter: match[1],
-				startIndex: match.index,
+		const violations = (data.violations || []).map((v) => ({
+			...v,
+			status: v.status || "new",
+		}));
+
+		if (violations.length === 0 && data.status === "fail") {
+			violations.push({
+				file: "unknown",
+				line: "?",
+				issue: "Previous run failed but no violations found in JSON",
+				status: "new",
 			});
 		}
 
-		if (sections.length === 0) {
-			return null;
-		}
+		if (violations.length === 0) return null;
 
-		for (let i = 0; i < sections.length; i++) {
-			const currentSection = sections[i];
-			const nextSection = sections[i + 1];
-			const endIndex = nextSection ? nextSection.startIndex : content.length;
-			const sectionContent = content.substring(
-				currentSection.startIndex,
-				endIndex,
-			);
+		return {
+			jobId,
+			gateName: "",
+			entryPoint: "",
+			adapterFailures: [
+				{
+					adapterName: data.adapter,
+					reviewIndex: parsed?.reviewIndex,
+					violations,
+				},
+			],
+			logPath: jsonPath.replace(/\.json$/, ".log"),
+		};
+	} catch (error) {
+		console.warn("Warning: Failed to parse JSON review file:", jsonPath, error);
+		return null;
+	}
+}
 
-			const violations: PreviousViolation[] = [];
+/**
+ * Extract the log prefix (job ID) from a numbered log filename.
+ * Handles both patterns:
+ *   - Check: `check_src_test.2.log` -> `check_src_test`
+ *   - Review (new): `review_src_claude@1.2.log` -> `review_src_claude@1`
+ */
+export function extractPrefix(filename: string): string {
+	// Pattern: <prefix>.<number>.(log|json)
+	const m = filename.match(/^(.+)\.\d+\.(log|json)$/);
+	if (m) return m[1]!;
+	// Fallback for non-numbered files
+	return filename.replace(/\.(log|json)$/, "");
+}
 
-			// 1. Look for "--- Parsed Result ---"
-			const parsedResultMatch = sectionContent.match(
-				/---\s*Parsed Result(?:\s+\(([^)]+)\))?\s*---([\s\S]*?)(?:$|---)/,
-			);
+/**
+ * Parses a single log file to extract failures per adapter.
+ * Processes both review and check gates.
+ */
+export async function parseLogFile(
+	logPath: string,
+): Promise<GateFailures | null> {
+	try {
+		const content = await fs.readFile(logPath, "utf-8");
+		const filename = path.basename(logPath);
 
-			if (parsedResultMatch) {
-				const parsedContent = parsedResultMatch[2];
+		// Try to parse as review filename with @index pattern
+		const parsed = parseReviewFilename(filename);
+		const jobId = parsed ? parsed.jobId : extractPrefix(filename);
 
-				// Check status
-				if (parsedContent.includes("Status: PASS")) {
-					continue; // No violations for this adapter
-				}
+		// Check if it's a review log
+		if (content.includes("--- Review Output")) {
+			const adapterFailures: AdapterFailure[] = [];
+			const sectionRegex = /--- Review Output \(([^)]+)\) ---/g;
+
+			let match: RegExpExecArray | null;
+			const sections: { adapter: string; startIndex: number }[] = [];
 
-				// Extract violations
-				// Pattern: 1. src/app.ts:42 - Missing error handling
-				// Pattern: 1. src/app.ts:? - Missing error handling
-				// Pattern: 1. src/app.ts:NaN - Missing error handling
-				/**
-				 * Extract violations from the parsed result section.
-				 * Pattern matches "1. file:line - issue" where line can be a number, NaN, or ?.
-				 */
-				const violationRegex = /^\d+\.\s+(.+?):(\d+|NaN|\?)\s+-\s+(.+)$/gm;
-				let vMatch: RegExpExecArray | null;
-
-				for (;;) {
-					vMatch = violationRegex.exec(parsedContent);
-					if (!vMatch) break;
-					const file = vMatch[1].trim();
-					let line: number | string = vMatch[2];
-					if (line !== "NaN" && line !== "?") {
-						line = parseInt(line, 10);
+			for (;;) {
+				match = sectionRegex.exec(content);
+				if (!match) break;
+				sections.push({
+					adapter: match[1]!,
+					startIndex: match.index,
+				});
+			}
+
+			if (sections.length === 0) return null;
+
+			for (let i = 0; i < sections.length; i++) {
+				const currentSection = sections[i]!;
+				const nextSection = sections[i + 1];
+				const endIndex = nextSection ? nextSection.startIndex : content.length;
+				const sectionContent = content.substring(
+					currentSection.startIndex,
+					endIndex,
+				);
+
+				const violations: PreviousViolation[] = [];
+				const parsedResultMatch = sectionContent.match(
+					/---\s*Parsed Result(?:\s+\(([^)]+)\))?\s*---([\s\S]*?)(?:$|---)/,
+				);
+
+				if (parsedResultMatch) {
+					const parsedContent = parsedResultMatch[2]!;
+					if (parsedContent.includes("Status: PASS")) continue;
+					const violationRegex = /^\d+\.\s+(.+?):(\d+|NaN|\?)\s+-\s+(.+)$/gm;
+					let vMatch: RegExpExecArray | null;
+					for (;;) {
+						vMatch = violationRegex.exec(parsedContent);
+						if (!vMatch) break;
+						const file = vMatch[1]!.trim();
+						let line: number | string = vMatch[2]!;
+						if (line !== "NaN" && line !== "?")
+							line = parseInt(line as string, 10);
+						const issue = vMatch[3]!.trim();
+						let fix: string | undefined;
+						const remainder = parsedContent.substring(
+							vMatch.index + vMatch[0].length,
+						);
+						const fixMatch = remainder.match(/^\s+Fix:\s+(.+)$/m);
+						const nextViolationIndex = remainder.search(/^\d+\./m);
+						if (
+							fixMatch?.index !== undefined &&
+							(nextViolationIndex === -1 || fixMatch.index < nextViolationIndex)
+						) {
+							fix = fixMatch[1]!.trim();
+						}
+						violations.push({ file, line, issue, fix });
 					}
-					const issue = vMatch[3].trim();
+				} else {
+					// Fallback JSON
+					const firstBrace = sectionContent.indexOf("{");
+					const lastBrace = sectionContent.lastIndexOf("}");
+					if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
+						try {
+							const jsonStr = sectionContent.substring(
+								firstBrace,
+								lastBrace + 1,
+							);
+							const json = JSON.parse(jsonStr);
+							if (json.violations && Array.isArray(json.violations)) {
+								for (const v of json.violations) {
+									if (v.file && v.issue) {
+										violations.push({
+											file: v.file,
+											line: v.line || 0,
+											issue: v.issue,
+											fix: v.fix,
+											status: v.status,
+											result: v.result,
+										});
+									}
+								}
+							}
+						} catch (_e) {}
+					}
+				}
 
-					// Look for fix in the next line(s)
-					let fix: string | undefined;
-					const remainder = parsedContent.substring(
-						vMatch.index + vMatch[0].length,
-					);
+				if (violations.length > 0) {
+					adapterFailures.push({
+						adapterName: currentSection.adapter,
+						reviewIndex: parsed?.reviewIndex,
+						violations,
+					});
+				} else if (parsedResultMatch?.[2]?.includes("Status: FAIL")) {
+					adapterFailures.push({
+						adapterName: currentSection.adapter,
+						reviewIndex: parsed?.reviewIndex,
+						violations: [
+							{
+								file: "unknown",
+								line: "?",
+								issue:
+									"Previous run failed but specific violations could not be parsed",
+							},
+						],
+					});
+				}
+			}
 
-					const fixMatch = remainder.match(/^\s+Fix:\s+(.+)$/m);
-					const nextViolationIndex = remainder.search(/^\d+\./m);
+			if (adapterFailures.length === 0) return null;
+			return { jobId, gateName: "", entryPoint: "", adapterFailures, logPath };
+		} else {
+			// Check log
+			if (content.includes("Result: pass")) return null;
+
+			const hasFailure =
+				content.includes("Result: fail") ||
+				content.includes("Result: error") ||
+				content.includes("Command failed:");
+
+			if (!hasFailure) return null;
+
+			return {
+				jobId,
+				gateName: "",
+				entryPoint: "",
+				adapterFailures: [
+					{
+						adapterName: "check",
+						violations: [{ file: "check", line: 0, issue: "Check failed" }],
+					},
+				],
+				logPath,
+			};
+		}
+	} catch (_error) {
+		return null;
+	}
+}
 
-					const isFixBelongingToCurrentViolation =
-						fixMatch?.index !== undefined &&
-						(nextViolationIndex === -1 || fixMatch.index < nextViolationIndex);
+export interface RunIteration {
+	iteration: number;
+	fixed: Array<{
+		jobId: string;
+		adapter?: string;
+		details: string;
+	}>;
+	skipped: Array<{
+		jobId: string;
+		adapter?: string;
+		file: string;
+		line: number | string;
+		issue: string;
+		result?: string | null;
+	}>;
+}
 
-					if (isFixBelongingToCurrentViolation && fixMatch) {
-						fix = fixMatch[1].trim();
-					}
+/**
+ * Reconstructs the history of fixes and skips after all iterations.
+ */
+export async function reconstructHistory(
+	logDir: string,
+): Promise<RunIteration[]> {
+	try {
+		const files = await fs.readdir(logDir);
+		const runNumbers = new Set<number>();
+		for (const file of files) {
+			const m = file.match(/\.(\d+)\.(log|json)$/);
+			if (m) runNumbers.add(parseInt(m[1]!, 10));
+		}
 
-					violations.push({
-						file,
-						line,
-						issue,
-						fix,
-					});
+		const sortedRuns = Array.from(runNumbers).sort((a, b) => a - b);
+		const iterations: RunIteration[] = [];
+
+		let previousFailuresByJob = new Map<string, PreviousViolation[]>();
+
+		for (const runNum of sortedRuns) {
+			const currentFailuresByJob = new Map<string, PreviousViolation[]>();
+			const iteration: RunIteration = {
+				iteration: runNum,
+				fixed: [],
+				skipped: [],
+			};
+
+			const runFiles = files.filter((f) => f.includes(`.${runNum}.`));
+			const prefixes = new Set(runFiles.map((f) => extractPrefix(f)));
+
+			for (const prefix of prefixes) {
+				const jsonFile = runFiles.find(
+					(f) => f.startsWith(`${prefix}.${runNum}.`) && f.endsWith(".json"),
+				);
+				const logFile = runFiles.find(
+					(f) => f.startsWith(`${prefix}.${runNum}.`) && f.endsWith(".log"),
+				);
+
+				let failure: GateFailures | null = null;
+				if (jsonFile) {
+					failure = await parseJsonReviewFile(path.join(logDir, jsonFile));
+				} else if (logFile) {
+					failure = await parseLogFile(path.join(logDir, logFile));
 				}
-			} else {
-				// Fallback: Try to parse JSON
-				// Extract JSON using first '{' and last '}' to capture the full object
-				const firstBrace = sectionContent.indexOf("{");
-				const lastBrace = sectionContent.lastIndexOf("}");
-
-				if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
-					try {
-						const jsonStr = sectionContent.substring(firstBrace, lastBrace + 1);
-						// Try to find the valid JSON object
-						const json = JSON.parse(jsonStr);
-
-						if (json.violations && Array.isArray(json.violations)) {
-							for (const v of json.violations) {
-								if (v.file && v.issue) {
-									violations.push({
-										file: v.file,
-										line: v.line || 0,
-										issue: v.issue,
-										fix: v.fix,
-									});
-								}
+
+				if (failure) {
+					for (const af of failure.adapterFailures) {
+						const key = af.reviewIndex
+							? `${failure.jobId}:${af.reviewIndex}`
+							: `${failure.jobId}:${af.adapterName}`;
+						currentFailuresByJob.set(key, af.violations);
+
+						for (const v of af.violations) {
+							if (v.status === "skipped") {
+								iteration.skipped.push({
+									jobId: failure.jobId,
+									adapter: af.adapterName,
+									file: v.file,
+									line: v.line,
+									issue: v.issue,
+									result: v.result,
+								});
 							}
 						}
-					} catch (_e: unknown) {
-						// Log warning for debugging (commented out to reduce noise in production)
-						// console.warn(`Warning: Failed to parse JSON for ${currentSection.adapter} in ${jobId}: ${e.message}`);
 					}
 				}
 			}
-			if (violations.length > 0) {
-				adapterFailures.push({
-					adapterName: currentSection.adapter,
-					violations,
-				});
-			} else if (parsedResultMatch?.[1].includes("Status: FAIL")) {
-				// Track failure even if violations couldn't be parsed
-				adapterFailures.push({
-					adapterName: currentSection.adapter,
-					violations: [
-						{
-							file: "unknown",
-							line: "?",
-							issue:
-								"Previous run failed but specific violations could not be parsed",
-						},
-					],
+
+			for (const [key, prevViolations] of previousFailuresByJob.entries()) {
+				const current = currentFailuresByJob.get(key);
+				const sep = key.lastIndexOf(":");
+				const jobId = key.substring(0, sep);
+				const adapter = key.substring(sep + 1);
+
+				const trulyFixed = prevViolations.filter((pv) => {
+					if (pv.status === "skipped") return false;
+					return !current?.some(
+						(cv) =>
+							cv.file === pv.file &&
+							cv.line === pv.line &&
+							cv.issue === pv.issue,
+					);
 				});
+
+				if (trulyFixed.length > 0) {
+					if (jobId.startsWith("check_")) {
+						iteration.fixed.push({
+							jobId,
+							details: `${trulyFixed.length} violations resolved`,
+						});
+					} else {
+						for (const f of trulyFixed) {
+							iteration.fixed.push({
+								jobId,
+								adapter,
+								details: `${f.file}:${f.line} ${f.issue}`,
+							});
+						}
+					}
+				}
 			}
-		}
 
-		if (adapterFailures.length === 0) {
-			return null;
+			iterations.push(iteration);
+			previousFailuresByJob = currentFailuresByJob;
 		}
 
-		return {
-			jobId,
-			gateName,
-			entryPoint,
-			adapterFailures,
-			logPath,
-		};
-	} catch (_error) {
-		// console.warn(`Error parsing log file ${logPath}:`, error);
-		return null;
+		return iterations;
+	} catch (_e) {
+		return [];
 	}
 }
 
 /**
  * Finds all previous failures from the log directory.
+ * For review gates with the @<index> pattern, groups by (jobId, reviewIndex)
+ * and returns the highest-numbered run for each index.
+ * The resulting Map keys are the review index (as string) for lookup by the review gate.
  */
 export async function findPreviousFailures(
 	logDir: string,
@@ -214,26 +412,166 @@ export async function findPreviousFailures(
 		const files = await fs.readdir(logDir);
 		const gateFailures: GateFailures[] = [];
 
+		// Separate review files (with @index) from check files
+		// Group review files by (jobId, reviewIndex) -> highest run number
+		const reviewSlotMap = new Map<
+			string,
+			{ filename: string; runNumber: number; ext: string }
+		>();
+		const checkPrefixMap = new Map<string, Map<number, Set<string>>>();
+
 		for (const file of files) {
-			if (!file.endsWith(".log")) continue;
+			const isLog = file.endsWith(".log");
+			const isJson = file.endsWith(".json");
+			if (!isLog && !isJson) continue;
+			if (gateFilter && !file.includes(gateFilter)) continue;
+
+			const parsed = parseReviewFilename(file);
+			if (parsed) {
+				// Review file with @index pattern
+				const slotKey = `${parsed.jobId}:${parsed.reviewIndex}`;
+				const existing = reviewSlotMap.get(slotKey);
+				if (!existing || parsed.runNumber > existing.runNumber) {
+					reviewSlotMap.set(slotKey, {
+						filename: file,
+						runNumber: parsed.runNumber,
+						ext: parsed.ext,
+					});
+				}
+			} else {
+				// Check file or legacy review file
+				const m = file.match(/^(.+)\.(\d+)\.(log|json)$/);
+				if (!m) continue;
+
+				const prefix = m[1]!;
+				const runNum = parseInt(m[2]!, 10);
+				const ext = m[3]!;
+
+				let runMap = checkPrefixMap.get(prefix);
+				if (!runMap) {
+					runMap = new Map();
+					checkPrefixMap.set(prefix, runMap);
+				}
 
-			// If gate filter provided, check if filename matches
-			// filename is sanitized, so we do a loose check
-			if (gateFilter && !file.includes(gateFilter)) {
-				continue;
+				let exts = runMap.get(runNum);
+				if (!exts) {
+					exts = new Set();
+					runMap.set(runNum, exts);
+				}
+				exts.add(ext);
 			}
+		}
+
+		// Process review files grouped by slot (jobId + reviewIndex)
+		// Group by jobId to produce a single GateFailures per job
+		const jobReviewFailures = new Map<string, AdapterFailure[]>();
 
-			const logPath = path.join(logDir, file);
-			const failure = await parseLogFile(logPath);
+		for (const [slotKey, fileInfo] of reviewSlotMap.entries()) {
+			const sepIdx = slotKey.lastIndexOf(":");
+			const jobId = slotKey.substring(0, sepIdx);
+			const reviewIndex = parseInt(slotKey.substring(sepIdx + 1), 10);
+
+			let failure: GateFailures | null = null;
+			if (fileInfo.ext === "json") {
+				failure = await parseJsonReviewFile(
+					path.join(logDir, fileInfo.filename),
+				);
+			} else {
+				failure = await parseLogFile(path.join(logDir, fileInfo.filename));
+			}
 
 			if (failure) {
-				gateFailures.push(failure);
+				// Apply status filtering
+				for (const af of failure.adapterFailures) {
+					af.reviewIndex = reviewIndex;
+					const filteredViolations: PreviousViolation[] = [];
+					for (const v of af.violations) {
+						const status = v.status || "new";
+						if (status === "skipped") continue;
+						if (
+							status !== "new" &&
+							status !== "fixed" &&
+							status !== "skipped"
+						) {
+							console.warn(
+								`Warning: Unexpected status "${status}" for violation in ${jobId}. Treating as "new".`,
+							);
+							v.status = "new";
+						}
+						filteredViolations.push(v);
+					}
+					af.violations = filteredViolations;
+
+					if (af.violations.length > 0) {
+						if (!jobReviewFailures.has(jobId)) {
+							jobReviewFailures.set(jobId, []);
+						}
+						jobReviewFailures.get(jobId)!.push(af);
+					}
+				}
+			}
+		}
+
+		for (const [jobId, adapterFailures] of jobReviewFailures.entries()) {
+			gateFailures.push({
+				jobId,
+				gateName: "",
+				entryPoint: "",
+				adapterFailures,
+				logPath: path.join(logDir, `${jobId}.log`),
+			});
+		}
+
+		// Process check files (non-review)
+		for (const [prefix, runMap] of checkPrefixMap.entries()) {
+			const latestRun = Math.max(...runMap.keys());
+			const exts = runMap.get(latestRun);
+			if (!exts) continue;
+
+			let failure: GateFailures | null = null;
+			if (exts.has("json")) {
+				failure = await parseJsonReviewFile(
+					path.join(logDir, `${prefix}.${latestRun}.json`),
+				);
+			} else if (exts.has("log")) {
+				failure = await parseLogFile(
+					path.join(logDir, `${prefix}.${latestRun}.log`),
+				);
+			}
+
+			if (failure) {
+				for (const af of failure.adapterFailures) {
+					const filteredViolations: PreviousViolation[] = [];
+					for (const v of af.violations) {
+						const status = v.status || "new";
+						if (status === "skipped") continue;
+						if (
+							status !== "new" &&
+							status !== "fixed" &&
+							status !== "skipped"
+						) {
+							console.warn(
+								`Warning: Unexpected status "${status}" for violation in ${failure.jobId}. Treating as "new".`,
+							);
+							v.status = "new";
+						}
+						filteredViolations.push(v);
+					}
+					af.violations = filteredViolations;
+				}
+
+				const totalViolations = failure.adapterFailures.reduce(
+					(sum, af) => sum + af.violations.length,
+					0,
+				);
+				if (totalViolations > 0) {
+					gateFailures.push(failure);
+				}
 			}
 		}
 
 		return gateFailures;
 	} catch (error: unknown) {
-		// If directory doesn't exist, return empty
 		if (
 			typeof error === "object" &&
 			error !== null &&
@@ -242,8 +580,6 @@ export async function findPreviousFailures(
 		) {
 			return [];
 		}
-		// Otherwise log and return empty
-		// console.warn(`Error reading log directory ${logDir}:`, error instanceof Error ? error.message : String(error));
 		return [];
 	}
 }