agent-gauntlet 0.1.9 → 0.1.11

package/src/templates/run_gauntlet.template.md

@@ -0,0 +1,18 @@
+---
+description: Run the full verification gauntlet
+allowed-tools: Bash
+---
+# /gauntlet
+Execute the autonomous verification suite.
+
+1. Run `agent-gauntlet run`.
+2. If it fails, read the log files in `.gauntlet_logs/` to understand exactly what went wrong.
+3. Fix any code or logic errors found by the tools or AI reviewers, prioritizing higher-priority violations (critical > high > medium > low).
+4. If you disagree with AI reviewer feedback, briefly explain your reasoning in the code comments rather than ignoring it silently.
+5. Do NOT commit your changes yet—keep them uncommitted so the rerun command can review them.
+6. Run `agent-gauntlet rerun` to verify your fixes. The rerun command reviews only uncommitted changes and uses previous failures as context.
+7. Repeat steps 2-6 until one of the following termination conditions is met:
+   - All gates pass
+   - You disagree with remaining failures (ask the human how to proceed)
+   - Still failing after 3 rerun attempts
+8. Once all gates pass, do NOT commit or push your changes—await the human's review and explicit instruction to commit.

package/src/utils/diff-parser.ts

@@ -5,82 +5,84 @@ export type DiffFileRange = Set<number>;
  * Valid line numbers are those that appear in the diff as added or modified lines.
  */
 export function parseDiff(diff: string): Map<string, DiffFileRange> {
-  const fileRanges = new Map<string, DiffFileRange>();
-  const lines = diff.split('\n');
+	const fileRanges = new Map<string, DiffFileRange>();
+	const lines = diff.split("\n");
 
-  let currentFile: string | null = null;
-  let currentRanges: DiffFileRange | null = null;
-  let currentLineNumber = 0;
+	let currentFile: string | null = null;
+	let currentRanges: DiffFileRange | null = null;
+	let currentLineNumber = 0;
 
-  for (const line of lines) {
-    // Parse file header: diff --git a/path/to/file b/path/to/file
-    if (line.startsWith('diff --git')) {
-      const parts = line.split(' ');
-      if (parts.length >= 4) {
-        // Extract filename from b/path/to/file (target file)
-        const targetPath = parts[3];
-        // Remove 'b/' prefix
-        currentFile = targetPath.startsWith('b/') ? targetPath.substring(2) : targetPath;
-        
-        // Skip .git/ paths
-        if (currentFile.startsWith('.git/')) {
-          currentFile = null;
-          currentRanges = null;
-          continue;
-        }
+	for (const line of lines) {
+		// Parse file header: diff --git a/path/to/file b/path/to/file
+		if (line.startsWith("diff --git")) {
+			const parts = line.split(" ");
+			if (parts.length >= 4) {
+				// Extract filename from b/path/to/file (target file)
+				const targetPath = parts[3];
+				// Remove 'b/' prefix
+				currentFile = targetPath.startsWith("b/")
+					? targetPath.substring(2)
+					: targetPath;
 
-        currentRanges = new Set<number>();
-        fileRanges.set(currentFile, currentRanges);
-      }
-      continue;
-    }
+				// Skip .git/ paths
+				if (currentFile.startsWith(".git/")) {
+					currentFile = null;
+					currentRanges = null;
+					continue;
+				}
 
-    // Skip if we're ignoring this file (e.g. .git/)
-    if (!currentFile || !currentRanges) continue;
+				currentRanges = new Set<number>();
+				fileRanges.set(currentFile, currentRanges);
+			}
+			continue;
+		}
 
-    // Parse hunk header: @@ -old,count +new,count @@
-    if (line.startsWith('@@')) {
-      const match = line.match(/@@ \-\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
-      if (match && match[1]) {
-        currentLineNumber = parseInt(match[1], 10);
-      }
-      continue;
-    }
+		// Skip if we're ignoring this file (e.g. .git/)
+		if (!currentFile || !currentRanges) continue;
 
-    // Track added lines
-    if (line.startsWith('+') && !line.startsWith('+++')) {
-      currentRanges.add(currentLineNumber);
-      currentLineNumber++;
-    } 
-    // Track context lines (unchanged) to keep line count correct
-    else if (line.startsWith(' ')) {
-      currentLineNumber++;
-    }
-    // Removed lines (-) do not increment the new line counter
-  }
+		// Parse hunk header: @@ -old,count +new,count @@
+		if (line.startsWith("@@")) {
+			const match = line.match(/@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
+			if (match?.[1]) {
+				currentLineNumber = parseInt(match[1], 10);
+			}
+			continue;
+		}
 
-  return fileRanges;
+		// Track added lines
+		if (line.startsWith("+") && !line.startsWith("+++")) {
+			currentRanges.add(currentLineNumber);
+			currentLineNumber++;
+		}
+		// Track context lines (unchanged) to keep line count correct
+		else if (line.startsWith(" ")) {
+			currentLineNumber++;
+		}
+		// Removed lines (-) do not increment the new line counter
+	}
+
+	return fileRanges;
 }
 
 /**
  * Checks if a violation is valid based on the parsed diff ranges.
  */
 export function isValidViolationLocation(
-  file: string, 
-  line: number | undefined, 
-  diffRanges: Map<string, DiffFileRange> | undefined
+	file: string,
+	line: number | undefined,
+	diffRanges: Map<string, DiffFileRange> | undefined,
 ): boolean {
-  // If no diff ranges provided (e.g. full file review), assume valid
-  if (!diffRanges) return true;
-  
-  // Line is required for diff-scoped reviews
-  if (line === undefined) return false;
+	// If no diff ranges provided (e.g. full file review), assume valid
+	if (!diffRanges) return true;
+
+	// Line is required for diff-scoped reviews
+	if (line === undefined) return false;
 
-  const validLines = diffRanges.get(file);
-  if (!validLines) {
-    // File not in diff
-    return false;
-  }
+	const validLines = diffRanges.get(file);
+	if (!validLines) {
+		// File not in diff
+		return false;
+	}
 
-  return validLines.has(line);
+	return validLines.has(line);
 }

package/src/utils/log-parser.ts

@@ -1,228 +1,249 @@
-import fs from 'fs/promises';
-import path from 'path';
+import fs from "node:fs/promises";
+import path from "node:path";
 
 export interface PreviousViolation {
-  file: string;
-  line: number | string;
-  issue: string;
-  fix?: string;
+	file: string;
+	line: number | string;
+	issue: string;
+	fix?: string;
 }
 
 export interface AdapterFailure {
-  adapterName: string;          // e.g., 'claude', 'gemini'
-  violations: PreviousViolation[];
+	adapterName: string; // e.g., 'claude', 'gemini'
+	violations: PreviousViolation[];
 }
 
 export interface GateFailures {
-  jobId: string;                // This will be the sanitized Job ID (filename without extension)
-  gateName: string;             // Parsed or empty
-  entryPoint: string;           // Parsed or empty
-  adapterFailures: AdapterFailure[];  // Failures grouped by adapter
-  logPath: string;
+	jobId: string; // This will be the sanitized Job ID (filename without extension)
+	gateName: string; // Parsed or empty
+	entryPoint: string; // Parsed or empty
+	adapterFailures: AdapterFailure[]; // Failures grouped by adapter
+	logPath: string;
 }
 
 /**
  * Parses a single log file to extract failures per adapter.
  * Only processes review gates (ignores check gates).
  */
-export async function parseLogFile(logPath: string): Promise<GateFailures | null> {
-  try {
-    const content = await fs.readFile(logPath, 'utf-8');
-    const filename = path.basename(logPath);
-    
-    // Check if it's a review log by content marker
-    if (!content.includes('--- Review Output')) {
-        return null;
-    }
-    
-    // Use the sanitized filename as the Job ID key
-    const jobId = filename.replace(/\.log$/, '');
-    
-    // We can't reliably parse entryPoint/gateName from sanitized filename
-    // leaving them empty for now as they aren't critical for the map lookup
-    const gateName = '';
-    const entryPoint = '';
-
-    const adapterFailures: AdapterFailure[] = [];
-
-    // Split by sections using `--- Review Output (adapterName) ---` markers
-    const sectionRegex = /--- Review Output \(([^)]+)\) ---/g;
-    
-    let match;
-    const sections: { adapter: string, startIndex: number }[] = [];
-    
-    while ((match = sectionRegex.exec(content)) !== null) {
-        sections.push({
-            adapter: match[1],
-            startIndex: match.index
-        });
-    }
-
-    if (sections.length === 0) {
-        return null;
-    }
-
-    for (let i = 0; i < sections.length; i++) {
-        const currentSection = sections[i];
-        const nextSection = sections[i + 1];
-        const endIndex = nextSection ? nextSection.startIndex : content.length;
-        const sectionContent = content.substring(currentSection.startIndex, endIndex);
-
-        const violations: PreviousViolation[] = [];
-
-        // 1. Look for "--- Parsed Result ---"
-        const parsedResultMatch = sectionContent.match(/--- Parsed Result ---([\s\S]*?)(?:$|---)/);
-        
-        if (parsedResultMatch) {
-            const parsedContent = parsedResultMatch[1];
-            
-            // Check status
-            if (parsedContent.includes('Status: PASS')) {
-                continue; // No violations for this adapter
-            }
-
-            // Extract violations
-            // Pattern: 1. src/app.ts:42 - Missing error handling
-            // Pattern: 1. src/app.ts:? - Missing error handling
-            // Pattern: 1. src/app.ts:NaN - Missing error handling
-            /**
-             * Extract violations from the parsed result section.
-             * Pattern matches "1. file:line - issue" where line can be a number, NaN, or ?.
-             */
-            const violationRegex = /^\d+\.\s+(.+?):(\d+|NaN|\?)\s+-\s+(.+)$/gm;
-            let vMatch;
-            
-            while ((vMatch = violationRegex.exec(parsedContent)) !== null) {
-                const file = vMatch[1].trim();
-                let line: number | string = vMatch[2];
-                if (line !== 'NaN' && line !== '?') {
-                    line = parseInt(line, 10);
-                }
-                const issue = vMatch[3].trim();
-                
-                // Look for fix in the next line(s)
-                let fix = undefined;
-                const remainder = parsedContent.substring(vMatch.index + vMatch[0].length);
-                
-                const fixMatch = remainder.match(/^\s+Fix:\s+(.+)$/m);
-                const nextViolationIndex = remainder.search(/^\d+\./m);
-                
-                const isFixBelongingToCurrentViolation = fixMatch?.index !== undefined && 
-                                                       (nextViolationIndex === -1 || fixMatch.index < nextViolationIndex);
-
-                if (isFixBelongingToCurrentViolation && fixMatch) {
-                     fix = fixMatch[1].trim();
-                }
-
-                violations.push({
-                    file,
-                    line,
-                    issue,
-                    fix
-                });
-            }
-        } else {
-            // Fallback: Try to parse JSON
-            // Extract JSON using first '{' and last '}' to capture the full object
-            const firstBrace = sectionContent.indexOf('{');
-            const lastBrace = sectionContent.lastIndexOf('}');
-            
-            if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
-                try {
-                    const jsonStr = sectionContent.substring(firstBrace, lastBrace + 1);
-                    // Try to find the valid JSON object
-                    const json = JSON.parse(jsonStr);
-                    
-                    if (json.violations && Array.isArray(json.violations)) {
-                        for (const v of json.violations) {
-                            if (v.file && v.issue) {
-                                violations.push({
-                                    file: v.file,
-                                    line: v.line || 0,
-                                    issue: v.issue,
-                                    fix: v.fix
-                                });
-                            }
-                        }
-                    }
-                } catch (e: any) {
-                     // Log warning for debugging (commented out to reduce noise in production)
-                     // console.warn(`Warning: Failed to parse JSON for ${currentSection.adapter} in ${jobId}: ${e.message}`);
-                }
-            }
-        }
-        if (violations.length > 0) {
-            adapterFailures.push({
-                adapterName: currentSection.adapter,
-                violations
-            });
-        } else if (parsedResultMatch && parsedResultMatch[1].includes('Status: FAIL')) {
-            // Track failure even if violations couldn't be parsed
-            adapterFailures.push({
-                adapterName: currentSection.adapter,
-                violations: [{
-                    file: 'unknown',
-                    line: '?',
-                    issue: 'Previous run failed but specific violations could not be parsed'
-                }]
-            });
-        }
-    }
-
-    if (adapterFailures.length === 0) {
-        return null;
-    }
-
-    return {
-        jobId,
-        gateName,
-        entryPoint,
-        adapterFailures,
-        logPath
-    };
-
-  } catch (error) {
-    // console.warn(`Error parsing log file ${logPath}:`, error);
-    return null;
-  }
+export async function parseLogFile(
+	logPath: string,
+): Promise<GateFailures | null> {
+	try {
+		const content = await fs.readFile(logPath, "utf-8");
+		const filename = path.basename(logPath);
+
+		// Check if it's a review log by content marker
+		if (!content.includes("--- Review Output")) {
+			return null;
+		}
+
+		// Use the sanitized filename as the Job ID key
+		const jobId = filename.replace(/\.log$/, "");
+
+		// We can't reliably parse entryPoint/gateName from sanitized filename
+		// leaving them empty for now as they aren't critical for the map lookup
+		const gateName = "";
+		const entryPoint = "";
+
+		const adapterFailures: AdapterFailure[] = [];
+
+		// Split by sections using `--- Review Output (adapterName) ---` markers
+		const sectionRegex = /--- Review Output \(([^)]+)\) ---/g;
+
+		let match: RegExpExecArray | null;
+		const sections: { adapter: string; startIndex: number }[] = [];
+
+		for (;;) {
+			match = sectionRegex.exec(content);
+			if (!match) break;
+			sections.push({
+				adapter: match[1],
+				startIndex: match.index,
+			});
+		}
+
+		if (sections.length === 0) {
+			return null;
+		}
+
+		for (let i = 0; i < sections.length; i++) {
+			const currentSection = sections[i];
+			const nextSection = sections[i + 1];
+			const endIndex = nextSection ? nextSection.startIndex : content.length;
+			const sectionContent = content.substring(
+				currentSection.startIndex,
+				endIndex,
+			);
+
+			const violations: PreviousViolation[] = [];
+
+			// 1. Look for "--- Parsed Result ---"
+			const parsedResultMatch = sectionContent.match(
+				/---\s*Parsed Result(?:\s+\(([^)]+)\))?\s*---([\s\S]*?)(?:$|---)/,
+			);
+
+			if (parsedResultMatch) {
+				const parsedContent = parsedResultMatch[2];
+
+				// Check status
+				if (parsedContent.includes("Status: PASS")) {
+					continue; // No violations for this adapter
+				}
+
+				// Extract violations
+				// Pattern: 1. src/app.ts:42 - Missing error handling
+				// Pattern: 1. src/app.ts:? - Missing error handling
+				// Pattern: 1. src/app.ts:NaN - Missing error handling
+				/**
+				 * Extract violations from the parsed result section.
+				 * Pattern matches "1. file:line - issue" where line can be a number, NaN, or ?.
+				 */
+				const violationRegex = /^\d+\.\s+(.+?):(\d+|NaN|\?)\s+-\s+(.+)$/gm;
+				let vMatch: RegExpExecArray | null;
+
+				for (;;) {
+					vMatch = violationRegex.exec(parsedContent);
+					if (!vMatch) break;
+					const file = vMatch[1].trim();
+					let line: number | string = vMatch[2];
+					if (line !== "NaN" && line !== "?") {
+						line = parseInt(line, 10);
+					}
+					const issue = vMatch[3].trim();
+
+					// Look for fix in the next line(s)
+					let fix: string | undefined;
+					const remainder = parsedContent.substring(
+						vMatch.index + vMatch[0].length,
+					);
+
+					const fixMatch = remainder.match(/^\s+Fix:\s+(.+)$/m);
+					const nextViolationIndex = remainder.search(/^\d+\./m);
+
+					const isFixBelongingToCurrentViolation =
+						fixMatch?.index !== undefined &&
+						(nextViolationIndex === -1 || fixMatch.index < nextViolationIndex);
+
+					if (isFixBelongingToCurrentViolation && fixMatch) {
+						fix = fixMatch[1].trim();
+					}
+
+					violations.push({
+						file,
+						line,
+						issue,
+						fix,
+					});
+				}
+			} else {
+				// Fallback: Try to parse JSON
+				// Extract JSON using first '{' and last '}' to capture the full object
+				const firstBrace = sectionContent.indexOf("{");
+				const lastBrace = sectionContent.lastIndexOf("}");
+
+				if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
+					try {
+						const jsonStr = sectionContent.substring(firstBrace, lastBrace + 1);
+						// Try to find the valid JSON object
+						const json = JSON.parse(jsonStr);
+
+						if (json.violations && Array.isArray(json.violations)) {
+							for (const v of json.violations) {
+								if (v.file && v.issue) {
+									violations.push({
+										file: v.file,
+										line: v.line || 0,
+										issue: v.issue,
+										fix: v.fix,
+									});
+								}
+							}
+						}
+					} catch (_e: unknown) {
+						// Log warning for debugging (commented out to reduce noise in production)
+						// console.warn(`Warning: Failed to parse JSON for ${currentSection.adapter} in ${jobId}: ${e.message}`);
+					}
+				}
+			}
+			if (violations.length > 0) {
+				adapterFailures.push({
+					adapterName: currentSection.adapter,
+					violations,
+				});
+			} else if (parsedResultMatch?.[1].includes("Status: FAIL")) {
+				// Track failure even if violations couldn't be parsed
+				adapterFailures.push({
+					adapterName: currentSection.adapter,
+					violations: [
+						{
+							file: "unknown",
+							line: "?",
+							issue:
+								"Previous run failed but specific violations could not be parsed",
+						},
+					],
+				});
+			}
+		}
+
+		if (adapterFailures.length === 0) {
+			return null;
+		}
+
+		return {
+			jobId,
+			gateName,
+			entryPoint,
+			adapterFailures,
+			logPath,
+		};
+	} catch (_error) {
+		// console.warn(`Error parsing log file ${logPath}:`, error);
+		return null;
+	}
 }
 
 /**
  * Finds all previous failures from the log directory.
  */
 export async function findPreviousFailures(
-  logDir: string,
-  gateFilter?: string
+	logDir: string,
+	gateFilter?: string,
 ): Promise<GateFailures[]> {
-  try {
-    const files = await fs.readdir(logDir);
-    const gateFailures: GateFailures[] = [];
-
-    for (const file of files) {
-      if (!file.endsWith('.log')) continue;
-      
-      // If gate filter provided, check if filename matches
-      // filename is sanitized, so we do a loose check
-      if (gateFilter && !file.includes(gateFilter)) {
-          continue;
-      }
-
-      const logPath = path.join(logDir, file);
-      const failure = await parseLogFile(logPath);
-      
-      if (failure) {
-        gateFailures.push(failure);
-      }
-    }
-
-    return gateFailures;
-  } catch (error: any) {
-    // If directory doesn't exist, return empty
-    if (typeof error === 'object' && error !== null && 'code' in error && (error as any).code === 'ENOENT') {
-        return [];
-    }
-    // Otherwise log and return empty
-    // console.warn(`Error reading log directory ${logDir}:`, error instanceof Error ? error.message : String(error));
-    return [];
-  }
+	try {
+		const files = await fs.readdir(logDir);
+		const gateFailures: GateFailures[] = [];
+
+		for (const file of files) {
+			if (!file.endsWith(".log")) continue;
+
+			// If gate filter provided, check if filename matches
+			// filename is sanitized, so we do a loose check
+			if (gateFilter && !file.includes(gateFilter)) {
+				continue;
+			}
+
+			const logPath = path.join(logDir, file);
+			const failure = await parseLogFile(logPath);
+
+			if (failure) {
+				gateFailures.push(failure);
+			}
+		}
+
+		return gateFailures;
+	} catch (error: unknown) {
+		// If directory doesn't exist, return empty
+		if (
+			typeof error === "object" &&
+			error !== null &&
+			"code" in error &&
+			(error as { code: string }).code === "ENOENT"
+		) {
+			return [];
+		}
+		// Otherwise log and return empty
+		// console.warn(`Error reading log directory ${logDir}:`, error instanceof Error ? error.message : String(error));
+		return [];
+	}
 }

package/src/utils/sanitizer.ts

@@ -1,3 +1,3 @@
 export function sanitizeJobId(jobId: string): string {
-  return jobId.replace(/[^a-zA-Z0-9._-]/g, '_');
+	return jobId.replace(/[^a-zA-Z0-9._-]/g, "_");
 }