agent-gauntlet 0.1.4

package/src/output/console.ts

@@ -0,0 +1,201 @@
+import chalk from 'chalk';
+import fs from 'node:fs/promises';
+import { GateResult } from '../gates/result.js';
+import { Job } from '../core/job.js';
+
+export class ConsoleReporter {
+  onJobStart(job: Job) {
+    console.log(chalk.blue(`[START] ${job.id}`));
+  }
+
+  onJobComplete(job: Job, result: GateResult) {
+    const duration = (result.duration / 1000).toFixed(2) + 's';
+    const message = result.message ?? '';
+    
+    if (result.status === 'pass') {
+      console.log(chalk.green(`[PASS]  ${job.id} (${duration})`));
+    } else if (result.status === 'fail') {
+      console.log(chalk.red(`[FAIL]  ${job.id} (${duration}) - ${message}`));
+    } else {
+      console.log(chalk.magenta(`[ERROR] ${job.id} (${duration}) - ${message}`));
+    }
+  }
+
+  async printSummary(results: GateResult[]) {
+    console.log('\n' + chalk.bold('--- Gauntlet Summary ---'));
+    
+    const passed = results.filter(r => r.status === 'pass');
+    const failed = results.filter(r => r.status === 'fail');
+    const errored = results.filter(r => r.status === 'error');
+
+    console.log(`Total: ${results.length}`);
+    console.log(chalk.green(`Passed: ${passed.length}`));
+    if (failed.length > 0) console.log(chalk.red(`Failed: ${failed.length}`));
+    if (errored.length > 0) console.log(chalk.magenta(`Errored: ${errored.length}`));
+
+    if (failed.length > 0 || errored.length > 0) {
+      console.log('\n' + chalk.bold('=== Failure Details ===\n'));
+      
+      for (const result of [...failed, ...errored]) {
+        const details = await this.extractFailureDetails(result);
+        this.printFailureDetails(result, details);
+      }
+    }
+  }
+
+  private async extractFailureDetails(result: GateResult): Promise<string[]> {
+    if (!result.logPath) {
+      return [result.message ?? 'Unknown error'];
+    }
+
+    try {
+      const logContent = await fs.readFile(result.logPath, 'utf-8');
+      return this.parseLogContent(logContent, result.jobId);
+    } catch (error) {
+      return [result.message ?? 'Unknown error', `(Could not read log file: ${result.logPath})`];
+    }
+  }
+
+  private parseLogContent(logContent: string, jobId: string): string[] {
+    const lines = logContent.split('\n');
+    const details: string[] = [];
+
+    // Check if this is a review log
+    if (jobId.startsWith('review:')) {
+      // Look for parsed violations section (formatted output)
+      const violationsStart = logContent.indexOf('--- Parsed Result ---');
+      if (violationsStart !== -1) {
+        const violationsSection = logContent.substring(violationsStart);
+        const sectionLines = violationsSection.split('\n');
+        
+        for (let i = 0; i < sectionLines.length; i++) {
+          const line = sectionLines[i];
+          // Match numbered violation lines: "1. file:line - issue" (line can be a number or '?')
+          const violationMatch = line.match(/^\d+\.\s+(.+?):(\d+|\?)\s+-\s+(.+)$/);
+          if (violationMatch) {
+            const file = violationMatch[1];
+            const lineNum = violationMatch[2];
+            const issue = violationMatch[3];
+            details.push(`  ${chalk.cyan(file)}:${chalk.yellow(lineNum)} - ${issue}`);
+            
+            // Check next line for "Fix:" suggestion
+            if (i + 1 < sectionLines.length) {
+              const nextLine = sectionLines[i + 1].trim();
+              if (nextLine.startsWith('Fix:')) {
+                const fix = nextLine.substring(4).trim();
+                details.push(`    ${chalk.dim('Fix:')} ${fix}`);
+                i++; // Skip the fix line
+              }
+            }
+          }
+        }
+      }
+
+      // If no parsed violations, look for JSON violations (handles both minified and pretty-printed)
+      if (details.length === 0) {
+        // Find the first '{' and last '}' to extract JSON object
+        const jsonStart = logContent.indexOf('{');
+        const jsonEnd = logContent.lastIndexOf('}');
+        if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) {
+          try {
+            const jsonStr = logContent.substring(jsonStart, jsonEnd + 1);
+            const json = JSON.parse(jsonStr);
+            if (json.status === 'fail' && json.violations && Array.isArray(json.violations)) {
+              json.violations.forEach((v: any) => {
+                const file = v.file || 'unknown';
+                const line = v.line || '?';
+                const issue = v.issue || 'Unknown issue';
+                details.push(`  ${chalk.cyan(file)}:${chalk.yellow(line)} - ${issue}`);
+                if (v.fix) {
+                  details.push(`    ${chalk.dim('Fix:')} ${v.fix}`);
+                }
+              });
+            }
+          } catch {
+            // JSON parse failed, fall through to other parsing
+          }
+        }
+      }
+
+      // If still no details, look for error messages
+      if (details.length === 0) {
+        // Try to find the actual error message (first non-empty line after "Error:")
+        const errorIndex = logContent.indexOf('Error:');
+        if (errorIndex !== -1) {
+          const afterError = logContent.substring(errorIndex + 6).trim();
+          const firstErrorLine = afterError.split('\n')[0].trim();
+          if (firstErrorLine && !firstErrorLine.startsWith('Usage:') && !firstErrorLine.startsWith('Commands:')) {
+            details.push(`  ${firstErrorLine}`);
+          }
+        }
+        
+        // Also check for "Result: error" lines
+        if (details.length === 0) {
+          const resultMatch = logContent.match(/Result:\s*error(?:\s*-\s*(.+?))?(?:\n|$)/);
+          if (resultMatch && resultMatch[1]) {
+            details.push(`  ${resultMatch[1]}`);
+          }
+        }
+      }
+    } else {
+      // This is a check log
+      // Look for STDERR section
+      const stderrStart = logContent.indexOf('STDERR:');
+      if (stderrStart !== -1) {
+        const stderrSection = logContent.substring(stderrStart + 7).trim();
+        const stderrLines = stderrSection.split('\n').filter(line => {
+          // Skip empty lines and command output markers
+          return line.trim() && 
+                 !line.includes('STDOUT:') && 
+                 !line.includes('Command failed:') &&
+                 !line.includes('Result:');
+        });
+        if (stderrLines.length > 0) {
+          details.push(...stderrLines.slice(0, 10).map(line => `  ${line.trim()}`));
+        }
+      }
+
+      // If no STDERR, look for error messages
+      if (details.length === 0) {
+        const errorMatch = logContent.match(/Command failed:\s*(.+?)(?:\n|$)/);
+        if (errorMatch) {
+          details.push(`  ${errorMatch[1]}`);
+        } else {
+          // Look for any line with "Result: fail" or "Result: error"
+          const resultMatch = logContent.match(/Result:\s*(fail|error)\s*-\s*(.+?)(?:\n|$)/);
+          if (resultMatch) {
+            details.push(`  ${resultMatch[2]}`);
+          }
+        }
+      }
+    }
+
+    // If we still have no details, use the message from the result
+    if (details.length === 0) {
+      details.push('  (See log file for details)');
+    }
+
+    return details;
+  }
+
+  private printFailureDetails(result: GateResult, details: string[]) {
+    const statusColor = result.status === 'error' ? chalk.magenta : chalk.red;
+    const statusLabel = result.status === 'error' ? 'ERROR' : 'FAIL';
+    
+    console.log(statusColor(`[${statusLabel}] ${result.jobId}`));
+    if (result.message) {
+      console.log(chalk.dim(`  Summary: ${result.message}`));
+    }
+    
+    if (details.length > 0) {
+      console.log(chalk.dim('  Details:'));
+      details.forEach(detail => console.log(detail));
+    }
+    
+    if (result.logPath) {
+      console.log(chalk.dim(`  Log: ${result.logPath}`));
+    }
+    
+    console.log(''); // Empty line between failures
+  }
+}

package/src/output/logger.ts

@@ -0,0 +1,66 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { sanitizeJobId } from '../utils/sanitizer.js';
+
+function formatTimestamp(): string {
+  return new Date().toISOString();
+}
+
+export class Logger {
+  private initializedFiles: Set<string> = new Set();
+
+  constructor(private logDir: string) {}
+
+  async init() {
+    await fs.mkdir(this.logDir, { recursive: true });
+  }
+
+  async close() {
+    // No-op - using append mode
+  }
+
+  getLogPath(jobId: string, adapterName?: string): string {
+    const safeName = sanitizeJobId(jobId);
+    if (adapterName) {
+      return path.join(this.logDir, `${safeName}_${adapterName}.log`);
+    }
+    return path.join(this.logDir, `${safeName}.log`);
+  }
+
+  private async initFile(logPath: string): Promise<void> {
+    if (!this.initializedFiles.has(logPath)) {
+      await fs.writeFile(logPath, '');
+      this.initializedFiles.add(logPath);
+    }
+  }
+
+  async createJobLogger(jobId: string): Promise<(text: string) => Promise<void>> {
+    const logPath = this.getLogPath(jobId);
+    await this.initFile(logPath);
+
+    return async (text: string) => {
+      const timestamp = formatTimestamp();
+      const lines = text.split('\n');
+      if (lines.length > 0) {
+        lines[0] = `[${timestamp}] ${lines[0]}`;
+      }
+      await fs.appendFile(logPath, lines.join('\n') + (text.endsWith('\n') ? '' : '\n'));
+    };
+  }
+
+  createLoggerFactory(jobId: string): (adapterName?: string) => Promise<(text: string) => Promise<void>> {
+    return async (adapterName?: string) => {
+      const logPath = this.getLogPath(jobId, adapterName);
+      await this.initFile(logPath);
+
+      return async (text: string) => {
+        const timestamp = formatTimestamp();
+        const lines = text.split('\n');
+        if (lines.length > 0) {
+          lines[0] = `[${timestamp}] ${lines[0]}`;
+        }
+        await fs.appendFile(logPath, lines.join('\n') + (text.endsWith('\n') ? '' : '\n'));
+      };
+    };
+  }
+}

package/src/utils/log-parser.ts

@@ -0,0 +1,228 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+export interface PreviousViolation {
+  file: string;
+  line: number | string;
+  issue: string;
+  fix?: string;
+}
+
+export interface AdapterFailure {
+  adapterName: string;          // e.g., 'claude', 'gemini'
+  violations: PreviousViolation[];
+}
+
+export interface GateFailures {
+  jobId: string;                // This will be the sanitized Job ID (filename without extension)
+  gateName: string;             // Parsed or empty
+  entryPoint: string;           // Parsed or empty
+  adapterFailures: AdapterFailure[];  // Failures grouped by adapter
+  logPath: string;
+}
+
+/**
+ * Parses a single log file to extract failures per adapter.
+ * Only processes review gates (ignores check gates).
+ */
+export async function parseLogFile(logPath: string): Promise<GateFailures | null> {
+  try {
+    const content = await fs.readFile(logPath, 'utf-8');
+    const filename = path.basename(logPath);
+    
+    // Check if it's a review log by content marker
+    if (!content.includes('--- Review Output')) {
+        return null;
+    }
+    
+    // Use the sanitized filename as the Job ID key
+    const jobId = filename.replace(/\.log$/, '');
+    
+    // We can't reliably parse entryPoint/gateName from sanitized filename
+    // leaving them empty for now as they aren't critical for the map lookup
+    const gateName = '';
+    const entryPoint = '';
+
+    const adapterFailures: AdapterFailure[] = [];
+
+    // Split by sections using `--- Review Output (adapterName) ---` markers
+    const sectionRegex = /--- Review Output \(([^)]+)\) ---/g;
+    
+    let match;
+    const sections: { adapter: string, startIndex: number }[] = [];
+    
+    while ((match = sectionRegex.exec(content)) !== null) {
+        sections.push({
+            adapter: match[1],
+            startIndex: match.index
+        });
+    }
+
+    if (sections.length === 0) {
+        return null;
+    }
+
+    for (let i = 0; i < sections.length; i++) {
+        const currentSection = sections[i];
+        const nextSection = sections[i + 1];
+        const endIndex = nextSection ? nextSection.startIndex : content.length;
+        const sectionContent = content.substring(currentSection.startIndex, endIndex);
+
+        const violations: PreviousViolation[] = [];
+
+        // 1. Look for "--- Parsed Result ---"
+        const parsedResultMatch = sectionContent.match(/--- Parsed Result ---([\s\S]*?)(?:$|---)/);
+        
+        if (parsedResultMatch) {
+            const parsedContent = parsedResultMatch[1];
+            
+            // Check status
+            if (parsedContent.includes('Status: PASS')) {
+                continue; // No violations for this adapter
+            }
+
+            // Extract violations
+            // Pattern: 1. src/app.ts:42 - Missing error handling
+            // Pattern: 1. src/app.ts:? - Missing error handling
+            // Pattern: 1. src/app.ts:NaN - Missing error handling
+            /**
+             * Extract violations from the parsed result section.
+             * Pattern matches "1. file:line - issue" where line can be a number, NaN, or ?.
+             */
+            const violationRegex = /^\d+\.\s+(.+?):(\d+|NaN|\?)\s+-\s+(.+)$/gm;
+            let vMatch;
+            
+            while ((vMatch = violationRegex.exec(parsedContent)) !== null) {
+                const file = vMatch[1].trim();
+                let line: number | string = vMatch[2];
+                if (line !== 'NaN' && line !== '?') {
+                    line = parseInt(line, 10);
+                }
+                const issue = vMatch[3].trim();
+                
+                // Look for fix in the next line(s)
+                let fix = undefined;
+                const remainder = parsedContent.substring(vMatch.index + vMatch[0].length);
+                
+                const fixMatch = remainder.match(/^\s+Fix:\s+(.+)$/m);
+                const nextViolationIndex = remainder.search(/^\d+\./m);
+                
+                const isFixBelongingToCurrentViolation = fixMatch?.index !== undefined && 
+                                                       (nextViolationIndex === -1 || fixMatch.index < nextViolationIndex);
+
+                if (isFixBelongingToCurrentViolation && fixMatch) {
+                     fix = fixMatch[1].trim();
+                }
+
+                violations.push({
+                    file,
+                    line,
+                    issue,
+                    fix
+                });
+            }
+        } else {
+            // Fallback: Try to parse JSON
+            // Extract JSON using first '{' and last '}' to capture the full object
+            const firstBrace = sectionContent.indexOf('{');
+            const lastBrace = sectionContent.lastIndexOf('}');
+            
+            if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
+                try {
+                    const jsonStr = sectionContent.substring(firstBrace, lastBrace + 1);
+                    // Try to find the valid JSON object
+                    const json = JSON.parse(jsonStr);
+                    
+                    if (json.violations && Array.isArray(json.violations)) {
+                        for (const v of json.violations) {
+                            if (v.file && v.issue) {
+                                violations.push({
+                                    file: v.file,
+                                    line: v.line || 0,
+                                    issue: v.issue,
+                                    fix: v.fix
+                                });
+                            }
+                        }
+                    }
+                } catch (e: any) {
+                     // Log warning for debugging (commented out to reduce noise in production)
+                     // console.warn(`Warning: Failed to parse JSON for ${currentSection.adapter} in ${jobId}: ${e.message}`);
+                }
+            }
+        }
+        if (violations.length > 0) {
+            adapterFailures.push({
+                adapterName: currentSection.adapter,
+                violations
+            });
+        } else if (parsedResultMatch && parsedResultMatch[1].includes('Status: FAIL')) {
+            // Track failure even if violations couldn't be parsed
+            adapterFailures.push({
+                adapterName: currentSection.adapter,
+                violations: [{
+                    file: 'unknown',
+                    line: '?',
+                    issue: 'Previous run failed but specific violations could not be parsed'
+                }]
+            });
+        }
+    }
+
+    if (adapterFailures.length === 0) {
+        return null;
+    }
+
+    return {
+        jobId,
+        gateName,
+        entryPoint,
+        adapterFailures,
+        logPath
+    };
+
+  } catch (error) {
+    // console.warn(`Error parsing log file ${logPath}:`, error);
+    return null;
+  }
+}
+
+/**
+ * Finds all previous failures from the log directory.
+ */
+export async function findPreviousFailures(
+  logDir: string,
+  gateFilter?: string
+): Promise<GateFailures[]> {
+  try {
+    const files = await fs.readdir(logDir);
+    const gateFailures: GateFailures[] = [];
+
+    for (const file of files) {
+      if (!file.endsWith('.log')) continue;
+      
+      // If gate filter provided, check if filename matches
+      // filename is sanitized, so we do a loose check
+      if (gateFilter && !file.includes(gateFilter)) {
+          continue;
+      }
+
+      const logPath = path.join(logDir, file);
+      const failure = await parseLogFile(logPath);
+      
+      if (failure) {
+        gateFailures.push(failure);
+      }
+    }
+
+    return gateFailures;
+  } catch (error: any) {
+    // If directory doesn't exist, return empty
+    if (typeof error === 'object' && error !== null && 'code' in error && (error as any).code === 'ENOENT') {
+        return [];
+    }
+    // Otherwise log and return empty
+    // console.warn(`Error reading log directory ${logDir}:`, error instanceof Error ? error.message : String(error));
+    return [];
+  }
+}

package/src/utils/sanitizer.ts

@@ -0,0 +1,3 @@
+export function sanitizeJobId(jobId: string): string {
+  return jobId.replace(/[^a-zA-Z0-9._-]/g, '_');
+}