agent-gauntlet 0.1.4

package/src/cli-adapters/gemini.ts

@@ -0,0 +1,149 @@
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
+import { type CLIAdapter, isUsageLimit } from './index.js';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+
+const execAsync = promisify(exec);
+const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
+
+export class GeminiAdapter implements CLIAdapter {
+  name = 'gemini';
+
+  async isAvailable(): Promise<boolean> {
+    try {
+      await execAsync('which gemini');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async checkHealth(options?: { checkUsageLimit?: boolean }): Promise<{ available: boolean; status: 'healthy' | 'missing' | 'unhealthy'; message?: string }> {
+    const available = await this.isAvailable();
+    if (!available) {
+      return { available: false, status: 'missing', message: 'Command not found' };
+    }
+
+    if (options?.checkUsageLimit) {
+      try {
+        const { stdout, stderr } = await execAsync('echo "hello" | gemini --sandbox --output-format text', { timeout: 10000 });
+        
+        const combined = (stdout || '') + (stderr || '');
+        if (isUsageLimit(combined)) {
+           return { 
+            available: true, 
+            status: 'unhealthy', 
+            message: 'Usage limit exceeded' 
+          };
+        }
+  
+        return { available: true, status: 'healthy', message: 'Installed' };
+      } catch (error: any) {
+        const stderr = error.stderr || '';
+        const stdout = error.stdout || '';
+        const combined = (stderr + stdout);
+        
+        if (isUsageLimit(combined)) {
+          return { 
+            available: true, 
+            status: 'unhealthy', 
+            message: 'Usage limit exceeded' 
+          };
+        }
+        
+        // Since we sent a valid prompt ("hello"), any other error implies the tool is broken
+        const cleanError = combined.split('\n')[0]?.trim() || error.message || 'Command failed';
+        return { 
+          available: true, 
+          status: 'unhealthy', 
+          message: `Error: ${cleanError}` 
+        };
+      }
+    }
+
+    return {
+      available,
+      status: available ? 'healthy' : 'missing',
+      message: available ? 'Installed' : 'Command not found'
+    };
+  }
+
+  getProjectCommandDir(): string | null {
+    return '.gemini/commands';
+  }
+
+  getUserCommandDir(): string | null {
+    // Gemini supports user-level commands at ~/.gemini/commands
+    return path.join(os.homedir(), '.gemini', 'commands');
+  }
+
+  getCommandExtension(): string {
+    return '.toml';
+  }
+
+  canUseSymlink(): boolean {
+    // Gemini uses TOML format, needs transformation
+    return false;
+  }
+
+  transformCommand(markdownContent: string): string {
+    // Transform Markdown with YAML frontmatter to Gemini's TOML format
+    const { frontmatter, body } = this.parseMarkdownWithFrontmatter(markdownContent);
+    
+    const description = frontmatter.description || 'Run the gauntlet verification suite';
+    // Escape the body for TOML multi-line string
+    const escapedBody = body.trim();
+    
+    return `description = ${JSON.stringify(description)}
+prompt = """
+${escapedBody}
+"""
+`;
+  }
+
+  private parseMarkdownWithFrontmatter(content: string): { frontmatter: Record<string, string>; body: string } {
+    const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+    if (!frontmatterMatch) {
+      return { frontmatter: {}, body: content };
+    }
+    
+    const frontmatterStr = frontmatterMatch[1] ?? '';
+    const body = frontmatterMatch[2] ?? '';
+    
+    // Simple YAML parsing for key: value pairs
+    const frontmatter: Record<string, string> = {};
+    for (const line of frontmatterStr.split('\n')) {
+      const kvMatch = line.match(/^([^:]+):\s*(.*)$/);
+      if (kvMatch && kvMatch[1] && kvMatch[2] !== undefined) {
+        frontmatter[kvMatch[1].trim()] = kvMatch[2].trim();
+      }
+    }
+    
+    return { frontmatter, body };
+  }
+
+  async execute(opts: { prompt: string; diff: string; model?: string; timeoutMs?: number }): Promise<string> {
+    // Construct the full prompt content
+    const fullContent = opts.prompt + "\n\n--- DIFF ---\n" + opts.diff;
+
+    // Write to a temporary file to avoid shell escaping issues
+    const tmpDir = os.tmpdir();
+    const tmpFile = path.join(tmpDir, `gauntlet-gemini-${Date.now()}.txt`);
+    await fs.writeFile(tmpFile, fullContent);
+
+    try {
+      // Use gemini CLI with file input
+      // --sandbox: enables the execution sandbox
+      // --allowed-tools: whitelists read-only tools for non-interactive execution
+      // --output-format text: ensures plain text output
+      // Use < for stdin redirection instead of cat pipe (cleaner)
+      const cmd = `gemini --sandbox --allowed-tools read_file,list_directory,glob,search_file_content --output-format text < "${tmpFile}"`;
+      const { stdout } = await execAsync(cmd, { timeout: opts.timeoutMs, maxBuffer: MAX_BUFFER_BYTES });
+      return stdout;
+    } finally {
+      await fs.unlink(tmpFile).catch(() => {});
+    }
+  }
+}

package/src/cli-adapters/index.ts

@@ -0,0 +1,79 @@
+export interface CLIAdapterHealth {
+  available: boolean;
+  status: 'healthy' | 'missing' | 'unhealthy';
+  message?: string;
+}
+
+export function isUsageLimit(output: string): boolean {
+  const lower = output.toLowerCase();
+  return lower.includes('usage limit') || 
+         lower.includes('quota exceeded') ||
+         lower.includes('quota will reset') ||
+         lower.includes('credit balance is too low') ||
+         lower.includes('out of extra usage') ||
+         lower.includes('out of usage');
+}
+
+export interface CLIAdapter {
+  name: string;
+  isAvailable(): Promise<boolean>;
+  checkHealth(options?: { checkUsageLimit?: boolean }): Promise<CLIAdapterHealth>;
+  execute(opts: { prompt: string; diff: string; model?: string; timeoutMs?: number }): Promise<string>;
+  /**
+   * Returns the project-scoped command directory path (relative to project root).
+   * Returns null if the CLI only supports user-level commands.
+   */
+  getProjectCommandDir(): string | null;
+  /**
+   * Returns the user-level command directory path (absolute path).
+   * Returns null if the CLI doesn't support user-level commands.
+   */
+  getUserCommandDir(): string | null;
+  /**
+   * Returns the command file extension used by this CLI.
+   */
+  getCommandExtension(): string;
+  /**
+   * Returns true if this adapter can use symlinks (same format as source Markdown).
+   */
+  canUseSymlink(): boolean;
+  /**
+   * Transforms gauntlet command content to this CLI's format.
+   * The source content is always Markdown with YAML frontmatter.
+   */
+  transformCommand(markdownContent: string): string;
+}
+
+import { GeminiAdapter } from './gemini.js';
+import { CodexAdapter } from './codex.js';
+import { ClaudeAdapter } from './claude.js';
+
+export { GeminiAdapter, CodexAdapter, ClaudeAdapter };
+
+const adapters: Record<string, CLIAdapter> = {
+  gemini: new GeminiAdapter(),
+  codex: new CodexAdapter(),
+  claude: new ClaudeAdapter(),
+};
+
+export function getAdapter(name: string): CLIAdapter | undefined {
+  return adapters[name];
+}
+
+export function getAllAdapters(): CLIAdapter[] {
+  return Object.values(adapters);
+}
+
+/**
+ * Returns all adapters that support project-scoped commands.
+ */
+export function getProjectCommandAdapters(): CLIAdapter[] {
+  return Object.values(adapters).filter(a => a.getProjectCommandDir() !== null);
+}
+
+/**
+ * Returns all adapters that support user-level commands.
+ */
+export function getUserCommandAdapters(): CLIAdapter[] {
+  return Object.values(adapters).filter(a => a.getUserCommandDir() !== null);
+}

package/src/commands/check.test.ts

@@ -0,0 +1,25 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { Command } from 'commander';
+import { registerCheckCommand } from './check.js';
+
+describe('Check Command', () => {
+  let program: Command;
+
+  beforeEach(() => {
+    program = new Command();
+    registerCheckCommand(program);
+  });
+
+  it('should register the check command', () => {
+    const checkCmd = program.commands.find(cmd => cmd.name() === 'check');
+    expect(checkCmd).toBeDefined();
+    expect(checkCmd?.description()).toBe('Run only applicable checks for detected changes');
+  });
+
+  it('should have correct options', () => {
+    const checkCmd = program.commands.find(cmd => cmd.name() === 'check');
+    expect(checkCmd?.options.some(opt => opt.long === '--gate')).toBe(true);
+    expect(checkCmd?.options.some(opt => opt.long === '--commit')).toBe(true);
+    expect(checkCmd?.options.some(opt => opt.long === '--uncommitted')).toBe(true);
+  });
+});

package/src/commands/check.ts

@@ -0,0 +1,67 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import { loadConfig } from '../config/loader.js';
+import { ChangeDetector } from '../core/change-detector.js';
+import { EntryPointExpander } from '../core/entry-point.js';
+import { JobGenerator } from '../core/job.js';
+import { Runner } from '../core/runner.js';
+import { Logger } from '../output/logger.js';
+import { ConsoleReporter } from '../output/console.js';
+
+export function registerCheckCommand(program: Command): void {
+  program
+    .command('check')
+    .description('Run only applicable checks for detected changes')
+    .option('-g, --gate <name>', 'Run specific check gate only')
+    .option('-c, --commit <sha>', 'Use diff for a specific commit')
+    .option('-u, --uncommitted', 'Use diff for current uncommitted changes (staged and unstaged)')
+    .action(async (options) => {
+      try {
+        const config = await loadConfig();
+        const changeDetector = new ChangeDetector(config.project.base_branch, {
+          commit: options.commit,
+          uncommitted: options.uncommitted
+        });
+        const expander = new EntryPointExpander();
+        const jobGen = new JobGenerator(config);
+        
+        console.log(chalk.dim('Detecting changes...'));
+        const changes = await changeDetector.getChangedFiles();
+        
+        if (changes.length === 0) {
+          console.log(chalk.green('No changes detected.'));
+          process.exit(0);
+        }
+        
+        console.log(chalk.dim(`Found ${changes.length} changed files.`));
+
+        const entryPoints = await expander.expand(config.project.entry_points, changes);
+        let jobs = jobGen.generateJobs(entryPoints);
+
+        // Filter to only checks
+        jobs = jobs.filter(j => j.type === 'check');
+
+        if (options.gate) {
+          jobs = jobs.filter(j => j.name === options.gate);
+        }
+
+        if (jobs.length === 0) {
+          console.log(chalk.yellow('No applicable checks for these changes.'));
+          process.exit(0);
+        }
+
+        console.log(chalk.dim(`Running ${jobs.length} check(s)...`));
+
+        const logger = new Logger(config.project.log_dir);
+        const reporter = new ConsoleReporter();
+        const runner = new Runner(config, logger, reporter);
+
+        const success = await runner.run(jobs);
+        process.exit(success ? 0 : 1);
+
+      } catch (error: any) {
+        console.error(chalk.red('Error:'), error.message);
+        process.exit(1);
+      }
+    });
+}

package/src/commands/detect.test.ts

@@ -0,0 +1,37 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { Command } from 'commander';
+import { registerDetectCommand } from './detect.js';
+
+describe('Detect Command', () => {
+  let program: Command;
+  const originalConsoleLog = console.log;
+  const originalConsoleError = console.error;
+  let logs: string[];
+  let errors: string[];
+
+  beforeEach(() => {
+    program = new Command();
+    registerDetectCommand(program);
+    logs = [];
+    errors = [];
+    console.log = (...args: any[]) => {
+      logs.push(args.join(' '));
+    };
+    console.error = (...args: any[]) => {
+      errors.push(args.join(' '));
+    };
+  });
+
+  afterEach(() => {
+    console.log = originalConsoleLog;
+    console.error = originalConsoleError;
+  });
+
+  it('should register the detect command', () => {
+    const detectCmd = program.commands.find(cmd => cmd.name() === 'detect');
+    expect(detectCmd).toBeDefined();
+    expect(detectCmd?.description()).toBe('Show what gates would run for detected changes (without executing them)');
+    expect(detectCmd?.options.some(opt => opt.long === '--commit')).toBe(true);
+    expect(detectCmd?.options.some(opt => opt.long === '--uncommitted')).toBe(true);
+  });
+});

package/src/commands/detect.ts

@@ -0,0 +1,69 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import { loadConfig } from '../config/loader.js';
+import { ChangeDetector } from '../core/change-detector.js';
+import { EntryPointExpander } from '../core/entry-point.js';
+import { JobGenerator, type Job } from '../core/job.js';
+
+export function registerDetectCommand(program: Command): void {
+  program
+    .command('detect')
+    .description('Show what gates would run for detected changes (without executing them)')
+    .option('-c, --commit <sha>', 'Use diff for a specific commit')
+    .option('-u, --uncommitted', 'Use diff for current uncommitted changes (staged and unstaged)')
+    .action(async (options) => {
+      try {
+        const config = await loadConfig();
+        const changeDetector = new ChangeDetector(config.project.base_branch, {
+          commit: options.commit,
+          uncommitted: options.uncommitted
+        });
+        const expander = new EntryPointExpander();
+        const jobGen = new JobGenerator(config);
+        
+        console.log(chalk.dim('Detecting changes...'));
+        const changes = await changeDetector.getChangedFiles();
+        
+        if (changes.length === 0) {
+          console.log(chalk.green('No changes detected.'));
+          return;
+        }
+        
+        console.log(chalk.dim(`Found ${changes.length} changed files:`));
+        changes.forEach(file => console.log(chalk.dim(`  - ${file}`)));
+        console.log();
+
+        const entryPoints = await expander.expand(config.project.entry_points, changes);
+        const jobs = jobGen.generateJobs(entryPoints);
+
+        if (jobs.length === 0) {
+          console.log(chalk.yellow('No applicable gates for these changes.'));
+          return;
+        }
+
+        console.log(chalk.bold(`Would run ${jobs.length} gate(s):\n`));
+        
+        // Group jobs by entry point for better display
+        const jobsByEntryPoint = new Map<string, Job[]>();
+        for (const job of jobs) {
+          if (!jobsByEntryPoint.has(job.entryPoint)) {
+            jobsByEntryPoint.set(job.entryPoint, []);
+          }
+          jobsByEntryPoint.get(job.entryPoint)!.push(job);
+        }
+
+        for (const [entryPoint, entryJobs] of jobsByEntryPoint.entries()) {
+          console.log(chalk.cyan(`Entry point: ${entryPoint}`));
+          for (const job of entryJobs) {
+            const typeLabel = job.type === 'check' ? chalk.yellow('check') : chalk.blue('review');
+            console.log(`  ${typeLabel} ${chalk.bold(job.name)}`);
+          }
+          console.log();
+        }
+
+      } catch (error: any) {
+        console.error(chalk.red('Error:'), error.message);
+        process.exit(1);
+      }
+    });
+}

package/src/commands/health.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from 'bun:test';
+import { Command } from 'commander';
+import { registerHealthCommand } from './health.js';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const TEST_DIR = path.join(process.cwd(), 'test-health-' + Date.now());
+const GAUNTLET_DIR = path.join(TEST_DIR, '.gauntlet');
+const REVIEWS_DIR = path.join(GAUNTLET_DIR, 'reviews');
+
+describe('Health Command', () => {
+  let program: Command;
+  const originalConsoleLog = console.log;
+  const originalCwd = process.cwd();
+  let logs: string[];
+
+  beforeAll(async () => {
+    // Setup test directory structure
+    await fs.mkdir(TEST_DIR, { recursive: true });
+    await fs.mkdir(GAUNTLET_DIR, { recursive: true });
+    await fs.mkdir(REVIEWS_DIR, { recursive: true });
+
+    // Write config.yml
+    await fs.writeFile(path.join(GAUNTLET_DIR, 'config.yml'), `
+base_branch: origin/main
+log_dir: .gauntlet_logs
+cli:
+  default_preference:
+    - gemini
+  check_usage_limit: false
+entry_points:
+  - path: .
+`);
+
+    // Write review definition with CLI preference
+    await fs.writeFile(path.join(REVIEWS_DIR, 'security.md'), `---
+cli_preference:
+  - gemini
+---
+
+# Security Review
+Review for security.
+`);
+  });
+
+  afterAll(async () => {
+    await fs.rm(TEST_DIR, { recursive: true, force: true });
+  });
+
+  beforeEach(() => {
+    program = new Command();
+    registerHealthCommand(program);
+    logs = [];
+    console.log = (...args: any[]) => {
+      logs.push(args.join(' '));
+    };
+    process.chdir(TEST_DIR);
+  });
+
+  afterEach(() => {
+    console.log = originalConsoleLog;
+    process.chdir(originalCwd);
+  });
+
+  it('should register the health command', () => {
+    const healthCmd = program.commands.find(cmd => cmd.name() === 'health');
+    expect(healthCmd).toBeDefined();
+    expect(healthCmd?.description()).toBe('Check CLI tool availability');
+  });
+
+  it('should run health check', async () => {
+    const healthCmd = program.commands.find(cmd => cmd.name() === 'health');
+    await healthCmd?.parseAsync(['health']);
+
+    const output = logs.join('\n');
+    expect(output).toContain('Config validation:');
+    expect(output).toContain('CLI Tool Health Check:');
+  });
+});

package/src/commands/health.ts

@@ -0,0 +1,148 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import path from 'node:path';
+import { loadConfig } from '../config/loader.js';
+import { validateConfig } from '../config/validator.js';
+import { getAllAdapters, getAdapter } from '../cli-adapters/index.js';
+
+export function registerHealthCommand(program: Command): void {
+  program
+    .command('health')
+    .description('Check CLI tool availability')
+    .action(async () => {
+      // 1. Config validation
+      console.log(chalk.bold('Config validation:'));
+      const validationResult = await validateConfig();
+      
+      if (validationResult.filesChecked.length === 0) {
+        console.log(chalk.yellow('  No config files found'));
+      } else {
+        // List all files checked
+        for (const file of validationResult.filesChecked) {
+          const relativePath = path.relative(process.cwd(), file);
+          console.log(chalk.dim(`  ${relativePath}`));
+        }
+        
+        // Show validation results
+        if (validationResult.valid && validationResult.issues.length === 0) {
+          console.log(chalk.green('  ✓ All config files are valid'));
+        } else {
+          // Group issues by file
+          const issuesByFile = new Map<string, typeof validationResult.issues>();
+          for (const issue of validationResult.issues) {
+            const relativeFile = path.relative(process.cwd(), issue.file);
+            if (!issuesByFile.has(relativeFile)) {
+              issuesByFile.set(relativeFile, []);
+            }
+            issuesByFile.get(relativeFile)!.push(issue);
+          }
+          
+          // Display issues
+          for (const [file, issues] of issuesByFile.entries()) {
+            for (const issue of issues) {
+              const icon = issue.severity === 'error' ? chalk.red('✗') : chalk.yellow('⚠');
+              const fieldInfo = issue.field ? chalk.dim(` (${issue.field})`) : '';
+              console.log(`  ${icon} ${file}${fieldInfo}`);
+              console.log(`    ${issue.message}`);
+            }
+          }
+        }
+      }
+      
+      console.log();
+      
+      // 2. CLI Tool Health Check
+      console.log(chalk.bold('CLI Tool Health Check:'));
+      
+      try {
+        const config = await loadConfig();
+        const checkUsageLimit = config.project.cli.check_usage_limit;
+        
+        // Check for reviews configuration
+        const reviewEntries = Object.entries(config.reviews);
+        
+        if (reviewEntries.length === 0) {
+          console.log(chalk.yellow('  No CLI tools configured'));
+          console.log(chalk.dim('  No review gates found. Add review gates with cli_preference to check tool availability.'));
+          return;
+        }
+        
+        // Collect all unique agent names from review gate cli_preference settings
+        const preferredAgents = new Set<string>();
+        const reviewsWithEmptyPreference: string[] = [];
+        
+        reviewEntries.forEach(([reviewName, review]) => {
+          if (!review.cli_preference || review.cli_preference.length === 0) {
+            reviewsWithEmptyPreference.push(reviewName);
+          } else {
+            review.cli_preference.forEach(agent => {
+              preferredAgents.add(agent);
+            });
+          }
+        });
+        
+        // Report Empty Preferences (Loader should handle this via default merging, but good to check)
+        if (reviewsWithEmptyPreference.length > 0) {
+          console.log(chalk.yellow('  ⚠️  Misconfiguration detected:'));
+          reviewsWithEmptyPreference.forEach(name => {
+            console.log(chalk.yellow(`     Review gate "${name}" has empty cli_preference`));
+          });
+          console.log();
+        }
+        
+        // If no agents are configured, show message
+        if (preferredAgents.size === 0) {
+          console.log(chalk.yellow('  No CLI tools configured'));
+          console.log(chalk.dim('  All review gates have empty cli_preference. Add tools to cli_preference to check availability.'));
+          return;
+        }
+        
+        // Check the configured agents
+        for (const agentName of Array.from(preferredAgents).sort()) {
+          const adapter = getAdapter(agentName);
+          if (adapter) {
+            const health = await adapter.checkHealth({ checkUsageLimit });
+            let statusStr = '';
+            
+            switch (health.status) {
+              case 'healthy':
+                statusStr = chalk.green('Installed');
+                break;
+              case 'missing':
+                statusStr = chalk.red('Missing');
+                break;
+              case 'unhealthy':
+                statusStr = chalk.red(`${health.message || 'Unhealthy'}`);
+                break;
+            }
+            
+            console.log(`  ${adapter.name.padEnd(10)} : ${statusStr}`);
+          } else {
+            console.log(`  ${agentName.padEnd(10)} : ${chalk.yellow('Unknown')}`);
+          }
+        }
+      } catch (error: any) {
+        // If config can't be loaded, fall back to checking all adapters
+        const adapters = getAllAdapters();
+        console.log(chalk.dim('  (Config not found, checking all supported agents)'));
+        
+        for (const adapter of adapters) {
+          const health = await adapter.checkHealth();
+          let statusStr = '';
+          
+          switch (health.status) {
+            case 'healthy':
+              statusStr = chalk.green('Installed');
+              break;
+            case 'missing':
+              statusStr = chalk.red('Missing');
+              break;
+            case 'unhealthy':
+              statusStr = chalk.red(`${health.message || 'Unhealthy'}`);
+              break;
+          }
+          console.log(`  ${adapter.name.padEnd(10)} : ${statusStr}`);
+        }
+      }
+    });
+}