agent-gauntlet 0.1.4

package/src/commands/rerun.ts

@@ -0,0 +1,118 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import { loadConfig } from '../config/loader.js';
+import { ChangeDetector } from '../core/change-detector.js';
+import { EntryPointExpander } from '../core/entry-point.js';
+import { JobGenerator } from '../core/job.js';
+import { Runner } from '../core/runner.js';
+import { Logger } from '../output/logger.js';
+import { ConsoleReporter } from '../output/console.js';
+import { findPreviousFailures, GateFailures, PreviousViolation } from '../utils/log-parser.js';
+
+export function registerRerunCommand(program: Command): void {
+  program
+    .command('rerun')
+    .description('Rerun gates (checks & reviews) with previous failures as context (defaults to uncommitted changes)')
+    .option('-g, --gate <name>', 'Run specific gate only')
+    .option('-c, --commit <sha>', 'Use diff for a specific commit (overrides default uncommitted mode)')
+    .action(async (options) => {
+      try {
+        const config = await loadConfig();
+
+        // Parse previous failures from log files (only for review gates)
+        console.log(chalk.dim('Analyzing previous runs...'));
+        
+        // findPreviousFailures handles errors internally and returns empty array on failure
+        const previousFailures = await findPreviousFailures(
+          config.project.log_dir,
+          options.gate
+        );
+
+        // Create a map: jobId -> (adapterName -> violations)
+        const failuresMap = new Map<string, Map<string, PreviousViolation[]>>();
+        for (const gateFailure of previousFailures) {
+          const adapterMap = new Map<string, PreviousViolation[]>();
+          for (const adapterFailure of gateFailure.adapterFailures) {
+            adapterMap.set(adapterFailure.adapterName, adapterFailure.violations);
+          }
+          failuresMap.set(gateFailure.jobId, adapterMap);
+        }
+
+        if (previousFailures.length > 0) {
+          const totalViolations = previousFailures.reduce(
+            (sum, gf) => sum + gf.adapterFailures.reduce(
+              (s, af) => s + af.violations.length, 0
+            ), 0
+          );
+          console.log(chalk.yellow(
+            `Found ${previousFailures.length} gate(s) with ${totalViolations} previous violation(s)`
+          ));
+        } else {
+          console.log(chalk.dim('No previous failures found. Running as normal...'));
+        }
+
+        // Detect changes (default to uncommitted unless --commit is specified)
+        // Note: Rerun defaults to uncommitted changes for faster iteration loops,
+        // unlike 'run' which defaults to base_branch comparison.
+        const changeOptions = {
+          commit: options.commit,
+          uncommitted: !options.commit  // Default to uncommitted unless commit is specified
+        };
+
+        const changeDetector = new ChangeDetector(
+          config.project.base_branch,
+          changeOptions
+        );
+        const expander = new EntryPointExpander();
+        const jobGen = new JobGenerator(config);
+
+        const modeDesc = options.commit
+          ? `commit ${options.commit}`
+          : 'uncommitted changes';
+        console.log(chalk.dim(`Detecting changes (${modeDesc})...`));
+
+        const changes = await changeDetector.getChangedFiles();
+
+        if (changes.length === 0) {
+          console.log(chalk.green('No changes detected.'));
+          process.exit(0);
+        }
+
+        console.log(chalk.dim(`Found ${changes.length} changed files.`));
+
+        const entryPoints = await expander.expand(config.project.entry_points, changes);
+        let jobs = jobGen.generateJobs(entryPoints);
+
+        if (options.gate) {
+          jobs = jobs.filter(j => j.name === options.gate);
+        }
+
+        if (jobs.length === 0) {
+          console.log(chalk.yellow('No applicable gates for these changes.'));
+          process.exit(0);
+        }
+
+        console.log(chalk.dim(`Running ${jobs.length} gates...`));
+        if (previousFailures.length > 0) {
+          console.log(chalk.dim('Previous failures will be injected as context for matching reviewers.'));
+        }
+
+        const logger = new Logger(config.project.log_dir);
+        const reporter = new ConsoleReporter();
+        const runner = new Runner(
+          config,
+          logger,
+          reporter,
+          failuresMap,      // Pass previous failures map
+          changeOptions     // Pass change detection options
+        );
+
+        const success = await runner.run(jobs);
+        process.exit(success ? 0 : 1);
+
+      } catch (error: any) {
+        console.error(chalk.red('Error:'), error.message);
+        process.exit(1);
+      }
+    });
+}

package/src/commands/review.test.ts

@@ -0,0 +1,25 @@
+import { describe, it, expect, beforeEach } from 'bun:test';
+import { Command } from 'commander';
+import { registerReviewCommand } from './review.js';
+
+describe('Review Command', () => {
+  let program: Command;
+
+  beforeEach(() => {
+    program = new Command();
+    registerReviewCommand(program);
+  });
+
+  it('should register the review command', () => {
+    const reviewCmd = program.commands.find(cmd => cmd.name() === 'review');
+    expect(reviewCmd).toBeDefined();
+    expect(reviewCmd?.description()).toBe('Run only applicable reviews for detected changes');
+  });
+
+  it('should have correct options', () => {
+    const reviewCmd = program.commands.find(cmd => cmd.name() === 'review');
+    expect(reviewCmd?.options.some(opt => opt.long === '--gate')).toBe(true);
+    expect(reviewCmd?.options.some(opt => opt.long === '--commit')).toBe(true);
+    expect(reviewCmd?.options.some(opt => opt.long === '--uncommitted')).toBe(true);
+  });
+});

package/src/commands/review.ts

@@ -0,0 +1,67 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import { loadConfig } from '../config/loader.js';
+import { ChangeDetector } from '../core/change-detector.js';
+import { EntryPointExpander } from '../core/entry-point.js';
+import { JobGenerator } from '../core/job.js';
+import { Runner } from '../core/runner.js';
+import { Logger } from '../output/logger.js';
+import { ConsoleReporter } from '../output/console.js';
+
+export function registerReviewCommand(program: Command): void {
+  program
+    .command('review')
+    .description('Run only applicable reviews for detected changes')
+    .option('-g, --gate <name>', 'Run specific review gate only')
+    .option('-c, --commit <sha>', 'Use diff for a specific commit')
+    .option('-u, --uncommitted', 'Use diff for current uncommitted changes (staged and unstaged)')
+    .action(async (options) => {
+      try {
+        const config = await loadConfig();
+        const changeDetector = new ChangeDetector(config.project.base_branch, {
+          commit: options.commit,
+          uncommitted: options.uncommitted
+        });
+        const expander = new EntryPointExpander();
+        const jobGen = new JobGenerator(config);
+        
+        console.log(chalk.dim('Detecting changes...'));
+        const changes = await changeDetector.getChangedFiles();
+        
+        if (changes.length === 0) {
+          console.log(chalk.green('No changes detected.'));
+          process.exit(0);
+        }
+        
+        console.log(chalk.dim(`Found ${changes.length} changed files.`));
+
+        const entryPoints = await expander.expand(config.project.entry_points, changes);
+        let jobs = jobGen.generateJobs(entryPoints);
+
+        // Filter to only reviews
+        jobs = jobs.filter(j => j.type === 'review');
+
+        if (options.gate) {
+          jobs = jobs.filter(j => j.name === options.gate);
+        }
+
+        if (jobs.length === 0) {
+          console.log(chalk.yellow('No applicable reviews for these changes.'));
+          process.exit(0);
+        }
+
+        console.log(chalk.dim(`Running ${jobs.length} review(s)...`));
+
+        const logger = new Logger(config.project.log_dir);
+        const reporter = new ConsoleReporter();
+        const runner = new Runner(config, logger, reporter);
+
+        const success = await runner.run(jobs);
+        process.exit(success ? 0 : 1);
+
+      } catch (error: any) {
+        console.error(chalk.red('Error:'), error.message);
+        process.exit(1);
+      }
+    });
+}

package/src/commands/run.test.ts

@@ -0,0 +1,25 @@
+import { describe, it, expect, beforeEach } from 'bun:test';
+import { Command } from 'commander';
+import { registerRunCommand } from './run.js';
+
+describe('Run Command', () => {
+  let program: Command;
+
+  beforeEach(() => {
+    program = new Command();
+    registerRunCommand(program);
+  });
+
+  it('should register the run command', () => {
+    const runCmd = program.commands.find(cmd => cmd.name() === 'run');
+    expect(runCmd).toBeDefined();
+    expect(runCmd?.description()).toBe('Run gates for detected changes');
+  });
+
+  it('should have correct options', () => {
+    const runCmd = program.commands.find(cmd => cmd.name() === 'run');
+    expect(runCmd?.options.some(opt => opt.long === '--gate')).toBe(true);
+    expect(runCmd?.options.some(opt => opt.long === '--commit')).toBe(true);
+    expect(runCmd?.options.some(opt => opt.long === '--uncommitted')).toBe(true);
+  });
+});

package/src/commands/run.ts

@@ -0,0 +1,64 @@
+import type { Command } from 'commander';
+import chalk from 'chalk';
+import { loadConfig } from '../config/loader.js';
+import { ChangeDetector } from '../core/change-detector.js';
+import { EntryPointExpander } from '../core/entry-point.js';
+import { JobGenerator } from '../core/job.js';
+import { Runner } from '../core/runner.js';
+import { Logger } from '../output/logger.js';
+import { ConsoleReporter } from '../output/console.js';
+
+export function registerRunCommand(program: Command): void {
+  program
+    .command('run')
+    .description('Run gates for detected changes')
+    .option('-g, --gate <name>', 'Run specific gate only')
+    .option('-c, --commit <sha>', 'Use diff for a specific commit')
+    .option('-u, --uncommitted', 'Use diff for current uncommitted changes (staged and unstaged)')
+    .action(async (options) => {
+      try {
+        const config = await loadConfig();
+        const changeDetector = new ChangeDetector(config.project.base_branch, {
+          commit: options.commit,
+          uncommitted: options.uncommitted
+        });
+        const expander = new EntryPointExpander();
+        const jobGen = new JobGenerator(config);
+        
+        console.log(chalk.dim('Detecting changes...'));
+        const changes = await changeDetector.getChangedFiles();
+        
+        if (changes.length === 0) {
+          console.log(chalk.green('No changes detected.'));
+          process.exit(0);
+        }
+        
+        console.log(chalk.dim(`Found ${changes.length} changed files.`));
+
+        const entryPoints = await expander.expand(config.project.entry_points, changes);
+        let jobs = jobGen.generateJobs(entryPoints);
+
+        if (options.gate) {
+          jobs = jobs.filter(j => j.name === options.gate);
+        }
+
+        if (jobs.length === 0) {
+          console.log(chalk.yellow('No applicable gates for these changes.'));
+          process.exit(0);
+        }
+
+        console.log(chalk.dim(`Running ${jobs.length} gates...`));
+
+        const logger = new Logger(config.project.log_dir);
+        const reporter = new ConsoleReporter();
+        const runner = new Runner(config, logger, reporter);
+
+        const success = await runner.run(jobs);
+        process.exit(success ? 0 : 1);
+
+      } catch (error: any) {
+        console.error(chalk.red('Error:'), error.message);
+        process.exit(1);
+      }
+    });
+}

package/src/commands/shared.ts

@@ -0,0 +1,10 @@
+import fs from 'node:fs/promises';
+
+export async function exists(path: string): Promise<boolean> {
+  try {
+    await fs.stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/config/loader.test.ts

@@ -0,0 +1,129 @@
+import { describe, it, expect, beforeAll, afterAll } from 'bun:test';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { loadConfig } from './loader.js';
+
+const TEST_DIR = path.join(process.cwd(), 'test-env-' + Date.now());
+const GAUNTLET_DIR = path.join(TEST_DIR, '.gauntlet');
+const CHECKS_DIR = path.join(GAUNTLET_DIR, 'checks');
+const REVIEWS_DIR = path.join(GAUNTLET_DIR, 'reviews');
+
+describe('Config Loader', () => {
+  beforeAll(async () => {
+    // Setup directory structure
+    await fs.mkdir(TEST_DIR);
+    await fs.mkdir(GAUNTLET_DIR);
+    await fs.mkdir(CHECKS_DIR);
+    await fs.mkdir(REVIEWS_DIR);
+
+    // Write config.yml
+    await fs.writeFile(path.join(GAUNTLET_DIR, 'config.yml'), `
+base_branch: origin/dev
+log_dir: test_logs
+cli:
+  default_preference:
+    - claude
+    - gemini
+  check_usage_limit: false
+entry_points:
+  - path: src/
+    checks:
+      - lint
+    reviews:
+      - security
+`);
+
+    // Write a check definition
+    await fs.writeFile(path.join(CHECKS_DIR, 'lint.yml'), `
+name: lint
+command: npm run lint
+working_directory: .
+`);
+
+    // Write a review definition
+    await fs.writeFile(path.join(REVIEWS_DIR, 'security.md'), `---
+cli_preference:
+  - gemini
+---
+
+# Security Review
+Check for vulnerabilities.
+`);
+
+    // Write a review definition without preference
+    await fs.writeFile(path.join(REVIEWS_DIR, 'style.md'), `---
+num_reviews: 1
+---
+
+# Style Review
+Check style.
+`);
+  });
+
+  afterAll(async () => {
+    // Cleanup
+    await fs.rm(TEST_DIR, { recursive: true, force: true });
+  });
+
+  it('should load project configuration correctly', async () => {
+    const config = await loadConfig(TEST_DIR);
+    
+    expect(config.project.base_branch).toBe('origin/dev');
+    expect(config.project.log_dir).toBe('test_logs');
+    expect(config.project.entry_points).toHaveLength(1);
+    expect(config.project.entry_points[0].path).toBe('src/');
+  });
+
+  it('should load check gates correctly', async () => {
+    const config = await loadConfig(TEST_DIR);
+    
+    expect(Object.keys(config.checks)).toContain('lint');
+    expect(config.checks['lint'].command).toBe('npm run lint');
+  });
+
+  it('should load review gates correctly', async () => {
+    const config = await loadConfig(TEST_DIR);
+    
+    expect(Object.keys(config.reviews)).toContain('security');
+    expect(config.reviews['security'].name).toBe('security');
+    expect(config.reviews['security'].cli_preference).toEqual(['gemini']);
+    expect(config.reviews['security'].promptContent).toContain('Check for vulnerabilities.');
+  });
+
+  it('should merge default cli preference', async () => {
+    const config = await loadConfig(TEST_DIR);
+    
+    expect(Object.keys(config.reviews)).toContain('style');
+    expect(config.reviews['style'].cli_preference).toEqual(['claude', 'gemini']);
+  });
+
+  it('should reject check gate with fail_fast when parallel is true', async () => {
+    await fs.writeFile(path.join(CHECKS_DIR, 'invalid.yml'), `
+name: invalid
+command: echo test
+parallel: true
+fail_fast: true
+`);
+
+    await expect(loadConfig(TEST_DIR)).rejects.toThrow(/fail_fast can only be used when parallel is false/);
+  });
+
+  it('should accept check gate with fail_fast when parallel is false', async () => {
+    // Clean up the invalid file first
+    try {
+      await fs.unlink(path.join(CHECKS_DIR, 'invalid.yml'));
+    } catch {}
+
+    await fs.writeFile(path.join(CHECKS_DIR, 'valid.yml'), `
+name: valid
+command: echo test
+parallel: false
+fail_fast: true
+`);
+
+    const config = await loadConfig(TEST_DIR);
+    expect(config.checks['valid']).toBeDefined();
+    expect(config.checks['valid'].fail_fast).toBe(true);
+    expect(config.checks['valid'].parallel).toBe(false);
+  });
+});

package/src/config/loader.ts

@@ -0,0 +1,130 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import YAML from 'yaml';
+import matter from 'gray-matter';
+import { 
+  gauntletConfigSchema, 
+  checkGateSchema, 
+  reviewPromptFrontmatterSchema 
+} from './schema.js';
+import { LoadedConfig, CheckGateConfig } from './types.js';
+
+const GAUNTLET_DIR = '.gauntlet';
+const CONFIG_FILE = 'config.yml';
+const CHECKS_DIR = 'checks';
+const REVIEWS_DIR = 'reviews';
+
+export async function loadConfig(rootDir: string = process.cwd()): Promise<LoadedConfig> {
+  const gauntletPath = path.join(rootDir, GAUNTLET_DIR);
+  const configPath = path.join(gauntletPath, CONFIG_FILE);
+
+  // 1. Load project config
+  if (!(await fileExists(configPath))) {
+    throw new Error(`Configuration file not found at ${configPath}`);
+  }
+
+  const configContent = await fs.readFile(configPath, 'utf-8');
+  const projectConfigRaw = YAML.parse(configContent);
+  const projectConfig = gauntletConfigSchema.parse(projectConfigRaw);
+
+  // 2. Load checks
+  const checksPath = path.join(gauntletPath, CHECKS_DIR);
+  const checks: Record<string, CheckGateConfig> = {};
+  
+  if (await dirExists(checksPath)) {
+    const checkFiles = await fs.readdir(checksPath);
+    for (const file of checkFiles) {
+      if (file.endsWith('.yml') || file.endsWith('.yaml')) {
+        const filePath = path.join(checksPath, file);
+        const content = await fs.readFile(filePath, 'utf-8');
+        const raw = YAML.parse(content);
+        // Ensure name matches filename if not provided or just use filename as key
+        const parsed = checkGateSchema.parse(raw);
+        checks[parsed.name] = parsed;
+      }
+    }
+  }
+
+  // 3. Load reviews (prompts + frontmatter)
+  const reviewsPath = path.join(gauntletPath, REVIEWS_DIR);
+  const reviews: Record<string, any> = {};
+
+  if (await dirExists(reviewsPath)) {
+    const reviewFiles = await fs.readdir(reviewsPath);
+    for (const file of reviewFiles) {
+      if (file.endsWith('.md')) {
+        const filePath = path.join(reviewsPath, file);
+        const content = await fs.readFile(filePath, 'utf-8');
+        const { data: frontmatter, content: promptBody } = matter(content);
+        
+        const parsedFrontmatter = reviewPromptFrontmatterSchema.parse(frontmatter);
+        const name = path.basename(file, '.md');
+
+        reviews[name] = {
+          name,
+          prompt: file, // Store filename relative to reviews dir
+          promptContent: promptBody, // Store the actual prompt content for easy access
+          ...parsedFrontmatter
+        };
+
+        // Merge default CLI preference if not specified
+        if (!reviews[name].cli_preference) {
+          reviews[name].cli_preference = projectConfig.cli.default_preference;
+        } else {
+          // Validate that specified preferences are allowed by project config
+          const allowedTools = new Set(projectConfig.cli.default_preference);
+          for (const tool of reviews[name].cli_preference) {
+            if (!allowedTools.has(tool)) {
+              throw new Error(`Review "${name}" uses CLI tool "${tool}" which is not in the project-level allowed list (cli.default_preference).`);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // 4. Validate entry point references
+  const checkNames = new Set(Object.keys(checks));
+  const reviewNames = new Set(Object.keys(reviews));
+
+  for (const entryPoint of projectConfig.entry_points) {
+    if (entryPoint.checks) {
+      for (const checkName of entryPoint.checks) {
+        if (!checkNames.has(checkName)) {
+          throw new Error(`Entry point "${entryPoint.path}" references non-existent check gate: "${checkName}"`);
+        }
+      }
+    }
+    if (entryPoint.reviews) {
+      for (const reviewName of entryPoint.reviews) {
+        if (!reviewNames.has(reviewName)) {
+          throw new Error(`Entry point "${entryPoint.path}" references non-existent review gate: "${reviewName}"`);
+        }
+      }
+    }
+  }
+
+  return {
+    project: projectConfig,
+    checks,
+    reviews,
+  };
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    const stat = await fs.stat(path);
+    return stat.isFile();
+  } catch {
+    return false;
+  }
+}
+
+async function dirExists(path: string): Promise<boolean> {
+  try {
+    const stat = await fs.stat(path);
+    return stat.isDirectory();
+  } catch {
+    return false;
+  }
+}

package/src/config/schema.ts

@@ -0,0 +1,63 @@
+import { z } from 'zod';
+
+export const cliConfigSchema = z.object({
+  default_preference: z.array(z.string().min(1)).min(1),
+  check_usage_limit: z.boolean().default(false),
+});
+
+export const checkGateSchema = z.object({
+  name: z.string().min(1),
+  command: z.string().min(1),
+  working_directory: z.string().optional(),
+  parallel: z.boolean().default(false),
+  run_in_ci: z.boolean().default(true),
+  run_locally: z.boolean().default(true),
+  timeout: z.number().optional(),
+  fail_fast: z.boolean().optional(),
+}).refine(
+  (data) => {
+    // fail_fast can only be used when parallel is false
+    if (data.fail_fast === true && data.parallel === true) {
+      return false;
+    }
+    return true;
+  },
+  {
+    message: "fail_fast can only be used when parallel is false",
+  }
+);
+
+export const reviewGateSchema = z.object({
+  name: z.string().min(1),
+  prompt: z.string().min(1), // Path relative to .gauntlet/reviews/
+  cli_preference: z.array(z.string().min(1)).optional(),
+  num_reviews: z.number().default(1),
+  parallel: z.boolean().default(true),
+  run_in_ci: z.boolean().default(true),
+  run_locally: z.boolean().default(true),
+  timeout: z.number().optional(),
+});
+
+export const reviewPromptFrontmatterSchema = z.object({
+  model: z.string().optional(),
+  cli_preference: z.array(z.string().min(1)).optional(),
+  num_reviews: z.number().default(1),
+  parallel: z.boolean().default(true),
+  run_in_ci: z.boolean().default(true),
+  run_locally: z.boolean().default(true),
+  timeout: z.number().optional(),
+});
+
+export const entryPointSchema = z.object({
+  path: z.string().min(1),
+  checks: z.array(z.string().min(1)).optional(),
+  reviews: z.array(z.string().min(1)).optional(),
+});
+
+export const gauntletConfigSchema = z.object({
+  base_branch: z.string().min(1).default('origin/main'),
+  log_dir: z.string().min(1).default('.gauntlet_logs'),
+  allow_parallel: z.boolean().default(true),
+  cli: cliConfigSchema,
+  entry_points: z.array(entryPointSchema).min(1),
+});

package/src/config/types.ts

@@ -0,0 +1,23 @@
+import { z } from 'zod';
+import { 
+  checkGateSchema, 
+  reviewGateSchema, 
+  reviewPromptFrontmatterSchema, 
+  entryPointSchema, 
+  gauntletConfigSchema,
+  cliConfigSchema
+} from './schema.js';
+
+export type CheckGateConfig = z.infer<typeof checkGateSchema>;
+export type ReviewGateConfig = z.infer<typeof reviewGateSchema>;
+export type ReviewPromptFrontmatter = z.infer<typeof reviewPromptFrontmatterSchema>;
+export type EntryPointConfig = z.infer<typeof entryPointSchema>;
+export type GauntletConfig = z.infer<typeof gauntletConfigSchema>;
+export type CLIConfig = z.infer<typeof cliConfigSchema>;
+
+// Combined type for the fully loaded configuration
+export interface LoadedConfig {
+  project: GauntletConfig;
+  checks: Record<string, CheckGateConfig>;
+  reviews: Record<string, ReviewGateConfig & ReviewPromptFrontmatter>; // Merged with frontmatter
+}