agent-gauntlet 0.1.4

package/src/config/validator.ts

@@ -0,0 +1,493 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import YAML from 'yaml';
+import matter from 'gray-matter';
+import { ZodError } from 'zod';
+import {
+  gauntletConfigSchema,
+  checkGateSchema,
+  reviewPromptFrontmatterSchema,
+  entryPointSchema,
+} from './schema.js';
+
+// Valid CLI tool names (must match cli-adapters/index.ts)
+const VALID_CLI_TOOLS = ['gemini', 'codex', 'claude'];
+
+const GAUNTLET_DIR = '.gauntlet';
+const CONFIG_FILE = 'config.yml';
+const CHECKS_DIR = 'checks';
+const REVIEWS_DIR = 'reviews';
+
+export interface ValidationIssue {
+  file: string;
+  severity: 'error' | 'warning';
+  message: string;
+  field?: string;
+}
+
+export interface ValidationResult {
+  valid: boolean;
+  issues: ValidationIssue[];
+  filesChecked: string[];
+}
+
+export async function validateConfig(rootDir: string = process.cwd()): Promise<ValidationResult> {
+  const issues: ValidationIssue[] = [];
+  const filesChecked: string[] = [];
+  const gauntletPath = path.join(rootDir, GAUNTLET_DIR);
+  const existingCheckNames = new Set<string>(); // Track all check files that exist (even if invalid)
+  const existingReviewNames = new Set<string>(); // Track all review files that exist (even if invalid)
+
+  // 1. Validate project config
+  const configPath = path.join(gauntletPath, CONFIG_FILE);
+  let projectConfig: any = null;
+  let checks: Record<string, any> = {};
+  let reviews: Record<string, any> = {};
+
+  try {
+    if (await fileExists(configPath)) {
+      filesChecked.push(configPath);
+      const configContent = await fs.readFile(configPath, 'utf-8');
+      try {
+        const raw = YAML.parse(configContent);
+        projectConfig = gauntletConfigSchema.parse(raw);
+      } catch (error: any) {
+        if (error instanceof ZodError) {
+          error.errors.forEach(err => {
+            issues.push({
+              file: configPath,
+              severity: 'error',
+              message: err.message,
+              field: err.path.join('.'),
+            });
+          });
+        } else if (error.name === 'YAMLSyntaxError' || error.message?.includes('YAML')) {
+          issues.push({
+            file: configPath,
+            severity: 'error',
+            message: `Malformed YAML: ${error.message}`,
+          });
+        } else {
+          issues.push({
+            file: configPath,
+            severity: 'error',
+            message: `Parse error: ${error.message}`,
+          });
+        }
+      }
+    } else {
+      issues.push({
+        file: configPath,
+        severity: 'error',
+        message: 'Config file not found',
+      });
+    }
+  } catch (error: any) {
+    issues.push({
+      file: configPath,
+      severity: 'error',
+      message: `Error reading file: ${error.message}`,
+    });
+  }
+
+  // 2. Validate check gates
+  const checksPath = path.join(gauntletPath, CHECKS_DIR);
+  if (await dirExists(checksPath)) {
+    try {
+      const checkFiles = await fs.readdir(checksPath);
+      for (const file of checkFiles) {
+        if (file.endsWith('.yml') || file.endsWith('.yaml')) {
+          const filePath = path.join(checksPath, file);
+          filesChecked.push(filePath);
+          try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            const raw = YAML.parse(content);
+            const parsed = checkGateSchema.parse(raw);
+            existingCheckNames.add(parsed.name); // Track that this check exists
+            checks[parsed.name] = parsed;
+
+            // Semantic validation
+            if (!parsed.command || parsed.command.trim() === '') {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: 'command field is required and cannot be empty',
+                field: 'command',
+              });
+            }
+          } catch (error: any) {
+            // Try to extract check name from raw YAML even if parsing failed
+            try {
+              const content = await fs.readFile(filePath, 'utf-8');
+              const raw = YAML.parse(content);
+              if (raw.name && typeof raw.name === 'string') {
+                existingCheckNames.add(raw.name); // Track that this check file exists
+              }
+            } catch {
+              // If we can't even parse the name, that's okay - we'll just skip tracking it
+            }
+            
+            if (error instanceof ZodError) {
+              error.errors.forEach(err => {
+                issues.push({
+                  file: filePath,
+                  severity: 'error',
+                  message: err.message,
+                  field: err.path.join('.'),
+                });
+              });
+            } else if (error.name === 'YAMLSyntaxError' || error.message?.includes('YAML')) {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: `Malformed YAML: ${error.message}`,
+              });
+            } else {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: `Parse error: ${error.message}`,
+              });
+            }
+          }
+        }
+      }
+    } catch (error: any) {
+      issues.push({
+        file: checksPath,
+        severity: 'error',
+        message: `Error reading checks directory: ${error.message}`,
+      });
+    }
+  }
+
+  // 3. Validate review gates
+  const reviewsPath = path.join(gauntletPath, REVIEWS_DIR);
+  if (await dirExists(reviewsPath)) {
+    try {
+      const reviewFiles = await fs.readdir(reviewsPath);
+      for (const file of reviewFiles) {
+        if (file.endsWith('.md')) {
+          const filePath = path.join(reviewsPath, file);
+          const reviewName = path.basename(file, '.md');
+          existingReviewNames.add(reviewName); // Track that this review file exists
+          filesChecked.push(filePath);
+          try {
+            const content = await fs.readFile(filePath, 'utf-8');
+            const { data: frontmatter, content: promptBody } = matter(content);
+
+            // Check if frontmatter exists
+            if (!frontmatter || Object.keys(frontmatter).length === 0) {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: 'Review gate must have YAML frontmatter',
+              });
+              continue;
+            }
+
+            // Validate CLI tools even if schema validation fails
+            if (frontmatter.cli_preference && Array.isArray(frontmatter.cli_preference)) {
+              for (let i = 0; i < frontmatter.cli_preference.length; i++) {
+                const toolName = frontmatter.cli_preference[i];
+                if (typeof toolName === 'string' && !VALID_CLI_TOOLS.includes(toolName)) {
+                  issues.push({
+                    file: filePath,
+                    severity: 'error',
+                    message: `Invalid CLI tool "${toolName}" in cli_preference. Valid options are: ${VALID_CLI_TOOLS.join(', ')}`,
+                    field: `cli_preference[${i}]`,
+                  });
+                }
+              }
+            }
+
+            const parsedFrontmatter = reviewPromptFrontmatterSchema.parse(frontmatter);
+            const name = path.basename(file, '.md');
+            reviews[name] = parsedFrontmatter;
+
+            // Semantic validation
+            // cli_preference is optional; if missing, it defaults to project-level default_preference during load.
+            // See src/config/loader.ts loadConfig() for default merging logic.
+            // However, if explicitly provided, it must not be empty.
+            if (parsedFrontmatter.cli_preference !== undefined) {
+              if (parsedFrontmatter.cli_preference.length === 0) {
+                issues.push({
+                  file: filePath,
+                  severity: 'error',
+                  message: 'cli_preference if provided cannot be an empty array. Remove it to use defaults.',
+                  field: 'cli_preference',
+                });
+              } else {
+                // Validate each CLI tool name (double-check after parsing)
+                for (let i = 0; i < parsedFrontmatter.cli_preference.length; i++) {
+                  const toolName = parsedFrontmatter.cli_preference[i];
+                  if (!VALID_CLI_TOOLS.includes(toolName)) {
+                    issues.push({
+                      file: filePath,
+                      severity: 'error',
+                      message: `Invalid CLI tool "${toolName}" in cli_preference. Valid options are: ${VALID_CLI_TOOLS.join(', ')}`,
+                      field: `cli_preference[${i}]`,
+                    });
+                  }
+                }
+              }
+            }
+
+            if (parsedFrontmatter.num_reviews !== undefined && parsedFrontmatter.num_reviews < 1) {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: 'num_reviews must be at least 1',
+                field: 'num_reviews',
+              });
+            }
+
+            if (parsedFrontmatter.timeout !== undefined && parsedFrontmatter.timeout <= 0) {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: 'timeout must be greater than 0',
+                field: 'timeout',
+              });
+            }
+          } catch (error: any) {
+            if (error instanceof ZodError && error.errors && Array.isArray(error.errors)) {
+              error.errors.forEach((err: any) => {
+                const fieldPath = err.path && Array.isArray(err.path) ? err.path.join('.') : undefined;
+                const message = err.message || `Invalid value for ${fieldPath || 'field'}`;
+                issues.push({
+                  file: filePath,
+                  severity: 'error',
+                  message: message,
+                  field: fieldPath,
+                });
+              });
+            } else if (error.name === 'YAMLSyntaxError' || error.message?.includes('YAML')) {
+              issues.push({
+                file: filePath,
+                severity: 'error',
+                message: `Malformed YAML frontmatter: ${error.message || 'Unknown YAML error'}`,
+              });
+            } else {
+              // Try to parse error message from stringified error
+              let errorMessage = error.message || String(error);
+              try {
+                const parsed = JSON.parse(errorMessage);
+                if (Array.isArray(parsed)) {
+                  // Handle array of Zod errors
+                  parsed.forEach((err: any) => {
+                    const fieldPath = err.path && Array.isArray(err.path) ? err.path.join('.') : undefined;
+                    issues.push({
+                      file: filePath,
+                      severity: 'error',
+                      message: err.message || `Invalid value for ${fieldPath || 'field'}`,
+                      field: fieldPath,
+                    });
+                  });
+                } else {
+                  issues.push({
+                    file: filePath,
+                    severity: 'error',
+                    message: errorMessage,
+                  });
+                }
+              } catch {
+                issues.push({
+                  file: filePath,
+                  severity: 'error',
+                  message: errorMessage,
+                });
+              }
+            }
+          }
+        }
+      }
+    } catch (error: any) {
+      issues.push({
+        file: reviewsPath,
+        severity: 'error',
+        message: `Error reading reviews directory: ${error.message || String(error)}`,
+      });
+    }
+  }
+
+  // 4. Cross-reference validation (entry points referencing gates)
+  if (projectConfig && projectConfig.entry_points) {
+    for (let i = 0; i < projectConfig.entry_points.length; i++) {
+      const entryPoint = projectConfig.entry_points[i];
+      const entryPointPath = `entry_points[${i}]`;
+
+      // Validate entry point schema
+      try {
+        entryPointSchema.parse(entryPoint);
+      } catch (error: any) {
+        if (error instanceof ZodError) {
+          error.errors.forEach(err => {
+            issues.push({
+              file: configPath,
+              severity: 'error',
+              message: err.message,
+              field: `${entryPointPath}.${err.path.join('.')}`,
+            });
+          });
+        }
+      }
+
+      // Check referenced checks exist
+      if (entryPoint.checks) {
+        for (const checkName of entryPoint.checks) {
+          // Only report as "non-existent" if the file doesn't exist at all
+          // If the file exists but has validation errors, those are already reported
+          if (!existingCheckNames.has(checkName)) {
+            issues.push({
+              file: configPath,
+              severity: 'error',
+              message: `Entry point references non-existent check gate: "${checkName}"`,
+              field: `${entryPointPath}.checks`,
+            });
+          }
+          // If the check file exists but wasn't successfully parsed (has errors),
+          // we don't report it here - the validation errors for that file are already shown
+        }
+      }
+
+      // Check referenced reviews exist
+      if (entryPoint.reviews) {
+        for (const reviewName of entryPoint.reviews) {
+          // Only report as "non-existent" if the file doesn't exist at all
+          // If the file exists but has validation errors, those are already reported
+          if (!existingReviewNames.has(reviewName)) {
+            issues.push({
+              file: configPath,
+              severity: 'error',
+              message: `Entry point references non-existent review gate: "${reviewName}"`,
+              field: `${entryPointPath}.reviews`,
+            });
+          }
+          // If the review file exists but wasn't successfully parsed (has errors),
+          // we don't report it here - the validation errors for that file are already shown
+        }
+      }
+
+      // Validate entry point has at least one gate
+      if ((!entryPoint.checks || entryPoint.checks.length === 0) &&
+          (!entryPoint.reviews || entryPoint.reviews.length === 0)) {
+        issues.push({
+          file: configPath,
+          severity: 'warning',
+          message: `Entry point at "${entryPoint.path}" has no checks or reviews configured`,
+          field: `${entryPointPath}`,
+        });
+      }
+
+      // Validate path format (basic check)
+      if (!entryPoint.path || entryPoint.path.trim() === '') {
+        issues.push({
+          file: configPath,
+          severity: 'error',
+          message: 'Entry point path cannot be empty',
+          field: `${entryPointPath}.path`,
+        });
+      }
+    }
+  }
+
+  // 5. Validate project-level config values
+  if (projectConfig) {
+    if (projectConfig.log_dir !== undefined && projectConfig.log_dir.trim() === '') {
+      issues.push({
+        file: configPath,
+        severity: 'error',
+        message: 'log_dir cannot be empty',
+        field: 'log_dir',
+      });
+    }
+
+    if (projectConfig.base_branch !== undefined && projectConfig.base_branch.trim() === '') {
+      issues.push({
+        file: configPath,
+        severity: 'error',
+        message: 'base_branch cannot be empty',
+        field: 'base_branch',
+      });
+    }
+
+    if (projectConfig.entry_points === undefined || projectConfig.entry_points.length === 0) {
+      issues.push({
+        file: configPath,
+        severity: 'error',
+        message: 'entry_points is required and cannot be empty',
+        field: 'entry_points',
+      });
+    }
+
+    // Validate CLI config
+    if (projectConfig.cli) {
+      const defaults = projectConfig.cli.default_preference;
+      if (!defaults || !Array.isArray(defaults) || defaults.length === 0) {
+        issues.push({
+          file: configPath,
+          severity: 'error',
+          message: 'cli.default_preference is required and cannot be empty',
+          field: 'cli.default_preference',
+        });
+      } else {
+        // Validate defaults are valid tools
+        for (let i = 0; i < defaults.length; i++) {
+          const toolName = defaults[i];
+          if (!VALID_CLI_TOOLS.includes(toolName)) {
+            issues.push({
+              file: configPath,
+              severity: 'error',
+              message: `Invalid CLI tool "${toolName}" in default_preference. Valid options are: ${VALID_CLI_TOOLS.join(', ')}`,
+              field: `cli.default_preference[${i}]`,
+            });
+          }
+        }
+
+        // Validate review preferences against defaults
+        // We need to re-scan reviews here because we need the project config to be loaded first
+        // Ideally we would do this in the review loop, but we didn't have project config then.
+        // Instead, we'll iterate over the parsed reviews we collected.
+        const allowedTools = new Set(defaults);
+        for (const [reviewName, reviewConfig] of Object.entries(reviews)) {
+          const pref = (reviewConfig as any).cli_preference;
+          if (pref && Array.isArray(pref)) {
+             for (let i = 0; i < pref.length; i++) {
+               const tool = pref[i];
+               if (!allowedTools.has(tool)) {
+                 issues.push({
+                   file: path.join(reviewsPath, `${reviewName}.md`),
+                   severity: 'error',
+                   message: `CLI tool "${tool}" is not in project-level default_preference. Review gates can only use tools enabled in config.yml`,
+                   field: `cli_preference[${i}]`,
+                 });
+               }
+             }
+          }
+        }
+      }
+    }
+  }
+
+  const valid = issues.filter(i => i.severity === 'error').length === 0;
+  return { valid, issues, filesChecked };
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    const stat = await fs.stat(path);
+    return stat.isFile();
+  } catch {
+    return false;
+  }
+}
+
+async function dirExists(path: string): Promise<boolean> {
+  try {
+    const stat = await fs.stat(path);
+    return stat.isDirectory();
+  } catch {
+    return false;
+  }
+}

package/src/core/change-detector.ts

@@ -0,0 +1,112 @@
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execAsync = promisify(exec);
+
+export interface ChangeDetectorOptions {
+  commit?: string; // If provided, get diff for this commit vs its parent
+  uncommitted?: boolean; // If true, only get uncommitted changes (staged + unstaged)
+}
+
+export class ChangeDetector {
+  constructor(
+    private baseBranch: string = 'origin/main',
+    private options: ChangeDetectorOptions = {}
+  ) {}
+
+  async getChangedFiles(): Promise<string[]> {
+    // If commit option is provided, use that
+    if (this.options.commit) {
+      return this.getCommitChangedFiles(this.options.commit);
+    }
+
+    // If uncommitted option is provided, only get uncommitted changes
+    if (this.options.uncommitted) {
+      return this.getUncommittedChangedFiles();
+    }
+
+    const isCI = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true';
+
+    if (isCI) {
+      return this.getCIChangedFiles();
+    } else {
+      return this.getLocalChangedFiles();
+    }
+  }
+
+  private async getCIChangedFiles(): Promise<string[]> {
+    // In GitHub Actions, GITHUB_BASE_REF is the target branch (e.g., main)
+    // GITHUB_SHA is the commit being built
+    const baseRef = process.env.GITHUB_BASE_REF || this.baseBranch;
+    const headRef = process.env.GITHUB_SHA || 'HEAD';
+    
+    // We might need to fetch first in some shallow clones, but assuming strictly for now
+    // git diff --name-only base...head
+    try {
+      const { stdout } = await execAsync(`git diff --name-only ${baseRef}...${headRef}`);
+      return this.parseOutput(stdout);
+    } catch (error) {
+      console.warn('Failed to detect changes via git diff in CI, falling back to HEAD^...HEAD', error);
+      // Fallback for push events where base ref might not be available
+      const { stdout } = await execAsync('git diff --name-only HEAD^...HEAD');
+      return this.parseOutput(stdout);
+    }
+  }
+
+  private async getLocalChangedFiles(): Promise<string[]> {
+    // 1. Committed changes relative to base branch
+    const { stdout: committed } = await execAsync(`git diff --name-only ${this.baseBranch}...HEAD`);
+    
+    // 2. Uncommitted changes (staged and unstaged)
+    const { stdout: uncommitted } = await execAsync('git diff --name-only HEAD');
+    
+    // 3. Untracked files
+    const { stdout: untracked } = await execAsync('git ls-files --others --exclude-standard');
+
+    const files = new Set([
+      ...this.parseOutput(committed),
+      ...this.parseOutput(uncommitted),
+      ...this.parseOutput(untracked)
+    ]);
+
+    return Array.from(files);
+  }
+
+  private async getCommitChangedFiles(commit: string): Promise<string[]> {
+    // Get diff for commit vs its parent
+    try {
+      const { stdout } = await execAsync(`git diff --name-only ${commit}^..${commit}`);
+      return this.parseOutput(stdout);
+    } catch (error) {
+      // If commit has no parent (initial commit), just get files in that commit
+      try {
+        const { stdout } = await execAsync(`git diff --name-only --root ${commit}`);
+        return this.parseOutput(stdout);
+      } catch {
+        throw new Error(`Failed to get changes for commit ${commit}`);
+      }
+    }
+  }
+
+  private async getUncommittedChangedFiles(): Promise<string[]> {
+    // Get uncommitted changes (staged + unstaged) and untracked files
+    const { stdout: staged } = await execAsync('git diff --name-only --cached');
+    const { stdout: unstaged } = await execAsync('git diff --name-only');
+    const { stdout: untracked } = await execAsync('git ls-files --others --exclude-standard');
+
+    const files = new Set([
+      ...this.parseOutput(staged),
+      ...this.parseOutput(unstaged),
+      ...this.parseOutput(untracked)
+    ]);
+
+    return Array.from(files);
+  }
+
+  private parseOutput(stdout: string): string[] {
+    return stdout
+      .split('\n')
+      .map(line => line.trim())
+      .filter(line => line.length > 0);
+  }
+}

package/src/core/entry-point.test.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect } from 'bun:test';
+import { EntryPointExpander } from './entry-point.js';
+import { EntryPointConfig } from '../config/types.js';
+
+describe('EntryPointExpander', () => {
+  const expander = new EntryPointExpander();
+
+  it('should include root entry point if there are any changes', async () => {
+    const entryPoints: EntryPointConfig[] = [{ path: '.' }];
+    const changes = ['some/file.ts'];
+
+    const result = await expander.expand(entryPoints, changes);
+    
+    expect(result).toHaveLength(1);
+    expect(result[0].path).toBe('.');
+  });
+
+  it('should match fixed directory entry points', async () => {
+    const entryPoints: EntryPointConfig[] = [
+      { path: 'apps/api' },
+      { path: 'apps/web' }
+    ];
+    const changes = ['apps/api/src/index.ts'];
+
+    const result = await expander.expand(entryPoints, changes);
+    
+    // Result should have root (implicit or explicit fallback in code) + matched
+    // Looking at code: "if (changedFiles.length > 0) ... results.push({ path: '.', ... })"
+    // Wait, the code creates a default root config if one isn't provided in the list?
+    // Code: "const rootConfig = rootEntryPoint ?? { path: '.' }; results.push({ path: '.', config: rootConfig });"
+    // Yes, it always pushes root if changes > 0.
+
+    expect(result.some(r => r.path === 'apps/api')).toBe(true);
+    expect(result.some(r => r.path === 'apps/web')).toBe(false);
+  });
+
+  it('should match wildcard entry points', async () => {
+    const entryPoints: EntryPointConfig[] = [
+      { path: 'packages/*' }
+    ];
+    const changes = [
+      'packages/ui/button.ts',
+      'packages/utils/helper.ts',
+      'other/file.ts'
+    ];
+
+    const result = await expander.expand(entryPoints, changes);
+
+    const paths = result.map(r => r.path);
+    expect(paths).toContain('packages/ui');
+    expect(paths).toContain('packages/utils');
+    expect(paths).not.toContain('packages/other');
+  });
+
+  it('should handle no changes', async () => {
+    const entryPoints: EntryPointConfig[] = [{ path: '.' }];
+    const changes: string[] = [];
+
+    const result = await expander.expand(entryPoints, changes);
+    
+    expect(result).toHaveLength(0);
+  });
+});