agent-gauntlet 0.1.10 → 0.1.12

package/src/config/loader.test.ts

@@ -1,23 +1,25 @@
-import { describe, it, expect, beforeAll, afterAll } from 'bun:test';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { loadConfig } from './loader.js';
-
-const TEST_DIR = path.join(process.cwd(), 'test-env-' + Date.now());
-const GAUNTLET_DIR = path.join(TEST_DIR, '.gauntlet');
-const CHECKS_DIR = path.join(GAUNTLET_DIR, 'checks');
-const REVIEWS_DIR = path.join(GAUNTLET_DIR, 'reviews');
-
-describe('Config Loader', () => {
-  beforeAll(async () => {
-    // Setup directory structure
-    await fs.mkdir(TEST_DIR);
-    await fs.mkdir(GAUNTLET_DIR);
-    await fs.mkdir(CHECKS_DIR);
-    await fs.mkdir(REVIEWS_DIR);
-
-    // Write config.yml
-    await fs.writeFile(path.join(GAUNTLET_DIR, 'config.yml'), `
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { loadConfig } from "./loader.js";
+
+const TEST_DIR = path.join(process.cwd(), `test-env-${Date.now()}`);
+const GAUNTLET_DIR = path.join(TEST_DIR, ".gauntlet");
+const CHECKS_DIR = path.join(GAUNTLET_DIR, "checks");
+const REVIEWS_DIR = path.join(GAUNTLET_DIR, "reviews");
+
+describe("Config Loader", () => {
+	beforeAll(async () => {
+		// Setup directory structure
+		await fs.mkdir(TEST_DIR);
+		await fs.mkdir(GAUNTLET_DIR);
+		await fs.mkdir(CHECKS_DIR);
+		await fs.mkdir(REVIEWS_DIR);
+
+		// Write config.yml
+		await fs.writeFile(
+			path.join(GAUNTLET_DIR, "config.yml"),
+			`
 base_branch: origin/dev
 log_dir: test_logs
 cli:
@@ -31,99 +33,119 @@ entry_points:
       - lint
     reviews:
       - security
-`);
+`,
+		);
 
-    // Write a check definition
-    await fs.writeFile(path.join(CHECKS_DIR, 'lint.yml'), `
+		// Write a check definition
+		await fs.writeFile(
+			path.join(CHECKS_DIR, "lint.yml"),
+			`
 name: lint
 command: npm run lint
 working_directory: .
-`);
+`,
+		);
 
-    // Write a review definition
-    await fs.writeFile(path.join(REVIEWS_DIR, 'security.md'), `---
+		// Write a review definition
+		await fs.writeFile(
+			path.join(REVIEWS_DIR, "security.md"),
+			`---
 cli_preference:
   - gemini
 ---
 
 # Security Review
 Check for vulnerabilities.
-`);
+`,
+		);
 
-    // Write a review definition without preference
-    await fs.writeFile(path.join(REVIEWS_DIR, 'style.md'), `---
+		// Write a review definition without preference
+		await fs.writeFile(
+			path.join(REVIEWS_DIR, "style.md"),
+			`---
 num_reviews: 1
 ---
 
 # Style Review
 Check style.
-`);
-  });
-
-  afterAll(async () => {
-    // Cleanup
-    await fs.rm(TEST_DIR, { recursive: true, force: true });
-  });
-
-  it('should load project configuration correctly', async () => {
-    const config = await loadConfig(TEST_DIR);
-    
-    expect(config.project.base_branch).toBe('origin/dev');
-    expect(config.project.log_dir).toBe('test_logs');
-    expect(config.project.entry_points).toHaveLength(1);
-    expect(config.project.entry_points[0].path).toBe('src/');
-  });
-
-  it('should load check gates correctly', async () => {
-    const config = await loadConfig(TEST_DIR);
-    
-    expect(Object.keys(config.checks)).toContain('lint');
-    expect(config.checks['lint'].command).toBe('npm run lint');
-  });
-
-  it('should load review gates correctly', async () => {
-    const config = await loadConfig(TEST_DIR);
-    
-    expect(Object.keys(config.reviews)).toContain('security');
-    expect(config.reviews['security'].name).toBe('security');
-    expect(config.reviews['security'].cli_preference).toEqual(['gemini']);
-    expect(config.reviews['security'].promptContent).toContain('Check for vulnerabilities.');
-  });
-
-  it('should merge default cli preference', async () => {
-    const config = await loadConfig(TEST_DIR);
-    
-    expect(Object.keys(config.reviews)).toContain('style');
-    expect(config.reviews['style'].cli_preference).toEqual(['claude', 'gemini']);
-  });
-
-  it('should reject check gate with fail_fast when parallel is true', async () => {
-    await fs.writeFile(path.join(CHECKS_DIR, 'invalid.yml'), `
+`,
+		);
+	});
+
+	afterAll(async () => {
+		// Cleanup
+		await fs.rm(TEST_DIR, { recursive: true, force: true });
+	});
+
+	it("should load project configuration correctly", async () => {
+		const config = await loadConfig(TEST_DIR);
+
+		expect(config.project.base_branch).toBe("origin/dev");
+		expect(config.project.log_dir).toBe("test_logs");
+		expect(config.project.entry_points).toHaveLength(1);
+		expect(config.project.entry_points[0].path).toBe("src/");
+	});
+
+	it("should load check gates correctly", async () => {
+		const config = await loadConfig(TEST_DIR);
+
+		expect(Object.keys(config.checks)).toContain("lint");
+		expect(config.checks.lint.command).toBe("npm run lint");
+	});
+
+	it("should load review gates correctly", async () => {
+		const config = await loadConfig(TEST_DIR);
+
+		expect(Object.keys(config.reviews)).toContain("security");
+		expect(config.reviews.security.name).toBe("security");
+		expect(config.reviews.security.cli_preference).toEqual(["gemini"]);
+		expect(config.reviews.security.promptContent).toContain(
+			"Check for vulnerabilities.",
+		);
+	});
+
+	it("should merge default cli preference", async () => {
+		const config = await loadConfig(TEST_DIR);
+
+		expect(Object.keys(config.reviews)).toContain("style");
+		expect(config.reviews.style.cli_preference).toEqual(["claude", "gemini"]);
+	});
+
+	it("should reject check gate with fail_fast when parallel is true", async () => {
+		await fs.writeFile(
+			path.join(CHECKS_DIR, "invalid.yml"),
+			`
 name: invalid
 command: echo test
 parallel: true
 fail_fast: true
-`);
-
-    await expect(loadConfig(TEST_DIR)).rejects.toThrow(/fail_fast can only be used when parallel is false/);
-  });
-
-  it('should accept check gate with fail_fast when parallel is false', async () => {
-    // Clean up the invalid file first
-    try {
-      await fs.unlink(path.join(CHECKS_DIR, 'invalid.yml'));
-    } catch {}
-
-    await fs.writeFile(path.join(CHECKS_DIR, 'valid.yml'), `
+`,
+		);
+
+		await expect(loadConfig(TEST_DIR)).rejects.toThrow(
+			/fail_fast can only be used when parallel is false/,
+		);
+	});
+
+	it("should accept check gate with fail_fast when parallel is false", async () => {
+		// Clean up the invalid file first
+		try {
+			await fs.unlink(path.join(CHECKS_DIR, "invalid.yml"));
+		} catch {}
+
+		await fs.writeFile(
+			path.join(CHECKS_DIR, "valid.yml"),
+			`
 name: valid
 command: echo test
 parallel: false
 fail_fast: true
-`);
-
-    const config = await loadConfig(TEST_DIR);
-    expect(config.checks['valid']).toBeDefined();
-    expect(config.checks['valid'].fail_fast).toBe(true);
-    expect(config.checks['valid'].parallel).toBe(false);
-  });
+`,
+		);
+
+		const config = await loadConfig(TEST_DIR);
+		expect(config.checks.valid).toBeDefined();
+		expect(config.checks.valid.fail_fast).toBe(true);
+		expect(config.checks.valid.parallel).toBe(false);
+	});
 });

package/src/config/loader.ts

@@ -1,130 +1,139 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import YAML from 'yaml';
-import matter from 'gray-matter';
-import { 
-  gauntletConfigSchema, 
-  checkGateSchema, 
-  reviewPromptFrontmatterSchema 
-} from './schema.js';
-import { LoadedConfig, CheckGateConfig } from './types.js';
-
-const GAUNTLET_DIR = '.gauntlet';
-const CONFIG_FILE = 'config.yml';
-const CHECKS_DIR = 'checks';
-const REVIEWS_DIR = 'reviews';
-
-export async function loadConfig(rootDir: string = process.cwd()): Promise<LoadedConfig> {
-  const gauntletPath = path.join(rootDir, GAUNTLET_DIR);
-  const configPath = path.join(gauntletPath, CONFIG_FILE);
-
-  // 1. Load project config
-  if (!(await fileExists(configPath))) {
-    throw new Error(`Configuration file not found at ${configPath}`);
-  }
-
-  const configContent = await fs.readFile(configPath, 'utf-8');
-  const projectConfigRaw = YAML.parse(configContent);
-  const projectConfig = gauntletConfigSchema.parse(projectConfigRaw);
-
-  // 2. Load checks
-  const checksPath = path.join(gauntletPath, CHECKS_DIR);
-  const checks: Record<string, CheckGateConfig> = {};
-  
-  if (await dirExists(checksPath)) {
-    const checkFiles = await fs.readdir(checksPath);
-    for (const file of checkFiles) {
-      if (file.endsWith('.yml') || file.endsWith('.yaml')) {
-        const filePath = path.join(checksPath, file);
-        const content = await fs.readFile(filePath, 'utf-8');
-        const raw = YAML.parse(content);
-        // Ensure name matches filename if not provided or just use filename as key
-        const parsed = checkGateSchema.parse(raw);
-        checks[parsed.name] = parsed;
-      }
-    }
-  }
-
-  // 3. Load reviews (prompts + frontmatter)
-  const reviewsPath = path.join(gauntletPath, REVIEWS_DIR);
-  const reviews: Record<string, any> = {};
-
-  if (await dirExists(reviewsPath)) {
-    const reviewFiles = await fs.readdir(reviewsPath);
-    for (const file of reviewFiles) {
-      if (file.endsWith('.md')) {
-        const filePath = path.join(reviewsPath, file);
-        const content = await fs.readFile(filePath, 'utf-8');
-        const { data: frontmatter, content: promptBody } = matter(content);
-        
-        const parsedFrontmatter = reviewPromptFrontmatterSchema.parse(frontmatter);
-        const name = path.basename(file, '.md');
-
-        reviews[name] = {
-          name,
-          prompt: file, // Store filename relative to reviews dir
-          promptContent: promptBody, // Store the actual prompt content for easy access
-          ...parsedFrontmatter
-        };
-
-        // Merge default CLI preference if not specified
-        if (!reviews[name].cli_preference) {
-          reviews[name].cli_preference = projectConfig.cli.default_preference;
-        } else {
-          // Validate that specified preferences are allowed by project config
-          const allowedTools = new Set(projectConfig.cli.default_preference);
-          for (const tool of reviews[name].cli_preference) {
-            if (!allowedTools.has(tool)) {
-              throw new Error(`Review "${name}" uses CLI tool "${tool}" which is not in the project-level allowed list (cli.default_preference).`);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // 4. Validate entry point references
-  const checkNames = new Set(Object.keys(checks));
-  const reviewNames = new Set(Object.keys(reviews));
-
-  for (const entryPoint of projectConfig.entry_points) {
-    if (entryPoint.checks) {
-      for (const checkName of entryPoint.checks) {
-        if (!checkNames.has(checkName)) {
-          throw new Error(`Entry point "${entryPoint.path}" references non-existent check gate: "${checkName}"`);
-        }
-      }
-    }
-    if (entryPoint.reviews) {
-      for (const reviewName of entryPoint.reviews) {
-        if (!reviewNames.has(reviewName)) {
-          throw new Error(`Entry point "${entryPoint.path}" references non-existent review gate: "${reviewName}"`);
-        }
-      }
-    }
-  }
-
-  return {
-    project: projectConfig,
-    checks,
-    reviews,
-  };
+import fs from "node:fs/promises";
+import path from "node:path";
+import matter from "gray-matter";
+import YAML from "yaml";
+import {
+	checkGateSchema,
+	gauntletConfigSchema,
+	reviewPromptFrontmatterSchema,
+} from "./schema.js";
+import type { CheckGateConfig, LoadedConfig } from "./types.js";
+
+const GAUNTLET_DIR = ".gauntlet";
+const CONFIG_FILE = "config.yml";
+const CHECKS_DIR = "checks";
+const REVIEWS_DIR = "reviews";
+
+export async function loadConfig(
+	rootDir: string = process.cwd(),
+): Promise<LoadedConfig> {
+	const gauntletPath = path.join(rootDir, GAUNTLET_DIR);
+	const configPath = path.join(gauntletPath, CONFIG_FILE);
+
+	// 1. Load project config
+	if (!(await fileExists(configPath))) {
+		throw new Error(`Configuration file not found at ${configPath}`);
+	}
+
+	const configContent = await fs.readFile(configPath, "utf-8");
+	const projectConfigRaw = YAML.parse(configContent);
+	const projectConfig = gauntletConfigSchema.parse(projectConfigRaw);
+
+	// 2. Load checks
+	const checksPath = path.join(gauntletPath, CHECKS_DIR);
+	const checks: Record<string, CheckGateConfig> = {};
+
+	if (await dirExists(checksPath)) {
+		const checkFiles = await fs.readdir(checksPath);
+		for (const file of checkFiles) {
+			if (file.endsWith(".yml") || file.endsWith(".yaml")) {
+				const filePath = path.join(checksPath, file);
+				const content = await fs.readFile(filePath, "utf-8");
+				const raw = YAML.parse(content);
+				// Ensure name matches filename if not provided or just use filename as key
+				const parsed = checkGateSchema.parse(raw);
+				checks[parsed.name] = parsed;
+			}
+		}
+	}
+
+	// 3. Load reviews (prompts + frontmatter)
+	const reviewsPath = path.join(gauntletPath, REVIEWS_DIR);
+	const reviews: LoadedConfig["reviews"] = {};
+
+	if (await dirExists(reviewsPath)) {
+		const reviewFiles = await fs.readdir(reviewsPath);
+		for (const file of reviewFiles) {
+			if (file.endsWith(".md")) {
+				const filePath = path.join(reviewsPath, file);
+				const content = await fs.readFile(filePath, "utf-8");
+				const { data: frontmatter, content: promptBody } = matter(content);
+
+				const parsedFrontmatter =
+					reviewPromptFrontmatterSchema.parse(frontmatter);
+				const name = path.basename(file, ".md");
+
+				reviews[name] = {
+					name,
+					prompt: file, // Store filename relative to reviews dir
+					promptContent: promptBody, // Store the actual prompt content for easy access
+					...parsedFrontmatter,
+				};
+
+				// Merge default CLI preference if not specified
+				if (!reviews[name].cli_preference) {
+					reviews[name].cli_preference = projectConfig.cli.default_preference;
+				} else {
+					// Validate that specified preferences are allowed by project config
+					const allowedTools = new Set(projectConfig.cli.default_preference);
+					for (const tool of reviews[name].cli_preference) {
+						if (!allowedTools.has(tool)) {
+							throw new Error(
+								`Review "${name}" uses CLI tool "${tool}" which is not in the project-level allowed list (cli.default_preference).`,
+							);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// 4. Validate entry point references
+	const checkNames = new Set(Object.keys(checks));
+	const reviewNames = new Set(Object.keys(reviews));
+
+	for (const entryPoint of projectConfig.entry_points) {
+		if (entryPoint.checks) {
+			for (const checkName of entryPoint.checks) {
+				if (!checkNames.has(checkName)) {
+					throw new Error(
+						`Entry point "${entryPoint.path}" references non-existent check gate: "${checkName}"`,
+					);
+				}
+			}
+		}
+		if (entryPoint.reviews) {
+			for (const reviewName of entryPoint.reviews) {
+				if (!reviewNames.has(reviewName)) {
+					throw new Error(
+						`Entry point "${entryPoint.path}" references non-existent review gate: "${reviewName}"`,
+					);
+				}
+			}
+		}
+	}
+
+	return {
+		project: projectConfig,
+		checks,
+		reviews,
+	};
 }
 
 async function fileExists(path: string): Promise<boolean> {
-  try {
-    const stat = await fs.stat(path);
-    return stat.isFile();
-  } catch {
-    return false;
-  }
+	try {
+		const stat = await fs.stat(path);
+		return stat.isFile();
+	} catch {
+		return false;
+	}
 }
 
 async function dirExists(path: string): Promise<boolean> {
-  try {
-    const stat = await fs.stat(path);
-    return stat.isDirectory();
-  } catch {
-    return false;
-  }
+	try {
+		const stat = await fs.stat(path);
+		return stat.isDirectory();
+	} catch {
+		return false;
+	}
 }

package/src/config/schema.ts

@@ -1,63 +1,64 @@
-import { z } from 'zod';
+import { z } from "zod";
 
 export const cliConfigSchema = z.object({
-  default_preference: z.array(z.string().min(1)).min(1),
-  check_usage_limit: z.boolean().default(false),
+	default_preference: z.array(z.string().min(1)).min(1),
+	check_usage_limit: z.boolean().default(false),
 });
 
-export const checkGateSchema = z.object({
-  name: z.string().min(1),
-  command: z.string().min(1),
-  working_directory: z.string().optional(),
-  parallel: z.boolean().default(false),
-  run_in_ci: z.boolean().default(true),
-  run_locally: z.boolean().default(true),
-  timeout: z.number().optional(),
-  fail_fast: z.boolean().optional(),
-}).refine(
-  (data) => {
-    // fail_fast can only be used when parallel is false
-    if (data.fail_fast === true && data.parallel === true) {
-      return false;
-    }
-    return true;
-  },
-  {
-    message: "fail_fast can only be used when parallel is false",
-  }
-);
+export const checkGateSchema = z
+	.object({
+		name: z.string().min(1),
+		command: z.string().min(1),
+		working_directory: z.string().optional(),
+		parallel: z.boolean().default(false),
+		run_locally: z.boolean().default(true),
+		timeout: z.number().optional(),
+		fail_fast: z.boolean().optional(),
+	})
+	.refine(
+		(data) => {
+			// fail_fast can only be used when parallel is false
+			if (data.fail_fast === true && data.parallel === true) {
+				return false;
+			}
+			return true;
+		},
+		{
+			message: "fail_fast can only be used when parallel is false",
+		},
+	);
 
 export const reviewGateSchema = z.object({
-  name: z.string().min(1),
-  prompt: z.string().min(1), // Path relative to .gauntlet/reviews/
-  cli_preference: z.array(z.string().min(1)).optional(),
-  num_reviews: z.number().default(1),
-  parallel: z.boolean().default(true),
-  run_in_ci: z.boolean().default(true),
-  run_locally: z.boolean().default(true),
-  timeout: z.number().optional(),
+	name: z.string().min(1),
+	prompt: z.string().min(1), // Path relative to .gauntlet/reviews/
+	cli_preference: z.array(z.string().min(1)).optional(),
+	num_reviews: z.number().default(1),
+	parallel: z.boolean().default(true),
+	run_in_ci: z.boolean().default(true),
+	run_locally: z.boolean().default(true),
+	timeout: z.number().optional(),
 });
 
 export const reviewPromptFrontmatterSchema = z.object({
-  model: z.string().optional(),
-  cli_preference: z.array(z.string().min(1)).optional(),
-  num_reviews: z.number().default(1),
-  parallel: z.boolean().default(true),
-  run_in_ci: z.boolean().default(true),
-  run_locally: z.boolean().default(true),
-  timeout: z.number().optional(),
+	model: z.string().optional(),
+	cli_preference: z.array(z.string().min(1)).optional(),
+	num_reviews: z.number().default(1),
+	parallel: z.boolean().default(true),
+	run_in_ci: z.boolean().default(true),
+	run_locally: z.boolean().default(true),
+	timeout: z.number().optional(),
 });
 
 export const entryPointSchema = z.object({
-  path: z.string().min(1),
-  checks: z.array(z.string().min(1)).optional(),
-  reviews: z.array(z.string().min(1)).optional(),
+	path: z.string().min(1),
+	checks: z.array(z.string().min(1)).optional(),
+	reviews: z.array(z.string().min(1)).optional(),
 });
 
 export const gauntletConfigSchema = z.object({
-  base_branch: z.string().min(1).default('origin/main'),
-  log_dir: z.string().min(1).default('.gauntlet_logs'),
-  allow_parallel: z.boolean().default(true),
-  cli: cliConfigSchema,
-  entry_points: z.array(entryPointSchema).min(1),
+	base_branch: z.string().min(1).default("origin/main"),
+	log_dir: z.string().min(1).default(".gauntlet_logs"),
+	allow_parallel: z.boolean().default(true),
+	cli: cliConfigSchema,
+	entry_points: z.array(entryPointSchema).min(1),
 });