agent-gauntlet 0.1.10 → 0.1.12

package/src/gates/check.ts

@@ -1,82 +1,91 @@
-import { exec } from 'node:child_process';
-import { promisify } from 'node:util';
-import { CheckGateConfig } from '../config/types.js';
-import { GateResult } from './result.js';
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+import type { CheckGateConfig } from "../config/types.js";
+import type { GateResult } from "./result.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
 export class CheckGateExecutor {
-  async execute(
-    jobId: string, 
-    config: CheckGateConfig, 
-    workingDirectory: string,
-    logger: (output: string) => Promise<void>
-  ): Promise<GateResult> {
-    const startTime = Date.now();
-    
-    try {
-      await logger(`[${new Date().toISOString()}] Starting check: ${config.name}\n`);
-      await logger(`Executing command: ${config.command}\n`);
-      await logger(`Working directory: ${workingDirectory}\n\n`);
+	async execute(
+		jobId: string,
+		config: CheckGateConfig,
+		workingDirectory: string,
+		logger: (output: string) => Promise<void>,
+	): Promise<GateResult> {
+		const startTime = Date.now();
 
-      const { stdout, stderr } = await execAsync(config.command, { 
-        cwd: workingDirectory,
-        timeout: config.timeout ? config.timeout * 1000 : undefined,
-        maxBuffer: MAX_BUFFER_BYTES
-      });
+		try {
+			await logger(
+				`[${new Date().toISOString()}] Starting check: ${config.name}\n`,
+			);
+			await logger(`Executing command: ${config.command}\n`);
+			await logger(`Working directory: ${workingDirectory}\n\n`);
 
-      if (stdout) await logger(stdout);
-      if (stderr) await logger(`\nSTDERR:\n${stderr}`);
+			const { stdout, stderr } = await execAsync(config.command, {
+				cwd: workingDirectory,
+				timeout: config.timeout ? config.timeout * 1000 : undefined,
+				maxBuffer: MAX_BUFFER_BYTES,
+			});
 
-      const result: GateResult = {
-        jobId,
-        status: 'pass',
-        duration: Date.now() - startTime,
-        message: 'Command exited with code 0'
-      };
+			if (stdout) await logger(stdout);
+			if (stderr) await logger(`\nSTDERR:\n${stderr}`);
 
-      await logger(`Result: ${result.status} - ${result.message}\n`);
-      return result;
-    } catch (error: any) {
-      if (error.stdout) await logger(error.stdout);
-      if (error.stderr) await logger(`\nSTDERR:\n${error.stderr}`);
-      
-      await logger(`\nCommand failed: ${error.message}`);
+			const result: GateResult = {
+				jobId,
+				status: "pass",
+				duration: Date.now() - startTime,
+				message: "Command exited with code 0",
+			};
 
-      // If it's a timeout
-      if (error.signal === 'SIGTERM' && config.timeout) {
-        const result: GateResult = {
-          jobId,
-          status: 'fail',
-          duration: Date.now() - startTime,
-          message: `Timed out after ${config.timeout}s`
-        };
-        await logger(`Result: ${result.status} - ${result.message}\n`);
-        return result;
-      }
+			await logger(`Result: ${result.status} - ${result.message}\n`);
+			return result;
+		} catch (error: unknown) {
+			const err = error as {
+				stdout?: string;
+				stderr?: string;
+				message?: string;
+				signal?: string;
+				code?: number;
+			};
+			if (err.stdout) await logger(err.stdout);
+			if (err.stderr) await logger(`\nSTDERR:\n${err.stderr}`);
 
-      // If it's a non-zero exit code
-      if (typeof error.code === 'number') {
-        const result: GateResult = {
-          jobId,
-          status: 'fail',
-          duration: Date.now() - startTime,
-          message: `Exited with code ${error.code}`
-        };
-        await logger(`Result: ${result.status} - ${result.message}\n`);
-        return result;
-      }
+			await logger(`\nCommand failed: ${err.message}`);
 
-      // Other errors
-      const result: GateResult = {
-        jobId,
-        status: 'error',
-        duration: Date.now() - startTime,
-        message: error.message || 'Unknown error'
-      };
-      await logger(`Result: ${result.status} - ${result.message}\n`);
-      return result;
-    }
-  }
+			// If it's a timeout
+			if (err.signal === "SIGTERM" && config.timeout) {
+				const result: GateResult = {
+					jobId,
+					status: "fail",
+					duration: Date.now() - startTime,
+					message: `Timed out after ${config.timeout}s`,
+				};
+				await logger(`Result: ${result.status} - ${result.message}\n`);
+				return result;
+			}
+
+			// If it's a non-zero exit code
+			if (typeof err.code === "number") {
+				const result: GateResult = {
+					jobId,
+					status: "fail",
+					duration: Date.now() - startTime,
+					message: `Exited with code ${err.code}`,
+				};
+				await logger(`Result: ${result.status} - ${result.message}\n`);
+				return result;
+			}
+
+			// Other errors
+			const result: GateResult = {
+				jobId,
+				status: "error",
+				duration: Date.now() - startTime,
+				message: err.message || "Unknown error",
+			};
+			await logger(`Result: ${result.status} - ${result.message}\n`);
+			return result;
+		}
+	}
 }

package/src/gates/result.ts

@@ -1,10 +1,10 @@
-export type GateStatus = 'pass' | 'fail' | 'error';
+export type GateStatus = "pass" | "fail" | "error";
 
 export interface GateResult {
-  jobId: string;
-  status: GateStatus;
-  duration: number; // ms
-  message?: string; // summary message
-  logPath?: string; // path to full log
-  logPaths?: string[]; // paths to multiple logs (e.g. per-agent logs)
+	jobId: string;
+	status: GateStatus;
+	duration: number; // ms
+	message?: string; // summary message
+	logPath?: string; // path to full log
+	logPaths?: string[]; // paths to multiple logs (e.g. per-agent logs)
 }

package/src/gates/review.test.ts

@@ -1,152 +1,291 @@
-import { describe, it, expect, beforeEach, afterEach, mock } from 'bun:test';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { ReviewGateExecutor } from './review.js';
-import { Logger } from '../output/logger.js';
-import * as cliAdapters from '../cli-adapters/index.js';
-import type { CLIAdapter } from '../cli-adapters/index.js';
-
-const TEST_DIR = path.join(process.cwd(), 'test-review-logs-' + Date.now());
-const LOG_DIR = path.join(TEST_DIR, 'logs');
-
-describe('ReviewGateExecutor Logging', () => {
-  let logger: Logger;
-  let executor: ReviewGateExecutor;
-
-  beforeEach(async () => {
-    await fs.mkdir(TEST_DIR, { recursive: true });
-    await fs.mkdir(LOG_DIR, { recursive: true });
-    logger = new Logger(LOG_DIR);
-    executor = new ReviewGateExecutor();
-
-    // Mock getAdapter
-    mock.module('../cli-adapters/index.js', () => ({
-      getAdapter: (name: string) => ({
-        name,
-        isAvailable: async () => true,
-        checkHealth: async () => ({ status: 'healthy' }),
-        // execute returns the raw string output from the LLM, which is then parsed by the executor.
-        // The real adapter returns a string. In this test, we return a JSON string to simulate
-        // the LLM returning structured data. This IS intentional and matches the expected contract
-        // where execute() -> Promise<string>.
-        execute: async () => {
-          await new Promise(r => setTimeout(r, 1)); // Simulate async work
-          return JSON.stringify({ status: 'pass', message: 'OK' });
-        },
-        getProjectCommandDir: () => null,
-        getUserCommandDir: () => null,
-        getCommandExtension: () => 'md',
-        canUseSymlink: () => false,
-        transformCommand: (c: string) => c
-      } as unknown as CLIAdapter)
-    }));
-
-    // Mock git commands via util.promisify(exec)
-    mock.module('node:util', () => ({
-      promisify: (fn: Function) => {
-        // Only mock exec, let others pass (though in this test env we likely only use exec)
-        if (fn.name === 'exec') {
-          return async (cmd: string) => {
-            if (/^git diff/.test(cmd)) return 'diff content';
-            if (/^git ls-files/.test(cmd)) return 'file.ts';
-            return { stdout: '', stderr: '' };
-          };
-        }
-        // Fallback for other functions if needed
-        return async () => {}; 
-      }
-    }));
-  });
-
-  afterEach(async () => {
-    await fs.rm(TEST_DIR, { recursive: true, force: true });
-    mock.restore();
-  });
-
-  it('should only create adapter-specific logs and no generic log', async () => {
-    const jobId = 'review:src:code-quality';
-    const config = {
-      name: 'code-quality',
-      cli_preference: ['codex', 'claude'],
-      num_reviews: 2
-    };
-
-    const loggerFactory = logger.createLoggerFactory(jobId);
-    
-    // We need to mock getDiff since it uses execAsync which we mocked
-    // Actually ReviewGateExecutor is a class, we can mock its private method if needed
-    // or just let it run if the mock promisify works.
-    
-    const result = await executor.execute(
-      jobId,
-      config as any,
-      'src/',
-      loggerFactory,
-      'main'
-    );
-
-    expect(result.status).toBe('pass');
-    expect(result.logPaths).toBeDefined();
-    expect(result.logPaths).toHaveLength(2);
-    expect(result.logPaths?.[0]).toContain('review_src_code-quality_codex.log');
-    expect(result.logPaths?.[1]).toContain('review_src_code-quality_claude.log');
-
-    const files = await fs.readdir(LOG_DIR);
-    expect(files).toContain('review_src_code-quality_codex.log');
-    expect(files).toContain('review_src_code-quality_claude.log');
-    expect(files).not.toContain('review_src_code-quality.log');
-
-    // Verify multiplexed content
-    const codexLog = await fs.readFile(path.join(LOG_DIR, 'review_src_code-quality_codex.log'), 'utf-8');
-    expect(codexLog).toContain('Starting review: code-quality');
-    expect(codexLog).toContain('Review result (codex): pass');
-
-    const claudeLog = await fs.readFile(path.join(LOG_DIR, 'review_src_code-quality_claude.log'), 'utf-8');
-    expect(claudeLog).toContain('Starting review: code-quality');
-    expect(claudeLog).toContain('Review result (claude): pass');
-  });
-
-  it('should be handled correctly by ConsoleReporter', async () => {
-    const jobId = 'review:src:code-quality';
-    const codexPath = path.join(LOG_DIR, 'review_src_code-quality_codex.log');
-    const claudePath = path.join(LOG_DIR, 'review_src_code-quality_claude.log');
-
-    await fs.writeFile(codexPath, `
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { CLIAdapter } from "../cli-adapters/index.js";
+import type {
+	ReviewGateConfig,
+	ReviewPromptFrontmatter,
+} from "../config/types.js";
+import { Logger } from "../output/logger.js";
+import type { ReviewGateExecutor } from "./review.js";
+
+const TEST_DIR = path.join(process.cwd(), `test-review-logs-${Date.now()}`);
+
+describe("ReviewGateExecutor Logging", () => {
+	let logger: Logger;
+	let executor: ReviewGateExecutor;
+	let originalCI: string | undefined;
+	let originalGithubActions: string | undefined;
+	let originalCwd: string;
+
+	beforeEach(async () => {
+		await fs.mkdir(TEST_DIR, { recursive: true });
+
+		// Save and disable CI mode for this test to avoid complex git ref issues
+		originalCI = process.env.CI;
+		originalGithubActions = process.env.GITHUB_ACTIONS;
+		originalCwd = process.cwd();
+		delete process.env.CI;
+		delete process.env.GITHUB_ACTIONS;
+
+		// Change to test directory with its own git repo to avoid issues with the main repo
+		process.chdir(TEST_DIR);
+		// Initialize a minimal git repo for the test
+		const { exec } = await import("node:child_process");
+		const { promisify } = await import("node:util");
+		const execAsync = promisify(exec);
+		await execAsync("git init");
+		await execAsync('git config user.email "test@test.com"');
+		await execAsync('git config user.name "Test"');
+		// Create an initial commit so we have a history
+		await fs.writeFile("test.txt", "initial");
+		await execAsync("git add test.txt");
+		await execAsync('git commit -m "initial"');
+		// Create a "main" branch
+		await execAsync("git branch -M main");
+		// Create src directory for our test
+		await fs.mkdir("src", { recursive: true });
+		await fs.writeFile("src/test.ts", "test content");
+		await execAsync("git add src/test.ts");
+		await execAsync('git commit -m "add src"');
+
+		// Make uncommitted changes so the diff isn't empty
+		await fs.writeFile("src/test.ts", "modified test content");
+
+		// Now create the log directory and logger in the test directory
+		await fs.mkdir("logs", { recursive: true });
+		logger = new Logger(path.join(process.cwd(), "logs"));
+
+		// Create a factory function for mock adapters that returns the correct name
+		const createMockAdapter = (name: string): CLIAdapter =>
+			({
+				name,
+				isAvailable: async () => true,
+				checkHealth: async () => ({ status: "healthy" }),
+				// execute returns the raw string output from the LLM, which is then parsed by the executor.
+				// The real adapter returns a string. In this test, we return a JSON string to simulate
+				// the LLM returning structured data. This IS intentional and matches the expected contract
+				// where execute() -> Promise<string>.
+				execute: async () => {
+					await new Promise((r) => setTimeout(r, 1)); // Simulate async work
+					return JSON.stringify({ status: "pass", message: "OK" });
+				},
+				getProjectCommandDir: () => null,
+				getUserCommandDir: () => null,
+				getCommandExtension: () => "md",
+				canUseSymlink: () => false,
+				transformCommand: (c: string) => c,
+			}) as unknown as CLIAdapter;
+
+		// Mock getAdapter and other exports that may be imported by other modules
+		mock.module("../cli-adapters/index.js", () => ({
+			getAdapter: (name: string) => createMockAdapter(name),
+			getAllAdapters: () => [
+				createMockAdapter("codex"),
+				createMockAdapter("claude"),
+			],
+			getProjectCommandAdapters: () => [
+				createMockAdapter("codex"),
+				createMockAdapter("claude"),
+			],
+			getUserCommandAdapters: () => [
+				createMockAdapter("codex"),
+				createMockAdapter("claude"),
+			],
+			getValidCLITools: () => ["codex", "claude", "gemini"],
+		}));
+
+		const { ReviewGateExecutor } = await import("./review.js");
+		executor = new ReviewGateExecutor();
+	});
+
+	afterEach(async () => {
+		// Restore working directory first
+		process.chdir(originalCwd);
+
+		await fs.rm(TEST_DIR, { recursive: true, force: true });
+		mock.restore();
+
+		// Restore CI env vars
+		if (originalCI !== undefined) {
+			process.env.CI = originalCI;
+		}
+		if (originalGithubActions !== undefined) {
+			process.env.GITHUB_ACTIONS = originalGithubActions;
+		}
+	});
+
+	it("should only create adapter-specific logs and no generic log", async () => {
+		const jobId = "review:src:code-quality";
+		const config: ReviewGateConfig & ReviewPromptFrontmatter = {
+			name: "code-quality",
+			cli_preference: ["codex", "claude"],
+			num_reviews: 2,
+		};
+
+		const loggerFactory = logger.createLoggerFactory(jobId);
+
+		// We need to mock getDiff since it uses execAsync which we mocked
+		// Actually ReviewGateExecutor is a class, we can mock its private method if needed
+		// or just let it run if the mock promisify works.
+
+		const result = await executor.execute(
+			jobId,
+			config,
+			"src/",
+			loggerFactory,
+			"main",
+		);
+
+		// Enhanced error messages for better debugging
+		if (result.status !== "pass") {
+			throw new Error(
+				`Expected result.status to be "pass" but got "${result.status}". Message: ${result.message || "none"}. Duration: ${result.duration}ms`,
+			);
+		}
+
+		if (!result.logPaths) {
+			throw new Error(
+				`Expected result.logPaths to be defined but got ${JSON.stringify(result.logPaths)}`,
+			);
+		}
+
+		if (result.logPaths.length !== 2) {
+			throw new Error(
+				`Expected result.logPaths to have length 2 but got ${result.logPaths.length}. Paths: ${JSON.stringify(result.logPaths)}`,
+			);
+		}
+
+		if (!result.logPaths[0]?.includes("review_src_code-quality_codex.log")) {
+			throw new Error(
+				`Expected result.logPaths[0] to contain "review_src_code-quality_codex.log" but got "${result.logPaths[0]}"`,
+			);
+		}
+
+		if (!result.logPaths[1]?.includes("review_src_code-quality_claude.log")) {
+			throw new Error(
+				`Expected result.logPaths[1] to contain "review_src_code-quality_claude.log" but got "${result.logPaths[1]}"`,
+			);
+		}
+
+		const files = await fs.readdir("logs");
+		const filesList = files.join(", ");
+
+		if (!files.includes("review_src_code-quality_codex.log")) {
+			throw new Error(
+				`Expected log directory to contain "review_src_code-quality_codex.log" but only found: [${filesList}]`,
+			);
+		}
+
+		if (!files.includes("review_src_code-quality_claude.log")) {
+			throw new Error(
+				`Expected log directory to contain "review_src_code-quality_claude.log" but only found: [${filesList}]`,
+			);
+		}
+
+		if (files.includes("review_src_code-quality.log")) {
+			throw new Error(
+				`Expected log directory NOT to contain generic log "review_src_code-quality.log" but it was found. All files: [${filesList}]`,
+			);
+		}
+
+		// Verify multiplexed content
+		const codexLog = await fs.readFile(
+			"logs/review_src_code-quality_codex.log",
+			"utf-8",
+		);
+		if (!codexLog.includes("Starting review: code-quality")) {
+			throw new Error(
+				`Expected codex log to contain "Starting review: code-quality" but got: ${codexLog.substring(0, 200)}...`,
+			);
+		}
+		if (!codexLog.includes("Review result (codex): pass")) {
+			throw new Error(
+				`Expected codex log to contain "Review result (codex): pass" but got: ${codexLog.substring(0, 200)}...`,
+			);
+		}
+
+		const claudeLog = await fs.readFile(
+			"logs/review_src_code-quality_claude.log",
+			"utf-8",
+		);
+		if (!claudeLog.includes("Starting review: code-quality")) {
+			throw new Error(
+				`Expected claude log to contain "Starting review: code-quality" but got: ${claudeLog.substring(0, 200)}...`,
+			);
+		}
+		if (!claudeLog.includes("Review result (claude): pass")) {
+			throw new Error(
+				`Expected claude log to contain "Review result (claude): pass" but got: ${claudeLog.substring(0, 200)}...`,
+			);
+		}
+	});
+
+	it("should be handled correctly by ConsoleReporter", async () => {
+		const jobId = "review:src:code-quality";
+		const codexPath = "logs/review_src_code-quality_codex.log";
+		const claudePath = "logs/review_src_code-quality_claude.log";
+
+		await fs.writeFile(
+			codexPath,
+			`
 [2026-01-14T10:00:00.000Z] Starting review: code-quality
 --- Parsed Result (codex) ---
 Status: FAIL
 Violations:
 1. src/index.ts:10 - Security risk
    Fix: Use a safer method
-`);
+`,
+		);
 
-    await fs.writeFile(claudePath, `
+		await fs.writeFile(
+			claudePath,
+			`
 [2026-01-14T10:00:00.000Z] Starting review: code-quality
 --- Parsed Result (claude) ---
 Status: FAIL
 Violations:
 1. src/main.ts:20 - Style issue
    Fix: Rename variable
-`);
-
-    const result = {
-      jobId,
-      status: 'fail' as const,
-      duration: 1000,
-      message: 'Found violations',
-      logPaths: [codexPath, claudePath]
-    };
-
-    const { ConsoleReporter } = await import('../output/console.js');
-    const reporter = new ConsoleReporter();
-    
-    // We can access extractFailureDetails directly as it is public
-    const details = await reporter.extractFailureDetails(result);
-    
-    // Check for presence of key information rather than exact counts
-    expect(details.some((d: string) => d.includes('src/index.ts') && d.includes('10') && d.includes('Security risk'))).toBe(true);
-    expect(details.some((d: string) => d.includes('Use a safer method'))).toBe(true);
-    expect(details.some((d: string) => d.includes('src/main.ts') && d.includes('20') && d.includes('Style issue'))).toBe(true);
-    expect(details.some((d: string) => d.includes('Rename variable'))).toBe(true);
-  });
+`,
+		);
+
+		const result = {
+			jobId,
+			status: "fail" as const,
+			duration: 1000,
+			message: "Found violations",
+			logPaths: [codexPath, claudePath],
+		};
+
+		const { ConsoleReporter } = await import("../output/console.js");
+		const reporter = new ConsoleReporter();
+
+		// We can access extractFailureDetails directly as it is public
+		const details = await reporter.extractFailureDetails(result);
+
+		// Check for presence of key information rather than exact counts
+		expect(
+			details.some(
+				(d: string) =>
+					d.includes("src/index.ts") &&
+					d.includes("10") &&
+					d.includes("Security risk"),
+			),
+		).toBe(true);
+		expect(details.some((d: string) => d.includes("Use a safer method"))).toBe(
+			true,
+		);
+		expect(
+			details.some(
+				(d: string) =>
+					d.includes("src/main.ts") &&
+					d.includes("20") &&
+					d.includes("Style issue"),
+			),
+		).toBe(true);
+		expect(details.some((d: string) => d.includes("Rename variable"))).toBe(
+			true,
+		);
+	});
 });