agent-gauntlet 0.1.9 → 0.1.11

package/src/gates/check.ts

@@ -1,82 +1,91 @@
-import { exec } from 'node:child_process';
-import { promisify } from 'node:util';
-import { CheckGateConfig } from '../config/types.js';
-import { GateResult } from './result.js';
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+import type { CheckGateConfig } from "../config/types.js";
+import type { GateResult } from "./result.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
 export class CheckGateExecutor {
-  async execute(
-    jobId: string, 
-    config: CheckGateConfig, 
-    workingDirectory: string,
-    logger: (output: string) => Promise<void>
-  ): Promise<GateResult> {
-    const startTime = Date.now();
-    
-    try {
-      await logger(`[${new Date().toISOString()}] Starting check: ${config.name}\n`);
-      await logger(`Executing command: ${config.command}\n`);
-      await logger(`Working directory: ${workingDirectory}\n\n`);
+	async execute(
+		jobId: string,
+		config: CheckGateConfig,
+		workingDirectory: string,
+		logger: (output: string) => Promise<void>,
+	): Promise<GateResult> {
+		const startTime = Date.now();
 
-      const { stdout, stderr } = await execAsync(config.command, { 
-        cwd: workingDirectory,
-        timeout: config.timeout ? config.timeout * 1000 : undefined,
-        maxBuffer: MAX_BUFFER_BYTES
-      });
+		try {
+			await logger(
+				`[${new Date().toISOString()}] Starting check: ${config.name}\n`,
+			);
+			await logger(`Executing command: ${config.command}\n`);
+			await logger(`Working directory: ${workingDirectory}\n\n`);
 
-      if (stdout) await logger(stdout);
-      if (stderr) await logger(`\nSTDERR:\n${stderr}`);
+			const { stdout, stderr } = await execAsync(config.command, {
+				cwd: workingDirectory,
+				timeout: config.timeout ? config.timeout * 1000 : undefined,
+				maxBuffer: MAX_BUFFER_BYTES,
+			});
 
-      const result: GateResult = {
-        jobId,
-        status: 'pass',
-        duration: Date.now() - startTime,
-        message: 'Command exited with code 0'
-      };
+			if (stdout) await logger(stdout);
+			if (stderr) await logger(`\nSTDERR:\n${stderr}`);
 
-      await logger(`Result: ${result.status} - ${result.message}\n`);
-      return result;
-    } catch (error: any) {
-      if (error.stdout) await logger(error.stdout);
-      if (error.stderr) await logger(`\nSTDERR:\n${error.stderr}`);
-      
-      await logger(`\nCommand failed: ${error.message}`);
+			const result: GateResult = {
+				jobId,
+				status: "pass",
+				duration: Date.now() - startTime,
+				message: "Command exited with code 0",
+			};
 
-      // If it's a timeout
-      if (error.signal === 'SIGTERM' && config.timeout) {
-        const result: GateResult = {
-          jobId,
-          status: 'fail',
-          duration: Date.now() - startTime,
-          message: `Timed out after ${config.timeout}s`
-        };
-        await logger(`Result: ${result.status} - ${result.message}\n`);
-        return result;
-      }
+			await logger(`Result: ${result.status} - ${result.message}\n`);
+			return result;
+		} catch (error: unknown) {
+			const err = error as {
+				stdout?: string;
+				stderr?: string;
+				message?: string;
+				signal?: string;
+				code?: number;
+			};
+			if (err.stdout) await logger(err.stdout);
+			if (err.stderr) await logger(`\nSTDERR:\n${err.stderr}`);
 
-      // If it's a non-zero exit code
-      if (typeof error.code === 'number') {
-        const result: GateResult = {
-          jobId,
-          status: 'fail',
-          duration: Date.now() - startTime,
-          message: `Exited with code ${error.code}`
-        };
-        await logger(`Result: ${result.status} - ${result.message}\n`);
-        return result;
-      }
+			await logger(`\nCommand failed: ${err.message}`);
 
-      // Other errors
-      const result: GateResult = {
-        jobId,
-        status: 'error',
-        duration: Date.now() - startTime,
-        message: error.message || 'Unknown error'
-      };
-      await logger(`Result: ${result.status} - ${result.message}\n`);
-      return result;
-    }
-  }
+			// If it's a timeout
+			if (err.signal === "SIGTERM" && config.timeout) {
+				const result: GateResult = {
+					jobId,
+					status: "fail",
+					duration: Date.now() - startTime,
+					message: `Timed out after ${config.timeout}s`,
+				};
+				await logger(`Result: ${result.status} - ${result.message}\n`);
+				return result;
+			}
+
+			// If it's a non-zero exit code
+			if (typeof err.code === "number") {
+				const result: GateResult = {
+					jobId,
+					status: "fail",
+					duration: Date.now() - startTime,
+					message: `Exited with code ${err.code}`,
+				};
+				await logger(`Result: ${result.status} - ${result.message}\n`);
+				return result;
+			}
+
+			// Other errors
+			const result: GateResult = {
+				jobId,
+				status: "error",
+				duration: Date.now() - startTime,
+				message: err.message || "Unknown error",
+			};
+			await logger(`Result: ${result.status} - ${result.message}\n`);
+			return result;
+		}
+	}
 }

package/src/gates/result.ts

@@ -1,9 +1,10 @@
-export type GateStatus = 'pass' | 'fail' | 'error';
+export type GateStatus = "pass" | "fail" | "error";
 
 export interface GateResult {
-  jobId: string;
-  status: GateStatus;
-  duration: number; // ms
-  message?: string; // summary message
-  logPath?: string; // path to full log
+	jobId: string;
+	status: GateStatus;
+	duration: number; // ms
+	message?: string; // summary message
+	logPath?: string; // path to full log
+	logPaths?: string[]; // paths to multiple logs (e.g. per-agent logs)
 }

package/src/gates/review.test.ts

@@ -0,0 +1,188 @@
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { CLIAdapter } from "../cli-adapters/index.js";
+import type {
+	ReviewGateConfig,
+	ReviewPromptFrontmatter,
+} from "../config/types.js";
+import { Logger } from "../output/logger.js";
+import { ReviewGateExecutor } from "./review.js";
+
+const TEST_DIR = path.join(process.cwd(), `test-review-logs-${Date.now()}`);
+const LOG_DIR = path.join(TEST_DIR, "logs");
+
+describe("ReviewGateExecutor Logging", () => {
+	let logger: Logger;
+	let executor: ReviewGateExecutor;
+
+	beforeEach(async () => {
+		await fs.mkdir(TEST_DIR, { recursive: true });
+		await fs.mkdir(LOG_DIR, { recursive: true });
+		logger = new Logger(LOG_DIR);
+		executor = new ReviewGateExecutor();
+
+		// Mock getAdapter
+		mock.module("../cli-adapters/index.js", () => ({
+			getAdapter: (name: string) =>
+				({
+					name,
+					isAvailable: async () => true,
+					checkHealth: async () => ({ status: "healthy" }),
+					// execute returns the raw string output from the LLM, which is then parsed by the executor.
+					// The real adapter returns a string. In this test, we return a JSON string to simulate
+					// the LLM returning structured data. This IS intentional and matches the expected contract
+					// where execute() -> Promise<string>.
+					execute: async () => {
+						await new Promise((r) => setTimeout(r, 1)); // Simulate async work
+						return JSON.stringify({ status: "pass", message: "OK" });
+					},
+					getProjectCommandDir: () => null,
+					getUserCommandDir: () => null,
+					getCommandExtension: () => "md",
+					canUseSymlink: () => false,
+					transformCommand: (c: string) => c,
+				}) as unknown as CLIAdapter,
+		}));
+
+		// Mock git commands via util.promisify(exec)
+		mock.module("node:util", () => ({
+			promisify: (fn: (...args: unknown[]) => unknown) => {
+				// Only mock exec, let others pass (though in this test env we likely only use exec)
+				if (fn.name === "exec") {
+					return async (cmd: string) => {
+						if (/^git diff/.test(cmd)) return "diff content";
+						if (/^git ls-files/.test(cmd)) return "file.ts";
+						return { stdout: "", stderr: "" };
+					};
+				}
+				// Fallback for other functions if needed
+				return async () => {};
+			},
+		}));
+	});
+
+	afterEach(async () => {
+		await fs.rm(TEST_DIR, { recursive: true, force: true });
+		mock.restore();
+	});
+
+	it("should only create adapter-specific logs and no generic log", async () => {
+		const jobId = "review:src:code-quality";
+		const config: ReviewGateConfig & ReviewPromptFrontmatter = {
+			name: "code-quality",
+			cli_preference: ["codex", "claude"],
+			num_reviews: 2,
+		};
+
+		const loggerFactory = logger.createLoggerFactory(jobId);
+
+		// We need to mock getDiff since it uses execAsync which we mocked
+		// Actually ReviewGateExecutor is a class, we can mock its private method if needed
+		// or just let it run if the mock promisify works.
+
+		const result = await executor.execute(
+			jobId,
+			config,
+			"src/",
+			loggerFactory,
+			"main",
+		);
+
+		expect(result.status).toBe("pass");
+		expect(result.logPaths).toBeDefined();
+		expect(result.logPaths).toHaveLength(2);
+		expect(result.logPaths?.[0]).toContain("review_src_code-quality_codex.log");
+		expect(result.logPaths?.[1]).toContain(
+			"review_src_code-quality_claude.log",
+		);
+
+		const files = await fs.readdir(LOG_DIR);
+		expect(files).toContain("review_src_code-quality_codex.log");
+		expect(files).toContain("review_src_code-quality_claude.log");
+		expect(files).not.toContain("review_src_code-quality.log");
+
+		// Verify multiplexed content
+		const codexLog = await fs.readFile(
+			path.join(LOG_DIR, "review_src_code-quality_codex.log"),
+			"utf-8",
+		);
+		expect(codexLog).toContain("Starting review: code-quality");
+		expect(codexLog).toContain("Review result (codex): pass");
+
+		const claudeLog = await fs.readFile(
+			path.join(LOG_DIR, "review_src_code-quality_claude.log"),
+			"utf-8",
+		);
+		expect(claudeLog).toContain("Starting review: code-quality");
+		expect(claudeLog).toContain("Review result (claude): pass");
+	});
+
+	it("should be handled correctly by ConsoleReporter", async () => {
+		const jobId = "review:src:code-quality";
+		const codexPath = path.join(LOG_DIR, "review_src_code-quality_codex.log");
+		const claudePath = path.join(LOG_DIR, "review_src_code-quality_claude.log");
+
+		await fs.writeFile(
+			codexPath,
+			`
+[2026-01-14T10:00:00.000Z] Starting review: code-quality
+--- Parsed Result (codex) ---
+Status: FAIL
+Violations:
+1. src/index.ts:10 - Security risk
+   Fix: Use a safer method
+`,
+		);
+
+		await fs.writeFile(
+			claudePath,
+			`
+[2026-01-14T10:00:00.000Z] Starting review: code-quality
+--- Parsed Result (claude) ---
+Status: FAIL
+Violations:
+1. src/main.ts:20 - Style issue
+   Fix: Rename variable
+`,
+		);
+
+		const result = {
+			jobId,
+			status: "fail" as const,
+			duration: 1000,
+			message: "Found violations",
+			logPaths: [codexPath, claudePath],
+		};
+
+		const { ConsoleReporter } = await import("../output/console.js");
+		const reporter = new ConsoleReporter();
+
+		// We can access extractFailureDetails directly as it is public
+		const details = await reporter.extractFailureDetails(result);
+
+		// Check for presence of key information rather than exact counts
+		expect(
+			details.some(
+				(d: string) =>
+					d.includes("src/index.ts") &&
+					d.includes("10") &&
+					d.includes("Security risk"),
+			),
+		).toBe(true);
+		expect(details.some((d: string) => d.includes("Use a safer method"))).toBe(
+			true,
+		);
+		expect(
+			details.some(
+				(d: string) =>
+					d.includes("src/main.ts") &&
+					d.includes("20") &&
+					d.includes("Style issue"),
+			),
+		).toBe(true);
+		expect(details.some((d: string) => d.includes("Rename variable"))).toBe(
+			true,
+		);
+	});
+});