agent-gauntlet 0.1.10 → 0.1.11

package/README.md

@@ -20,7 +20,7 @@ Agent Gauntlet is designed to be "tool-agnostic" by leveraging the AI CLI tools
 
 - **Bun** (Required runtime, v1.0.0+)
 - **git** (change detection and diffs)
-- For review gates: one or more supported AI CLIs installed (`gemini`, `codex`, `claude`). For the full list of tools and how they are used, see [CLI Invocation Details](docs/cli-invocation-details.md)
+- For review gates: one or more supported AI CLIs installed (`gemini`, `codex`, `claude`, `github-copilot`, `cursor`). For the full list of tools and how they are used, see [CLI Invocation Details](docs/cli-invocation-details.md)
 
 ### Installation
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-gauntlet",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "description": "A CLI tool for testing AI coding agents",
   "license": "Apache-2.0",
   "author": "Paul Caplan",
@@ -33,9 +33,11 @@
   },
   "scripts": {
     "build": "bun build --compile --minify --sourcemap ./src/index.ts --outfile bin/agent-gauntlet",
-    "test": "bun test"
+    "test": "bun test",
+    "lint": "biome check src"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.3.11",
     "@types/bun": "latest"
   },
   "peerDependencies": {

package/src/cli-adapters/claude.ts

@@ -1,114 +1,145 @@
-import { exec } from 'node:child_process';
-import { promisify } from 'node:util';
-import { type CLIAdapter, isUsageLimit } from './index.js';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import os from 'node:os';
+import { exec } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { type CLIAdapter, isUsageLimit } from "./index.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
 export class ClaudeAdapter implements CLIAdapter {
-  name = 'claude';
-
-  async isAvailable(): Promise<boolean> {
-    try {
-      await execAsync('which claude');
-      return true;
-    } catch {
-      return false;
-    }
-  }
-
-  async checkHealth(options?: { checkUsageLimit?: boolean }): Promise<{ available: boolean; status: 'healthy' | 'missing' | 'unhealthy'; message?: string }> {
-    const available = await this.isAvailable();
-    if (!available) {
-      return { available: false, status: 'missing', message: 'Command not found' };
-    }
-
-    if (options?.checkUsageLimit) {
-      try {
-        // Try a lightweight command to check if we're rate limited
-        // We use a simple "hello" prompt to avoid "No messages returned" errors from empty input
-        const { stdout, stderr } = await execAsync('echo "hello" | claude -p --max-turns 1', { timeout: 10000 });
-        
-        const combined = (stdout || '') + (stderr || '');
-        if (isUsageLimit(combined)) {
-           return { 
-            available: true, 
-            status: 'unhealthy', 
-            message: 'Usage limit exceeded' 
-          };
-        }
-  
-        return { available: true, status: 'healthy', message: 'Ready' };
-      } catch (error: any) {
-        const stderr = error.stderr || '';
-        const stdout = error.stdout || '';
-        const combined = (stderr + stdout);
-        
-        if (isUsageLimit(combined)) {
-          return { 
-            available: true, 
-            status: 'unhealthy', 
-            message: 'Usage limit exceeded' 
-          };
-        }
-        
-        // Since we sent a valid prompt ("hello"), any other error implies the tool is broken
-        // Extract a brief error message if possible
-        const cleanError = combined.split('\n')[0]?.trim() || error.message || 'Command failed';
-        return { 
-          available: true, 
-          status: 'unhealthy', 
-          message: `Error: ${cleanError}` 
-        };
-      }
-    }
-
-    return { available: true, status: 'healthy', message: 'Ready' };
-  }
-
-  getProjectCommandDir(): string | null {
-    return '.claude/commands';
-  }
-
-  getUserCommandDir(): string | null {
-    // Claude supports user-level commands at ~/.claude/commands
-    return path.join(os.homedir(), '.claude', 'commands');
-  }
-
-  getCommandExtension(): string {
-    return '.md';
-  }
-
-  canUseSymlink(): boolean {
-    // Claude uses the same Markdown format as our canonical file
-    return true;
-  }
-
-  transformCommand(markdownContent: string): string {
-    // Claude uses the same Markdown format, no transformation needed
-    return markdownContent;
-  }
-
-  async execute(opts: { prompt: string; diff: string; model?: string; timeoutMs?: number }): Promise<string> {
-    const fullContent = opts.prompt + "\n\n--- DIFF ---\n" + opts.diff;
-
-    const tmpDir = os.tmpdir();
-    const tmpFile = path.join(tmpDir, `gauntlet-claude-${Date.now()}.txt`);
-    await fs.writeFile(tmpFile, fullContent);
-
-    try {
-      // Recommended invocation per spec:
-      // -p: non-interactive print mode
-      // --allowedTools: explicitly restricts to read-only tools
-      // --max-turns: caps agentic turns
-      const cmd = `cat "${tmpFile}" | claude -p --allowedTools "Read,Glob,Grep" --max-turns 10`;
-      const { stdout } = await execAsync(cmd, { timeout: opts.timeoutMs, maxBuffer: MAX_BUFFER_BYTES });
-      return stdout;
-    } finally {
-      await fs.unlink(tmpFile).catch(() => {});
-    }
-  }
+	name = "claude";
+
+	async isAvailable(): Promise<boolean> {
+		try {
+			await execAsync("which claude");
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	async checkHealth(options?: { checkUsageLimit?: boolean }): Promise<{
+		available: boolean;
+		status: "healthy" | "missing" | "unhealthy";
+		message?: string;
+	}> {
+		const available = await this.isAvailable();
+		if (!available) {
+			return {
+				available: false,
+				status: "missing",
+				message: "Command not found",
+			};
+		}
+
+		if (options?.checkUsageLimit) {
+			try {
+				// Try a lightweight command to check if we're rate limited
+				// We use a simple "hello" prompt to avoid "No messages returned" errors from empty input
+				const { stdout, stderr } = await execAsync(
+					'echo "hello" | claude -p --max-turns 1',
+					{ timeout: 10000 },
+				);
+
+				const combined = (stdout || "") + (stderr || "");
+				if (isUsageLimit(combined)) {
+					return {
+						available: true,
+						status: "unhealthy",
+						message: "Usage limit exceeded",
+					};
+				}
+
+				return { available: true, status: "healthy", message: "Ready" };
+			} catch (error: unknown) {
+				const execError = error as {
+					stderr?: string;
+					stdout?: string;
+					message?: string;
+				};
+				const stderr = execError.stderr || "";
+				const stdout = execError.stdout || "";
+				const combined = stderr + stdout;
+
+				if (isUsageLimit(combined)) {
+					return {
+						available: true,
+						status: "unhealthy",
+						message: "Usage limit exceeded",
+					};
+				}
+
+				// Since we sent a valid prompt ("hello"), any other error implies the tool is broken
+				// Extract a brief error message if possible
+				const cleanError =
+					combined.split("\n")[0]?.trim() ||
+					execError.message ||
+					"Command failed";
+				return {
+					available: true,
+					status: "unhealthy",
+					message: `Error: ${cleanError}`,
+				};
+			}
+		}
+
+		return { available: true, status: "healthy", message: "Ready" };
+	}
+
+	getProjectCommandDir(): string | null {
+		return ".claude/commands";
+	}
+
+	getUserCommandDir(): string | null {
+		// Claude supports user-level commands at ~/.claude/commands
+		return path.join(os.homedir(), ".claude", "commands");
+	}
+
+	getCommandExtension(): string {
+		return ".md";
+	}
+
+	canUseSymlink(): boolean {
+		// Claude uses the same Markdown format as our canonical file
+		return true;
+	}
+
+	transformCommand(markdownContent: string): string {
+		// Claude uses the same Markdown format, no transformation needed
+		return markdownContent;
+	}
+
+	async execute(opts: {
+		prompt: string;
+		diff: string;
+		model?: string;
+		timeoutMs?: number;
+	}): Promise<string> {
+		const fullContent = `${opts.prompt}\n\n--- DIFF ---\n${opts.diff}`;
+
+		const tmpDir = os.tmpdir();
+		// Include process.pid for uniqueness across concurrent processes
+		const tmpFile = path.join(
+			tmpDir,
+			`gauntlet-claude-${process.pid}-${Date.now()}.txt`,
+		);
+		await fs.writeFile(tmpFile, fullContent);
+
+		try {
+			// Recommended invocation per spec:
+			// -p: non-interactive print mode
+			// --allowedTools: explicitly restricts to read-only tools
+			// --max-turns: caps agentic turns
+			const cmd = `cat "${tmpFile}" | claude -p --allowedTools "Read,Glob,Grep" --max-turns 10`;
+			const { stdout } = await execAsync(cmd, {
+				timeout: opts.timeoutMs,
+				maxBuffer: MAX_BUFFER_BYTES,
+			});
+			return stdout;
+		} finally {
+			await fs.unlink(tmpFile).catch(() => {});
+		}
+	}
 }

package/src/cli-adapters/codex.ts

@@ -1,123 +1,147 @@
-import { exec } from 'node:child_process';
-import { promisify } from 'node:util';
-import { type CLIAdapter, isUsageLimit } from './index.js';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import os from 'node:os';
+import { exec } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { type CLIAdapter, isUsageLimit } from "./index.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
 export class CodexAdapter implements CLIAdapter {
-  name = 'codex';
-
-  async isAvailable(): Promise<boolean> {
-    try {
-      await execAsync('which codex');
-      return true;
-    } catch {
-      return false;
-    }
-  }
-
-  async checkHealth(options?: { checkUsageLimit?: boolean }): Promise<{ available: boolean; status: 'healthy' | 'missing' | 'unhealthy'; message?: string }> {
-    const available = await this.isAvailable();
-    if (!available) {
-      return { available: false, status: 'missing', message: 'Command not found' };
-    }
-
-    if (options?.checkUsageLimit) {
-      try {
-        const repoRoot = process.cwd();
-        // Try a lightweight command to check if we're rate limited
-        const cmd = `echo "hello" | codex exec --cd "${repoRoot}" --sandbox read-only -c 'ask_for_approval="never"' -`;
-        const { stdout, stderr } = await execAsync(cmd, { timeout: 10000 });
-        
-        const combined = (stdout || '') + (stderr || '');
-        if (isUsageLimit(combined)) {
-           return { 
-            available: true, 
-            status: 'unhealthy', 
-            message: 'Usage limit exceeded' 
-          };
-        }
-  
-        return { available: true, status: 'healthy', message: 'Installed' };
-      } catch (error: any) {
-        const stderr = error.stderr || '';
-        const stdout = error.stdout || '';
-        const combined = (stderr + stdout);
-        
-        if (isUsageLimit(combined)) {
-          return { 
-            available: true, 
-            status: 'unhealthy', 
-            message: 'Usage limit exceeded' 
-          };
-        }
-        
-        const cleanError = combined.split('\n')[0]?.trim() || error.message || 'Command failed';
-        return { 
-          available: true, 
-          status: 'unhealthy', 
-          message: `Error: ${cleanError}` 
-        };
-      }
-    }
-
-    return {
-      available,
-      status: available ? 'healthy' : 'missing',
-      message: available ? 'Installed' : 'Command not found'
-    };
-  }
-
-  getProjectCommandDir(): string | null {
-    // Codex only supports user-level prompts at ~/.codex/prompts/
-    // No project-scoped commands available
-    return null;
-  }
-
-  getUserCommandDir(): string | null {
-    // Codex uses user-level prompts at ~/.codex/prompts/
-    return path.join(os.homedir(), '.codex', 'prompts');
-  }
-
-  getCommandExtension(): string {
-    return '.md';
-  }
-
-  canUseSymlink(): boolean {
-    // Codex uses the same Markdown format as our canonical file
-    return true;
-  }
-
-  transformCommand(markdownContent: string): string {
-    // Codex uses the same Markdown format as Claude, no transformation needed
-    return markdownContent;
-  }
-
-  async execute(opts: { prompt: string; diff: string; model?: string; timeoutMs?: number }): Promise<string> {
-    const fullContent = opts.prompt + "\n\n--- DIFF ---\n" + opts.diff;
-
-    const tmpDir = os.tmpdir();
-    const tmpFile = path.join(tmpDir, `gauntlet-codex-${Date.now()}.txt`);
-    await fs.writeFile(tmpFile, fullContent);
-    
-    // Get absolute path to repo root (CWD)
-    const repoRoot = process.cwd();
-
-    try {
-      // Recommended invocation per spec:
-      // --cd: sets working directory to repo root
-      // --sandbox read-only: prevents file modifications
-      // -c ask_for_approval="never": prevents blocking on prompts
-      // -: reads prompt from stdin
-      const cmd = `cat "${tmpFile}" | codex exec --cd "${repoRoot}" --sandbox read-only -c 'ask_for_approval="never"' -`;
-      const { stdout } = await execAsync(cmd, { timeout: opts.timeoutMs, maxBuffer: MAX_BUFFER_BYTES });
-      return stdout;
-    } finally {
-      await fs.unlink(tmpFile).catch(() => {});
-    }
-  }
+	name = "codex";
+
+	async isAvailable(): Promise<boolean> {
+		try {
+			await execAsync("which codex");
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	async checkHealth(options?: { checkUsageLimit?: boolean }): Promise<{
+		available: boolean;
+		status: "healthy" | "missing" | "unhealthy";
+		message?: string;
+	}> {
+		const available = await this.isAvailable();
+		if (!available) {
+			return {
+				available: false,
+				status: "missing",
+				message: "Command not found",
+			};
+		}
+
+		if (options?.checkUsageLimit) {
+			try {
+				const repoRoot = process.cwd();
+				// Try a lightweight command to check if we're rate limited
+				const cmd = `echo "hello" | codex exec --cd "${repoRoot}" --sandbox read-only -c 'ask_for_approval="never"' -`;
+				const { stdout, stderr } = await execAsync(cmd, { timeout: 10000 });
+
+				const combined = (stdout || "") + (stderr || "");
+				if (isUsageLimit(combined)) {
+					return {
+						available: true,
+						status: "unhealthy",
+						message: "Usage limit exceeded",
+					};
+				}
+
+				return { available: true, status: "healthy", message: "Installed" };
+			} catch (error: unknown) {
+				const execError = error as {
+					stderr?: string;
+					stdout?: string;
+					message?: string;
+				};
+				const stderr = execError.stderr || "";
+				const stdout = execError.stdout || "";
+				const combined = stderr + stdout;
+
+				if (isUsageLimit(combined)) {
+					return {
+						available: true,
+						status: "unhealthy",
+						message: "Usage limit exceeded",
+					};
+				}
+
+				const cleanError =
+					combined.split("\n")[0]?.trim() ||
+					execError.message ||
+					"Command failed";
+				return {
+					available: true,
+					status: "unhealthy",
+					message: `Error: ${cleanError}`,
+				};
+			}
+		}
+
+		return {
+			available,
+			status: available ? "healthy" : "missing",
+			message: available ? "Installed" : "Command not found",
+		};
+	}
+
+	getProjectCommandDir(): string | null {
+		// Codex only supports user-level prompts at ~/.codex/prompts/
+		// No project-scoped commands available
+		return null;
+	}
+
+	getUserCommandDir(): string | null {
+		// Codex uses user-level prompts at ~/.codex/prompts/
+		return path.join(os.homedir(), ".codex", "prompts");
+	}
+
+	getCommandExtension(): string {
+		return ".md";
+	}
+
+	canUseSymlink(): boolean {
+		// Codex uses the same Markdown format as our canonical file
+		return true;
+	}
+
+	transformCommand(markdownContent: string): string {
+		// Codex uses the same Markdown format as Claude, no transformation needed
+		return markdownContent;
+	}
+
+	async execute(opts: {
+		prompt: string;
+		diff: string;
+		model?: string;
+		timeoutMs?: number;
+	}): Promise<string> {
+		const fullContent = `${opts.prompt}\n\n--- DIFF ---\n${opts.diff}`;
+
+		const tmpDir = os.tmpdir();
+		const tmpFile = path.join(tmpDir, `gauntlet-codex-${Date.now()}.txt`);
+		await fs.writeFile(tmpFile, fullContent);
+
+		// Get absolute path to repo root (CWD)
+		const repoRoot = process.cwd();
+
+		try {
+			// Recommended invocation per spec:
+			// --cd: sets working directory to repo root
+			// --sandbox read-only: prevents file modifications
+			// -c ask_for_approval="never": prevents blocking on prompts
+			// -: reads prompt from stdin
+			const cmd = `cat "${tmpFile}" | codex exec --cd "${repoRoot}" --sandbox read-only -c 'ask_for_approval="never"' -`;
+			const { stdout } = await execAsync(cmd, {
+				timeout: opts.timeoutMs,
+				maxBuffer: MAX_BUFFER_BYTES,
+			});
+			return stdout;
+		} finally {
+			await fs.unlink(tmpFile).catch(() => {});
+		}
+	}
 }