agent-gauntlet 0.8.0 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-gauntlet",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "A CLI tool for testing AI coding agents",
   "license": "Apache-2.0",
   "author": "Paul Caplan",
@@ -35,10 +35,10 @@
     "build": "bun build --compile --minify --sourcemap ./src/index.ts --outfile bin/agent-gauntlet",
     "test": "bun test",
     "lint": "biome check src",
+    "typecheck": "tsc --noEmit && tsc --noEmit -p test/tsconfig.json",
     "changeset": "changeset",
     "version": "changeset version",
-    "release": "npm publish",
-    "gen-changeset": "bun scripts/gen-changeset.ts"
+    "release": "changeset publish"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.11",
@@ -57,4 +57,4 @@
     "yaml": "^2.8.2",
     "zod": "^4.3.5"
   }
-}
+}

package/src/built-in-reviews/code-quality.md

@@ -0,0 +1,25 @@
+# Code Quality Review
+
+You are a senior software engineer performing a code review. Your primary goal is to identify **real problems** that could cause bugs, security vulnerabilities, or performance issues in production. Do not report style, formatting, naming conventions, or maintainability suggestions unless you see something egregious.
+
+## Focus Areas (in priority order)
+
+1. **Bugs** — Logic errors, null/undefined issues, race conditions, unhandled edge cases, resource leaks
+2. **Security** — Injection vulnerabilities, auth/authz flaws, sensitive data exposure, input validation gaps
+3. **Performance** — Algorithmic complexity issues, N+1 queries, blocking operations, memory problems
+4. **Maintainability** — Unclear code, missing error handling, duplication
+
+## Do NOT Report
+
+- Style, formatting, or naming preferences
+- Missing documentation, comments, or type annotations
+- Suggestions for "better" abstractions or patterns that aren't broken
+- Hypothetical issues that require unlikely preconditions
+- Issues in code that wasn't changed in this diff
+
+## Guidelines
+
+- **Threshold**: only report issues you would block a PR over
+- Explain **why** each issue is a problem with a concrete failure scenario
+- Provide a **concrete fix** with corrected code
+- If the status quo works correctly, it's not a violation

package/src/built-in-reviews/index.ts

@@ -0,0 +1,28 @@
+// @ts-expect-error Bun text import
+import codeQualityContent from "./code-quality.md" with { type: "text" };
+
+const BUILT_IN_PREFIX = "built-in:";
+
+const builtInSources: Record<string, string> = {
+	"code-quality": codeQualityContent,
+};
+
+/**
+ * Check if a review name uses the built-in prefix.
+ */
+export function isBuiltInReview(name: string): boolean {
+	return name.startsWith(BUILT_IN_PREFIX);
+}
+
+/**
+ * Load a built-in review prompt by name. Returns the raw markdown content.
+ */
+export function loadBuiltInReview(name: string): string {
+	const source = builtInSources[name];
+
+	if (!source) {
+		throw new Error(`Unknown built-in review: "${name}"`);
+	}
+
+	return source;
+}

package/src/cli-adapters/claude.ts

@@ -1,14 +1,197 @@
-import { exec, spawn } from "node:child_process";
+import { exec } from "node:child_process";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
 import { GAUNTLET_STOP_HOOK_ACTIVE_ENV } from "../commands/stop-hook.js";
-import type { CLIAdapter } from "./index.js";
+import { getDebugLogger } from "../utils/debug-log.js";
+import { type CLIAdapter, runStreamingCommand } from "./index.js";
+import { CLAUDE_THINKING_TOKENS } from "./thinking-budget.js";
 
 const execAsync = promisify(exec);
 const MAX_BUFFER_BYTES = 10 * 1024 * 1024;
 
+// Matches OTel console exporter metric blocks dumped to stdout at process exit.
+// Requires `descriptor`, `dataPointType`, and `dataPoints` fields which are
+// unique to OTel SDK output and won't appear in normal code review content.
+// Optionally matches [otel] prefix that some exporters add.
+const OTEL_METRIC_BLOCK_RE =
+	/(?:\[otel\]\s*)?\{\s*\n\s*descriptor:\s*\{[\s\S]*?dataPointType:\s*\d+[\s\S]*?dataPoints:\s*\[[\s\S]*?\]\s*,?\s*\n\}/g;
+
+interface OtelUsage {
+	cost?: number;
+	input?: number;
+	output?: number;
+	cacheRead?: number;
+	cacheCreation?: number;
+	toolCalls?: number;
+	toolContentBytes?: number;
+	apiRequests?: number;
+}
+
+const TOKEN_TYPES = ["input", "output", "cacheRead", "cacheCreation"] as const;
+
+function parseCostBlock(block: string): number | undefined {
+	const match = block.match(/value:\s*([\d.]+)/);
+	return match ? Number.parseFloat(match[1]!) : undefined;
+}
+
+function parseTokenBlock(block: string): Partial<OtelUsage> {
+	const result: Partial<OtelUsage> = {};
+	const re = /type:\s*"(\w+)"[\s\S]*?value:\s*(\d+)(?:,|\s*\})/g;
+	for (const match of block.matchAll(re)) {
+		const type = match[1]! as (typeof TOKEN_TYPES)[number];
+		if (TOKEN_TYPES.includes(type)) {
+			result[type] = Number.parseInt(match[2]!, 10);
+		}
+	}
+	return result;
+}
+
+function parseOtelMetrics(blocks: string[]): OtelUsage {
+	const usage: OtelUsage = {};
+	for (const block of blocks) {
+		const nameMatch = block.match(/name:\s*"([^"]+)"/);
+		if (!nameMatch) continue;
+
+		if (nameMatch[1] === "claude_code.cost.usage") {
+			usage.cost = parseCostBlock(block);
+		} else if (nameMatch[1] === "claude_code.token.usage") {
+			Object.assign(usage, parseTokenBlock(block));
+		}
+	}
+	return usage;
+}
+
+// Matches OTel console log exporter event records emitted by Claude Code.
+// The Node.js SDK console exporter uses util.inspect() format with unquoted keys
+// and single-quoted strings. Blocks start with `resource:` and contain a `body:`
+// field with the event name (e.g. 'claude_code.tool_result').
+const OTEL_LOG_BLOCK_RE =
+	/\{\s*\n\s*resource:\s*\{[\s\S]*?body:\s*'claude_code\.\w+'[\s\S]*?\n\}/g;
+
+/** Pre-compiled regexes for extracting single-quoted attribute values from OTel log blocks. */
+const OTEL_ATTR_RE = {
+	body: /body:\s*'([^']*)'/,
+	tool_result_size_bytes: /tool_result_size_bytes:\s*'([^']*)'/,
+	input_tokens: /input_tokens:\s*'([^']*)'/,
+	output_tokens: /output_tokens:\s*'([^']*)'/,
+	cache_read_tokens: /cache_read_tokens:\s*'([^']*)'/,
+	cache_creation_tokens: /cache_creation_tokens:\s*'([^']*)'/,
+	cost_usd: /cost_usd:\s*'([^']*)'/,
+} as const;
+
+/** Maps OTel api_request attribute regexes to OtelUsage fields. */
+const API_REQUEST_FIELDS: Array<[RegExp, keyof OtelUsage]> = [
+	[OTEL_ATTR_RE.input_tokens, "input"],
+	[OTEL_ATTR_RE.output_tokens, "output"],
+	[OTEL_ATTR_RE.cache_read_tokens, "cacheRead"],
+	[OTEL_ATTR_RE.cache_creation_tokens, "cacheCreation"],
+	[OTEL_ATTR_RE.cost_usd, "cost"],
+];
+
+/** Accumulate a tool_result log block into usage. */
+function accumulateToolResult(block: string, usage: OtelUsage): void {
+	usage.toolCalls = (usage.toolCalls || 0) + 1;
+	const bytes = block.match(OTEL_ATTR_RE.tool_result_size_bytes)?.[1];
+	if (bytes !== undefined) {
+		usage.toolContentBytes = (usage.toolContentBytes || 0) + Number(bytes);
+	}
+}
+
+/** Accumulate an api_request log block into usage. */
+function accumulateApiRequest(block: string, usage: OtelUsage): void {
+	usage.apiRequests = (usage.apiRequests || 0) + 1;
+	for (const [re, field] of API_REQUEST_FIELDS) {
+		const val = block.match(re)?.[1];
+		if (val !== undefined) {
+			usage[field] = (usage[field] || 0) + Number(val);
+		}
+	}
+}
+
+/** Accumulate tool_result and api_request event data from OTel log blocks. */
+function parseOtelLogEvents(raw: string, usage: OtelUsage): void {
+	const blocks = raw.match(OTEL_LOG_BLOCK_RE);
+	if (!blocks) return;
+	for (const block of blocks) {
+		const body = block.match(OTEL_ATTR_RE.body)?.[1];
+		if (body === "claude_code.tool_result") {
+			accumulateToolResult(block, usage);
+		} else if (body === "claude_code.api_request") {
+			accumulateApiRequest(block, usage);
+		}
+	}
+}
+
+const OTEL_SUMMARY_FIELDS: Array<[keyof OtelUsage, string]> = [
+	["input", "in"],
+	["output", "out"],
+	["cacheRead", "cacheRead"],
+	["cacheCreation", "cacheWrite"],
+	["toolCalls", "tool_calls"],
+	["toolContentBytes", "tool_content_bytes"],
+	["apiRequests", "api_requests"],
+];
+
+function formatOtelSummary(usage: OtelUsage): string | null {
+	if (usage.cost === undefined && usage.input === undefined) return null;
+
+	const parts: string[] = [];
+	if (usage.cost !== undefined) parts.push(`cost=$${usage.cost.toFixed(4)}`);
+	for (const [key, label] of OTEL_SUMMARY_FIELDS) {
+		if (usage[key] !== undefined) parts.push(`${label}=${usage[key]}`);
+	}
+
+	return `[otel] ${parts.join(" ")}`;
+}
+
+function extractOtelMetrics(
+	raw: string,
+	onLog?: (msg: string) => void,
+): string {
+	const metricBlocks = raw.match(OTEL_METRIC_BLOCK_RE);
+	const usage = metricBlocks ? parseOtelMetrics(metricBlocks) : {};
+
+	// Also parse log events for tool call and API request counts
+	parseOtelLogEvents(raw, usage);
+
+	const summary = formatOtelSummary(usage);
+	if (summary) {
+		onLog?.(`\n${summary}\n`);
+		process.stderr.write(`${summary}\n`);
+		getDebugLogger()?.logTelemetry({ adapter: "claude", summary });
+	}
+
+	return raw
+		.replace(OTEL_METRIC_BLOCK_RE, "")
+		.replace(OTEL_LOG_BLOCK_RE, "")
+		.trimEnd();
+}
+
+/** Build OTel environment overrides for console export. */
+function buildOtelEnv(): Record<string, string> {
+	const env: Record<string, string> = {};
+	if (!process.env.CLAUDE_CODE_ENABLE_TELEMETRY) {
+		env.CLAUDE_CODE_ENABLE_TELEMETRY = "1";
+	}
+	if (!process.env.OTEL_METRICS_EXPORTER) {
+		env.OTEL_METRICS_EXPORTER = "console";
+	}
+	if (!process.env.OTEL_LOGS_EXPORTER) {
+		env.OTEL_LOGS_EXPORTER = "console";
+	}
+	return env;
+}
+
+/** Strip OTel metric and log blocks from raw output. */
+function stripOtelBlocks(raw: string): string {
+	return raw
+		.replace(OTEL_METRIC_BLOCK_RE, "")
+		.replace(OTEL_LOG_BLOCK_RE, "")
+		.trimEnd();
+}
+
 export class ClaudeAdapter implements CLIAdapter {
 	name = "claude";
 
@@ -43,21 +226,26 @@ export class ClaudeAdapter implements CLIAdapter {
 	}
 
 	getUserCommandDir(): string | null {
-		// Claude supports user-level commands at ~/.claude/commands
 		return path.join(os.homedir(), ".claude", "commands");
 	}
 
+	getProjectSkillDir(): string | null {
+		return ".claude/skills";
+	}
+
+	getUserSkillDir(): string | null {
+		return path.join(os.homedir(), ".claude", "skills");
+	}
+
 	getCommandExtension(): string {
 		return ".md";
 	}
 
 	canUseSymlink(): boolean {
-		// Claude uses the same Markdown format as our canonical file
 		return true;
 	}
 
 	transformCommand(markdownContent: string): string {
-		// Claude uses the same Markdown format, no transformation needed
 		return markdownContent;
 	}
 
@@ -67,107 +255,71 @@ export class ClaudeAdapter implements CLIAdapter {
 		model?: string;
 		timeoutMs?: number;
 		onOutput?: (chunk: string) => void;
+		allowToolUse?: boolean;
+		thinkingBudget?: string;
 	}): Promise<string> {
 		const fullContent = `${opts.prompt}\n\n--- DIFF ---\n${opts.diff}`;
 
 		const tmpDir = os.tmpdir();
-		// Include process.pid for uniqueness across concurrent processes
 		const tmpFile = path.join(
 			tmpDir,
 			`gauntlet-claude-${process.pid}-${Date.now()}.txt`,
 		);
 		await fs.writeFile(tmpFile, fullContent);
 
-		// Recommended invocation per spec:
-		// -p: non-interactive print mode
-		// --allowedTools: explicitly restricts to read-only tools
-		// --max-turns: caps agentic turns
-		const args = [
-			"-p",
-			"--allowedTools",
-			"Read,Glob,Grep",
-			"--max-turns",
-			"10",
-		];
+		const args = ["-p"];
+		if (opts.allowToolUse === false) {
+			args.push("--tools", "");
+		} else {
+			args.push("--allowedTools", "Read,Glob,Grep");
+		}
+		args.push("--max-turns", "10");
+
+		const otelEnv = buildOtelEnv();
+		const thinkingEnv: Record<string, string> = {};
+		if (opts.thinkingBudget && opts.thinkingBudget in CLAUDE_THINKING_TOKENS) {
+			thinkingEnv.MAX_THINKING_TOKENS = String(
+				CLAUDE_THINKING_TOKENS[opts.thinkingBudget],
+			);
+		}
 
 		const cleanup = () => fs.unlink(tmpFile).catch(() => {});
+		const execEnv = {
+			...process.env,
+			[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
+			...otelEnv,
+			...thinkingEnv,
+		};
 
-		// If onOutput callback is provided, use spawn for real-time streaming
 		if (opts.onOutput) {
-			return new Promise((resolve, reject) => {
-				const chunks: string[] = [];
-				const inputStream = fs.open(tmpFile, "r").then((handle) => {
-					const stream = handle.createReadStream();
-					return { stream, handle };
-				});
-
-				inputStream
-					.then(({ stream, handle }) => {
-						const child = spawn("claude", args, {
-							stdio: ["pipe", "pipe", "pipe"],
-							env: {
-								...process.env,
-								[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
-							},
-						});
-
-						stream.pipe(child.stdin);
-
-						let timeoutId: ReturnType<typeof setTimeout> | undefined;
-						if (opts.timeoutMs) {
-							timeoutId = setTimeout(() => {
-								child.kill("SIGTERM");
-								reject(new Error("Command timed out"));
-							}, opts.timeoutMs);
-						}
-
-						child.stdout.on("data", (data: Buffer) => {
-							const chunk = data.toString();
-							chunks.push(chunk);
-							opts.onOutput?.(chunk);
-						});
-
-						child.stderr.on("data", (data: Buffer) => {
-							// Only log stderr, don't include in return value
-							opts.onOutput?.(data.toString());
-						});
-
-						child.on("close", (code) => {
-							if (timeoutId) clearTimeout(timeoutId);
-							handle.close().catch(() => {});
-							cleanup().then(() => {
-								if (code === 0 || code === null) {
-									resolve(chunks.join(""));
-								} else {
-									reject(new Error(`Process exited with code ${code}`));
-								}
-							});
-						});
-
-						child.on("error", (err) => {
-							if (timeoutId) clearTimeout(timeoutId);
-							handle.close().catch(() => {});
-							cleanup().then(() => reject(err));
-						});
-					})
-					.catch((err) => {
-						cleanup().then(() => reject(err));
-					});
+			const outputBuffer: string[] = [];
+			const raw = await runStreamingCommand({
+				command: "claude",
+				args,
+				tmpFile,
+				timeoutMs: opts.timeoutMs,
+				onOutput: (chunk: string) => {
+					outputBuffer.push(chunk);
+				},
+				cleanup,
+				env: execEnv,
 			});
+			const cleanedOutput = extractOtelMetrics(
+				outputBuffer.join(""),
+				opts.onOutput,
+			);
+			opts.onOutput(cleanedOutput);
+			return stripOtelBlocks(raw);
 		}
 
-		// Otherwise use exec for buffered output
 		try {
-			const cmd = `cat "${tmpFile}" | claude -p --allowedTools "Read,Glob,Grep" --max-turns 10`;
+			const cmd = `cat "${tmpFile}" | claude ${args.map((a) => (a === "" ? '""' : a)).join(" ")}`;
 			const { stdout } = await execAsync(cmd, {
 				timeout: opts.timeoutMs,
 				maxBuffer: MAX_BUFFER_BYTES,
-				env: {
-					...process.env,
-					[GAUNTLET_STOP_HOOK_ACTIVE_ENV]: "1",
-				},
+				env: execEnv,
 			});
-			return stdout;
+			return extractOtelMetrics(stdout);
 		} finally {
 			await cleanup();
 		}